Files
pages/internal/store/feeds.go
T
Anya Tokugawa bf9a9c2847
Build and publish container images / images (push) Successful in 10m33s
feat(feeds): inherit FreshRSS folders in cached subscription tree
2026-11-08 13:34:22 +09:00

570 lines
28 KiB
Go

package store
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"unicode/utf8"
"github.com/google/uuid"
"pageswiki/internal/db"
)
const FeedSyncEventType = "feed.source.sync"
var feedMigrations = []db.SQLMigration{{Version: 1, Name: "private_feed_reader", Statements: []string{
`CREATE TABLE feed_sources(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,provider TEXT NOT NULL CHECK(provider IN ('local','freshrss')),title TEXT NOT NULL,folder TEXT NOT NULL,url TEXT NOT NULL,credential TEXT NOT NULL DEFAULT '',version INTEGER NOT NULL,enabled INTEGER NOT NULL,interval_seconds INTEGER NOT NULL,next_ms INTEGER NOT NULL DEFAULT 0,lease TEXT NOT NULL DEFAULT '',lease_until INTEGER NOT NULL DEFAULT 0,last_ms INTEGER NOT NULL DEFAULT 0,error TEXT NOT NULL DEFAULT '',etag TEXT NOT NULL DEFAULT '',modified TEXT NOT NULL DEFAULT '')`,
`CREATE INDEX feed_sources_owner ON feed_sources(domain_uuid,owner_uuid,uuid)`,
`CREATE INDEX feed_sources_due ON feed_sources(enabled,next_ms,uuid)`,
`CREATE TABLE feed_items(source_uuid TEXT NOT NULL REFERENCES feed_sources(uuid) ON DELETE CASCADE,id TEXT NOT NULL,title TEXT NOT NULL,url TEXT NOT NULL,body TEXT NOT NULL,published_ms INTEGER NOT NULL,feed TEXT NOT NULL,folder TEXT NOT NULL,is_read INTEGER NOT NULL,starred INTEGER NOT NULL,PRIMARY KEY(source_uuid,id))`,
`CREATE INDEX feed_items_date ON feed_items(source_uuid,published_ms DESC,id DESC)`,
`CREATE TABLE feed_states(source_uuid TEXT NOT NULL REFERENCES feed_sources(uuid) ON DELETE CASCADE,id TEXT NOT NULL,is_read INTEGER NOT NULL,starred INTEGER NOT NULL,version INTEGER NOT NULL,PRIMARY KEY(source_uuid,id))`,
`CREATE TRIGGER feed_source_delete AFTER DELETE ON feed_sources BEGIN DELETE FROM feed_items WHERE source_uuid=old.uuid; DELETE FROM feed_states WHERE source_uuid=old.uuid; END`,
`CREATE TABLE feed_note_receipts(domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,request_id TEXT NOT NULL,input_hash TEXT NOT NULL,page_uuid TEXT NOT NULL,PRIMARY KEY(domain_uuid,owner_uuid,request_id))`,
}, Assertions: []string{schemaColumns("feed_sources", "uuid,domain_uuid,owner_uuid,provider,title,folder,url,credential,version,enabled,interval_seconds,next_ms,lease,lease_until,last_ms,error,etag,modified"), schemaColumns("feed_items", "source_uuid,id,title,url,body,published_ms,feed,folder,is_read,starred"), schemaUnique("feed_items", "source_uuid,id"), schemaUnique("feed_states", "source_uuid,id"), schemaUnique("feed_note_receipts", "domain_uuid,owner_uuid,request_id"), `SELECT 1 WHERE NOT EXISTS(SELECT 1 FROM sqlite_master WHERE type='trigger' AND name='feed_source_delete')`}}, {
Version: 2, Name: "feed_html_body", Statements: []string{
`ALTER TABLE feed_items ADD COLUMN body_html TEXT NOT NULL DEFAULT ''`,
// Text-only caches cannot recover markup from a 304. Request one full
// background fetch while fencing workers started before this migration.
`UPDATE feed_sources SET etag='',modified='',next_ms=0,lease='',lease_until=0,version=version+1`,
}, Assertions: []string{schemaColumns("feed_items", "body_html")},
}, {
Version: 3, Name: "feed_import_receipts", Statements: []string{
`CREATE TABLE feed_import_receipts(domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,request_id TEXT NOT NULL,input_hash TEXT NOT NULL,result TEXT NOT NULL,PRIMARY KEY(domain_uuid,owner_uuid,request_id))`,
}, Assertions: []string{schemaUnique("feed_import_receipts", "domain_uuid,owner_uuid,request_id")},
}, {
Version: 4, Name: "freshrss_catalog", Statements: []string{
`ALTER TABLE feed_sources ADD COLUMN remote_catalog TEXT NOT NULL DEFAULT '[]'`,
`ALTER TABLE feed_items ADD COLUMN remote_feed TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE feed_items ADD COLUMN remote_categories TEXT NOT NULL DEFAULT '[]'`,
`UPDATE feed_sources SET next_ms=0,lease='',lease_until=0,version=version+1 WHERE provider='freshrss'`,
}, Assertions: []string{schemaColumns("feed_sources", "remote_catalog"), schemaColumns("feed_items", "remote_feed,remote_categories")},
}, {
Version: 5, Name: "feed_explicit_remote_actions", Statements: []string{
`ALTER TABLE feed_sources ADD COLUMN write_enabled INTEGER NOT NULL DEFAULT 0`,
`CREATE TABLE feed_actions(domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,request_id TEXT NOT NULL,source_uuid TEXT NOT NULL,item_id TEXT NOT NULL,field TEXT NOT NULL,desired INTEGER NOT NULL,source_version INTEGER NOT NULL,status TEXT NOT NULL,claim TEXT NOT NULL DEFAULT '',until_ms INTEGER NOT NULL DEFAULT 0,created_ms INTEGER NOT NULL,error TEXT NOT NULL DEFAULT '',PRIMARY KEY(domain_uuid,owner_uuid,request_id))`,
`CREATE INDEX feed_actions_due ON feed_actions(status,created_ms)`,
`CREATE INDEX feed_actions_item ON feed_actions(domain_uuid,owner_uuid,source_uuid,item_id,created_ms DESC)`,
`CREATE UNIQUE INDEX feed_actions_active ON feed_actions(source_uuid,item_id) WHERE status IN ('queued','preparing','sending','unknown','check_queued','checking')`,
`CREATE TRIGGER feed_actions_source_change AFTER UPDATE OF version ON feed_sources BEGIN UPDATE feed_actions SET status=CASE WHEN status IN ('sending','unknown','check_queued','checking') THEN 'unknown' ELSE 'cancelled' END,error='source_changed' WHERE source_uuid=old.uuid AND status IN ('queued','preparing','sending','unknown','check_queued','checking'); END`,
`CREATE TRIGGER feed_actions_source_delete AFTER DELETE ON feed_sources BEGIN UPDATE feed_actions SET status=CASE WHEN status IN ('sending','unknown','check_queued','checking') THEN 'unknown' ELSE 'cancelled' END,error='source_deleted' WHERE source_uuid=old.uuid AND status IN ('queued','preparing','sending','unknown','check_queued','checking'); END`,
}, Assertions: []string{schemaColumns("feed_sources", "write_enabled"), schemaUnique("feed_actions", "domain_uuid,owner_uuid,request_id")},
}, {
Version: 6, Name: "freshrss_empty_folders", Statements: []string{
`ALTER TABLE feed_sources ADD COLUMN remote_folders TEXT NOT NULL DEFAULT '[]'`,
}, Assertions: []string{schemaColumns("feed_sources", "remote_folders")},
}}
// CaptureFeedNote creates an inert Wiki note and receipt atomically. It does not
// mutate the feed or create a task, and rechecks grants under the writer lock.
func (s *Store) CaptureFeedNote(ctx context.Context, domain, owner, request, source, id, target, title, body string) (string, error) {
if _, e := uuid.Parse(request); e != nil {
return "", ErrTaskInvalid
}
if !ValidTaskWikiPrefix(target) || target == "/" || len(target) > 500 || len(title) > 300 || title == "" || len(body) > 65536 {
return "", ErrTaskInvalid
}
// User/group spaces have additional ownership/publication contracts. Initial
// capture supports ordinary Wiki paths only, rather than bypassing those caps.
first := strings.Split(strings.TrimPrefix(target, "/"), "/")[0]
if first == "u" || first == "g" {
return "", ErrAccessDenied
}
raw, _ := json.Marshal([]string{source, id, target, title, body})
hash := fmt.Sprintf("%x", sha256.Sum256(raw))
page := ""
e := s.feedWrite(ctx, domain, owner, func(tx *sql.Tx) error {
caps, e := loadLocalityAccess(tx, domain, owner)
if e != nil {
return e
}
for _, capability := range []string{"read", "create", "write"} {
basis := authorizeGrantFrom(tx, AuthorizationRequest{DomainUUID: domain, PrincipalUUID: owner, Capability: capability, TargetType: "path", TargetID: target}, false)
if basis != nil && !errors.Is(basis, ErrAccessDenied) {
return basis
}
if !caps.Decide(target, capability, basis == nil) {
return ErrAccessDenied
}
}
var oldHash string
e = tx.QueryRowContext(ctx, `SELECT input_hash,page_uuid FROM feed_note_receipts WHERE domain_uuid=? AND owner_uuid=? AND request_id=?`, domain, owner, request).Scan(&oldHash, &page)
if e == nil {
if oldHash != hash {
return ErrTaskConflict
}
return nil
}
if !errors.Is(e, sql.ErrNoRows) {
return e
}
var exists bool
if e = tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid WHERE s.domain_uuid=? AND s.owner_uuid=? AND s.uuid=? AND i.id=?)`, domain, owner, source, id).Scan(&exists); e != nil {
return e
}
if !exists {
return sql.ErrNoRows
}
item := TaskRecord{Title: title}
if e = s.createCapturedTaskWiki(ctx, tx, domain, owner, &item, target, body); e != nil {
return e
}
page = item.PageUUID
_, e = tx.ExecContext(ctx, `INSERT INTO feed_note_receipts VALUES(?,?,?,?,?)`, domain, owner, request, hash, page)
return e
})
if e == nil {
s.InvalidatePathCaches()
}
return page, e
}
type FeedSource struct {
UUID string `json:"uuid"`
Domain string `json:"-"`
Owner string `json:"-"`
Provider string `json:"provider"`
Title string `json:"title"`
Folder string `json:"folder"`
URL string `json:"url"`
Credential string `json:"-"`
Version int64 `json:"version"`
Enabled bool `json:"enabled"`
Interval int `json:"interval_seconds"`
NextMS int64 `json:"next_ms"`
Lease string `json:"-"`
LeaseUntil int64 `json:"-"`
LastMS int64 `json:"last_ms"`
Error string `json:"error"`
ETag string `json:"-"`
Modified string `json:"-"`
WriteEnabled bool `json:"write_enabled"`
}
type FeedItem struct {
Source string `json:"source"`
ID string `json:"id"`
Title string `json:"title"`
URL string `json:"url"`
Body string `json:"body,omitempty"`
BodyHTML string `json:"body_html,omitempty"`
PublishedMS int64 `json:"published_ms"`
Feed string `json:"feed"`
Folder string `json:"folder"`
Read bool `json:"read"`
Starred bool `json:"starred"`
Version int64 `json:"version"`
Provider string `json:"provider"`
RemoteFeed string `json:"remote_feed,omitempty"`
RemoteCategories []string `json:"remote_categories,omitempty"`
}
const feedSourceColumns = `uuid,domain_uuid,owner_uuid,provider,title,folder,url,credential,version,enabled,interval_seconds,next_ms,lease,lease_until,last_ms,error,etag,modified,write_enabled`
func scanFeedSource(row interface{ Scan(...any) error }) (v FeedSource, e error) {
e = row.Scan(&v.UUID, &v.Domain, &v.Owner, &v.Provider, &v.Title, &v.Folder, &v.URL, &v.Credential, &v.Version, &v.Enabled, &v.Interval, &v.NextMS, &v.Lease, &v.LeaseUntil, &v.LastMS, &v.Error, &v.ETag, &v.Modified, &v.WriteEnabled)
return
}
func feedActor(ctx context.Context, q interface {
QueryRowContext(context.Context, string, ...any) *sql.Row
}, domain, owner string, write bool) error {
if e := taskActor(ctx, q, domain, owner, write); e != nil {
return e
}
var raw string
e := q.QueryRowContext(ctx, `SELECT value FROM config WHERE domain_uuid=? AND key='system_extension_states'`, domain).Scan(&raw)
if e != nil && !errors.Is(e, sql.ErrNoRows) {
return e
}
states := map[string]bool{}
if raw != "" && json.Unmarshal([]byte(raw), &states) != nil {
return ErrAccessDenied
}
if enabled, ok := states["organizer.feeds"]; ok && !enabled {
return ErrAccessDenied
}
return nil
}
func (s *Store) feedWrite(ctx context.Context, domain, owner string, fn func(*sql.Tx) error) error {
if s.ReadOnly {
return ErrReadOnly
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
tx, e := s.DB.BeginTx(ctx, nil)
if e != nil {
return e
}
defer tx.Rollback()
if _, e = tx.ExecContext(ctx, `UPDATE domain_memberships SET state=state WHERE domain_uuid=? AND principal_uuid=?`, domain, owner); e != nil {
return e
}
if e = feedActor(ctx, tx, domain, owner, true); e != nil {
return e
}
if e = fn(tx); e != nil {
return e
}
return tx.Commit()
}
func (s *Store) FeedSources(ctx context.Context, domain, owner string) ([]FeedSource, error) {
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
return nil, e
}
rows, e := s.DB.QueryContext(ctx, `SELECT `+feedSourceColumns+` FROM feed_sources WHERE domain_uuid=? AND owner_uuid=? ORDER BY folder,title,uuid LIMIT 32`, domain, owner)
if e != nil {
return nil, e
}
defer rows.Close()
out := []FeedSource{}
for rows.Next() {
v, e := scanFeedSource(rows)
if e != nil {
return nil, e
}
out = append(out, v)
}
return out, rows.Err()
}
func (s *Store) SaveFeedSource(ctx context.Context, domain, owner string, v FeedSource) (FeedSource, error) {
if v.WriteEnabled && v.Provider != "freshrss" {
return v, ErrTaskInvalid
}
if _, e := uuid.Parse(v.UUID); e != nil {
return v, ErrTaskInvalid
}
v.Title = strings.TrimSpace(v.Title)
v.Folder = strings.TrimSpace(v.Folder)
if v.Title == "" || len(v.Title) > 200 || len(v.Folder) > 200 || len(v.URL) > 4096 || !utf8.ValidString(v.Title+v.Folder) || v.Interval < 300 || v.Interval > 86400 || (v.Provider != "local" && v.Provider != "freshrss") {
return v, ErrTaskInvalid
}
e := s.feedWrite(ctx, domain, owner, func(tx *sql.Tx) error {
if v.Version == 0 {
old, lookup := scanFeedSource(tx.QueryRowContext(ctx, `SELECT `+feedSourceColumns+` FROM feed_sources WHERE uuid=?`, v.UUID))
if lookup == nil {
if old.Domain != domain || old.Owner != owner || old.Version != 1 || old.Provider != v.Provider || old.Title != v.Title || old.Folder != v.Folder || old.URL != v.URL || old.Interval != v.Interval || old.Enabled != v.Enabled || old.WriteEnabled != v.WriteEnabled {
return ErrTaskConflict
}
if old.Credential != v.Credential {
candidate := v
candidate.Domain = domain
candidate.Owner = owner
u, p, e := s.OpenFeedCredential(candidate)
if e != nil {
return ErrTaskConflict
}
ou, op, e := s.OpenFeedCredential(old)
if e != nil || u != ou || p != op {
return ErrTaskConflict
}
}
return nil
}
if !errors.Is(lookup, sql.ErrNoRows) {
return lookup
}
// Do not rebind retained receipts to a different remote identity.
// Both predicates use existing indexes: owner history and the global
// partial index of unresolved operations. Other owners' terminal
// receipts remain isolated by domain/owner in metadata lookups.
var retired bool
if e := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM feed_actions WHERE domain_uuid=? AND owner_uuid=? AND source_uuid=?) OR EXISTS(SELECT 1 FROM feed_actions WHERE source_uuid=? AND status IN ('queued','preparing','sending','unknown','check_queued','checking'))`, domain, owner, v.UUID, v.UUID).Scan(&retired); e != nil {
return e
}
if retired {
return ErrTaskConflict
}
var n int
if e := tx.QueryRowContext(ctx, `SELECT count(*) FROM feed_sources WHERE domain_uuid=? AND owner_uuid=?`, domain, owner).Scan(&n); e != nil {
return e
}
if n >= 32 {
return ErrTaskDAVLimit
}
_, e := tx.ExecContext(ctx, `INSERT INTO feed_sources(uuid,domain_uuid,owner_uuid,provider,title,folder,url,credential,version,enabled,interval_seconds,write_enabled) VALUES(?,?,?,?,?,?,?,?,1,?,?,?)`, v.UUID, domain, owner, v.Provider, v.Title, v.Folder, v.URL, v.Credential, v.Enabled, v.Interval, v.WriteEnabled)
return e
}
old, e := scanFeedSource(tx.QueryRowContext(ctx, `SELECT `+feedSourceColumns+` FROM feed_sources WHERE uuid=? AND domain_uuid=? AND owner_uuid=?`, v.UUID, domain, owner))
if e != nil {
return e
}
// Changing remote identity requires a new source, not reassignment of cached IDs.
if old.Provider != v.Provider || old.URL != v.URL || v.Credential != "" {
return ErrTaskInvalid
}
result, e := tx.ExecContext(ctx, `UPDATE feed_sources SET title=?,folder=?,enabled=?,interval_seconds=?,write_enabled=?,version=version+1,next_ms=0,lease='',lease_until=0 WHERE uuid=? AND version=?`, v.Title, v.Folder, v.Enabled, v.Interval, v.WriteEnabled, v.UUID, v.Version)
if e != nil {
return e
}
n, _ := result.RowsAffected()
if n != 1 {
return ErrTaskConflict
}
return nil
})
v.Version++
v.Credential = ""
return v, e
}
func (s *Store) FeedSourceAction(ctx context.Context, domain, owner, id, action string, version int64) error {
return s.feedWrite(ctx, domain, owner, func(tx *sql.Tx) error {
var result sql.Result
var e error
switch action {
case "refresh":
result, e = tx.ExecContext(ctx, `UPDATE feed_sources SET next_ms=0 WHERE uuid=? AND domain_uuid=? AND owner_uuid=? AND version=? AND enabled=1 AND lease_until<? AND (last_ms=0 OR last_ms<?)`, id, domain, owner, version, time.Now().UnixMilli(), time.Now().Add(-time.Minute).UnixMilli())
case "delete":
result, e = tx.ExecContext(ctx, `DELETE FROM feed_sources WHERE uuid=? AND domain_uuid=? AND owner_uuid=? AND version=?`, id, domain, owner, version)
default:
return ErrTaskInvalid
}
if e != nil {
return e
}
n, _ := result.RowsAffected()
if n != 1 {
return ErrTaskConflict
}
return nil
})
}
func (s *Store) FeedItems(ctx context.Context, domain, owner, source, filter, query string, before int64, beforeID string, scope ...string) ([]FeedItem, error) {
remoteFeed, category := "", ""
if len(scope) > 3 {
return nil, ErrTaskInvalid
}
if len(scope) > 0 {
remoteFeed = scope[0]
}
if len(scope) > 1 {
category = scope[1]
}
mode := ""
if len(scope) > 2 {
mode = scope[2]
}
if mode != "" && (mode != "uncategorized" || source == "" || category != "") {
return nil, ErrTaskInvalid
}
if (remoteFeed != "" && !ValidFeedRemoteID(remoteFeed)) || (category != "" && !ValidFeedRemoteID(category)) || ((remoteFeed != "" || category != "") && source == "") {
return nil, ErrTaskInvalid
}
if len(query) > 200 || len(beforeID) > 1200 {
return nil, ErrTaskInvalid
}
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
return nil, e
}
args := []any{domain, owner, source, source, query, query, query, before, before, before, beforeID, filter, filter, remoteFeed, remoteFeed, category, category}
extra := ""
if mode == "uncategorized" {
// The uncorrelated list is built once, not a catalog scan per article.
extra = ` AND i.remote_feed IN (SELECT json_extract(c.value,'$.id') FROM feed_sources cs,json_each(cs.remote_catalog) c WHERE cs.uuid=? AND cs.domain_uuid=? AND cs.owner_uuid=? AND json_array_length(c.value,'$.categories')=0)`
args = append(args, source, domain, owner)
}
rows, e := s.DB.QueryContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.published_ms,i.feed,CASE WHEN s.folder<>'' THEN s.folder ELSE i.folder END,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=? AND s.owner_uuid=? AND (?='' OR s.uuid=?) AND (?='' OR instr(lower(i.title),lower(?))>0 OR instr(lower(i.feed),lower(?))>0) AND (?<0 OR i.published_ms<? OR (i.published_ms=? AND (i.source_uuid||':'||i.id)<?)) AND (?<>'unread' OR (CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END)=0) AND (?<>'starred' OR (CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END)=1) AND (?='' OR i.remote_feed=?) AND (?='' OR EXISTS(SELECT 1 FROM json_each(i.remote_categories) WHERE value=?))`+extra+` ORDER BY i.published_ms DESC,i.source_uuid||':'||i.id DESC LIMIT 101`, args...)
if e != nil {
return nil, e
}
defer rows.Close()
out := []FeedItem{}
for rows.Next() {
var v FeedItem
if e = rows.Scan(&v.Source, &v.ID, &v.Title, &v.URL, &v.PublishedMS, &v.Feed, &v.Folder, &v.Read, &v.Starred, &v.Version, &v.Provider); e != nil {
return nil, e
}
out = append(out, v)
}
return out, rows.Err()
}
func (s *Store) FeedItem(ctx context.Context, domain, owner, source, id string) (FeedItem, error) {
var v FeedItem
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
return v, e
}
e := s.DB.QueryRowContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.body,i.body_html,i.published_ms,i.feed,i.folder,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=? AND s.owner_uuid=? AND s.uuid=? AND i.id=?`, domain, owner, source, id).Scan(&v.Source, &v.ID, &v.Title, &v.URL, &v.Body, &v.BodyHTML, &v.PublishedMS, &v.Feed, &v.Folder, &v.Read, &v.Starred, &v.Version, &v.Provider)
return v, e
}
func (s *Store) SetFeedState(ctx context.Context, domain, owner, source, id string, read, star bool, version int64) error {
return s.feedWrite(ctx, domain, owner, func(tx *sql.Tx) error {
var provider string
if e := tx.QueryRowContext(ctx, `SELECT s.provider FROM feed_sources s JOIN feed_items i ON i.source_uuid=s.uuid WHERE s.uuid=? AND s.domain_uuid=? AND s.owner_uuid=? AND i.id=?`, source, domain, owner, id).Scan(&provider); e != nil {
return e
}
if provider != "local" {
return ErrAccessDenied
}
if version == 0 {
var n int
if e := tx.QueryRowContext(ctx, `SELECT count(*) FROM feed_states WHERE source_uuid=?`, source).Scan(&n); e != nil {
return e
}
if n >= 10000 {
return ErrTaskDAVLimit
}
}
result, e := tx.ExecContext(ctx, `INSERT INTO feed_states(source_uuid,id,is_read,starred,version) SELECT ?,?,?,?,1 WHERE ?=0 ON CONFLICT(source_uuid,id) DO NOTHING`, source, id, read, star, version)
if e != nil {
return e
}
n, _ := result.RowsAffected()
if n == 1 {
return nil
}
result, e = tx.ExecContext(ctx, `UPDATE feed_states SET is_read=?,starred=?,version=version+1 WHERE source_uuid=? AND id=? AND version=?`, read, star, source, id, version)
if e != nil {
return e
}
n, _ = result.RowsAffected()
if n != 1 {
return ErrTaskConflict
}
return nil
})
}
func (s *Store) SealFeedCredential(domain, owner, id, user, password string) (string, error) {
b, _ := json.Marshal([]string{"feed-source-v1", domain, owner, id, user, password})
return s.encryptAuthSecret(b)
}
func (s *Store) OpenFeedCredential(v FeedSource) (string, string, error) {
b, e := s.decryptAuthSecret(v.Credential)
if e != nil {
return "", "", e
}
var f []string
if json.Unmarshal(b, &f) != nil || len(f) != 6 || f[0] != "feed-source-v1" || f[1] != v.Domain || f[2] != v.Owner || f[3] != v.UUID {
return "", "", ErrAccessDenied
}
return f[4], f[5], nil
}
func (s *Store) EnsureFeedEvent() error {
if s.ReadOnly {
return ErrReadOnly
}
now := time.Now().UnixMilli()
_, e := s.DB.Exec(`INSERT OR IGNORE INTO event_definitions(uuid,name,event_type,event_key,payload,schedule_kind,interval_seconds,next_run_at_ms,enabled,priority,max_attempts,created_at_ms,updated_at_ms) VALUES('builtin.feed.sync','RSS source synchronization',?,'local','{}','interval',60,?,1,-5,1,?,?)`, FeedSyncEventType, now, now, now)
return e
}
func (s *Store) DueFeedSources(ctx context.Context) ([]FeedSource, error) {
rows, e := s.DB.QueryContext(ctx, `SELECT `+feedSourceColumns+` FROM feed_sources WHERE enabled=1 AND next_ms<=? AND lease_until<? AND EXISTS(SELECT 1 FROM principals p JOIN domain_memberships m ON m.principal_uuid=p.uuid JOIN domains d ON d.uuid=m.domain_uuid WHERE p.uuid=feed_sources.owner_uuid AND m.domain_uuid=feed_sources.domain_uuid AND p.state='active' AND m.state='active' AND d.enabled=1 AND COALESCE(d.placement,'local')='local') AND NOT EXISTS(SELECT 1 FROM config c WHERE c.domain_uuid=feed_sources.domain_uuid AND c.key='system_extension_states' AND COALESCE(json_extract(CASE WHEN json_valid(c.value) THEN c.value ELSE '{"organizer.feeds":false}' END,'$."organizer.feeds"'),1)=0) ORDER BY next_ms,uuid LIMIT 4`, time.Now().UnixMilli(), time.Now().UnixMilli())
if e != nil {
return nil, e
}
defer rows.Close()
out := []FeedSource{}
for rows.Next() {
v, e := scanFeedSource(rows)
if e != nil {
return nil, e
}
out = append(out, v)
}
return out, rows.Err()
}
func (s *Store) ClaimFeed(ctx context.Context, v FeedSource) (FeedSource, error) {
v.Lease = uuid.NewString()
e := s.feedWrite(ctx, v.Domain, v.Owner, func(tx *sql.Tx) error {
now := time.Now().UnixMilli()
result, e := tx.ExecContext(ctx, `UPDATE feed_sources SET lease=?,lease_until=?,next_ms=? WHERE uuid=? AND version=? AND enabled=1 AND lease_until<? AND next_ms<=?`, v.Lease, now+60000, now+int64(v.Interval)*1000, v.UUID, v.Version, now, now)
if e != nil {
return e
}
n, _ := result.RowsAffected()
if n != 1 {
return ErrTaskConflict
}
return nil
})
return v, e
}
// Failed fetches preserve the previous cache. Version and claim token fence stale workers.
func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem, etag, modified, code string, unchanged bool, catalogs ...FeedCatalogSnapshot) error {
if len(catalogs) > 1 {
return ErrTaskInvalid
}
var catalog, folders string
if len(catalogs) == 1 {
var e error
catalog, folders, e = catalogs[0].Encode()
if e != nil {
return e
}
if v.Provider == "freshrss" {
membership := map[string][]string{}
for _, sub := range catalogs[0].Subscriptions {
for _, c := range sub.Categories {
membership[sub.ID] = append(membership[sub.ID], c.ID)
}
}
items = append([]FeedItem(nil), items...)
for n := range items {
items[n].RemoteCategories = membership[items[n].RemoteFeed]
}
}
}
if len(items) > 500 || len(etag) > 1024 || len(modified) > 1024 {
return ErrTaskInvalid
}
switch code {
case "", "network", "invalid_feed", "credentials", "remote_error", "limit":
default:
return ErrTaskInvalid
}
for _, i := range items {
if (i.RemoteFeed != "" && !ValidFeedRemoteID(i.RemoteFeed)) || len(i.RemoteCategories) > 64 {
return ErrTaskInvalid
}
for _, c := range i.RemoteCategories {
if !ValidFeedRemoteID(c) {
return ErrTaskInvalid
}
}
if i.ID == "" || len(i.ID) > 1024 || len(i.Title) > 2048 || len(i.URL) > 4096 || len(i.Body) > 32768 || len(i.BodyHTML) > 128<<10 || len(i.Feed) > 512 || len(i.Folder) > 512 || i.PublishedMS < 0 {
return ErrTaskInvalid
}
}
return s.feedWrite(ctx, v.Domain, v.Owner, func(tx *sql.Tx) error {
result, e := tx.ExecContext(ctx, `UPDATE feed_sources SET lease='',lease_until=0,last_ms=CASE WHEN ?='' THEN ? ELSE last_ms END,error=?,etag=CASE WHEN ?='' THEN ? ELSE etag END,modified=CASE WHEN ?='' THEN ? ELSE modified END WHERE uuid=? AND version=? AND lease=? AND enabled=1`, code, time.Now().UnixMilli(), code, code, etag, code, modified, v.UUID, v.Version, v.Lease)
if e != nil {
return e
}
n, _ := result.RowsAffected()
if n != 1 {
return ErrTaskConflict
}
if code != "" || unchanged {
return nil
}
if catalog != "" {
if _, e = tx.ExecContext(ctx, `UPDATE feed_sources SET remote_catalog=?,remote_folders=? WHERE uuid=?`, catalog, folders, v.UUID); e != nil {
return e
}
}
// FreshRSS is authoritative: retain exactly this bounded recent window,
// not stale rows from unsubscribed feeds or old external read/star state.
if v.Provider == "freshrss" {
if _, e = tx.ExecContext(ctx, `DELETE FROM feed_items WHERE source_uuid=?`, v.UUID); e != nil {
return e
}
}
for _, i := range items {
categories := i.RemoteCategories
if categories == nil {
categories = []string{}
}
raw, _ := json.Marshal(categories)
_, e = tx.ExecContext(ctx, `INSERT INTO feed_items(source_uuid,id,title,url,body,body_html,published_ms,feed,folder,is_read,starred,remote_feed,remote_categories) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_uuid,id) DO UPDATE SET title=excluded.title,url=excluded.url,body=excluded.body,body_html=excluded.body_html,published_ms=excluded.published_ms,feed=excluded.feed,folder=excluded.folder,is_read=excluded.is_read,starred=excluded.starred,remote_feed=excluded.remote_feed,remote_categories=excluded.remote_categories`, v.UUID, i.ID, i.Title, i.URL, i.Body, i.BodyHTML, i.PublishedMS, i.Feed, i.Folder, i.Read, i.Starred, i.RemoteFeed, string(raw))
if e != nil {
return e
}
}
_, e = tx.ExecContext(ctx, `DELETE FROM feed_items WHERE source_uuid=? AND id NOT IN(SELECT id FROM feed_items WHERE source_uuid=? ORDER BY published_ms DESC,id DESC LIMIT 500)`, v.UUID, v.UUID)
return e
})
}