Files
pages/internal/store/feeds.go
T
2026-11-15 05:47:14 +09:00

599 lines
32 KiB
Go

package store
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"unicode/utf8"
"github.com/google/uuid"
"pageswiki/internal/db"
)
const FeedSyncEventType = "feed.source.sync"
var feedMigrations = []db.SQLMigration{{Version: 1, Name: "private_feed_reader", Statements: []string{
`CREATE TABLE feed_sources(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,provider TEXT NOT NULL CHECK(provider IN ('local','freshrss')),title TEXT NOT NULL,folder TEXT NOT NULL,url TEXT NOT NULL,credential TEXT NOT NULL DEFAULT '',version INTEGER NOT NULL,enabled INTEGER NOT NULL,interval_seconds INTEGER NOT NULL,next_ms INTEGER NOT NULL DEFAULT 0,lease TEXT NOT NULL DEFAULT '',lease_until INTEGER NOT NULL DEFAULT 0,last_ms INTEGER NOT NULL DEFAULT 0,error TEXT NOT NULL DEFAULT '',etag TEXT NOT NULL DEFAULT '',modified TEXT NOT NULL DEFAULT '')`,
`CREATE INDEX feed_sources_owner ON feed_sources(domain_uuid,owner_uuid,uuid)`,
`CREATE INDEX feed_sources_due ON feed_sources(enabled,next_ms,uuid)`,
`CREATE TABLE feed_items(source_uuid TEXT NOT NULL REFERENCES feed_sources(uuid) ON DELETE CASCADE,id TEXT NOT NULL,title TEXT NOT NULL,url TEXT NOT NULL,body TEXT NOT NULL,published_ms INTEGER NOT NULL,feed TEXT NOT NULL,folder TEXT NOT NULL,is_read INTEGER NOT NULL,starred INTEGER NOT NULL,PRIMARY KEY(source_uuid,id))`,
`CREATE INDEX feed_items_date ON feed_items(source_uuid,published_ms DESC,id DESC)`,
`CREATE TABLE feed_states(source_uuid TEXT NOT NULL REFERENCES feed_sources(uuid) ON DELETE CASCADE,id TEXT NOT NULL,is_read INTEGER NOT NULL,starred INTEGER NOT NULL,version INTEGER NOT NULL,PRIMARY KEY(source_uuid,id))`,
`CREATE TRIGGER feed_source_delete AFTER DELETE ON feed_sources BEGIN DELETE FROM feed_items WHERE source_uuid=old.uuid; DELETE FROM feed_states WHERE source_uuid=old.uuid; END`,
`CREATE TABLE feed_note_receipts(domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,request_id TEXT NOT NULL,input_hash TEXT NOT NULL,page_uuid TEXT NOT NULL,PRIMARY KEY(domain_uuid,owner_uuid,request_id))`,
}, Assertions: []string{schemaColumns("feed_sources", "uuid,domain_uuid,owner_uuid,provider,title,folder,url,credential,version,enabled,interval_seconds,next_ms,lease,lease_until,last_ms,error,etag,modified"), schemaColumns("feed_items", "source_uuid,id,title,url,body,published_ms,feed,folder,is_read,starred"), schemaUnique("feed_items", "source_uuid,id"), schemaUnique("feed_states", "source_uuid,id"), schemaUnique("feed_note_receipts", "domain_uuid,owner_uuid,request_id"), `SELECT 1 WHERE NOT EXISTS(SELECT 1 FROM sqlite_master WHERE type='trigger' AND name='feed_source_delete')`}}, {
Version: 2, Name: "feed_html_body", Statements: []string{
`ALTER TABLE feed_items ADD COLUMN body_html TEXT NOT NULL DEFAULT ''`,
// Text-only caches cannot recover markup from a 304. Request one full
// background fetch while fencing workers started before this migration.
`UPDATE feed_sources SET etag='',modified='',next_ms=0,lease='',lease_until=0,version=version+1`,
}, Assertions: []string{schemaColumns("feed_items", "body_html")},
}, {
Version: 3, Name: "feed_import_receipts", Statements: []string{
`CREATE TABLE feed_import_receipts(domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,request_id TEXT NOT NULL,input_hash TEXT NOT NULL,result TEXT NOT NULL,PRIMARY KEY(domain_uuid,owner_uuid,request_id))`,
}, Assertions: []string{schemaUnique("feed_import_receipts", "domain_uuid,owner_uuid,request_id")},
}, {
Version: 4, Name: "freshrss_catalog", Statements: []string{
`ALTER TABLE feed_sources ADD COLUMN remote_catalog TEXT NOT NULL DEFAULT '[]'`,
`ALTER TABLE feed_items ADD COLUMN remote_feed TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE feed_items ADD COLUMN remote_categories TEXT NOT NULL DEFAULT '[]'`,
`UPDATE feed_sources SET next_ms=0,lease='',lease_until=0,version=version+1 WHERE provider='freshrss'`,
}, Assertions: []string{schemaColumns("feed_sources", "remote_catalog"), schemaColumns("feed_items", "remote_feed,remote_categories")},
}, {
Version: 5, Name: "feed_explicit_remote_actions", Statements: []string{
`ALTER TABLE feed_sources ADD COLUMN write_enabled INTEGER NOT NULL DEFAULT 0`,
`CREATE TABLE feed_actions(domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,request_id TEXT NOT NULL,source_uuid TEXT NOT NULL,item_id TEXT NOT NULL,field TEXT NOT NULL,desired INTEGER NOT NULL,source_version INTEGER NOT NULL,status TEXT NOT NULL,claim TEXT NOT NULL DEFAULT '',until_ms INTEGER NOT NULL DEFAULT 0,created_ms INTEGER NOT NULL,error TEXT NOT NULL DEFAULT '',PRIMARY KEY(domain_uuid,owner_uuid,request_id))`,
`CREATE INDEX feed_actions_due ON feed_actions(status,created_ms)`,
`CREATE INDEX feed_actions_item ON feed_actions(domain_uuid,owner_uuid,source_uuid,item_id,created_ms DESC)`,
`CREATE UNIQUE INDEX feed_actions_active ON feed_actions(source_uuid,item_id) WHERE status IN ('queued','preparing','sending','unknown','check_queued','checking')`,
`CREATE TRIGGER feed_actions_source_change AFTER UPDATE OF version ON feed_sources BEGIN UPDATE feed_actions SET status=CASE WHEN status IN ('sending','unknown','check_queued','checking') THEN 'unknown' ELSE 'cancelled' END,error='source_changed' WHERE source_uuid=old.uuid AND status IN ('queued','preparing','sending','unknown','check_queued','checking'); END`,
`CREATE TRIGGER feed_actions_source_delete AFTER DELETE ON feed_sources BEGIN UPDATE feed_actions SET status=CASE WHEN status IN ('sending','unknown','check_queued','checking') THEN 'unknown' ELSE 'cancelled' END,error='source_deleted' WHERE source_uuid=old.uuid AND status IN ('queued','preparing','sending','unknown','check_queued','checking'); END`,
}, Assertions: []string{schemaColumns("feed_sources", "write_enabled"), schemaUnique("feed_actions", "domain_uuid,owner_uuid,request_id")},
}, {
Version: 6, Name: "freshrss_empty_folders", Statements: []string{
`ALTER TABLE feed_sources ADD COLUMN remote_folders TEXT NOT NULL DEFAULT '[]'`,
}, Assertions: []string{schemaColumns("feed_sources", "remote_folders")},
}}
// CaptureFeedNote creates an inert Wiki note and receipt atomically. It does not
// mutate the feed or create a task, and rechecks grants under the writer lock.
func (s *Store) CaptureFeedNote(ctx context.Context, domain, owner, request, source, id, target, title, body string) (string, error) {
if _, e := uuid.Parse(request); e != nil {
return "", ErrTaskInvalid
}
if !ValidTaskWikiPrefix(target) || target == "/" || len(target) > 500 || len(title) > 300 || title == "" || len(body) > 65536 {
return "", ErrTaskInvalid
}
// User/group spaces have additional ownership/publication contracts. Initial
// capture supports ordinary Wiki paths only, rather than bypassing those caps.
first := strings.Split(strings.TrimPrefix(target, "/"), "/")[0]
if first == "u" || first == "g" {
return "", ErrAccessDenied
}
raw, _ := json.Marshal([]string{source, id, target, title, body})
hash := fmt.Sprintf("%x", sha256.Sum256(raw))
page := ""
e := s.feedWrite(ctx, domain, owner, func(tx *sql.Tx) error {
caps, e := s.loadLocalityAccess(tx, domain, owner)
if e != nil {
return e
}
for _, capability := range []string{"read", "create", "write"} {
basis := s.authorizeGrantContext(ctx, tx, AuthorizationRequest{DomainUUID: domain, PrincipalUUID: owner, Capability: capability, TargetType: "path", TargetID: target}, false)
if basis != nil && !errors.Is(basis, ErrAccessDenied) {
return basis
}
if !caps.Decide(target, capability, basis == nil) {
return ErrAccessDenied
}
}
var oldHash string
e = tx.QueryRowContext(ctx, `SELECT input_hash,page_uuid FROM feed_note_receipts WHERE domain_uuid=`+s.sqlParam(1)+` AND owner_uuid=`+s.sqlParam(2)+` AND request_id=`+s.sqlParam(3), domain, owner, request).Scan(&oldHash, &page)
if e == nil {
if oldHash != hash {
return ErrTaskConflict
}
return nil
}
if !errors.Is(e, sql.ErrNoRows) {
return e
}
var exists bool
if e = tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid WHERE s.domain_uuid=`+s.sqlParam(1)+` AND s.owner_uuid=`+s.sqlParam(2)+` AND s.uuid=`+s.sqlParam(3)+` AND i.id=`+s.sqlParam(4)+`)`, domain, owner, source, id).Scan(&exists); e != nil {
return e
}
if !exists {
return sql.ErrNoRows
}
item := TaskRecord{Title: title}
if e = s.createCapturedTaskWiki(ctx, tx, domain, owner, &item, target, body); e != nil {
return e
}
page = item.PageUUID
_, e = tx.ExecContext(ctx, `INSERT INTO feed_note_receipts VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`,`+s.sqlParam(5)+`)`, domain, owner, request, hash, page)
return e
})
if e == nil {
s.InvalidatePathCaches()
}
return page, e
}
type FeedSource struct {
UUID string `json:"uuid"`
Domain string `json:"-"`
Owner string `json:"-"`
Provider string `json:"provider"`
Title string `json:"title"`
Folder string `json:"folder"`
URL string `json:"url"`
Credential string `json:"-"`
Version int64 `json:"version"`
Enabled bool `json:"enabled"`
Interval int `json:"interval_seconds"`
NextMS int64 `json:"next_ms"`
Lease string `json:"-"`
LeaseUntil int64 `json:"-"`
LastMS int64 `json:"last_ms"`
Error string `json:"error"`
ETag string `json:"-"`
Modified string `json:"-"`
WriteEnabled bool `json:"write_enabled"`
}
type FeedItem struct {
Source string `json:"source"`
ID string `json:"id"`
Title string `json:"title"`
URL string `json:"url"`
Body string `json:"body,omitempty"`
BodyHTML string `json:"body_html,omitempty"`
PublishedMS int64 `json:"published_ms"`
Feed string `json:"feed"`
Folder string `json:"folder"`
Read bool `json:"read"`
Starred bool `json:"starred"`
Version int64 `json:"version"`
Provider string `json:"provider"`
RemoteFeed string `json:"remote_feed,omitempty"`
RemoteCategories []string `json:"remote_categories,omitempty"`
}
const feedSourceColumns = `uuid,domain_uuid,owner_uuid,provider,title,folder,url,credential,version,enabled,interval_seconds,next_ms,lease,lease_until,last_ms,error,etag,modified,write_enabled`
func scanFeedSource(row interface{ Scan(...any) error }) (v FeedSource, e error) {
e = row.Scan(&v.UUID, &v.Domain, &v.Owner, &v.Provider, &v.Title, &v.Folder, &v.URL, &v.Credential, &v.Version, &v.Enabled, &v.Interval, &v.NextMS, &v.Lease, &v.LeaseUntil, &v.LastMS, &v.Error, &v.ETag, &v.Modified, &v.WriteEnabled)
return
}
func (s *Store) feedActor(ctx context.Context, q interface {
QueryRowContext(context.Context, string, ...any) *sql.Row
}, domain, owner string, write bool) error {
if e := s.taskActor(ctx, q, domain, owner, write); e != nil {
return e
}
var raw string
e := q.QueryRowContext(ctx, `SELECT value FROM config WHERE domain_uuid=`+s.sqlParam(1)+` AND key='system_extension_states'`, domain).Scan(&raw)
if e != nil && !errors.Is(e, sql.ErrNoRows) {
return e
}
states := map[string]bool{}
if raw != "" && json.Unmarshal([]byte(raw), &states) != nil {
return ErrAccessDenied
}
if enabled, ok := states["organizer.feeds"]; ok && !enabled {
return ErrAccessDenied
}
return nil
}
func (s *Store) feedWrite(ctx context.Context, domain, owner string, fn func(*sql.Tx) error) (resultErr error) {
defer func() { resultErr = s.sqlError("feed write", resultErr) }()
if s.ReadOnly {
return ErrReadOnly
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
tx, e := s.DB.BeginTx(ctx, nil)
if e != nil {
return e
}
defer tx.Rollback()
if e = s.lockTaskWriter(ctx, tx); e != nil {
return e
}
if _, e = tx.ExecContext(ctx, `UPDATE domain_memberships SET state=state WHERE domain_uuid=`+s.sqlParam(1)+` AND principal_uuid=`+s.sqlParam(2), domain, owner); e != nil {
return e
}
if e = s.feedActor(ctx, tx, domain, owner, true); e != nil {
return e
}
if e = fn(tx); e != nil {
return e
}
return tx.Commit()
}
func (s *Store) FeedSources(ctx context.Context, domain, owner string) (result []FeedSource, resultErr error) {
defer func() { resultErr = s.sqlError("feeds repository", resultErr) }()
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
if e := s.feedActor(ctx, s.DB, domain, owner, false); e != nil {
return nil, e
}
rows, e := s.DB.QueryContext(ctx, `SELECT `+feedSourceColumns+` FROM feed_sources WHERE domain_uuid=`+s.sqlParam(1)+` AND owner_uuid=`+s.sqlParam(2)+` ORDER BY folder,title,uuid LIMIT 32`, domain, owner)
if e != nil {
return nil, e
}
defer rows.Close()
out := []FeedSource{}
for rows.Next() {
v, e := scanFeedSource(rows)
if e != nil {
return nil, e
}
out = append(out, v)
}
return out, rows.Err()
}
func (s *Store) SaveFeedSource(ctx context.Context, domain, owner string, v FeedSource) (FeedSource, error) {
if v.WriteEnabled && v.Provider != "freshrss" {
return v, ErrTaskInvalid
}
if _, e := uuid.Parse(v.UUID); e != nil {
return v, ErrTaskInvalid
}
v.Title = strings.TrimSpace(v.Title)
v.Folder = strings.TrimSpace(v.Folder)
if v.Title == "" || len(v.Title) > 200 || len(v.Folder) > 200 || len(v.URL) > 4096 || !utf8.ValidString(v.Title+v.Folder) || v.Interval < 300 || v.Interval > 86400 || (v.Provider != "local" && v.Provider != "freshrss") {
return v, ErrTaskInvalid
}
e := s.feedWrite(ctx, domain, owner, func(tx *sql.Tx) error {
if v.Version == 0 {
old, lookup := scanFeedSource(tx.QueryRowContext(ctx, `SELECT `+feedSourceColumns+` FROM feed_sources WHERE uuid=`+s.sqlParam(1), v.UUID))
if lookup == nil {
if old.Domain != domain || old.Owner != owner || old.Version != 1 || old.Provider != v.Provider || old.Title != v.Title || old.Folder != v.Folder || old.URL != v.URL || old.Interval != v.Interval || old.Enabled != v.Enabled || old.WriteEnabled != v.WriteEnabled {
return ErrTaskConflict
}
if old.Credential != v.Credential {
candidate := v
candidate.Domain = domain
candidate.Owner = owner
u, p, e := s.OpenFeedCredential(candidate)
if e != nil {
return ErrTaskConflict
}
ou, op, e := s.OpenFeedCredential(old)
if e != nil || u != ou || p != op {
return ErrTaskConflict
}
}
return nil
}
if !errors.Is(lookup, sql.ErrNoRows) {
return lookup
}
// Do not rebind retained receipts to a different remote identity.
// Both predicates use existing indexes: owner history and the global
// partial index of unresolved operations. Other owners' terminal
// receipts remain isolated by domain/owner in metadata lookups.
var retired bool
if e := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM feed_actions WHERE domain_uuid=`+s.sqlParam(1)+` AND owner_uuid=`+s.sqlParam(2)+` AND source_uuid=`+s.sqlParam(3)+`) OR EXISTS(SELECT 1 FROM feed_actions WHERE source_uuid=`+s.sqlParam(4)+` AND status IN ('queued','preparing','sending','unknown','check_queued','checking'))`, domain, owner, v.UUID, v.UUID).Scan(&retired); e != nil {
return e
}
if retired {
return ErrTaskConflict
}
var n int
if e := tx.QueryRowContext(ctx, `SELECT count(*) FROM feed_sources WHERE domain_uuid=`+s.sqlParam(1)+` AND owner_uuid=`+s.sqlParam(2), domain, owner).Scan(&n); e != nil {
return e
}
if n >= 32 {
return ErrTaskDAVLimit
}
_, e := tx.ExecContext(ctx, `INSERT INTO feed_sources(uuid,domain_uuid,owner_uuid,provider,title,folder,url,credential,version,enabled,interval_seconds,write_enabled) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`,`+s.sqlParam(5)+`,`+s.sqlParam(6)+`,`+s.sqlParam(7)+`,`+s.sqlParam(8)+`,1,`+s.sqlParam(9)+`,`+s.sqlParam(10)+`,`+s.sqlParam(11)+`)`, v.UUID, domain, owner, v.Provider, v.Title, v.Folder, v.URL, v.Credential, boolToInt(v.Enabled), v.Interval, boolToInt(v.WriteEnabled))
return e
}
old, e := scanFeedSource(tx.QueryRowContext(ctx, `SELECT `+feedSourceColumns+` FROM feed_sources WHERE uuid=`+s.sqlParam(1)+` AND domain_uuid=`+s.sqlParam(2)+` AND owner_uuid=`+s.sqlParam(3), v.UUID, domain, owner))
if e != nil {
return e
}
// Changing remote identity requires a new source, not reassignment of cached IDs.
if old.Provider != v.Provider || old.URL != v.URL || v.Credential != "" {
return ErrTaskInvalid
}
result, e := tx.ExecContext(ctx, `UPDATE feed_sources SET title=`+s.sqlParam(1)+`,folder=`+s.sqlParam(2)+`,enabled=`+s.sqlParam(3)+`,interval_seconds=`+s.sqlParam(4)+`,write_enabled=`+s.sqlParam(5)+`,version=version+1,next_ms=0,lease='',lease_until=0 WHERE uuid=`+s.sqlParam(6)+` AND version=`+s.sqlParam(7), v.Title, v.Folder, boolToInt(v.Enabled), v.Interval, boolToInt(v.WriteEnabled), v.UUID, v.Version)
if e != nil {
return e
}
n, _ := result.RowsAffected()
if n != 1 {
return ErrTaskConflict
}
return nil
})
v.Version++
v.Credential = ""
return v, e
}
func (s *Store) FeedSourceAction(ctx context.Context, domain, owner, id, action string, version int64) error {
return s.feedWrite(ctx, domain, owner, func(tx *sql.Tx) error {
var result sql.Result
var e error
switch action {
case "refresh":
result, e = tx.ExecContext(ctx, `UPDATE feed_sources SET next_ms=0 WHERE uuid=`+s.sqlParam(1)+` AND domain_uuid=`+s.sqlParam(2)+` AND owner_uuid=`+s.sqlParam(3)+` AND version=`+s.sqlParam(4)+` AND enabled=1 AND lease_until<`+s.sqlParam(5)+` AND (last_ms=0 OR last_ms<`+s.sqlParam(6)+`)`, id, domain, owner, version, time.Now().UnixMilli(), time.Now().Add(-time.Minute).UnixMilli())
case "delete":
result, e = tx.ExecContext(ctx, `DELETE FROM feed_sources WHERE uuid=`+s.sqlParam(1)+` AND domain_uuid=`+s.sqlParam(2)+` AND owner_uuid=`+s.sqlParam(3)+` AND version=`+s.sqlParam(4), id, domain, owner, version)
default:
return ErrTaskInvalid
}
if e != nil {
return e
}
n, _ := result.RowsAffected()
if n != 1 {
return ErrTaskConflict
}
return nil
})
}
func (s *Store) FeedItems(ctx context.Context, domain, owner, source, filter, query string, before int64, beforeID string, scope ...string) (result []FeedItem, resultErr error) {
defer func() { resultErr = s.sqlError("feeds repository", resultErr) }()
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
remoteFeed, category := "", ""
if len(scope) > 3 {
return nil, ErrTaskInvalid
}
if len(scope) > 0 {
remoteFeed = scope[0]
}
if len(scope) > 1 {
category = scope[1]
}
mode := ""
if len(scope) > 2 {
mode = scope[2]
}
if mode != "" && (mode != "uncategorized" || source == "" || category != "") {
return nil, ErrTaskInvalid
}
if (remoteFeed != "" && !ValidFeedRemoteID(remoteFeed)) || (category != "" && !ValidFeedRemoteID(category)) || ((remoteFeed != "" || category != "") && source == "") {
return nil, ErrTaskInvalid
}
if len(query) > 200 || len(beforeID) > 1200 {
return nil, ErrTaskInvalid
}
if e := s.feedActor(ctx, s.DB, domain, owner, false); e != nil {
return nil, e
}
args := []any{domain, owner, source, source, query, query, query, before, before, before, beforeID, filter, filter, remoteFeed, remoteFeed, category, category}
position := "instr"
categories := "json_each(i.remote_categories)"
if s.sqlBackend == storePostgreSQL {
position = "strpos"
categories = "jsonb_array_elements_text(i.remote_categories::jsonb)"
}
extra := ""
if mode == "uncategorized" {
// The uncorrelated list is built once, not a catalog scan per article.
idExpr, catalogExpr, categoryCount := `json_extract(c.value,'$.id')`, `json_each(cs.remote_catalog)`, `json_array_length(c.value,'$.categories')`
if s.sqlBackend == storePostgreSQL {
idExpr, catalogExpr, categoryCount = `c.value->>'id'`, `jsonb_array_elements(cs.remote_catalog::jsonb)`, `jsonb_array_length(CASE WHEN jsonb_typeof(c.value->'categories')='array' THEN c.value->'categories' ELSE '[]'::jsonb END)`
}
extra = ` AND i.remote_feed IN (SELECT ` + idExpr + ` FROM feed_sources cs,` + catalogExpr + ` c WHERE cs.uuid=` + s.sqlParam(18) + ` AND cs.domain_uuid=` + s.sqlParam(19) + ` AND cs.owner_uuid=` + s.sqlParam(20) + ` AND ` + categoryCount + `=0)`
args = append(args, source, domain, owner)
}
rows, e := s.DB.QueryContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.published_ms,i.feed,CASE WHEN s.folder<>'' THEN s.folder ELSE i.folder END,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=`+s.sqlParam(1)+` AND s.owner_uuid=`+s.sqlParam(2)+` AND (`+s.sqlParam(3)+`='' OR s.uuid=`+s.sqlParam(4)+`) AND (`+s.sqlParam(5)+`='' OR `+position+`(lower(i.title),lower(`+s.sqlParam(6)+`))>0 OR `+position+`(lower(i.feed),lower(`+s.sqlParam(7)+`))>0) AND (CAST(`+s.sqlParam(8)+` AS BIGINT)<0 OR i.published_ms<`+s.sqlParam(9)+` OR (i.published_ms=`+s.sqlParam(10)+` AND (i.source_uuid||':'||i.id)<`+s.sqlParam(11)+`)) AND (`+s.sqlParam(12)+`<>'unread' OR (CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END)=0) AND (`+s.sqlParam(13)+`<>'starred' OR (CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END)=1) AND (`+s.sqlParam(14)+`='' OR i.remote_feed=`+s.sqlParam(15)+`) AND (`+s.sqlParam(16)+`='' OR EXISTS(SELECT 1 FROM `+categories+` WHERE value=`+s.sqlParam(17)+`))`+extra+` ORDER BY i.published_ms DESC,i.source_uuid||':'||i.id DESC LIMIT 101`, args...)
if e != nil {
return nil, e
}
defer rows.Close()
out := []FeedItem{}
for rows.Next() {
var v FeedItem
if e = rows.Scan(&v.Source, &v.ID, &v.Title, &v.URL, &v.PublishedMS, &v.Feed, &v.Folder, &v.Read, &v.Starred, &v.Version, &v.Provider); e != nil {
return nil, e
}
out = append(out, v)
}
return out, rows.Err()
}
func (s *Store) FeedItem(ctx context.Context, domain, owner, source, id string) (result FeedItem, resultErr error) {
defer func() { resultErr = s.sqlError("feeds repository", resultErr) }()
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
var v FeedItem
if e := s.feedActor(ctx, s.DB, domain, owner, false); e != nil {
return v, e
}
e := s.DB.QueryRowContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.body,i.body_html,i.published_ms,i.feed,i.folder,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=`+s.sqlParam(1)+` AND s.owner_uuid=`+s.sqlParam(2)+` AND s.uuid=`+s.sqlParam(3)+` AND i.id=`+s.sqlParam(4), domain, owner, source, id).Scan(&v.Source, &v.ID, &v.Title, &v.URL, &v.Body, &v.BodyHTML, &v.PublishedMS, &v.Feed, &v.Folder, &v.Read, &v.Starred, &v.Version, &v.Provider)
return v, e
}
func (s *Store) SetFeedState(ctx context.Context, domain, owner, source, id string, read, star bool, version int64) error {
return s.feedWrite(ctx, domain, owner, func(tx *sql.Tx) error {
var provider string
if e := tx.QueryRowContext(ctx, `SELECT s.provider FROM feed_sources s JOIN feed_items i ON i.source_uuid=s.uuid WHERE s.uuid=`+s.sqlParam(1)+` AND s.domain_uuid=`+s.sqlParam(2)+` AND s.owner_uuid=`+s.sqlParam(3)+` AND i.id=`+s.sqlParam(4), source, domain, owner, id).Scan(&provider); e != nil {
return e
}
if provider != "local" {
return ErrAccessDenied
}
if version == 0 {
var n int
if e := tx.QueryRowContext(ctx, `SELECT count(*) FROM feed_states WHERE source_uuid=`+s.sqlParam(1), source).Scan(&n); e != nil {
return e
}
if n >= 10000 {
return ErrTaskDAVLimit
}
}
result, e := tx.ExecContext(ctx, `INSERT INTO feed_states(source_uuid,id,is_read,starred,version) SELECT `+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`,1 WHERE CAST(`+s.sqlParam(5)+` AS BIGINT)=0 ON CONFLICT(source_uuid,id) DO NOTHING`, source, id, boolToInt(read), boolToInt(star), version)
if e != nil {
return e
}
n, _ := result.RowsAffected()
if n == 1 {
return nil
}
result, e = tx.ExecContext(ctx, `UPDATE feed_states SET is_read=`+s.sqlParam(1)+`,starred=`+s.sqlParam(2)+`,version=version+1 WHERE source_uuid=`+s.sqlParam(3)+` AND id=`+s.sqlParam(4)+` AND version=`+s.sqlParam(5), boolToInt(read), boolToInt(star), source, id, version)
if e != nil {
return e
}
n, _ = result.RowsAffected()
if n != 1 {
return ErrTaskConflict
}
return nil
})
}
func (s *Store) SealFeedCredential(domain, owner, id, user, password string) (string, error) {
b, _ := json.Marshal([]string{"feed-source-v1", domain, owner, id, user, password})
return s.encryptAuthSecret(b)
}
func (s *Store) OpenFeedCredential(v FeedSource) (string, string, error) {
b, e := s.decryptAuthSecret(v.Credential)
if e != nil {
return "", "", e
}
var f []string
if json.Unmarshal(b, &f) != nil || len(f) != 6 || f[0] != "feed-source-v1" || f[1] != v.Domain || f[2] != v.Owner || f[3] != v.UUID {
return "", "", ErrAccessDenied
}
return f[4], f[5], nil
}
func (s *Store) EnsureFeedEvent() error {
if s.sqlBackend == storePostgreSQL {
return s.ensureNativeBuiltinEvent("builtin.feed.sync", "RSS source synchronization", FeedSyncEventType, 60, -5, 1)
}
if s.ReadOnly {
return ErrReadOnly
}
now := time.Now().UnixMilli()
_, e := s.DB.Exec(`INSERT OR IGNORE INTO event_definitions(uuid,name,event_type,event_key,payload,schedule_kind,interval_seconds,next_run_at_ms,enabled,priority,max_attempts,created_at_ms,updated_at_ms) VALUES('builtin.feed.sync','RSS source synchronization',`+s.sqlParam(1)+`,'local','{}','interval',60,`+s.sqlParam(2)+`,1,-5,1,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`)`, FeedSyncEventType, now, now, now)
return e
}
func (s *Store) DueFeedSources(ctx context.Context) (result []FeedSource, resultErr error) {
defer func() { resultErr = s.sqlError("feeds repository", resultErr) }()
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
rows, e := s.DB.QueryContext(ctx, `SELECT `+feedSourceColumns+` FROM feed_sources WHERE enabled=1 AND next_ms<=`+s.sqlParam(1)+` AND lease_until<`+s.sqlParam(2)+` AND EXISTS(SELECT 1 FROM principals p JOIN domain_memberships m ON m.principal_uuid=p.uuid JOIN domains d ON d.uuid=m.domain_uuid WHERE p.uuid=feed_sources.owner_uuid AND m.domain_uuid=feed_sources.domain_uuid AND p.state='active' AND m.state='active' AND d.enabled=1 AND COALESCE(d.placement,'local')='local'`+s.feedCampusSQL()+`) AND NOT EXISTS(SELECT 1 FROM config c WHERE c.domain_uuid=feed_sources.domain_uuid AND c.key='system_extension_states' AND `+s.feedDisabledConfigSQL()+`) ORDER BY next_ms,uuid LIMIT 4`, time.Now().UnixMilli(), time.Now().UnixMilli())
if e != nil {
return nil, e
}
defer rows.Close()
out := []FeedSource{}
for rows.Next() {
v, e := scanFeedSource(rows)
if e != nil {
return nil, e
}
out = append(out, v)
}
return out, rows.Err()
}
func (s *Store) ClaimFeed(ctx context.Context, v FeedSource) (FeedSource, error) {
v.Lease = uuid.NewString()
e := s.feedWrite(ctx, v.Domain, v.Owner, func(tx *sql.Tx) error {
now := time.Now().UnixMilli()
result, e := tx.ExecContext(ctx, `UPDATE feed_sources SET lease=`+s.sqlParam(1)+`,lease_until=`+s.sqlParam(2)+`,next_ms=`+s.sqlParam(3)+` WHERE uuid=`+s.sqlParam(4)+` AND version=`+s.sqlParam(5)+` AND enabled=1 AND lease_until<`+s.sqlParam(6)+` AND next_ms<=`+s.sqlParam(7), v.Lease, now+60000, now+int64(v.Interval)*1000, v.UUID, v.Version, now, now)
if e != nil {
return e
}
n, _ := result.RowsAffected()
if n != 1 {
return ErrTaskConflict
}
return nil
})
return v, e
}
// Failed fetches preserve the previous cache. Version and claim token fence stale workers.
func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem, etag, modified, code string, unchanged bool, catalogs ...FeedCatalogSnapshot) error {
if len(catalogs) > 1 {
return ErrTaskInvalid
}
var catalog, folders string
if len(catalogs) == 1 {
var e error
catalog, folders, e = catalogs[0].Encode()
if e != nil {
return e
}
if v.Provider == "freshrss" {
membership := map[string][]string{}
for _, sub := range catalogs[0].Subscriptions {
for _, c := range sub.Categories {
membership[sub.ID] = append(membership[sub.ID], c.ID)
}
}
items = append([]FeedItem(nil), items...)
for n := range items {
items[n].RemoteCategories = membership[items[n].RemoteFeed]
}
}
}
if len(items) > 500 || len(etag) > 1024 || len(modified) > 1024 {
return ErrTaskInvalid
}
switch code {
case "", "network", "invalid_feed", "credentials", "remote_error", "limit":
default:
return ErrTaskInvalid
}
for _, i := range items {
if (i.RemoteFeed != "" && !ValidFeedRemoteID(i.RemoteFeed)) || len(i.RemoteCategories) > 64 {
return ErrTaskInvalid
}
for _, c := range i.RemoteCategories {
if !ValidFeedRemoteID(c) {
return ErrTaskInvalid
}
}
if i.ID == "" || len(i.ID) > 1024 || len(i.Title) > 2048 || len(i.URL) > 4096 || len(i.Body) > 32768 || len(i.BodyHTML) > 128<<10 || len(i.Feed) > 512 || len(i.Folder) > 512 || i.PublishedMS < 0 {
return ErrTaskInvalid
}
}
return s.feedWrite(ctx, v.Domain, v.Owner, func(tx *sql.Tx) error {
result, e := tx.ExecContext(ctx, `UPDATE feed_sources SET lease='',lease_until=0,last_ms=CASE WHEN `+s.sqlParam(1)+`='' THEN `+s.sqlParam(2)+` ELSE last_ms END,error=`+s.sqlParam(3)+`,etag=CASE WHEN `+s.sqlParam(4)+`='' THEN `+s.sqlParam(5)+` ELSE etag END,modified=CASE WHEN `+s.sqlParam(6)+`='' THEN `+s.sqlParam(7)+` ELSE modified END WHERE uuid=`+s.sqlParam(8)+` AND version=`+s.sqlParam(9)+` AND lease=`+s.sqlParam(10)+` AND enabled=1`, code, time.Now().UnixMilli(), code, code, etag, code, modified, v.UUID, v.Version, v.Lease)
if e != nil {
return e
}
n, _ := result.RowsAffected()
if n != 1 {
return ErrTaskConflict
}
if code != "" || unchanged {
return nil
}
if catalog != "" {
if _, e = tx.ExecContext(ctx, `UPDATE feed_sources SET remote_catalog=`+s.sqlParam(1)+`,remote_folders=`+s.sqlParam(2)+` WHERE uuid=`+s.sqlParam(3), catalog, folders, v.UUID); e != nil {
return e
}
}
// FreshRSS is authoritative: retain exactly this bounded recent window,
// not stale rows from unsubscribed feeds or old external read/star state.
if v.Provider == "freshrss" {
if _, e = tx.ExecContext(ctx, `DELETE FROM feed_items WHERE source_uuid=`+s.sqlParam(1), v.UUID); e != nil {
return e
}
}
for _, i := range items {
categories := i.RemoteCategories
if categories == nil {
categories = []string{}
}
raw, _ := json.Marshal(categories)
_, e = tx.ExecContext(ctx, `INSERT INTO feed_items(source_uuid,id,title,url,body,body_html,published_ms,feed,folder,is_read,starred,remote_feed,remote_categories) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`,`+s.sqlParam(5)+`,`+s.sqlParam(6)+`,`+s.sqlParam(7)+`,`+s.sqlParam(8)+`,`+s.sqlParam(9)+`,`+s.sqlParam(10)+`,`+s.sqlParam(11)+`,`+s.sqlParam(12)+`,`+s.sqlParam(13)+`) ON CONFLICT(source_uuid,id) DO UPDATE SET title=excluded.title,url=excluded.url,body=excluded.body,body_html=excluded.body_html,published_ms=excluded.published_ms,feed=excluded.feed,folder=excluded.folder,is_read=excluded.is_read,starred=excluded.starred,remote_feed=excluded.remote_feed,remote_categories=excluded.remote_categories`, v.UUID, i.ID, i.Title, i.URL, i.Body, i.BodyHTML, i.PublishedMS, i.Feed, i.Folder, boolToInt(i.Read), boolToInt(i.Starred), i.RemoteFeed, string(raw))
if e != nil {
return e
}
}
_, e = tx.ExecContext(ctx, `DELETE FROM feed_items WHERE source_uuid=`+s.sqlParam(1)+` AND id NOT IN(SELECT id FROM feed_items WHERE source_uuid=`+s.sqlParam(2)+` ORDER BY published_ms DESC,id DESC LIMIT 500)`, v.UUID, v.UUID)
return e
})
}