464 lines
22 KiB
Go
464 lines
22 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"github.com/google/uuid"
|
|
"pageswiki/internal/db"
|
|
)
|
|
|
|
const FeedSyncEventType = "feed.source.sync"
|
|
|
|
var feedMigrations = []db.SQLMigration{{Version: 1, Name: "private_feed_reader", Statements: []string{
|
|
`CREATE TABLE feed_sources(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,provider TEXT NOT NULL CHECK(provider IN ('local','freshrss')),title TEXT NOT NULL,folder TEXT NOT NULL,url TEXT NOT NULL,credential TEXT NOT NULL DEFAULT '',version INTEGER NOT NULL,enabled INTEGER NOT NULL,interval_seconds INTEGER NOT NULL,next_ms INTEGER NOT NULL DEFAULT 0,lease TEXT NOT NULL DEFAULT '',lease_until INTEGER NOT NULL DEFAULT 0,last_ms INTEGER NOT NULL DEFAULT 0,error TEXT NOT NULL DEFAULT '',etag TEXT NOT NULL DEFAULT '',modified TEXT NOT NULL DEFAULT '')`,
|
|
`CREATE INDEX feed_sources_owner ON feed_sources(domain_uuid,owner_uuid,uuid)`,
|
|
`CREATE INDEX feed_sources_due ON feed_sources(enabled,next_ms,uuid)`,
|
|
`CREATE TABLE feed_items(source_uuid TEXT NOT NULL REFERENCES feed_sources(uuid) ON DELETE CASCADE,id TEXT NOT NULL,title TEXT NOT NULL,url TEXT NOT NULL,body TEXT NOT NULL,published_ms INTEGER NOT NULL,feed TEXT NOT NULL,folder TEXT NOT NULL,is_read INTEGER NOT NULL,starred INTEGER NOT NULL,PRIMARY KEY(source_uuid,id))`,
|
|
`CREATE INDEX feed_items_date ON feed_items(source_uuid,published_ms DESC,id DESC)`,
|
|
`CREATE TABLE feed_states(source_uuid TEXT NOT NULL REFERENCES feed_sources(uuid) ON DELETE CASCADE,id TEXT NOT NULL,is_read INTEGER NOT NULL,starred INTEGER NOT NULL,version INTEGER NOT NULL,PRIMARY KEY(source_uuid,id))`,
|
|
`CREATE TRIGGER feed_source_delete AFTER DELETE ON feed_sources BEGIN DELETE FROM feed_items WHERE source_uuid=old.uuid; DELETE FROM feed_states WHERE source_uuid=old.uuid; END`,
|
|
`CREATE TABLE feed_note_receipts(domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,request_id TEXT NOT NULL,input_hash TEXT NOT NULL,page_uuid TEXT NOT NULL,PRIMARY KEY(domain_uuid,owner_uuid,request_id))`,
|
|
}, Assertions: []string{schemaColumns("feed_sources", "uuid,domain_uuid,owner_uuid,provider,title,folder,url,credential,version,enabled,interval_seconds,next_ms,lease,lease_until,last_ms,error,etag,modified"), schemaColumns("feed_items", "source_uuid,id,title,url,body,published_ms,feed,folder,is_read,starred"), schemaUnique("feed_items", "source_uuid,id"), schemaUnique("feed_states", "source_uuid,id"), schemaUnique("feed_note_receipts", "domain_uuid,owner_uuid,request_id"), `SELECT 1 WHERE NOT EXISTS(SELECT 1 FROM sqlite_master WHERE type='trigger' AND name='feed_source_delete')`}}, {
|
|
Version: 2, Name: "feed_html_body", Statements: []string{
|
|
`ALTER TABLE feed_items ADD COLUMN body_html TEXT NOT NULL DEFAULT ''`,
|
|
// Text-only caches cannot recover markup from a 304. Request one full
|
|
// background fetch while fencing workers started before this migration.
|
|
`UPDATE feed_sources SET etag='',modified='',next_ms=0,lease='',lease_until=0,version=version+1`,
|
|
}, Assertions: []string{schemaColumns("feed_items", "body_html")},
|
|
}, {
|
|
Version: 3, Name: "feed_import_receipts", Statements: []string{
|
|
`CREATE TABLE feed_import_receipts(domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,request_id TEXT NOT NULL,input_hash TEXT NOT NULL,result TEXT NOT NULL,PRIMARY KEY(domain_uuid,owner_uuid,request_id))`,
|
|
}, Assertions: []string{schemaUnique("feed_import_receipts", "domain_uuid,owner_uuid,request_id")},
|
|
}}
|
|
|
|
// CaptureFeedNote creates an inert Wiki note and receipt atomically. It does not
|
|
// mutate the feed or create a task, and rechecks grants under the writer lock.
|
|
func (s *Store) CaptureFeedNote(ctx context.Context, domain, owner, request, source, id, target, title, body string) (string, error) {
|
|
if _, e := uuid.Parse(request); e != nil {
|
|
return "", ErrTaskInvalid
|
|
}
|
|
if !ValidTaskWikiPrefix(target) || target == "/" || len(target) > 500 || len(title) > 300 || title == "" || len(body) > 65536 {
|
|
return "", ErrTaskInvalid
|
|
}
|
|
// User/group spaces have additional ownership/publication contracts. Initial
|
|
// capture supports ordinary Wiki paths only, rather than bypassing those caps.
|
|
first := strings.Split(strings.TrimPrefix(target, "/"), "/")[0]
|
|
if first == "u" || first == "g" {
|
|
return "", ErrAccessDenied
|
|
}
|
|
raw, _ := json.Marshal([]string{source, id, target, title, body})
|
|
hash := fmt.Sprintf("%x", sha256.Sum256(raw))
|
|
page := ""
|
|
e := s.feedWrite(ctx, domain, owner, func(tx *sql.Tx) error {
|
|
caps, e := loadLocalityAccess(tx, domain, owner)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
for _, capability := range []string{"read", "create", "write"} {
|
|
basis := authorizeGrantFrom(tx, AuthorizationRequest{DomainUUID: domain, PrincipalUUID: owner, Capability: capability, TargetType: "path", TargetID: target}, false)
|
|
if basis != nil && !errors.Is(basis, ErrAccessDenied) {
|
|
return basis
|
|
}
|
|
if !caps.Decide(target, capability, basis == nil) {
|
|
return ErrAccessDenied
|
|
}
|
|
}
|
|
var oldHash string
|
|
e = tx.QueryRowContext(ctx, `SELECT input_hash,page_uuid FROM feed_note_receipts WHERE domain_uuid=? AND owner_uuid=? AND request_id=?`, domain, owner, request).Scan(&oldHash, &page)
|
|
if e == nil {
|
|
if oldHash != hash {
|
|
return ErrTaskConflict
|
|
}
|
|
return nil
|
|
}
|
|
if !errors.Is(e, sql.ErrNoRows) {
|
|
return e
|
|
}
|
|
var exists bool
|
|
if e = tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid WHERE s.domain_uuid=? AND s.owner_uuid=? AND s.uuid=? AND i.id=?)`, domain, owner, source, id).Scan(&exists); e != nil {
|
|
return e
|
|
}
|
|
if !exists {
|
|
return sql.ErrNoRows
|
|
}
|
|
item := TaskRecord{Title: title}
|
|
if e = s.createCapturedTaskWiki(ctx, tx, domain, owner, &item, target, body); e != nil {
|
|
return e
|
|
}
|
|
page = item.PageUUID
|
|
_, e = tx.ExecContext(ctx, `INSERT INTO feed_note_receipts VALUES(?,?,?,?,?)`, domain, owner, request, hash, page)
|
|
return e
|
|
})
|
|
if e == nil {
|
|
s.InvalidatePathCaches()
|
|
}
|
|
return page, e
|
|
}
|
|
|
|
type FeedSource struct {
|
|
UUID string `json:"uuid"`
|
|
Domain string `json:"-"`
|
|
Owner string `json:"-"`
|
|
Provider string `json:"provider"`
|
|
Title string `json:"title"`
|
|
Folder string `json:"folder"`
|
|
URL string `json:"url"`
|
|
Credential string `json:"-"`
|
|
Version int64 `json:"version"`
|
|
Enabled bool `json:"enabled"`
|
|
Interval int `json:"interval_seconds"`
|
|
NextMS int64 `json:"next_ms"`
|
|
Lease string `json:"-"`
|
|
LeaseUntil int64 `json:"-"`
|
|
LastMS int64 `json:"last_ms"`
|
|
Error string `json:"error"`
|
|
ETag string `json:"-"`
|
|
Modified string `json:"-"`
|
|
}
|
|
type FeedItem struct {
|
|
Source string `json:"source"`
|
|
ID string `json:"id"`
|
|
Title string `json:"title"`
|
|
URL string `json:"url"`
|
|
Body string `json:"body,omitempty"`
|
|
BodyHTML string `json:"body_html,omitempty"`
|
|
PublishedMS int64 `json:"published_ms"`
|
|
Feed string `json:"feed"`
|
|
Folder string `json:"folder"`
|
|
Read bool `json:"read"`
|
|
Starred bool `json:"starred"`
|
|
Version int64 `json:"version"`
|
|
Provider string `json:"provider"`
|
|
}
|
|
|
|
const feedSourceColumns = `uuid,domain_uuid,owner_uuid,provider,title,folder,url,credential,version,enabled,interval_seconds,next_ms,lease,lease_until,last_ms,error,etag,modified`
|
|
|
|
func scanFeedSource(row interface{ Scan(...any) error }) (v FeedSource, e error) {
|
|
e = row.Scan(&v.UUID, &v.Domain, &v.Owner, &v.Provider, &v.Title, &v.Folder, &v.URL, &v.Credential, &v.Version, &v.Enabled, &v.Interval, &v.NextMS, &v.Lease, &v.LeaseUntil, &v.LastMS, &v.Error, &v.ETag, &v.Modified)
|
|
return
|
|
}
|
|
func feedActor(ctx context.Context, q interface {
|
|
QueryRowContext(context.Context, string, ...any) *sql.Row
|
|
}, domain, owner string, write bool) error {
|
|
if e := taskActor(ctx, q, domain, owner, write); e != nil {
|
|
return e
|
|
}
|
|
var raw string
|
|
e := q.QueryRowContext(ctx, `SELECT value FROM config WHERE domain_uuid=? AND key='system_extension_states'`, domain).Scan(&raw)
|
|
if e != nil && !errors.Is(e, sql.ErrNoRows) {
|
|
return e
|
|
}
|
|
states := map[string]bool{}
|
|
if raw != "" && json.Unmarshal([]byte(raw), &states) != nil {
|
|
return ErrAccessDenied
|
|
}
|
|
if enabled, ok := states["organizer.feeds"]; ok && !enabled {
|
|
return ErrAccessDenied
|
|
}
|
|
return nil
|
|
}
|
|
func (s *Store) feedWrite(ctx context.Context, domain, owner string, fn func(*sql.Tx) error) error {
|
|
if s.ReadOnly {
|
|
return ErrReadOnly
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
tx, e := s.DB.BeginTx(ctx, nil)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
defer tx.Rollback()
|
|
if _, e = tx.ExecContext(ctx, `UPDATE domain_memberships SET state=state WHERE domain_uuid=? AND principal_uuid=?`, domain, owner); e != nil {
|
|
return e
|
|
}
|
|
if e = feedActor(ctx, tx, domain, owner, true); e != nil {
|
|
return e
|
|
}
|
|
if e = fn(tx); e != nil {
|
|
return e
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
func (s *Store) FeedSources(ctx context.Context, domain, owner string) ([]FeedSource, error) {
|
|
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
|
|
return nil, e
|
|
}
|
|
rows, e := s.DB.QueryContext(ctx, `SELECT `+feedSourceColumns+` FROM feed_sources WHERE domain_uuid=? AND owner_uuid=? ORDER BY folder,title,uuid LIMIT 32`, domain, owner)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
defer rows.Close()
|
|
out := []FeedSource{}
|
|
for rows.Next() {
|
|
v, e := scanFeedSource(rows)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
out = append(out, v)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
func (s *Store) SaveFeedSource(ctx context.Context, domain, owner string, v FeedSource) (FeedSource, error) {
|
|
if _, e := uuid.Parse(v.UUID); e != nil {
|
|
return v, ErrTaskInvalid
|
|
}
|
|
v.Title = strings.TrimSpace(v.Title)
|
|
v.Folder = strings.TrimSpace(v.Folder)
|
|
if v.Title == "" || len(v.Title) > 200 || len(v.Folder) > 200 || len(v.URL) > 4096 || !utf8.ValidString(v.Title+v.Folder) || v.Interval < 300 || v.Interval > 86400 || (v.Provider != "local" && v.Provider != "freshrss") {
|
|
return v, ErrTaskInvalid
|
|
}
|
|
e := s.feedWrite(ctx, domain, owner, func(tx *sql.Tx) error {
|
|
if v.Version == 0 {
|
|
old, lookup := scanFeedSource(tx.QueryRowContext(ctx, `SELECT `+feedSourceColumns+` FROM feed_sources WHERE uuid=?`, v.UUID))
|
|
if lookup == nil {
|
|
if old.Domain != domain || old.Owner != owner || old.Version != 1 || old.Provider != v.Provider || old.Title != v.Title || old.Folder != v.Folder || old.URL != v.URL || old.Interval != v.Interval || old.Enabled != v.Enabled {
|
|
return ErrTaskConflict
|
|
}
|
|
if old.Credential != v.Credential {
|
|
candidate := v
|
|
candidate.Domain = domain
|
|
candidate.Owner = owner
|
|
u, p, e := s.OpenFeedCredential(candidate)
|
|
if e != nil {
|
|
return ErrTaskConflict
|
|
}
|
|
ou, op, e := s.OpenFeedCredential(old)
|
|
if e != nil || u != ou || p != op {
|
|
return ErrTaskConflict
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
if !errors.Is(lookup, sql.ErrNoRows) {
|
|
return lookup
|
|
}
|
|
var n int
|
|
if e := tx.QueryRowContext(ctx, `SELECT count(*) FROM feed_sources WHERE domain_uuid=? AND owner_uuid=?`, domain, owner).Scan(&n); e != nil {
|
|
return e
|
|
}
|
|
if n >= 32 {
|
|
return ErrTaskDAVLimit
|
|
}
|
|
_, e := tx.ExecContext(ctx, `INSERT INTO feed_sources(uuid,domain_uuid,owner_uuid,provider,title,folder,url,credential,version,enabled,interval_seconds) VALUES(?,?,?,?,?,?,?,?,1,?,?)`, v.UUID, domain, owner, v.Provider, v.Title, v.Folder, v.URL, v.Credential, v.Enabled, v.Interval)
|
|
return e
|
|
}
|
|
old, e := scanFeedSource(tx.QueryRowContext(ctx, `SELECT `+feedSourceColumns+` FROM feed_sources WHERE uuid=? AND domain_uuid=? AND owner_uuid=?`, v.UUID, domain, owner))
|
|
if e != nil {
|
|
return e
|
|
}
|
|
// Changing remote identity requires a new source, not reassignment of cached IDs.
|
|
if old.Provider != v.Provider || old.URL != v.URL || v.Credential != "" {
|
|
return ErrTaskInvalid
|
|
}
|
|
result, e := tx.ExecContext(ctx, `UPDATE feed_sources SET title=?,folder=?,enabled=?,interval_seconds=?,version=version+1,next_ms=0,lease='',lease_until=0 WHERE uuid=? AND version=?`, v.Title, v.Folder, v.Enabled, v.Interval, v.UUID, v.Version)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
n, _ := result.RowsAffected()
|
|
if n != 1 {
|
|
return ErrTaskConflict
|
|
}
|
|
return nil
|
|
})
|
|
v.Version++
|
|
v.Credential = ""
|
|
return v, e
|
|
}
|
|
func (s *Store) FeedSourceAction(ctx context.Context, domain, owner, id, action string, version int64) error {
|
|
return s.feedWrite(ctx, domain, owner, func(tx *sql.Tx) error {
|
|
var result sql.Result
|
|
var e error
|
|
switch action {
|
|
case "refresh":
|
|
result, e = tx.ExecContext(ctx, `UPDATE feed_sources SET next_ms=0 WHERE uuid=? AND domain_uuid=? AND owner_uuid=? AND version=? AND enabled=1 AND lease_until<? AND (last_ms=0 OR last_ms<?)`, id, domain, owner, version, time.Now().UnixMilli(), time.Now().Add(-time.Minute).UnixMilli())
|
|
case "delete":
|
|
result, e = tx.ExecContext(ctx, `DELETE FROM feed_sources WHERE uuid=? AND domain_uuid=? AND owner_uuid=? AND version=?`, id, domain, owner, version)
|
|
default:
|
|
return ErrTaskInvalid
|
|
}
|
|
if e != nil {
|
|
return e
|
|
}
|
|
n, _ := result.RowsAffected()
|
|
if n != 1 {
|
|
return ErrTaskConflict
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
func (s *Store) FeedItems(ctx context.Context, domain, owner, source, filter, query string, before int64, beforeID string) ([]FeedItem, error) {
|
|
if len(query) > 200 || len(beforeID) > 1200 {
|
|
return nil, ErrTaskInvalid
|
|
}
|
|
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
|
|
return nil, e
|
|
}
|
|
rows, e := s.DB.QueryContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.published_ms,i.feed,CASE WHEN s.folder<>'' THEN s.folder ELSE i.folder END,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=? AND s.owner_uuid=? AND (?='' OR s.uuid=?) AND (?='' OR instr(lower(i.title),lower(?))>0 OR instr(lower(i.feed),lower(?))>0) AND (?<0 OR i.published_ms<? OR (i.published_ms=? AND (i.source_uuid||':'||i.id)<?)) AND (?<>'unread' OR (CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END)=0) AND (?<>'starred' OR (CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END)=1) ORDER BY i.published_ms DESC,i.source_uuid||':'||i.id DESC LIMIT 101`, domain, owner, source, source, query, query, query, before, before, before, beforeID, filter, filter)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
defer rows.Close()
|
|
out := []FeedItem{}
|
|
for rows.Next() {
|
|
var v FeedItem
|
|
if e = rows.Scan(&v.Source, &v.ID, &v.Title, &v.URL, &v.PublishedMS, &v.Feed, &v.Folder, &v.Read, &v.Starred, &v.Version, &v.Provider); e != nil {
|
|
return nil, e
|
|
}
|
|
out = append(out, v)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
func (s *Store) FeedItem(ctx context.Context, domain, owner, source, id string) (FeedItem, error) {
|
|
var v FeedItem
|
|
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
|
|
return v, e
|
|
}
|
|
e := s.DB.QueryRowContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.body,i.body_html,i.published_ms,i.feed,i.folder,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=? AND s.owner_uuid=? AND s.uuid=? AND i.id=?`, domain, owner, source, id).Scan(&v.Source, &v.ID, &v.Title, &v.URL, &v.Body, &v.BodyHTML, &v.PublishedMS, &v.Feed, &v.Folder, &v.Read, &v.Starred, &v.Version, &v.Provider)
|
|
return v, e
|
|
}
|
|
func (s *Store) SetFeedState(ctx context.Context, domain, owner, source, id string, read, star bool, version int64) error {
|
|
return s.feedWrite(ctx, domain, owner, func(tx *sql.Tx) error {
|
|
var provider string
|
|
if e := tx.QueryRowContext(ctx, `SELECT s.provider FROM feed_sources s JOIN feed_items i ON i.source_uuid=s.uuid WHERE s.uuid=? AND s.domain_uuid=? AND s.owner_uuid=? AND i.id=?`, source, domain, owner, id).Scan(&provider); e != nil {
|
|
return e
|
|
}
|
|
if provider != "local" {
|
|
return ErrAccessDenied
|
|
}
|
|
if version == 0 {
|
|
var n int
|
|
if e := tx.QueryRowContext(ctx, `SELECT count(*) FROM feed_states WHERE source_uuid=?`, source).Scan(&n); e != nil {
|
|
return e
|
|
}
|
|
if n >= 10000 {
|
|
return ErrTaskDAVLimit
|
|
}
|
|
}
|
|
result, e := tx.ExecContext(ctx, `INSERT INTO feed_states(source_uuid,id,is_read,starred,version) SELECT ?,?,?,?,1 WHERE ?=0 ON CONFLICT(source_uuid,id) DO NOTHING`, source, id, read, star, version)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
n, _ := result.RowsAffected()
|
|
if n == 1 {
|
|
return nil
|
|
}
|
|
result, e = tx.ExecContext(ctx, `UPDATE feed_states SET is_read=?,starred=?,version=version+1 WHERE source_uuid=? AND id=? AND version=?`, read, star, source, id, version)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
n, _ = result.RowsAffected()
|
|
if n != 1 {
|
|
return ErrTaskConflict
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
func (s *Store) SealFeedCredential(domain, owner, id, user, password string) (string, error) {
|
|
b, _ := json.Marshal([]string{"feed-source-v1", domain, owner, id, user, password})
|
|
return s.encryptAuthSecret(b)
|
|
}
|
|
func (s *Store) OpenFeedCredential(v FeedSource) (string, string, error) {
|
|
b, e := s.decryptAuthSecret(v.Credential)
|
|
if e != nil {
|
|
return "", "", e
|
|
}
|
|
var f []string
|
|
if json.Unmarshal(b, &f) != nil || len(f) != 6 || f[0] != "feed-source-v1" || f[1] != v.Domain || f[2] != v.Owner || f[3] != v.UUID {
|
|
return "", "", ErrAccessDenied
|
|
}
|
|
return f[4], f[5], nil
|
|
}
|
|
func (s *Store) EnsureFeedEvent() error {
|
|
if s.ReadOnly {
|
|
return ErrReadOnly
|
|
}
|
|
now := time.Now().UnixMilli()
|
|
_, e := s.DB.Exec(`INSERT OR IGNORE INTO event_definitions(uuid,name,event_type,event_key,payload,schedule_kind,interval_seconds,next_run_at_ms,enabled,priority,max_attempts,created_at_ms,updated_at_ms) VALUES('builtin.feed.sync','RSS source synchronization',?,'local','{}','interval',60,?,1,-5,1,?,?)`, FeedSyncEventType, now, now, now)
|
|
return e
|
|
}
|
|
func (s *Store) DueFeedSources(ctx context.Context) ([]FeedSource, error) {
|
|
rows, e := s.DB.QueryContext(ctx, `SELECT `+feedSourceColumns+` FROM feed_sources WHERE enabled=1 AND next_ms<=? AND lease_until<? AND EXISTS(SELECT 1 FROM principals p JOIN domain_memberships m ON m.principal_uuid=p.uuid JOIN domains d ON d.uuid=m.domain_uuid WHERE p.uuid=feed_sources.owner_uuid AND m.domain_uuid=feed_sources.domain_uuid AND p.state='active' AND m.state='active' AND d.enabled=1 AND COALESCE(d.placement,'local')='local') AND NOT EXISTS(SELECT 1 FROM config c WHERE c.domain_uuid=feed_sources.domain_uuid AND c.key='system_extension_states' AND COALESCE(json_extract(CASE WHEN json_valid(c.value) THEN c.value ELSE '{"organizer.feeds":false}' END,'$."organizer.feeds"'),1)=0) ORDER BY next_ms,uuid LIMIT 4`, time.Now().UnixMilli(), time.Now().UnixMilli())
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
defer rows.Close()
|
|
out := []FeedSource{}
|
|
for rows.Next() {
|
|
v, e := scanFeedSource(rows)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
out = append(out, v)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
func (s *Store) ClaimFeed(ctx context.Context, v FeedSource) (FeedSource, error) {
|
|
v.Lease = uuid.NewString()
|
|
e := s.feedWrite(ctx, v.Domain, v.Owner, func(tx *sql.Tx) error {
|
|
now := time.Now().UnixMilli()
|
|
result, e := tx.ExecContext(ctx, `UPDATE feed_sources SET lease=?,lease_until=?,next_ms=? WHERE uuid=? AND version=? AND enabled=1 AND lease_until<? AND next_ms<=?`, v.Lease, now+60000, now+int64(v.Interval)*1000, v.UUID, v.Version, now, now)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
n, _ := result.RowsAffected()
|
|
if n != 1 {
|
|
return ErrTaskConflict
|
|
}
|
|
return nil
|
|
})
|
|
return v, e
|
|
}
|
|
|
|
// Failed fetches preserve the previous cache. Version and claim token fence stale workers.
|
|
func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem, etag, modified, code string, unchanged bool) error {
|
|
if len(items) > 500 || len(etag) > 1024 || len(modified) > 1024 {
|
|
return ErrTaskInvalid
|
|
}
|
|
switch code {
|
|
case "", "network", "invalid_feed", "credentials", "remote_error", "limit":
|
|
default:
|
|
return ErrTaskInvalid
|
|
}
|
|
for _, i := range items {
|
|
if i.ID == "" || len(i.ID) > 1024 || len(i.Title) > 2048 || len(i.URL) > 4096 || len(i.Body) > 32768 || len(i.BodyHTML) > 128<<10 || len(i.Feed) > 512 || len(i.Folder) > 512 || i.PublishedMS < 0 {
|
|
return ErrTaskInvalid
|
|
}
|
|
}
|
|
return s.feedWrite(ctx, v.Domain, v.Owner, func(tx *sql.Tx) error {
|
|
result, e := tx.ExecContext(ctx, `UPDATE feed_sources SET lease='',lease_until=0,last_ms=CASE WHEN ?='' THEN ? ELSE last_ms END,error=?,etag=CASE WHEN ?='' THEN ? ELSE etag END,modified=CASE WHEN ?='' THEN ? ELSE modified END WHERE uuid=? AND version=? AND lease=? AND enabled=1`, code, time.Now().UnixMilli(), code, code, etag, code, modified, v.UUID, v.Version, v.Lease)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
n, _ := result.RowsAffected()
|
|
if n != 1 {
|
|
return ErrTaskConflict
|
|
}
|
|
if code != "" || unchanged {
|
|
return nil
|
|
}
|
|
// FreshRSS is authoritative: retain exactly this bounded recent window,
|
|
// not stale rows from unsubscribed feeds or old external read/star state.
|
|
if v.Provider == "freshrss" {
|
|
if _, e = tx.ExecContext(ctx, `DELETE FROM feed_items WHERE source_uuid=?`, v.UUID); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
for _, i := range items {
|
|
_, e = tx.ExecContext(ctx, `INSERT INTO feed_items(source_uuid,id,title,url,body,body_html,published_ms,feed,folder,is_read,starred) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_uuid,id) DO UPDATE SET title=excluded.title,url=excluded.url,body=excluded.body,body_html=excluded.body_html,published_ms=excluded.published_ms,feed=excluded.feed,folder=excluded.folder,is_read=excluded.is_read,starred=excluded.starred`, v.UUID, i.ID, i.Title, i.URL, i.Body, i.BodyHTML, i.PublishedMS, i.Feed, i.Folder, i.Read, i.Starred)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
}
|
|
_, e = tx.ExecContext(ctx, `DELETE FROM feed_items WHERE source_uuid=? AND id NOT IN(SELECT id FROM feed_items WHERE source_uuid=? ORDER BY published_ms DESC,id DESC LIMIT 500)`, v.UUID, v.UUID)
|
|
return e
|
|
})
|
|
}
|