feat(feeds): cache FreshRSS subscriptions and category navigation
Build and publish container images / images (push) Failing after 6m7s
Build and publish container images / images (push) Failing after 6m7s
This commit is contained in:
1 parent
7fe64113c7
commit
bfed3d20f5
12 files changed
+503
-34
No files matched your search
+68
-17
@@ -37,6 +37,13 @@ var feedMigrations = []db.SQLMigration{{Version: 1, Name: "private_feed_reader",
|
||||
Version: 3, Name: "feed_import_receipts", Statements: []string{
|
||||
`CREATE TABLE feed_import_receipts(domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,request_id TEXT NOT NULL,input_hash TEXT NOT NULL,result TEXT NOT NULL,PRIMARY KEY(domain_uuid,owner_uuid,request_id))`,
|
||||
}, Assertions: []string{schemaUnique("feed_import_receipts", "domain_uuid,owner_uuid,request_id")},
|
||||
}, {
|
||||
Version: 4, Name: "freshrss_catalog", Statements: []string{
|
||||
`ALTER TABLE feed_sources ADD COLUMN remote_catalog TEXT NOT NULL DEFAULT '[]'`,
|
||||
`ALTER TABLE feed_items ADD COLUMN remote_feed TEXT NOT NULL DEFAULT ''`,
|
||||
`ALTER TABLE feed_items ADD COLUMN remote_categories TEXT NOT NULL DEFAULT '[]'`,
|
||||
`UPDATE feed_sources SET next_ms=0,lease='',lease_until=0,version=version+1 WHERE provider='freshrss'`,
|
||||
}, Assertions: []string{schemaColumns("feed_sources", "remote_catalog"), schemaColumns("feed_items", "remote_feed,remote_categories")},
|
||||
}}
|
||||
|
||||
// CaptureFeedNote creates an inert Wiki note and receipt atomically. It does not
|
||||
@@ -124,19 +131,21 @@ type FeedSource struct {
|
||||
Modified string `json:"-"`
|
||||
}
|
||||
type FeedItem struct {
|
||||
Source string `json:"source"`
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Body string `json:"body,omitempty"`
|
||||
BodyHTML string `json:"body_html,omitempty"`
|
||||
PublishedMS int64 `json:"published_ms"`
|
||||
Feed string `json:"feed"`
|
||||
Folder string `json:"folder"`
|
||||
Read bool `json:"read"`
|
||||
Starred bool `json:"starred"`
|
||||
Version int64 `json:"version"`
|
||||
Provider string `json:"provider"`
|
||||
Source string `json:"source"`
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Body string `json:"body,omitempty"`
|
||||
BodyHTML string `json:"body_html,omitempty"`
|
||||
PublishedMS int64 `json:"published_ms"`
|
||||
Feed string `json:"feed"`
|
||||
Folder string `json:"folder"`
|
||||
Read bool `json:"read"`
|
||||
Starred bool `json:"starred"`
|
||||
Version int64 `json:"version"`
|
||||
Provider string `json:"provider"`
|
||||
RemoteFeed string `json:"remote_feed,omitempty"`
|
||||
RemoteCategories []string `json:"remote_categories,omitempty"`
|
||||
}
|
||||
|
||||
const feedSourceColumns = `uuid,domain_uuid,owner_uuid,provider,title,folder,url,credential,version,enabled,interval_seconds,next_ms,lease,lease_until,last_ms,error,etag,modified`
|
||||
@@ -294,14 +303,27 @@ func (s *Store) FeedSourceAction(ctx context.Context, domain, owner, id, action
|
||||
return nil
|
||||
})
|
||||
}
|
||||
func (s *Store) FeedItems(ctx context.Context, domain, owner, source, filter, query string, before int64, beforeID string) ([]FeedItem, error) {
|
||||
func (s *Store) FeedItems(ctx context.Context, domain, owner, source, filter, query string, before int64, beforeID string, scope ...string) ([]FeedItem, error) {
|
||||
remoteFeed, category := "", ""
|
||||
if len(scope) > 2 {
|
||||
return nil, ErrTaskInvalid
|
||||
}
|
||||
if len(scope) > 0 {
|
||||
remoteFeed = scope[0]
|
||||
}
|
||||
if len(scope) > 1 {
|
||||
category = scope[1]
|
||||
}
|
||||
if (remoteFeed != "" && !ValidFeedRemoteID(remoteFeed)) || (category != "" && !ValidFeedRemoteID(category)) || ((remoteFeed != "" || category != "") && source == "") {
|
||||
return nil, ErrTaskInvalid
|
||||
}
|
||||
if len(query) > 200 || len(beforeID) > 1200 {
|
||||
return nil, ErrTaskInvalid
|
||||
}
|
||||
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
rows, e := s.DB.QueryContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.published_ms,i.feed,CASE WHEN s.folder<>'' THEN s.folder ELSE i.folder END,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=? AND s.owner_uuid=? AND (?='' OR s.uuid=?) AND (?='' OR instr(lower(i.title),lower(?))>0 OR instr(lower(i.feed),lower(?))>0) AND (?<0 OR i.published_ms<? OR (i.published_ms=? AND (i.source_uuid||':'||i.id)<?)) AND (?<>'unread' OR (CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END)=0) AND (?<>'starred' OR (CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END)=1) ORDER BY i.published_ms DESC,i.source_uuid||':'||i.id DESC LIMIT 101`, domain, owner, source, source, query, query, query, before, before, before, beforeID, filter, filter)
|
||||
rows, e := s.DB.QueryContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.published_ms,i.feed,CASE WHEN s.folder<>'' THEN s.folder ELSE i.folder END,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=? AND s.owner_uuid=? AND (?='' OR s.uuid=?) AND (?='' OR instr(lower(i.title),lower(?))>0 OR instr(lower(i.feed),lower(?))>0) AND (?<0 OR i.published_ms<? OR (i.published_ms=? AND (i.source_uuid||':'||i.id)<?)) AND (?<>'unread' OR (CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END)=0) AND (?<>'starred' OR (CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END)=1) AND (?='' OR i.remote_feed=?) AND (?='' OR EXISTS(SELECT 1 FROM json_each(i.remote_categories) WHERE value=?)) ORDER BY i.published_ms DESC,i.source_uuid||':'||i.id DESC LIMIT 101`, domain, owner, source, source, query, query, query, before, before, before, beforeID, filter, filter, remoteFeed, remoteFeed, category, category)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
@@ -418,7 +440,18 @@ func (s *Store) ClaimFeed(ctx context.Context, v FeedSource) (FeedSource, error)
|
||||
}
|
||||
|
||||
// Failed fetches preserve the previous cache. Version and claim token fence stale workers.
|
||||
func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem, etag, modified, code string, unchanged bool) error {
|
||||
func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem, etag, modified, code string, unchanged bool, catalogs ...[]FeedRemoteSubscription) error {
|
||||
if len(catalogs) > 1 {
|
||||
return ErrTaskInvalid
|
||||
}
|
||||
var catalog string
|
||||
if len(catalogs) == 1 {
|
||||
var e error
|
||||
catalog, e = FeedCatalogJSON(catalogs[0])
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
if len(items) > 500 || len(etag) > 1024 || len(modified) > 1024 {
|
||||
return ErrTaskInvalid
|
||||
}
|
||||
@@ -428,6 +461,14 @@ func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem
|
||||
return ErrTaskInvalid
|
||||
}
|
||||
for _, i := range items {
|
||||
if (i.RemoteFeed != "" && !ValidFeedRemoteID(i.RemoteFeed)) || len(i.RemoteCategories) > 64 {
|
||||
return ErrTaskInvalid
|
||||
}
|
||||
for _, c := range i.RemoteCategories {
|
||||
if !ValidFeedRemoteID(c) {
|
||||
return ErrTaskInvalid
|
||||
}
|
||||
}
|
||||
if i.ID == "" || len(i.ID) > 1024 || len(i.Title) > 2048 || len(i.URL) > 4096 || len(i.Body) > 32768 || len(i.BodyHTML) > 128<<10 || len(i.Feed) > 512 || len(i.Folder) > 512 || i.PublishedMS < 0 {
|
||||
return ErrTaskInvalid
|
||||
}
|
||||
@@ -444,6 +485,11 @@ func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem
|
||||
if code != "" || unchanged {
|
||||
return nil
|
||||
}
|
||||
if catalog != "" {
|
||||
if _, e = tx.ExecContext(ctx, `UPDATE feed_sources SET remote_catalog=? WHERE uuid=?`, catalog, v.UUID); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
// FreshRSS is authoritative: retain exactly this bounded recent window,
|
||||
// not stale rows from unsubscribed feeds or old external read/star state.
|
||||
if v.Provider == "freshrss" {
|
||||
@@ -452,7 +498,12 @@ func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem
|
||||
}
|
||||
}
|
||||
for _, i := range items {
|
||||
_, e = tx.ExecContext(ctx, `INSERT INTO feed_items(source_uuid,id,title,url,body,body_html,published_ms,feed,folder,is_read,starred) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_uuid,id) DO UPDATE SET title=excluded.title,url=excluded.url,body=excluded.body,body_html=excluded.body_html,published_ms=excluded.published_ms,feed=excluded.feed,folder=excluded.folder,is_read=excluded.is_read,starred=excluded.starred`, v.UUID, i.ID, i.Title, i.URL, i.Body, i.BodyHTML, i.PublishedMS, i.Feed, i.Folder, i.Read, i.Starred)
|
||||
categories := i.RemoteCategories
|
||||
if categories == nil {
|
||||
categories = []string{}
|
||||
}
|
||||
raw, _ := json.Marshal(categories)
|
||||
_, e = tx.ExecContext(ctx, `INSERT INTO feed_items(source_uuid,id,title,url,body,body_html,published_ms,feed,folder,is_read,starred,remote_feed,remote_categories) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_uuid,id) DO UPDATE SET title=excluded.title,url=excluded.url,body=excluded.body,body_html=excluded.body_html,published_ms=excluded.published_ms,feed=excluded.feed,folder=excluded.folder,is_read=excluded.is_read,starred=excluded.starred,remote_feed=excluded.remote_feed,remote_categories=excluded.remote_categories`, v.UUID, i.ID, i.Title, i.URL, i.Body, i.BodyHTML, i.PublishedMS, i.Feed, i.Folder, i.Read, i.Starred, i.RemoteFeed, string(raw))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Remote IDs are opaque identities, not URLs to fetch or display labels.
|
||||
type FeedCategory struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
}
|
||||
type FeedRemoteSubscription struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Categories []FeedCategory `json:"categories"`
|
||||
}
|
||||
|
||||
func ValidFeedRemoteID(id string) bool {
|
||||
return id != "" && len(id) <= 2048 && utf8.ValidString(id) && !strings.ContainsAny(id, "\x00\r\n")
|
||||
}
|
||||
|
||||
// Bound both the number of entries and the encoded snapshot independently of
|
||||
// the HTTP response limit. Invalid snapshots must not erase a good cache.
|
||||
func FeedCatalogJSON(catalog []FeedRemoteSubscription) (string, error) {
|
||||
if len(catalog) > 1000 {
|
||||
return "", ErrTaskDAVLimit
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
labels := map[string]string{}
|
||||
for _, sub := range catalog {
|
||||
if !ValidFeedRemoteID(sub.ID) || seen[sub.ID] || len(sub.Title) > 512 || !utf8.ValidString(sub.Title) || len(sub.Categories) > 64 {
|
||||
return "", ErrTaskInvalid
|
||||
}
|
||||
seen[sub.ID] = true
|
||||
cats := map[string]bool{}
|
||||
for _, c := range sub.Categories {
|
||||
if !ValidFeedRemoteID(c.ID) || cats[c.ID] || len(c.Label) > 512 || !utf8.ValidString(c.Label) {
|
||||
return "", ErrTaskInvalid
|
||||
}
|
||||
if label, ok := labels[c.ID]; ok && label != c.Label {
|
||||
return "", ErrTaskInvalid
|
||||
}
|
||||
cats[c.ID], labels[c.ID] = true, c.Label
|
||||
}
|
||||
}
|
||||
if catalog == nil {
|
||||
catalog = []FeedRemoteSubscription{}
|
||||
}
|
||||
raw, e := json.Marshal(catalog)
|
||||
if e != nil {
|
||||
return "", e
|
||||
}
|
||||
if len(raw) > 1<<20 {
|
||||
return "", ErrTaskDAVLimit
|
||||
}
|
||||
return string(raw), nil
|
||||
}
|
||||
|
||||
func (s *Store) FeedCatalog(ctx context.Context, domain, owner, source string) ([]FeedRemoteSubscription, error) {
|
||||
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
var raw string
|
||||
if e := s.DB.QueryRowContext(ctx, `SELECT remote_catalog FROM feed_sources WHERE uuid=? AND domain_uuid=? AND owner_uuid=?`, source, domain, owner).Scan(&raw); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
var catalog []FeedRemoteSubscription
|
||||
if e := json.Unmarshal([]byte(raw), &catalog); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
return catalog, nil
|
||||
}
|
||||
@@ -109,3 +109,44 @@ func TestFeedImportReceiptMigrationRollbackAndRepeat(t *testing.T) {
|
||||
t.Fatal("receipt must survive deletion", count, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedCatalogMigrationRollbackAndRepeat(t *testing.T) {
|
||||
st := newSchemaTestStore(t)
|
||||
ctx := context.Background()
|
||||
if e := db.ApplySQLiteMigrations(ctx, st.DB, "feeds", feedMigrations[:3]); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := st.DB.Exec(`INSERT INTO feed_sources(uuid,domain_uuid,owner_uuid,provider,title,folder,url,version,enabled,interval_seconds,lease,next_ms) VALUES('s','d','u','freshrss','Title','','https://example.org',7,1,300,'old',999); INSERT INTO feed_items(source_uuid,id,title,url,body,published_ms,feed,folder,is_read,starred,body_html) VALUES('s','i','Title','','Body',1,'','',1,1,'<p>Body</p>')`); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
broken := append([]db.SQLMigration(nil), feedMigrations...)
|
||||
broken[3].Statements = append(append([]string(nil), feedMigrations[3].Statements...), `INSERT INTO no_such_feed_table VALUES(1)`)
|
||||
if e := db.ApplySQLiteMigrations(ctx, st.DB, "feeds", broken); e == nil {
|
||||
t.Fatal("expected failure")
|
||||
}
|
||||
var count int
|
||||
if e := st.DB.QueryRow(`SELECT count(*) FROM pragma_table_info('feed_items') WHERE name='remote_feed'`).Scan(&count); e != nil || count != 0 {
|
||||
t.Fatal("partial migration", count, e)
|
||||
}
|
||||
if e := st.DB.QueryRow(`SELECT version FROM feed_sources WHERE uuid='s'`).Scan(&count); e != nil || count != 7 {
|
||||
t.Fatal("partial fence", count, e)
|
||||
}
|
||||
for range 2 {
|
||||
if e := st.ensureSchemaComponent("feeds"); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
var version, next int
|
||||
var catalog, lease, body string
|
||||
var read, star bool
|
||||
if e := st.DB.QueryRow(`SELECT version,next_ms,lease,remote_catalog FROM feed_sources WHERE uuid='s'`).Scan(&version, &next, &lease, &catalog); e != nil || version != 8 || next != 0 || lease != "" || catalog != "[]" {
|
||||
t.Fatal(version, next, lease, catalog, e)
|
||||
}
|
||||
if e := st.DB.QueryRow(`SELECT body_html,is_read,starred FROM feed_items WHERE source_uuid='s'`).Scan(&body, &read, &star); e != nil || body != "<p>Body</p>" || !read || !star {
|
||||
t.Fatal("lost old cache", body, read, star, e)
|
||||
}
|
||||
var integrity string
|
||||
if e := st.DB.QueryRow(`PRAGMA integrity_check`).Scan(&integrity); e != nil || integrity != "ok" {
|
||||
t.Fatal(integrity, e)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user