feat(feeds): inherit FreshRSS folders in cached subscription tree
Build and publish container images / images (push) Successful in 10m33s

This commit is contained in:
Anya Tokugawa committed 2026-11-08 13:34:22 +09:00
1 parent 1057be6d2b
commit bf9a9c2847
14 files changed
+408 -55

No files matched your search

+3 -3
View File
@@ -51,12 +51,12 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
}
switch {
case key == "catalog" && r.Method == "GET":
catalog, e := s.store.FeedCatalog(r.Context(), domain, actor, r.URL.Query().Get("source"))
catalog, e := s.store.FeedCatalogSnapshot(r.Context(), domain, actor, r.URL.Query().Get("source"))
if e != nil {
taskError(w, e)
return
}
writeEventJSON(w, 200, map[string]any{"subscriptions": catalog})
writeEventJSON(w, 200, catalog)
return
case key == "note" && r.Method == "POST":
var input struct {
@@ -170,7 +170,7 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
return
}
}
items, e := s.store.FeedItems(r.Context(), domain, actor, q.Get("source"), q.Get("filter"), q.Get("q"), before, q.Get("key"), q.Get("feed"), q.Get("category"))
items, e := s.store.FeedItems(r.Context(), domain, actor, q.Get("source"), q.Get("filter"), q.Get("q"), before, q.Get("key"), q.Get("feed"), q.Get("category"), q.Get("category_mode"))
if e != nil {
taskError(w, e)
return
+16 -3
View File
@@ -49,6 +49,7 @@ func TestFeedCatalogBackgroundSnapshotIsolation(t *testing.T) {
}
calls := 0
failCatalog := false
failFolders := false
client := &http.Client{Transport: feedTransportFunc(func(r *http.Request) (*http.Response, error) {
calls++
body := ""
@@ -63,7 +64,7 @@ func TestFeedCatalogBackgroundSnapshotIsolation(t *testing.T) {
t.Error("article request")
}
body = `{"items":[{"id":"a","title":"First","origin":{"title":"Same title","streamId":"feed/1"},"categories":["user/alice/label/Tech"]},{"id":"b","title":"Second","origin":{"title":"Same title","streamId":"feed/2"},"categories":["user/alice/label/Other"]}]}`
if failCatalog {
if failCatalog || failFolders {
body = `{"items":[]}`
}
case strings.HasSuffix(r.URL.Path, "/subscription/list"):
@@ -74,6 +75,11 @@ func TestFeedCatalogBackgroundSnapshotIsolation(t *testing.T) {
return nil, errors.New("synthetic timeout")
}
body = testFeedCatalog
case strings.HasSuffix(r.URL.Path, "/tag/list"):
if failFolders {
return nil, errors.New("synthetic folder failure")
}
body = `{"tags":[{"id":"user/-/label/Empty/Folder","type":"folder"},{"id":"user/-/label/Not a folder","type":"tag"}]}`
default:
t.Fatalf("unexpected external request %s %s", r.Method, r.URL.Path)
}
@@ -89,7 +95,7 @@ func TestFeedCatalogBackgroundSnapshotIsolation(t *testing.T) {
}
}
run()
if calls != 3 {
if calls != 4 {
t.Fatal("fetch count", calls)
}
w := feedRequestTest(s, token, "GET", "catalog?source="+v.UUID, nil)
@@ -127,9 +133,16 @@ func TestFeedCatalogBackgroundSnapshotIsolation(t *testing.T) {
if _, e := s.store.FeedCatalog(ctx, uuid.NewString(), actor, v.UUID); e == nil {
t.Fatal("cross-domain catalog leak")
}
if calls != 3 {
if calls != 4 {
t.Fatal("foreground external request")
}
failFolders = true
run()
kept, e := s.store.FeedCatalogSnapshot(ctx, domain, actor, v.UUID)
if e != nil || len(kept.Folders) != 3 || len(kept.Subscriptions) != 3 {
t.Fatal("folder failure erased snapshot", kept, e)
}
failFolders = false
failCatalog = true
run()
catalog, e := s.store.FeedCatalog(ctx, domain, actor, v.UUID)
+19 -3
View File
@@ -345,7 +345,7 @@ func freshRSSLogin(ctx context.Context, client *http.Client, v store.FeedSource,
}
return auth, nil
}
func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string, catalogs ...*[]store.FeedRemoteSubscription) ([]store.FeedItem, error) {
func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string, catalogs ...*store.FeedCatalogSnapshot) ([]store.FeedItem, error) {
base := strings.TrimRight(v.URL, "/")
auth, e := freshRSSLogin(ctx, client, v, user, password)
if e != nil {
@@ -368,7 +368,23 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
if e != nil {
return nil, e
}
*catalogs[0] = catalog
raw, _, _, e = feedRequest(ctx, client, "GET", base+"/reader/api/0/tag/list?output=json", auth, nil, nil)
if e != nil {
return nil, e
}
folders, e := parseFreshRSSFolders(raw)
if e != nil {
return nil, e
}
folders, e = store.FeedFolders(catalog, folders)
if e != nil {
return nil, errFeedInvalid
}
snapshot := store.FeedCatalogSnapshot{Subscriptions: catalog, Folders: folders}
if _, _, e = snapshot.Encode(); e != nil {
return nil, errFeedInvalid
}
*catalogs[0] = snapshot
}
return items, nil
}
@@ -504,7 +520,7 @@ func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.Event
}
child, stop := context.WithTimeout(ctx, 8*time.Second)
var items []store.FeedItem
var catalog []store.FeedRemoteSubscription
var catalog store.FeedCatalogSnapshot
etag, modified := v.ETag, v.Modified
unchanged := false
if !validCalendarSourceURL(v.URL) {
+42
View File
@@ -0,0 +1,42 @@
package server
import (
"encoding/json"
"strings"
"unicode/utf8"
"pageswiki/internal/store"
)
func parseFreshRSSFolders(raw []byte) ([]store.FeedCategory, error) {
if len(raw) > 4<<20 || !utf8.Valid(raw) {
return nil, errFeedInvalid
}
var response struct {
Tags []struct {
ID string `json:"id"`
Type string `json:"type"`
} `json:"tags"`
}
if json.Unmarshal(raw, &response) != nil || response.Tags == nil || len(response.Tags) > 10000 {
return nil, errFeedInvalid
}
folders := []store.FeedCategory{}
seen := map[string]bool{}
for _, tag := range response.Tags {
// Article tags and system state tags are not subscription folders.
if tag.Type != "folder" {
continue
}
prefix, label, ok := strings.Cut(tag.ID, "/label/")
if !ok || !strings.HasPrefix(prefix, "user/") || len(prefix) <= 5 || label == "" || !store.ValidFeedRemoteID(tag.ID) || seen[tag.ID] {
return nil, errFeedInvalid
}
seen[tag.ID] = true
folders = append(folders, store.FeedCategory{ID: tag.ID, Label: feedText(label, 512)})
}
if _, e := store.FeedFolders(nil, folders); e != nil {
return nil, errFeedInvalid
}
return folders, nil
}
+95
View File
@@ -0,0 +1,95 @@
package server
import (
"context"
"encoding/json"
"fmt"
"net/url"
"strings"
"testing"
"pageswiki/internal/store"
)
func TestFreshRSSFoldersParsing(t *testing.T) {
folders, e := parseFreshRSSFolders([]byte(`{"tags":[{"id":"user/-/state/com.google/starred"},{"id":"user/-/label/ArticleTag","type":"tag"},{"id":"user/-/label/技術/Linux","type":"folder"}]}`))
if e != nil || len(folders) != 1 || folders[0].Label != "技術/Linux" {
t.Fatal(folders, e)
}
for _, raw := range []string{`{}`, `{"tags":null}`, `{"tags":[{"id":"invalid","type":"folder"}]}`, `{"tags":[{"id":"user/-/label/","type":"folder"}]}`, `{"tags":[{"id":"user/-/label/A","type":"folder"},{"id":"user/-/label/A","type":"folder"}]}`} {
if _, e = parseFreshRSSFolders([]byte(raw)); e == nil {
t.Fatal("invalid folder accepted", raw)
}
}
tags := []map[string]string{}
for i := 0; i < 1001; i++ {
tags = append(tags, map[string]string{"id": fmt.Sprintf("user/-/label/%d", i), "type": "folder"})
}
raw, _ := json.Marshal(map[string]any{"tags": tags})
if _, e = parseFreshRSSFolders(raw); e == nil {
t.Fatal("unbounded folders")
}
if _, e = store.FeedFolders([]store.FeedRemoteSubscription{{ID: "feed/1", Categories: []store.FeedCategory{{ID: "cat", Label: "A"}}}}, []store.FeedCategory{{ID: "cat", Label: "B"}}); e == nil {
t.Fatal("conflicting labels")
}
large := store.FeedCatalogSnapshot{}
for i := 0; i < 1000; i++ {
large.Subscriptions = append(large.Subscriptions, store.FeedRemoteSubscription{ID: fmt.Sprintf("feed/%d", i), Title: strings.Repeat("s", 512)})
large.Folders = append(large.Folders, store.FeedCategory{ID: fmt.Sprintf("user/-/label/%d", i), Label: strings.Repeat("f", 512)})
}
if _, e := store.FeedCatalogJSON(large.Subscriptions); e != nil {
t.Fatal("catalog alone should fit", e)
}
if _, _, e := large.Encode(); e != store.ErrTaskDAVLimit {
t.Fatal("combined snapshot limit", e)
}
}
func TestFeedFolderMembershipAndUncategorized(t *testing.T) {
s, token, domain, actor, source := feedActionFixture(t)
ctx := context.Background()
if _, e := s.store.DB.Exec(`UPDATE feed_sources SET next_ms=0 WHERE uuid=?`, source.UUID); e != nil {
t.Fatal(e)
}
sources, e := s.store.FeedSources(ctx, domain, actor)
if e != nil {
t.Fatal(e)
}
for _, v := range sources {
if v.UUID == source.UUID {
source = v
}
}
claim, e := s.store.ClaimFeed(ctx, source)
if e != nil {
t.Fatal(e)
}
cat := store.FeedCategory{ID: "user/-/label/New/Folder", Label: "New/Folder"}
snapshot := store.FeedCatalogSnapshot{Subscriptions: []store.FeedRemoteSubscription{{ID: "feed/1", Title: "Moved", Categories: []store.FeedCategory{cat}}, {ID: "feed/2", Title: "Unclassified", Categories: []store.FeedCategory{}}}, Folders: []store.FeedCategory{{ID: "user/-/label/Empty", Label: "Empty"}}}
items := []store.FeedItem{{ID: "a", Title: "Moved", RemoteFeed: "feed/1", RemoteCategories: []string{"user/-/label/Old"}}, {ID: "b", Title: "Unclassified", RemoteFeed: "feed/2", RemoteCategories: []string{"user/-/label/ArticleTag"}}, {ID: "c", Title: "Unknown subscription", RemoteFeed: "feed/unknown"}}
if e = s.store.CompleteFeed(ctx, claim, items, "", "", "", false, snapshot); e != nil {
t.Fatal(e)
}
for _, test := range []struct {
query string
want string
}{{"category=" + url.QueryEscape(cat.ID), "a"}, {"category_mode=uncategorized", "b"}, {"category=user%2F-%2Flabel%2FOld", ""}, {"category=user%2F-%2Flabel%2FEmpty", ""}} {
w := feedRequestTest(s, token, "GET", "items?source="+source.UUID+"&"+test.query, nil)
var data struct {
Items []store.FeedItem `json:"items"`
}
json.Unmarshal(w.Body.Bytes(), &data)
if w.Code != 200 || (test.want == "" && len(data.Items) != 0) || (test.want != "" && (len(data.Items) != 1 || data.Items[0].ID != test.want)) {
t.Fatal(test.query, w.Code, w.Body)
}
}
for _, query := range []string{"category_mode=uncategorized", "source=" + source.UUID + "&category_mode=bad", "source=" + source.UUID + "&category_mode=uncategorized&category=" + url.QueryEscape(cat.ID)} {
if w := feedRequestTest(s, token, "GET", "items?"+query, nil); w.Code != 422 {
t.Fatal(query, w.Code)
}
}
got, e := s.store.FeedCatalogSnapshot(ctx, domain, actor, source.UUID)
if e != nil || len(got.Folders) != 2 {
t.Fatal(got, e)
}
}
+1 -1
View File
@@ -569,7 +569,7 @@ func TestFeedsChromium(t *testing.T) {
if e != nil {
t.Fatal(e)
}
if e = s.store.CompleteFeed(context.Background(), claim, []store.FeedItem{{ID: "catalog-a", Title: "Catalog first", RemoteFeed: "feed/1", RemoteCategories: []string{"user/alice/label/Tech"}}, {ID: "catalog-b", Title: "Catalog second", RemoteFeed: "feed/2", RemoteCategories: []string{"user/alice/label/Other"}}}, "", "", "", false, catalog); e != nil {
if e = s.store.CompleteFeed(context.Background(), claim, []store.FeedItem{{ID: "catalog-a", Title: "Catalog first", RemoteFeed: "feed/1", RemoteCategories: []string{"user/alice/label/Tech"}}, {ID: "catalog-b", Title: "Catalog second", RemoteFeed: "feed/2", RemoteCategories: []string{"user/alice/label/Other"}}}, "", "", "", false, store.FeedCatalogSnapshot{Subscriptions: catalog, Folders: []store.FeedCategory{{ID: "user/-/label/Empty/Folder", Label: "Empty/Folder"}}}); e != nil {
t.Fatal(e)
}
s.store.BackgroundJobsDisabled = true
+13 -5
View File
@@ -4,6 +4,7 @@
const frame=document.createElement('iframe');frame.style='display:block;width:100%;height:100vh;border:0';document.body.append(frame);
try{
await new Promise(resolve=>{frame.onload=resolve;frame.src='/v/feeds';});const d=frame.contentDocument,w=frame.contentWindow;
for(const key of Object.keys(w.localStorage))if(key.startsWith('pageswiki.feed-folders.v1:'))w.localStorage.removeItem(key);
await wait(()=>d.querySelectorAll('.feed-row').length===100);
assert(d.documentElement.scrollWidth<=w.innerWidth,'Viewport overflow');
assert(d.title.startsWith('RSS ·'),'RSS title');assert(d.getElementById('feed-status').textContent==='','Unrequested status copy');assert(w.getComputedStyle(d.getElementById('feed-status')).display==='none','Empty status takes space');
@@ -27,10 +28,15 @@
assert(d.documentElement.scrollWidth<=w.innerWidth,'OPML overflow');d.querySelector('.feed-import-options input[type=checkbox]').checked=false;d.getElementById('feed-opml-preview').requestSubmit();await wait(()=>!d.getElementById('feed-opml-preview'));
assert(d.getElementById('feed-sources').textContent.includes('Import '+w.innerWidth),'Imported source missing');d.getElementById('feed-opml').click();await wait(()=>d.getElementById('feed-opml-file'));choose();await wait(()=>d.querySelectorAll('#feed-opml-preview tbody tr').length===3);assert(d.querySelector('#feed-opml-preview tbody input').disabled,'Duplicate after import');
assert(d.documentElement.scrollWidth<=w.innerWidth,'OPML preview overflow');
[...d.querySelectorAll('.feed-source>button:first-child')].find(b=>b.textContent==='FreshRSS fixture').click();await wait(()=>d.querySelectorAll('.feed-subscriptions button').length===4&&d.querySelectorAll('.feed-row').length===2);
const category=d.getElementById('feed-category');category.value='user/alice/label/Tech';category.dispatchEvent(new w.Event('change'));await wait(()=>d.querySelectorAll('.feed-row').length===1&&d.querySelector('.feed-row').textContent.includes('Catalog first'));
assert(d.querySelectorAll('.feed-subscriptions button').length===2,'Category navigation');
d.getElementById('feed-category').value='';d.getElementById('feed-category').dispatchEvent(new w.Event('change'));await wait(()=>d.querySelectorAll('.feed-row').length===2);
const freshButton=()=>[...d.querySelectorAll('.feed-source>button:first-child')].find(b=>b.textContent==='FreshRSS fixture');freshButton().click();await wait(()=>d.querySelectorAll('.feed-folder').length===4&&d.querySelectorAll('.feed-row').length===2);
const folder=id=>[...d.querySelectorAll('.feed-folder')].find(n=>n.dataset.category===id),tech=()=>folder('user/alice/label/Tech');
tech().querySelector('.feed-folder-name').click();await wait(()=>d.querySelectorAll('.feed-row').length===1&&d.querySelector('.feed-row').textContent.includes('Catalog first'));
assert(tech().querySelector('.feed-folder-toggle').getAttribute('aria-expanded')==='false','Selecting folder expands it unexpectedly');tech().querySelector('.feed-folder-toggle').click();await wait(()=>tech().querySelector('button[data-feed="feed/1"]'));
const empty=folder('user/-/label/Empty/Folder');assert(empty.querySelector('.feed-folder-name span').textContent==='Empty/Folder','Slash label split');empty.querySelector('.feed-folder-toggle').click();assert(!empty.querySelector('button[data-feed]'),'Empty folder fabricated a feed');empty.querySelector('.feed-folder-name').click();await wait(()=>d.querySelectorAll('.feed-row').length===0);
d.querySelector('.feed-folder[data-mode="uncategorized"] .feed-folder-name').click();await wait(()=>d.querySelectorAll('.feed-row').length===0);
d.querySelector('.feed-catalog .feed-tree-all').click();await wait(()=>d.querySelectorAll('.feed-row').length===2);
[...d.querySelectorAll('.feed-source>button:first-child')].find(b=>b.textContent==='Synthetic news').click();await wait(()=>!d.querySelector('.feed-catalog'));freshButton().click();await wait(()=>tech()?.querySelector('button[data-feed="feed/1"]'));assert(tech().querySelector('.feed-folder-toggle').getAttribute('aria-expanded')==='true','Folder expansion not retained');
folder('user/alice/label/Other').querySelector('.feed-folder-toggle').click();await wait(()=>d.querySelector('.feed-subscriptions button[data-feed="feed/2"]'));
d.querySelector('.feed-subscriptions button[data-feed="feed/2"]').click();await wait(()=>d.querySelectorAll('.feed-row').length===1&&d.querySelector('.feed-row').textContent.includes('Catalog second'));
d.querySelector('.feed-row').click();await wait(()=>d.querySelector('#feed-reader .feed-body'));assert(d.querySelectorAll('#feed-reader .feed-actions button:disabled').length===2,'FreshRSS not read-only');
[...d.querySelectorAll('.feed-source')].find(n=>n.textContent.includes('FreshRSS fixture')).querySelector('button:last-child').click();await wait(()=>d.querySelector('#feed-reader form'));
@@ -52,6 +58,8 @@
const hs=d.getElementById('feed-history-state');hs.value='all';hs.dispatchEvent(new w.Event('change'));await wait(()=>receipt(seed.missing)?.querySelector('[data-state=confirmed]'));
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'Escape',bubbles:true}));assert(!d.querySelector('.feed-layout').classList.contains('action-history'),'History escape');
d.getElementById('feed-history').click();await wait(()=>receipt(seed.orphan));d.getElementById('feed-history-state').value='all';d.getElementById('feed-history-state').dispatchEvent(new w.Event('change'));await wait(()=>receipt(seed.missing)?.querySelector('[data-state=confirmed]'));
assert(d.documentElement.scrollWidth<=w.innerWidth,'History final overflow');document.body.dataset.vimReady='1';
assert(d.documentElement.scrollWidth<=w.innerWidth,'History final overflow');
const probe=document.createElement('iframe');probe.hidden=true;document.body.append(probe);await new Promise(resolve=>{probe.onload=resolve;probe.src='/v/feeds?source='+encodeURIComponent(source.uuid);});await wait(()=>probe.contentDocument.querySelector('.feed-folder[data-category="user/alice/label/Tech"] .feed-folder-toggle')?.getAttribute('aria-expanded')==='true');probe.remove();
freshButton().click();await wait(()=>tech()?.querySelector('button[data-feed="feed/1"]'));if(w.innerWidth<=700){d.querySelector('#feed-toolbar button').click();await wait(()=>d.querySelector('.feed-layout').classList.contains('sources-open'));assert(w.getComputedStyle(d.getElementById('feed-sources')).display!=='none','Mobile folders hidden');}assert(d.documentElement.scrollWidth<=w.innerWidth,'Folder tree overflow');document.body.dataset.vimReady='1';
}catch(e){const out=document.createElement('pre');out.id='error';out.textContent=e.stack;document.body.append(out);}
})();
+36 -6
View File
@@ -54,6 +54,10 @@ var feedMigrations = []db.SQLMigration{{Version: 1, Name: "private_feed_reader",
`CREATE TRIGGER feed_actions_source_change AFTER UPDATE OF version ON feed_sources BEGIN UPDATE feed_actions SET status=CASE WHEN status IN ('sending','unknown','check_queued','checking') THEN 'unknown' ELSE 'cancelled' END,error='source_changed' WHERE source_uuid=old.uuid AND status IN ('queued','preparing','sending','unknown','check_queued','checking'); END`,
`CREATE TRIGGER feed_actions_source_delete AFTER DELETE ON feed_sources BEGIN UPDATE feed_actions SET status=CASE WHEN status IN ('sending','unknown','check_queued','checking') THEN 'unknown' ELSE 'cancelled' END,error='source_deleted' WHERE source_uuid=old.uuid AND status IN ('queued','preparing','sending','unknown','check_queued','checking'); END`,
}, Assertions: []string{schemaColumns("feed_sources", "write_enabled"), schemaUnique("feed_actions", "domain_uuid,owner_uuid,request_id")},
}, {
Version: 6, Name: "freshrss_empty_folders", Statements: []string{
`ALTER TABLE feed_sources ADD COLUMN remote_folders TEXT NOT NULL DEFAULT '[]'`,
}, Assertions: []string{schemaColumns("feed_sources", "remote_folders")},
}}
// CaptureFeedNote creates an inert Wiki note and receipt atomically. It does not
@@ -330,7 +334,7 @@ func (s *Store) FeedSourceAction(ctx context.Context, domain, owner, id, action
}
func (s *Store) FeedItems(ctx context.Context, domain, owner, source, filter, query string, before int64, beforeID string, scope ...string) ([]FeedItem, error) {
remoteFeed, category := "", ""
if len(scope) > 2 {
if len(scope) > 3 {
return nil, ErrTaskInvalid
}
if len(scope) > 0 {
@@ -339,6 +343,13 @@ func (s *Store) FeedItems(ctx context.Context, domain, owner, source, filter, qu
if len(scope) > 1 {
category = scope[1]
}
mode := ""
if len(scope) > 2 {
mode = scope[2]
}
if mode != "" && (mode != "uncategorized" || source == "" || category != "") {
return nil, ErrTaskInvalid
}
if (remoteFeed != "" && !ValidFeedRemoteID(remoteFeed)) || (category != "" && !ValidFeedRemoteID(category)) || ((remoteFeed != "" || category != "") && source == "") {
return nil, ErrTaskInvalid
}
@@ -348,7 +359,14 @@ func (s *Store) FeedItems(ctx context.Context, domain, owner, source, filter, qu
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
return nil, e
}
rows, e := s.DB.QueryContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.published_ms,i.feed,CASE WHEN s.folder<>'' THEN s.folder ELSE i.folder END,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=? AND s.owner_uuid=? AND (?='' OR s.uuid=?) AND (?='' OR instr(lower(i.title),lower(?))>0 OR instr(lower(i.feed),lower(?))>0) AND (?<0 OR i.published_ms<? OR (i.published_ms=? AND (i.source_uuid||':'||i.id)<?)) AND (?<>'unread' OR (CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END)=0) AND (?<>'starred' OR (CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END)=1) AND (?='' OR i.remote_feed=?) AND (?='' OR EXISTS(SELECT 1 FROM json_each(i.remote_categories) WHERE value=?)) ORDER BY i.published_ms DESC,i.source_uuid||':'||i.id DESC LIMIT 101`, domain, owner, source, source, query, query, query, before, before, before, beforeID, filter, filter, remoteFeed, remoteFeed, category, category)
args := []any{domain, owner, source, source, query, query, query, before, before, before, beforeID, filter, filter, remoteFeed, remoteFeed, category, category}
extra := ""
if mode == "uncategorized" {
// The uncorrelated list is built once, not a catalog scan per article.
extra = ` AND i.remote_feed IN (SELECT json_extract(c.value,'$.id') FROM feed_sources cs,json_each(cs.remote_catalog) c WHERE cs.uuid=? AND cs.domain_uuid=? AND cs.owner_uuid=? AND json_array_length(c.value,'$.categories')=0)`
args = append(args, source, domain, owner)
}
rows, e := s.DB.QueryContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.published_ms,i.feed,CASE WHEN s.folder<>'' THEN s.folder ELSE i.folder END,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=? AND s.owner_uuid=? AND (?='' OR s.uuid=?) AND (?='' OR instr(lower(i.title),lower(?))>0 OR instr(lower(i.feed),lower(?))>0) AND (?<0 OR i.published_ms<? OR (i.published_ms=? AND (i.source_uuid||':'||i.id)<?)) AND (?<>'unread' OR (CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END)=0) AND (?<>'starred' OR (CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END)=1) AND (?='' OR i.remote_feed=?) AND (?='' OR EXISTS(SELECT 1 FROM json_each(i.remote_categories) WHERE value=?))`+extra+` ORDER BY i.published_ms DESC,i.source_uuid||':'||i.id DESC LIMIT 101`, args...)
if e != nil {
return nil, e
}
@@ -465,17 +483,29 @@ func (s *Store) ClaimFeed(ctx context.Context, v FeedSource) (FeedSource, error)
}
// Failed fetches preserve the previous cache. Version and claim token fence stale workers.
func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem, etag, modified, code string, unchanged bool, catalogs ...[]FeedRemoteSubscription) error {
func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem, etag, modified, code string, unchanged bool, catalogs ...FeedCatalogSnapshot) error {
if len(catalogs) > 1 {
return ErrTaskInvalid
}
var catalog string
var catalog, folders string
if len(catalogs) == 1 {
var e error
catalog, e = FeedCatalogJSON(catalogs[0])
catalog, folders, e = catalogs[0].Encode()
if e != nil {
return e
}
if v.Provider == "freshrss" {
membership := map[string][]string{}
for _, sub := range catalogs[0].Subscriptions {
for _, c := range sub.Categories {
membership[sub.ID] = append(membership[sub.ID], c.ID)
}
}
items = append([]FeedItem(nil), items...)
for n := range items {
items[n].RemoteCategories = membership[items[n].RemoteFeed]
}
}
}
if len(items) > 500 || len(etag) > 1024 || len(modified) > 1024 {
return ErrTaskInvalid
@@ -511,7 +541,7 @@ func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem
return nil
}
if catalog != "" {
if _, e = tx.ExecContext(ctx, `UPDATE feed_sources SET remote_catalog=? WHERE uuid=?`, catalog, v.UUID); e != nil {
if _, e = tx.ExecContext(ctx, `UPDATE feed_sources SET remote_catalog=?,remote_folders=? WHERE uuid=?`, catalog, folders, v.UUID); e != nil {
return e
}
}
+85 -10
View File
@@ -18,6 +18,51 @@ type FeedRemoteSubscription struct {
Categories []FeedCategory `json:"categories"`
}
type FeedCatalogSnapshot struct {
Subscriptions []FeedRemoteSubscription `json:"subscriptions"`
Folders []FeedCategory `json:"folders"`
}
// Merge membership-derived folders with explicit folders, including empty
// ones. IDs, not labels or slash-delimited paths, are the identity boundary.
func FeedFolders(catalog []FeedRemoteSubscription, folders []FeedCategory) ([]FeedCategory, error) {
out := []FeedCategory{}
seen := map[string]string{}
add := func(c FeedCategory) error {
if !ValidFeedRemoteID(c.ID) || len(c.Label) > 512 || !utf8.ValidString(c.Label) {
return ErrTaskInvalid
}
if label, ok := seen[c.ID]; ok {
if label != c.Label {
return ErrTaskInvalid
}
return nil
}
seen[c.ID] = c.Label
out = append(out, c)
if len(out) > 1000 {
return ErrTaskDAVLimit
}
return nil
}
if len(folders) > 1000 {
return nil, ErrTaskDAVLimit
}
for _, c := range folders {
if e := add(c); e != nil {
return nil, e
}
}
for _, sub := range catalog {
for _, c := range sub.Categories {
if e := add(c); e != nil {
return nil, e
}
}
}
return out, nil
}
func ValidFeedRemoteID(id string) bool {
return id != "" && len(id) <= 2048 && utf8.ValidString(id) && !strings.ContainsAny(id, "\x00\r\n")
}
@@ -59,17 +104,47 @@ func FeedCatalogJSON(catalog []FeedRemoteSubscription) (string, error) {
return string(raw), nil
}
func (s *Store) FeedCatalog(ctx context.Context, domain, owner, source string) ([]FeedRemoteSubscription, error) {
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
return nil, e
// Validate the complete snapshot before persisting or reporting fetch success.
func (v FeedCatalogSnapshot) Encode() (string, string, error) {
catalog, e := FeedCatalogJSON(v.Subscriptions)
if e != nil {
return "", "", e
}
var raw string
if e := s.DB.QueryRowContext(ctx, `SELECT remote_catalog FROM feed_sources WHERE uuid=? AND domain_uuid=? AND owner_uuid=?`, source, domain, owner).Scan(&raw); e != nil {
return nil, e
folders, e := FeedFolders(v.Subscriptions, v.Folders)
if e != nil {
return "", "", e
}
var catalog []FeedRemoteSubscription
if e := json.Unmarshal([]byte(raw), &catalog); e != nil {
return nil, e
raw, e := json.Marshal(folders)
if e != nil {
return "", "", e
}
return catalog, nil
if len(raw)+len(catalog) > 1<<20 {
return "", "", ErrTaskDAVLimit
}
return catalog, string(raw), nil
}
func (s *Store) FeedCatalog(ctx context.Context, domain, owner, source string) ([]FeedRemoteSubscription, error) {
v, e := s.FeedCatalogSnapshot(ctx, domain, owner, source)
return v.Subscriptions, e
}
func (s *Store) FeedCatalogSnapshot(ctx context.Context, domain, owner, source string) (FeedCatalogSnapshot, error) {
v := FeedCatalogSnapshot{}
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
return v, e
}
var raw, folders string
if e := s.DB.QueryRowContext(ctx, `SELECT remote_catalog,remote_folders FROM feed_sources WHERE uuid=? AND domain_uuid=? AND owner_uuid=?`, source, domain, owner).Scan(&raw, &folders); e != nil {
return v, e
}
if e := json.Unmarshal([]byte(raw), &v.Subscriptions); e != nil {
return v, e
}
if e := json.Unmarshal([]byte(folders), &v.Folders); e != nil {
return v, e
}
var e error
v.Folders, e = FeedFolders(v.Subscriptions, v.Folders)
return v, e
}
+38
View File
@@ -188,3 +188,41 @@ func TestFeedActionsMigrationRollbackAndRepeat(t *testing.T) {
t.Fatal(status, e)
}
}
func TestFeedFoldersMigrationRollbackAndRepeat(t *testing.T) {
st := newSchemaTestStore(t)
ctx := context.Background()
if e := db.ApplySQLiteMigrations(ctx, st.DB, "feeds", feedMigrations[:5]); e != nil {
t.Fatal(e)
}
const catalog = `[{"id":"feed/1","title":"Title","categories":[{"id":"category","label":"Folder"}]}]`
if _, e := st.DB.Exec(`INSERT INTO feed_sources(uuid,domain_uuid,owner_uuid,provider,title,folder,url,version,enabled,interval_seconds,lease,next_ms,remote_catalog) VALUES('s','d','u','freshrss','Title','','https://example.org',7,1,300,'worker',999,?);
INSERT INTO feed_actions(domain_uuid,owner_uuid,request_id,source_uuid,item_id,field,desired,source_version,status,created_ms) VALUES('d','u','r','s','i','read',1,7,'queued',1)`, catalog); e != nil {
t.Fatal(e)
}
broken := append([]db.SQLMigration(nil), feedMigrations...)
broken[5].Statements = append(append([]string(nil), feedMigrations[5].Statements...), `INSERT INTO no_such_feed_table VALUES(1)`)
if e := db.ApplySQLiteMigrations(ctx, st.DB, "feeds", broken); e == nil {
t.Fatal("expected failure")
}
var count int
if e := st.DB.QueryRow(`SELECT count(*) FROM pragma_table_info('feed_sources') WHERE name='remote_folders'`).Scan(&count); e != nil || count != 0 {
t.Fatal("partial schema", count, e)
}
for range 2 {
if e := st.ensureSchemaComponent("feeds"); e != nil {
t.Fatal(e)
}
}
var version, next int
var got, folders, lease, status string
if e := st.DB.QueryRow(`SELECT version,next_ms,lease,remote_catalog,remote_folders FROM feed_sources WHERE uuid='s'`).Scan(&version, &next, &lease, &got, &folders); e != nil || version != 7 || next != 999 || lease != "worker" || got != catalog || folders != "[]" {
t.Fatal("existing state changed", version, next, lease, got, folders, e)
}
if e := st.DB.QueryRow(`SELECT status FROM feed_actions WHERE request_id='r'`).Scan(&status); e != nil || status != "queued" {
t.Fatal("pending action changed", status, e)
}
if e := st.DB.QueryRow(`PRAGMA integrity_check`).Scan(&status); e != nil || status != "ok" {
t.Fatal(status, e)
}
}