feat(feeds): cache FreshRSS subscriptions and category navigation
Build and publish container images / images (push) Failing after 6m7s

This commit is contained in:
Anya Tokugawa committed 2026-11-08 04:48:55 +09:00
1 parent 7fe64113c7
commit bfed3d20f5
12 files changed
+503 -34

No files matched your search

+9 -1
View File
@@ -47,6 +47,14 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
return
}
switch {
case key == "catalog" && r.Method == "GET":
catalog, e := s.store.FeedCatalog(r.Context(), domain, actor, r.URL.Query().Get("source"))
if e != nil {
taskError(w, e)
return
}
writeEventJSON(w, 200, map[string]any{"subscriptions": catalog})
return
case key == "note" && r.Method == "POST":
var input struct {
RequestID string `json:"request_id"`
@@ -159,7 +167,7 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
return
}
}
items, e := s.store.FeedItems(r.Context(), domain, actor, q.Get("source"), q.Get("filter"), q.Get("q"), before, q.Get("key"))
items, e := s.store.FeedItems(r.Context(), domain, actor, q.Get("source"), q.Get("filter"), q.Get("q"), before, q.Get("key"), q.Get("feed"), q.Get("category"))
if e != nil {
taskError(w, e)
return
+149
View File
@@ -0,0 +1,149 @@
package server
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/url"
"strings"
"testing"
"github.com/google/uuid"
"pageswiki/internal/store"
)
const testFeedCatalog = `{"subscriptions":[{"id":"feed/1","title":"Same title","categories":[{"id":"user/alice/label/Tech","label":"技術"}]},{"id":"feed/2","title":"Same title","categories":[{"id":"user/alice/label/Other","label":"技術"}]},{"id":"feed/3","title":"Empty feed","categories":[]}]}`
func TestFreshRSSCatalogValidation(t *testing.T) {
catalog, e := parseFreshRSSCatalog([]byte(testFeedCatalog))
if e != nil || len(catalog) != 3 || catalog[0].Categories[0].ID == catalog[1].Categories[0].ID {
t.Fatal(catalog, e)
}
for _, raw := range []string{`{}`, `{"subscriptions":null}`, `{"subscriptions":[{"id":""}]}`, `{"subscriptions":[{"id":"feed/1"},{"id":"feed/1"}]}`, `{"subscriptions":[{"id":"feed/1","categories":[{"id":"cat","label":"A"},{"id":"cat","label":"A"}]}]}`, `{"subscriptions":[{"id":"feed/1","categories":[{"id":"cat","label":"A"}]},{"id":"feed/2","categories":[{"id":"cat","label":"B"}]}]}`} {
if _, e := parseFreshRSSCatalog([]byte(raw)); e == nil {
t.Fatal("accepted", raw)
}
}
if c, e := parseFreshRSSCatalog([]byte(`{"subscriptions":[]}`)); e != nil || len(c) != 0 {
t.Fatal(c, e)
}
big := make([]store.FeedRemoteSubscription, 1001)
raw, _ := json.Marshal(map[string]any{"subscriptions": big})
if _, e := parseFreshRSSCatalog(raw); e == nil {
t.Fatal("unbounded catalog")
}
}
func TestFeedCatalogBackgroundSnapshotIsolation(t *testing.T) {
s, token, domain, actor, local := feedFixture(t)
ctx := context.Background()
if e := s.store.FeedSourceAction(ctx, domain, actor, local.UUID, "delete", local.Version); e != nil {
t.Fatal(e)
}
v := store.FeedSource{UUID: uuid.NewString(), Provider: "freshrss", Title: "FreshRSS", URL: "https://example.org/api/greader.php", Enabled: true, Interval: 300}
v.Credential, _ = s.store.SealFeedCredential(domain, actor, v.UUID, "reader", "secret")
if _, e := s.store.SaveFeedSource(ctx, domain, actor, v); e != nil {
t.Fatal(e)
}
calls := 0
failCatalog := false
client := &http.Client{Transport: feedTransportFunc(func(r *http.Request) (*http.Response, error) {
calls++
body := ""
switch {
case strings.HasSuffix(r.URL.Path, "/accounts/ClientLogin"):
if r.Method != "POST" {
t.Error("login method")
}
body = "Auth=synthetic"
case strings.HasSuffix(r.URL.Path, "/stream/contents/reading-list"):
if r.Method != "GET" || r.URL.Query().Get("n") != "200" {
t.Error("article request")
}
body = `{"items":[{"id":"a","title":"First","origin":{"title":"Same title","streamId":"feed/1"},"categories":["user/alice/label/Tech"]},{"id":"b","title":"Second","origin":{"title":"Same title","streamId":"feed/2"},"categories":["user/alice/label/Other"]}]}`
if failCatalog {
body = `{"items":[]}`
}
case strings.HasSuffix(r.URL.Path, "/subscription/list"):
if r.Method != "GET" || r.Header.Get("Authorization") != "GoogleLogin auth=synthetic" {
t.Error("catalog request")
}
if failCatalog {
return nil, errors.New("synthetic timeout")
}
body = testFeedCatalog
default:
t.Fatalf("unexpected external request %s %s", r.Method, r.URL.Path)
}
return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body))}, nil
})}
run := func() {
t.Helper()
if _, e := s.store.DB.Exec(`UPDATE feed_sources SET next_ms=0 WHERE uuid=?`, v.UUID); e != nil {
t.Fatal(e)
}
if e := s.syncFeedSourcesWithClient(ctx, &store.EventJob{}, client); e != nil {
t.Fatal(e)
}
}
run()
if calls != 3 {
t.Fatal("fetch count", calls)
}
w := feedRequestTest(s, token, "GET", "catalog?source="+v.UUID, nil)
if w.Code != 200 || !strings.Contains(w.Body.String(), "Empty feed") || strings.Contains(w.Body.String(), "secret") {
t.Fatal(w.Code, w.Body)
}
for _, scope := range []string{"feed=feed%2F1", "category=" + url.QueryEscape("user/alice/label/Tech")} {
w = feedRequestTest(s, token, "GET", "items?source="+v.UUID+"&"+scope, nil)
var data struct {
Items []store.FeedItem `json:"items"`
}
json.Unmarshal(w.Body.Bytes(), &data)
if w.Code != 200 || len(data.Items) != 1 || data.Items[0].ID != "a" {
t.Fatal(scope, w.Code, w.Body)
}
}
w = feedRequestTest(s, token, "GET", "items?feed=feed%2F1", nil)
if w.Code != 422 {
t.Fatal("unscoped remote ID", w.Code, w.Body)
}
other, e := s.store.CreatePasswordAccount(domain, "catalogother", "long test password", "Other", "")
if e != nil {
t.Fatal(e)
}
otherToken, e := s.store.CreateAuthSession(domain, other.UUID)
if e != nil {
t.Fatal(e)
}
if w = feedRequestTest(s, otherToken, "GET", "catalog?source="+v.UUID, nil); w.Code == 200 {
t.Fatal("cross-owner catalog leak")
}
if w = feedRequestTest(s, "", "GET", "catalog?source="+v.UUID, nil); w.Code == 200 {
t.Fatal("anonymous catalog leak")
}
if _, e := s.store.FeedCatalog(ctx, uuid.NewString(), actor, v.UUID); e == nil {
t.Fatal("cross-domain catalog leak")
}
if calls != 3 {
t.Fatal("foreground external request")
}
failCatalog = true
run()
catalog, e := s.store.FeedCatalog(ctx, domain, actor, v.UUID)
if e != nil || len(catalog) != 3 {
t.Fatal("lost catalog", catalog, e)
}
items, e := s.store.FeedItems(ctx, domain, actor, v.UUID, "", "", -1, "")
if e != nil || len(items) != 2 {
t.Fatal("lost articles", items, e)
}
if e := s.setSystemExtensionEnabled(domain, "organizer.feeds", false); e != nil {
t.Fatal(e)
}
if _, e := s.store.FeedCatalog(ctx, domain, actor, v.UUID); !errors.Is(e, store.ErrAccessDenied) {
t.Fatal("disabled read", e)
}
}
+54 -5
View File
@@ -328,7 +328,7 @@ func feedRequest(ctx context.Context, client *http.Client, method, target, auth
}
return raw, res.Header, res.StatusCode, nil
}
func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string) ([]store.FeedItem, error) {
func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string, catalogs ...*[]store.FeedRemoteSubscription) ([]store.FeedItem, error) {
base := strings.TrimRight(v.URL, "/")
raw, _, _, e := feedRequest(ctx, client, "POST", base+"/accounts/ClientLogin", "", url.Values{"Email": {user}, "Passwd": {password}}, nil)
if e != nil {
@@ -354,7 +354,8 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
Published int64 `json:"published"`
Categories []string `json:"categories"`
Origin struct {
Title string `json:"title"`
Title string `json:"title"`
StreamID string `json:"streamId"`
} `json:"origin"`
Alternate []struct {
Href string `json:"href"`
@@ -377,7 +378,10 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
return nil, errFeedInvalid
}
seen[i.ID] = true
item := store.FeedItem{ID: i.ID, Title: feedText(i.Title, 2048), Feed: feedText(i.Origin.Title, 512)}
if (i.Origin.StreamID != "" && !store.ValidFeedRemoteID(i.Origin.StreamID)) || len(i.Categories) > 64 {
return nil, errFeedInvalid
}
item := store.FeedItem{ID: i.ID, Title: feedText(i.Title, 2048), Feed: feedText(i.Origin.Title, 512), RemoteFeed: i.Origin.StreamID, RemoteCategories: []string{}}
if i.Published > 0 && i.Published < time.Now().AddDate(1, 0, 0).Unix() {
item.PublishedMS = i.Published * 1000
}
@@ -391,6 +395,9 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
item.URL = feedLink(base, i.Alternate[0].Href)
}
for _, c := range i.Categories {
if !store.ValidFeedRemoteID(c) {
return nil, errFeedInvalid
}
if strings.HasSuffix(c, "/state/com.google/read") {
item.Read = true
}
@@ -399,12 +406,53 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
}
if pos := strings.Index(c, "/label/"); pos >= 0 {
item.Folder = feedText(c[pos+7:], 512)
item.RemoteCategories = append(item.RemoteCategories, c)
}
}
out = append(out, item)
}
if len(catalogs) > 0 {
raw, _, _, e = feedRequest(ctx, client, "GET", base+"/reader/api/0/subscription/list?output=json", auth, nil, nil)
if e != nil {
return nil, e
}
catalog, e := parseFreshRSSCatalog(raw)
if e != nil {
return nil, e
}
*catalogs[0] = catalog
}
return out, nil
}
func parseFreshRSSCatalog(raw []byte) ([]store.FeedRemoteSubscription, error) {
if len(raw) > 4<<20 || !utf8.Valid(raw) {
return nil, errFeedInvalid
}
var response struct {
Subscriptions []store.FeedRemoteSubscription `json:"subscriptions"`
}
if json.Unmarshal(raw, &response) != nil || response.Subscriptions == nil {
return nil, errFeedInvalid
}
if len(response.Subscriptions) > 1000 {
return nil, errors.New("limit")
}
for i := range response.Subscriptions {
sub := &response.Subscriptions[i]
sub.Title = feedText(sub.Title, 512)
if sub.Categories == nil {
sub.Categories = []store.FeedCategory{}
}
for j := range sub.Categories {
sub.Categories[j].Label = feedText(sub.Categories[j].Label, 512)
}
}
if _, e := store.FeedCatalogJSON(response.Subscriptions); e != nil {
return nil, errFeedInvalid
}
return response.Subscriptions, nil
}
func (s *Server) syncFeedSources(ctx context.Context, job *store.EventJob) error {
client := calendarSourceClient()
client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
@@ -435,6 +483,7 @@ func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.Event
}
child, stop := context.WithTimeout(ctx, 8*time.Second)
var items []store.FeedItem
var catalog []store.FeedRemoteSubscription
etag, modified := v.ETag, v.Modified
unchanged := false
if !validCalendarSourceURL(v.URL) {
@@ -443,7 +492,7 @@ func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.Event
var user, password string
user, password, e = s.store.OpenFeedCredential(v)
if e == nil {
items, e = fetchFreshRSS(child, client, v, user, password)
items, e = fetchFreshRSS(child, client, v, user, password, &catalog)
} else {
e = errors.New("credentials")
}
@@ -478,7 +527,7 @@ func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.Event
} else {
done++
}
if e = s.store.CompleteFeed(ctx, v, items, etag, modified, code, unchanged); e != nil && !errors.Is(e, store.ErrAccessDenied) && !errors.Is(e, store.ErrTaskConflict) {
if e = s.store.CompleteFeed(ctx, v, items, etag, modified, code, unchanged, catalog); e != nil && !errors.Is(e, store.ErrAccessDenied) && !errors.Is(e, store.ErrTaskConflict) {
return e
}
}
+24 -2
View File
@@ -543,8 +543,30 @@ func TestFeedsChromium(t *testing.T) {
if e = s.store.CompleteFeed(context.Background(), claim, items, "", "", "", false); e != nil {
t.Fatal(e)
}
_ = domain
_ = actor
fresh, e := s.store.SaveFeedSource(context.Background(), domain, actor, store.FeedSource{UUID: uuid.NewString(), Provider: "freshrss", Title: "FreshRSS fixture", URL: "https://example.org/api/greader.php", Enabled: true, Interval: 300})
if e != nil {
t.Fatal(e)
}
all, e := s.store.FeedSources(context.Background(), domain, actor)
if e != nil {
t.Fatal(e)
}
for _, v := range all {
if v.UUID == fresh.UUID {
fresh = v
}
}
claim, e = s.store.ClaimFeed(context.Background(), fresh)
if e != nil {
t.Fatal(e)
}
catalog, e := parseFreshRSSCatalog([]byte(testFeedCatalog))
if e != nil {
t.Fatal(e)
}
if e = s.store.CompleteFeed(context.Background(), claim, []store.FeedItem{{ID: "catalog-a", Title: "Catalog first", RemoteFeed: "feed/1", RemoteCategories: []string{"user/alice/label/Tech"}}, {ID: "catalog-b", Title: "Catalog second", RemoteFeed: "feed/2", RemoteCategories: []string{"user/alice/label/Other"}}}, "", "", "", false, catalog); e != nil {
t.Fatal(e)
}
s.store.BackgroundJobsDisabled = true
s = New(s.store)
script, e := os.ReadFile("internal/server/testdata/feeds.js")
+10 -2
View File
@@ -13,7 +13,7 @@
d.querySelectorAll('.feed-row')[1].click();await wait(()=>d.querySelector('#feed-reader .feed-body')&&!d.querySelector('#feed-reader .feed-body').shadowRoot);assert(d.querySelector('#feed-reader .feed-body').textContent.includes('<script>'),'Literal text not preserved');d.querySelector('.feed-row').click();await wait(()=>d.querySelector('#feed-reader .feed-body')?.shadowRoot);
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'*',bubbles:true}));await wait(()=>d.querySelector('.feed-row[aria-selected=true]').textContent.includes('★'));
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'Escape',bubbles:true}));await wait(()=>!d.querySelector('.feed-layout').classList.contains('reading'));
d.getElementById('feed-more').click();await wait(()=>d.querySelectorAll('.feed-row').length===120);
d.getElementById('feed-more').click();await wait(()=>d.querySelectorAll('.feed-row').length===122);
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'/',bubbles:true}));await wait(()=>d.activeElement.type==='search');d.activeElement.value='119';d.activeElement.dispatchEvent(new w.Event('input',{bubbles:true}));await wait(()=>d.querySelectorAll('.feed-row').length===1);
d.querySelector('.feed-row').click();await wait(()=>d.querySelector('#feed-reader .feed-body'));
d.querySelector('#feed-reader .feed-actions button:last-child').click();await wait(()=>d.querySelector('#feed-reader form'));
@@ -26,6 +26,14 @@
await wait(()=>d.querySelectorAll('#feed-opml-preview tbody tr').length===3);const checks=d.querySelectorAll('#feed-opml-preview tbody input');assert(checks[0].checked&&!checks[0].disabled&&checks[1].disabled&&checks[2].disabled,'Import eligibility');
assert(d.documentElement.scrollWidth<=w.innerWidth,'OPML overflow');d.querySelector('.feed-import-options input[type=checkbox]').checked=false;d.getElementById('feed-opml-preview').requestSubmit();await wait(()=>!d.getElementById('feed-opml-preview'));
assert(d.getElementById('feed-sources').textContent.includes('Import '+w.innerWidth),'Imported source missing');d.getElementById('feed-opml').click();await wait(()=>d.getElementById('feed-opml-file'));choose();await wait(()=>d.querySelectorAll('#feed-opml-preview tbody tr').length===3);assert(d.querySelector('#feed-opml-preview tbody input').disabled,'Duplicate after import');
assert(d.documentElement.scrollWidth<=w.innerWidth,'OPML preview overflow');document.body.dataset.vimReady='1';
assert(d.documentElement.scrollWidth<=w.innerWidth,'OPML preview overflow');
[...d.querySelectorAll('.feed-source>button:first-child')].find(b=>b.textContent==='FreshRSS fixture').click();await wait(()=>d.querySelectorAll('.feed-subscriptions button').length===4&&d.querySelectorAll('.feed-row').length===2);
const category=d.getElementById('feed-category');category.value='user/alice/label/Tech';category.dispatchEvent(new w.Event('change'));await wait(()=>d.querySelectorAll('.feed-row').length===1&&d.querySelector('.feed-row').textContent.includes('Catalog first'));
assert(d.querySelectorAll('.feed-subscriptions button').length===2,'Category navigation');
d.getElementById('feed-category').value='';d.getElementById('feed-category').dispatchEvent(new w.Event('change'));await wait(()=>d.querySelectorAll('.feed-row').length===2);
d.querySelector('.feed-subscriptions button[data-feed="feed/2"]').click();await wait(()=>d.querySelectorAll('.feed-row').length===1&&d.querySelector('.feed-row').textContent.includes('Catalog second'));
d.querySelector('.feed-row').click();await wait(()=>d.querySelector('#feed-reader .feed-body'));assert(d.querySelectorAll('#feed-reader .feed-actions button:disabled').length===2,'FreshRSS not read-only');
if(w.innerWidth<=600)d.querySelector('#feed-toolbar button').click();
assert(d.documentElement.scrollWidth<=w.innerWidth,'Catalog overflow');document.body.dataset.vimReady='1';
}catch(e){const out=document.createElement('pre');out.id='error';out.textContent=e.stack;document.body.append(out);}
})();
+68 -17
View File
@@ -37,6 +37,13 @@ var feedMigrations = []db.SQLMigration{{Version: 1, Name: "private_feed_reader",
Version: 3, Name: "feed_import_receipts", Statements: []string{
`CREATE TABLE feed_import_receipts(domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,request_id TEXT NOT NULL,input_hash TEXT NOT NULL,result TEXT NOT NULL,PRIMARY KEY(domain_uuid,owner_uuid,request_id))`,
}, Assertions: []string{schemaUnique("feed_import_receipts", "domain_uuid,owner_uuid,request_id")},
}, {
Version: 4, Name: "freshrss_catalog", Statements: []string{
`ALTER TABLE feed_sources ADD COLUMN remote_catalog TEXT NOT NULL DEFAULT '[]'`,
`ALTER TABLE feed_items ADD COLUMN remote_feed TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE feed_items ADD COLUMN remote_categories TEXT NOT NULL DEFAULT '[]'`,
`UPDATE feed_sources SET next_ms=0,lease='',lease_until=0,version=version+1 WHERE provider='freshrss'`,
}, Assertions: []string{schemaColumns("feed_sources", "remote_catalog"), schemaColumns("feed_items", "remote_feed,remote_categories")},
}}
// CaptureFeedNote creates an inert Wiki note and receipt atomically. It does not
@@ -124,19 +131,21 @@ type FeedSource struct {
Modified string `json:"-"`
}
type FeedItem struct {
Source string `json:"source"`
ID string `json:"id"`
Title string `json:"title"`
URL string `json:"url"`
Body string `json:"body,omitempty"`
BodyHTML string `json:"body_html,omitempty"`
PublishedMS int64 `json:"published_ms"`
Feed string `json:"feed"`
Folder string `json:"folder"`
Read bool `json:"read"`
Starred bool `json:"starred"`
Version int64 `json:"version"`
Provider string `json:"provider"`
Source string `json:"source"`
ID string `json:"id"`
Title string `json:"title"`
URL string `json:"url"`
Body string `json:"body,omitempty"`
BodyHTML string `json:"body_html,omitempty"`
PublishedMS int64 `json:"published_ms"`
Feed string `json:"feed"`
Folder string `json:"folder"`
Read bool `json:"read"`
Starred bool `json:"starred"`
Version int64 `json:"version"`
Provider string `json:"provider"`
RemoteFeed string `json:"remote_feed,omitempty"`
RemoteCategories []string `json:"remote_categories,omitempty"`
}
const feedSourceColumns = `uuid,domain_uuid,owner_uuid,provider,title,folder,url,credential,version,enabled,interval_seconds,next_ms,lease,lease_until,last_ms,error,etag,modified`
@@ -294,14 +303,27 @@ func (s *Store) FeedSourceAction(ctx context.Context, domain, owner, id, action
return nil
})
}
func (s *Store) FeedItems(ctx context.Context, domain, owner, source, filter, query string, before int64, beforeID string) ([]FeedItem, error) {
func (s *Store) FeedItems(ctx context.Context, domain, owner, source, filter, query string, before int64, beforeID string, scope ...string) ([]FeedItem, error) {
remoteFeed, category := "", ""
if len(scope) > 2 {
return nil, ErrTaskInvalid
}
if len(scope) > 0 {
remoteFeed = scope[0]
}
if len(scope) > 1 {
category = scope[1]
}
if (remoteFeed != "" && !ValidFeedRemoteID(remoteFeed)) || (category != "" && !ValidFeedRemoteID(category)) || ((remoteFeed != "" || category != "") && source == "") {
return nil, ErrTaskInvalid
}
if len(query) > 200 || len(beforeID) > 1200 {
return nil, ErrTaskInvalid
}
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
return nil, e
}
rows, e := s.DB.QueryContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.published_ms,i.feed,CASE WHEN s.folder<>'' THEN s.folder ELSE i.folder END,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=? AND s.owner_uuid=? AND (?='' OR s.uuid=?) AND (?='' OR instr(lower(i.title),lower(?))>0 OR instr(lower(i.feed),lower(?))>0) AND (?<0 OR i.published_ms<? OR (i.published_ms=? AND (i.source_uuid||':'||i.id)<?)) AND (?<>'unread' OR (CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END)=0) AND (?<>'starred' OR (CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END)=1) ORDER BY i.published_ms DESC,i.source_uuid||':'||i.id DESC LIMIT 101`, domain, owner, source, source, query, query, query, before, before, before, beforeID, filter, filter)
rows, e := s.DB.QueryContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.published_ms,i.feed,CASE WHEN s.folder<>'' THEN s.folder ELSE i.folder END,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=? AND s.owner_uuid=? AND (?='' OR s.uuid=?) AND (?='' OR instr(lower(i.title),lower(?))>0 OR instr(lower(i.feed),lower(?))>0) AND (?<0 OR i.published_ms<? OR (i.published_ms=? AND (i.source_uuid||':'||i.id)<?)) AND (?<>'unread' OR (CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END)=0) AND (?<>'starred' OR (CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END)=1) AND (?='' OR i.remote_feed=?) AND (?='' OR EXISTS(SELECT 1 FROM json_each(i.remote_categories) WHERE value=?)) ORDER BY i.published_ms DESC,i.source_uuid||':'||i.id DESC LIMIT 101`, domain, owner, source, source, query, query, query, before, before, before, beforeID, filter, filter, remoteFeed, remoteFeed, category, category)
if e != nil {
return nil, e
}
@@ -418,7 +440,18 @@ func (s *Store) ClaimFeed(ctx context.Context, v FeedSource) (FeedSource, error)
}
// Failed fetches preserve the previous cache. Version and claim token fence stale workers.
func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem, etag, modified, code string, unchanged bool) error {
func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem, etag, modified, code string, unchanged bool, catalogs ...[]FeedRemoteSubscription) error {
if len(catalogs) > 1 {
return ErrTaskInvalid
}
var catalog string
if len(catalogs) == 1 {
var e error
catalog, e = FeedCatalogJSON(catalogs[0])
if e != nil {
return e
}
}
if len(items) > 500 || len(etag) > 1024 || len(modified) > 1024 {
return ErrTaskInvalid
}
@@ -428,6 +461,14 @@ func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem
return ErrTaskInvalid
}
for _, i := range items {
if (i.RemoteFeed != "" && !ValidFeedRemoteID(i.RemoteFeed)) || len(i.RemoteCategories) > 64 {
return ErrTaskInvalid
}
for _, c := range i.RemoteCategories {
if !ValidFeedRemoteID(c) {
return ErrTaskInvalid
}
}
if i.ID == "" || len(i.ID) > 1024 || len(i.Title) > 2048 || len(i.URL) > 4096 || len(i.Body) > 32768 || len(i.BodyHTML) > 128<<10 || len(i.Feed) > 512 || len(i.Folder) > 512 || i.PublishedMS < 0 {
return ErrTaskInvalid
}
@@ -444,6 +485,11 @@ func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem
if code != "" || unchanged {
return nil
}
if catalog != "" {
if _, e = tx.ExecContext(ctx, `UPDATE feed_sources SET remote_catalog=? WHERE uuid=?`, catalog, v.UUID); e != nil {
return e
}
}
// FreshRSS is authoritative: retain exactly this bounded recent window,
// not stale rows from unsubscribed feeds or old external read/star state.
if v.Provider == "freshrss" {
@@ -452,7 +498,12 @@ func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem
}
}
for _, i := range items {
_, e = tx.ExecContext(ctx, `INSERT INTO feed_items(source_uuid,id,title,url,body,body_html,published_ms,feed,folder,is_read,starred) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_uuid,id) DO UPDATE SET title=excluded.title,url=excluded.url,body=excluded.body,body_html=excluded.body_html,published_ms=excluded.published_ms,feed=excluded.feed,folder=excluded.folder,is_read=excluded.is_read,starred=excluded.starred`, v.UUID, i.ID, i.Title, i.URL, i.Body, i.BodyHTML, i.PublishedMS, i.Feed, i.Folder, i.Read, i.Starred)
categories := i.RemoteCategories
if categories == nil {
categories = []string{}
}
raw, _ := json.Marshal(categories)
_, e = tx.ExecContext(ctx, `INSERT INTO feed_items(source_uuid,id,title,url,body,body_html,published_ms,feed,folder,is_read,starred,remote_feed,remote_categories) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_uuid,id) DO UPDATE SET title=excluded.title,url=excluded.url,body=excluded.body,body_html=excluded.body_html,published_ms=excluded.published_ms,feed=excluded.feed,folder=excluded.folder,is_read=excluded.is_read,starred=excluded.starred,remote_feed=excluded.remote_feed,remote_categories=excluded.remote_categories`, v.UUID, i.ID, i.Title, i.URL, i.Body, i.BodyHTML, i.PublishedMS, i.Feed, i.Folder, i.Read, i.Starred, i.RemoteFeed, string(raw))
if e != nil {
return e
}
+75
View File
@@ -0,0 +1,75 @@
package store
import (
"context"
"encoding/json"
"strings"
"unicode/utf8"
)
// Remote IDs are opaque identities, not URLs to fetch or display labels.
type FeedCategory struct {
ID string `json:"id"`
Label string `json:"label"`
}
type FeedRemoteSubscription struct {
ID string `json:"id"`
Title string `json:"title"`
Categories []FeedCategory `json:"categories"`
}
func ValidFeedRemoteID(id string) bool {
return id != "" && len(id) <= 2048 && utf8.ValidString(id) && !strings.ContainsAny(id, "\x00\r\n")
}
// Bound both the number of entries and the encoded snapshot independently of
// the HTTP response limit. Invalid snapshots must not erase a good cache.
func FeedCatalogJSON(catalog []FeedRemoteSubscription) (string, error) {
if len(catalog) > 1000 {
return "", ErrTaskDAVLimit
}
seen := map[string]bool{}
labels := map[string]string{}
for _, sub := range catalog {
if !ValidFeedRemoteID(sub.ID) || seen[sub.ID] || len(sub.Title) > 512 || !utf8.ValidString(sub.Title) || len(sub.Categories) > 64 {
return "", ErrTaskInvalid
}
seen[sub.ID] = true
cats := map[string]bool{}
for _, c := range sub.Categories {
if !ValidFeedRemoteID(c.ID) || cats[c.ID] || len(c.Label) > 512 || !utf8.ValidString(c.Label) {
return "", ErrTaskInvalid
}
if label, ok := labels[c.ID]; ok && label != c.Label {
return "", ErrTaskInvalid
}
cats[c.ID], labels[c.ID] = true, c.Label
}
}
if catalog == nil {
catalog = []FeedRemoteSubscription{}
}
raw, e := json.Marshal(catalog)
if e != nil {
return "", e
}
if len(raw) > 1<<20 {
return "", ErrTaskDAVLimit
}
return string(raw), nil
}
func (s *Store) FeedCatalog(ctx context.Context, domain, owner, source string) ([]FeedRemoteSubscription, error) {
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
return nil, e
}
var raw string
if e := s.DB.QueryRowContext(ctx, `SELECT remote_catalog FROM feed_sources WHERE uuid=? AND domain_uuid=? AND owner_uuid=?`, source, domain, owner).Scan(&raw); e != nil {
return nil, e
}
var catalog []FeedRemoteSubscription
if e := json.Unmarshal([]byte(raw), &catalog); e != nil {
return nil, e
}
return catalog, nil
}
+41
View File
@@ -109,3 +109,44 @@ func TestFeedImportReceiptMigrationRollbackAndRepeat(t *testing.T) {
t.Fatal("receipt must survive deletion", count, err)
}
}
func TestFeedCatalogMigrationRollbackAndRepeat(t *testing.T) {
st := newSchemaTestStore(t)
ctx := context.Background()
if e := db.ApplySQLiteMigrations(ctx, st.DB, "feeds", feedMigrations[:3]); e != nil {
t.Fatal(e)
}
if _, e := st.DB.Exec(`INSERT INTO feed_sources(uuid,domain_uuid,owner_uuid,provider,title,folder,url,version,enabled,interval_seconds,lease,next_ms) VALUES('s','d','u','freshrss','Title','','https://example.org',7,1,300,'old',999); INSERT INTO feed_items(source_uuid,id,title,url,body,published_ms,feed,folder,is_read,starred,body_html) VALUES('s','i','Title','','Body',1,'','',1,1,'<p>Body</p>')`); e != nil {
t.Fatal(e)
}
broken := append([]db.SQLMigration(nil), feedMigrations...)
broken[3].Statements = append(append([]string(nil), feedMigrations[3].Statements...), `INSERT INTO no_such_feed_table VALUES(1)`)
if e := db.ApplySQLiteMigrations(ctx, st.DB, "feeds", broken); e == nil {
t.Fatal("expected failure")
}
var count int
if e := st.DB.QueryRow(`SELECT count(*) FROM pragma_table_info('feed_items') WHERE name='remote_feed'`).Scan(&count); e != nil || count != 0 {
t.Fatal("partial migration", count, e)
}
if e := st.DB.QueryRow(`SELECT version FROM feed_sources WHERE uuid='s'`).Scan(&count); e != nil || count != 7 {
t.Fatal("partial fence", count, e)
}
for range 2 {
if e := st.ensureSchemaComponent("feeds"); e != nil {
t.Fatal(e)
}
}
var version, next int
var catalog, lease, body string
var read, star bool
if e := st.DB.QueryRow(`SELECT version,next_ms,lease,remote_catalog FROM feed_sources WHERE uuid='s'`).Scan(&version, &next, &lease, &catalog); e != nil || version != 8 || next != 0 || lease != "" || catalog != "[]" {
t.Fatal(version, next, lease, catalog, e)
}
if e := st.DB.QueryRow(`SELECT body_html,is_read,starred FROM feed_items WHERE source_uuid='s'`).Scan(&body, &read, &star); e != nil || body != "<p>Body</p>" || !read || !star {
t.Fatal("lost old cache", body, read, star, e)
}
var integrity string
if e := st.DB.QueryRow(`PRAGMA integrity_check`).Scan(&integrity); e != nil || integrity != "ok" {
t.Fatal(integrity, e)
}
}