feat(feeds): cache FreshRSS subscriptions and category navigation
Build and publish container images / images (push) Failing after 6m7s
Build and publish container images / images (push) Failing after 6m7s
This commit is contained in:
1 parent
7fe64113c7
commit
bfed3d20f5
12 files changed
+503
-34
No files matched your search
@@ -47,6 +47,14 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case key == "catalog" && r.Method == "GET":
|
||||
catalog, e := s.store.FeedCatalog(r.Context(), domain, actor, r.URL.Query().Get("source"))
|
||||
if e != nil {
|
||||
taskError(w, e)
|
||||
return
|
||||
}
|
||||
writeEventJSON(w, 200, map[string]any{"subscriptions": catalog})
|
||||
return
|
||||
case key == "note" && r.Method == "POST":
|
||||
var input struct {
|
||||
RequestID string `json:"request_id"`
|
||||
@@ -159,7 +167,7 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
items, e := s.store.FeedItems(r.Context(), domain, actor, q.Get("source"), q.Get("filter"), q.Get("q"), before, q.Get("key"))
|
||||
items, e := s.store.FeedItems(r.Context(), domain, actor, q.Get("source"), q.Get("filter"), q.Get("q"), before, q.Get("key"), q.Get("feed"), q.Get("category"))
|
||||
if e != nil {
|
||||
taskError(w, e)
|
||||
return
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
const testFeedCatalog = `{"subscriptions":[{"id":"feed/1","title":"Same title","categories":[{"id":"user/alice/label/Tech","label":"技術"}]},{"id":"feed/2","title":"Same title","categories":[{"id":"user/alice/label/Other","label":"技術"}]},{"id":"feed/3","title":"Empty feed","categories":[]}]}`
|
||||
|
||||
func TestFreshRSSCatalogValidation(t *testing.T) {
|
||||
catalog, e := parseFreshRSSCatalog([]byte(testFeedCatalog))
|
||||
if e != nil || len(catalog) != 3 || catalog[0].Categories[0].ID == catalog[1].Categories[0].ID {
|
||||
t.Fatal(catalog, e)
|
||||
}
|
||||
for _, raw := range []string{`{}`, `{"subscriptions":null}`, `{"subscriptions":[{"id":""}]}`, `{"subscriptions":[{"id":"feed/1"},{"id":"feed/1"}]}`, `{"subscriptions":[{"id":"feed/1","categories":[{"id":"cat","label":"A"},{"id":"cat","label":"A"}]}]}`, `{"subscriptions":[{"id":"feed/1","categories":[{"id":"cat","label":"A"}]},{"id":"feed/2","categories":[{"id":"cat","label":"B"}]}]}`} {
|
||||
if _, e := parseFreshRSSCatalog([]byte(raw)); e == nil {
|
||||
t.Fatal("accepted", raw)
|
||||
}
|
||||
}
|
||||
if c, e := parseFreshRSSCatalog([]byte(`{"subscriptions":[]}`)); e != nil || len(c) != 0 {
|
||||
t.Fatal(c, e)
|
||||
}
|
||||
big := make([]store.FeedRemoteSubscription, 1001)
|
||||
raw, _ := json.Marshal(map[string]any{"subscriptions": big})
|
||||
if _, e := parseFreshRSSCatalog(raw); e == nil {
|
||||
t.Fatal("unbounded catalog")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedCatalogBackgroundSnapshotIsolation(t *testing.T) {
|
||||
s, token, domain, actor, local := feedFixture(t)
|
||||
ctx := context.Background()
|
||||
if e := s.store.FeedSourceAction(ctx, domain, actor, local.UUID, "delete", local.Version); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
v := store.FeedSource{UUID: uuid.NewString(), Provider: "freshrss", Title: "FreshRSS", URL: "https://example.org/api/greader.php", Enabled: true, Interval: 300}
|
||||
v.Credential, _ = s.store.SealFeedCredential(domain, actor, v.UUID, "reader", "secret")
|
||||
if _, e := s.store.SaveFeedSource(ctx, domain, actor, v); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
calls := 0
|
||||
failCatalog := false
|
||||
client := &http.Client{Transport: feedTransportFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
body := ""
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/accounts/ClientLogin"):
|
||||
if r.Method != "POST" {
|
||||
t.Error("login method")
|
||||
}
|
||||
body = "Auth=synthetic"
|
||||
case strings.HasSuffix(r.URL.Path, "/stream/contents/reading-list"):
|
||||
if r.Method != "GET" || r.URL.Query().Get("n") != "200" {
|
||||
t.Error("article request")
|
||||
}
|
||||
body = `{"items":[{"id":"a","title":"First","origin":{"title":"Same title","streamId":"feed/1"},"categories":["user/alice/label/Tech"]},{"id":"b","title":"Second","origin":{"title":"Same title","streamId":"feed/2"},"categories":["user/alice/label/Other"]}]}`
|
||||
if failCatalog {
|
||||
body = `{"items":[]}`
|
||||
}
|
||||
case strings.HasSuffix(r.URL.Path, "/subscription/list"):
|
||||
if r.Method != "GET" || r.Header.Get("Authorization") != "GoogleLogin auth=synthetic" {
|
||||
t.Error("catalog request")
|
||||
}
|
||||
if failCatalog {
|
||||
return nil, errors.New("synthetic timeout")
|
||||
}
|
||||
body = testFeedCatalog
|
||||
default:
|
||||
t.Fatalf("unexpected external request %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||
})}
|
||||
run := func() {
|
||||
t.Helper()
|
||||
if _, e := s.store.DB.Exec(`UPDATE feed_sources SET next_ms=0 WHERE uuid=?`, v.UUID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := s.syncFeedSourcesWithClient(ctx, &store.EventJob{}, client); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
run()
|
||||
if calls != 3 {
|
||||
t.Fatal("fetch count", calls)
|
||||
}
|
||||
w := feedRequestTest(s, token, "GET", "catalog?source="+v.UUID, nil)
|
||||
if w.Code != 200 || !strings.Contains(w.Body.String(), "Empty feed") || strings.Contains(w.Body.String(), "secret") {
|
||||
t.Fatal(w.Code, w.Body)
|
||||
}
|
||||
for _, scope := range []string{"feed=feed%2F1", "category=" + url.QueryEscape("user/alice/label/Tech")} {
|
||||
w = feedRequestTest(s, token, "GET", "items?source="+v.UUID+"&"+scope, nil)
|
||||
var data struct {
|
||||
Items []store.FeedItem `json:"items"`
|
||||
}
|
||||
json.Unmarshal(w.Body.Bytes(), &data)
|
||||
if w.Code != 200 || len(data.Items) != 1 || data.Items[0].ID != "a" {
|
||||
t.Fatal(scope, w.Code, w.Body)
|
||||
}
|
||||
}
|
||||
w = feedRequestTest(s, token, "GET", "items?feed=feed%2F1", nil)
|
||||
if w.Code != 422 {
|
||||
t.Fatal("unscoped remote ID", w.Code, w.Body)
|
||||
}
|
||||
other, e := s.store.CreatePasswordAccount(domain, "catalogother", "long test password", "Other", "")
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
otherToken, e := s.store.CreateAuthSession(domain, other.UUID)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if w = feedRequestTest(s, otherToken, "GET", "catalog?source="+v.UUID, nil); w.Code == 200 {
|
||||
t.Fatal("cross-owner catalog leak")
|
||||
}
|
||||
if w = feedRequestTest(s, "", "GET", "catalog?source="+v.UUID, nil); w.Code == 200 {
|
||||
t.Fatal("anonymous catalog leak")
|
||||
}
|
||||
if _, e := s.store.FeedCatalog(ctx, uuid.NewString(), actor, v.UUID); e == nil {
|
||||
t.Fatal("cross-domain catalog leak")
|
||||
}
|
||||
if calls != 3 {
|
||||
t.Fatal("foreground external request")
|
||||
}
|
||||
failCatalog = true
|
||||
run()
|
||||
catalog, e := s.store.FeedCatalog(ctx, domain, actor, v.UUID)
|
||||
if e != nil || len(catalog) != 3 {
|
||||
t.Fatal("lost catalog", catalog, e)
|
||||
}
|
||||
items, e := s.store.FeedItems(ctx, domain, actor, v.UUID, "", "", -1, "")
|
||||
if e != nil || len(items) != 2 {
|
||||
t.Fatal("lost articles", items, e)
|
||||
}
|
||||
if e := s.setSystemExtensionEnabled(domain, "organizer.feeds", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := s.store.FeedCatalog(ctx, domain, actor, v.UUID); !errors.Is(e, store.ErrAccessDenied) {
|
||||
t.Fatal("disabled read", e)
|
||||
}
|
||||
}
|
||||
@@ -328,7 +328,7 @@ func feedRequest(ctx context.Context, client *http.Client, method, target, auth
|
||||
}
|
||||
return raw, res.Header, res.StatusCode, nil
|
||||
}
|
||||
func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string) ([]store.FeedItem, error) {
|
||||
func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string, catalogs ...*[]store.FeedRemoteSubscription) ([]store.FeedItem, error) {
|
||||
base := strings.TrimRight(v.URL, "/")
|
||||
raw, _, _, e := feedRequest(ctx, client, "POST", base+"/accounts/ClientLogin", "", url.Values{"Email": {user}, "Passwd": {password}}, nil)
|
||||
if e != nil {
|
||||
@@ -354,7 +354,8 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
|
||||
Published int64 `json:"published"`
|
||||
Categories []string `json:"categories"`
|
||||
Origin struct {
|
||||
Title string `json:"title"`
|
||||
Title string `json:"title"`
|
||||
StreamID string `json:"streamId"`
|
||||
} `json:"origin"`
|
||||
Alternate []struct {
|
||||
Href string `json:"href"`
|
||||
@@ -377,7 +378,10 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
seen[i.ID] = true
|
||||
item := store.FeedItem{ID: i.ID, Title: feedText(i.Title, 2048), Feed: feedText(i.Origin.Title, 512)}
|
||||
if (i.Origin.StreamID != "" && !store.ValidFeedRemoteID(i.Origin.StreamID)) || len(i.Categories) > 64 {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
item := store.FeedItem{ID: i.ID, Title: feedText(i.Title, 2048), Feed: feedText(i.Origin.Title, 512), RemoteFeed: i.Origin.StreamID, RemoteCategories: []string{}}
|
||||
if i.Published > 0 && i.Published < time.Now().AddDate(1, 0, 0).Unix() {
|
||||
item.PublishedMS = i.Published * 1000
|
||||
}
|
||||
@@ -391,6 +395,9 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
|
||||
item.URL = feedLink(base, i.Alternate[0].Href)
|
||||
}
|
||||
for _, c := range i.Categories {
|
||||
if !store.ValidFeedRemoteID(c) {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
if strings.HasSuffix(c, "/state/com.google/read") {
|
||||
item.Read = true
|
||||
}
|
||||
@@ -399,12 +406,53 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
|
||||
}
|
||||
if pos := strings.Index(c, "/label/"); pos >= 0 {
|
||||
item.Folder = feedText(c[pos+7:], 512)
|
||||
item.RemoteCategories = append(item.RemoteCategories, c)
|
||||
}
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
if len(catalogs) > 0 {
|
||||
raw, _, _, e = feedRequest(ctx, client, "GET", base+"/reader/api/0/subscription/list?output=json", auth, nil, nil)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
catalog, e := parseFreshRSSCatalog(raw)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
*catalogs[0] = catalog
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func parseFreshRSSCatalog(raw []byte) ([]store.FeedRemoteSubscription, error) {
|
||||
if len(raw) > 4<<20 || !utf8.Valid(raw) {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
var response struct {
|
||||
Subscriptions []store.FeedRemoteSubscription `json:"subscriptions"`
|
||||
}
|
||||
if json.Unmarshal(raw, &response) != nil || response.Subscriptions == nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
if len(response.Subscriptions) > 1000 {
|
||||
return nil, errors.New("limit")
|
||||
}
|
||||
for i := range response.Subscriptions {
|
||||
sub := &response.Subscriptions[i]
|
||||
sub.Title = feedText(sub.Title, 512)
|
||||
if sub.Categories == nil {
|
||||
sub.Categories = []store.FeedCategory{}
|
||||
}
|
||||
for j := range sub.Categories {
|
||||
sub.Categories[j].Label = feedText(sub.Categories[j].Label, 512)
|
||||
}
|
||||
}
|
||||
if _, e := store.FeedCatalogJSON(response.Subscriptions); e != nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
return response.Subscriptions, nil
|
||||
}
|
||||
func (s *Server) syncFeedSources(ctx context.Context, job *store.EventJob) error {
|
||||
client := calendarSourceClient()
|
||||
client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
|
||||
@@ -435,6 +483,7 @@ func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.Event
|
||||
}
|
||||
child, stop := context.WithTimeout(ctx, 8*time.Second)
|
||||
var items []store.FeedItem
|
||||
var catalog []store.FeedRemoteSubscription
|
||||
etag, modified := v.ETag, v.Modified
|
||||
unchanged := false
|
||||
if !validCalendarSourceURL(v.URL) {
|
||||
@@ -443,7 +492,7 @@ func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.Event
|
||||
var user, password string
|
||||
user, password, e = s.store.OpenFeedCredential(v)
|
||||
if e == nil {
|
||||
items, e = fetchFreshRSS(child, client, v, user, password)
|
||||
items, e = fetchFreshRSS(child, client, v, user, password, &catalog)
|
||||
} else {
|
||||
e = errors.New("credentials")
|
||||
}
|
||||
@@ -478,7 +527,7 @@ func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.Event
|
||||
} else {
|
||||
done++
|
||||
}
|
||||
if e = s.store.CompleteFeed(ctx, v, items, etag, modified, code, unchanged); e != nil && !errors.Is(e, store.ErrAccessDenied) && !errors.Is(e, store.ErrTaskConflict) {
|
||||
if e = s.store.CompleteFeed(ctx, v, items, etag, modified, code, unchanged, catalog); e != nil && !errors.Is(e, store.ErrAccessDenied) && !errors.Is(e, store.ErrTaskConflict) {
|
||||
return e
|
||||
}
|
||||
}
|
||||
|
||||
@@ -543,8 +543,30 @@ func TestFeedsChromium(t *testing.T) {
|
||||
if e = s.store.CompleteFeed(context.Background(), claim, items, "", "", "", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
_ = domain
|
||||
_ = actor
|
||||
fresh, e := s.store.SaveFeedSource(context.Background(), domain, actor, store.FeedSource{UUID: uuid.NewString(), Provider: "freshrss", Title: "FreshRSS fixture", URL: "https://example.org/api/greader.php", Enabled: true, Interval: 300})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
all, e := s.store.FeedSources(context.Background(), domain, actor)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, v := range all {
|
||||
if v.UUID == fresh.UUID {
|
||||
fresh = v
|
||||
}
|
||||
}
|
||||
claim, e = s.store.ClaimFeed(context.Background(), fresh)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
catalog, e := parseFreshRSSCatalog([]byte(testFeedCatalog))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.store.CompleteFeed(context.Background(), claim, []store.FeedItem{{ID: "catalog-a", Title: "Catalog first", RemoteFeed: "feed/1", RemoteCategories: []string{"user/alice/label/Tech"}}, {ID: "catalog-b", Title: "Catalog second", RemoteFeed: "feed/2", RemoteCategories: []string{"user/alice/label/Other"}}}, "", "", "", false, catalog); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
s.store.BackgroundJobsDisabled = true
|
||||
s = New(s.store)
|
||||
script, e := os.ReadFile("internal/server/testdata/feeds.js")
|
||||
|
||||
Vendored
+10
-2
@@ -13,7 +13,7 @@
|
||||
d.querySelectorAll('.feed-row')[1].click();await wait(()=>d.querySelector('#feed-reader .feed-body')&&!d.querySelector('#feed-reader .feed-body').shadowRoot);assert(d.querySelector('#feed-reader .feed-body').textContent.includes('<script>'),'Literal text not preserved');d.querySelector('.feed-row').click();await wait(()=>d.querySelector('#feed-reader .feed-body')?.shadowRoot);
|
||||
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'*',bubbles:true}));await wait(()=>d.querySelector('.feed-row[aria-selected=true]').textContent.includes('★'));
|
||||
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'Escape',bubbles:true}));await wait(()=>!d.querySelector('.feed-layout').classList.contains('reading'));
|
||||
d.getElementById('feed-more').click();await wait(()=>d.querySelectorAll('.feed-row').length===120);
|
||||
d.getElementById('feed-more').click();await wait(()=>d.querySelectorAll('.feed-row').length===122);
|
||||
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'/',bubbles:true}));await wait(()=>d.activeElement.type==='search');d.activeElement.value='119';d.activeElement.dispatchEvent(new w.Event('input',{bubbles:true}));await wait(()=>d.querySelectorAll('.feed-row').length===1);
|
||||
d.querySelector('.feed-row').click();await wait(()=>d.querySelector('#feed-reader .feed-body'));
|
||||
d.querySelector('#feed-reader .feed-actions button:last-child').click();await wait(()=>d.querySelector('#feed-reader form'));
|
||||
@@ -26,6 +26,14 @@
|
||||
await wait(()=>d.querySelectorAll('#feed-opml-preview tbody tr').length===3);const checks=d.querySelectorAll('#feed-opml-preview tbody input');assert(checks[0].checked&&!checks[0].disabled&&checks[1].disabled&&checks[2].disabled,'Import eligibility');
|
||||
assert(d.documentElement.scrollWidth<=w.innerWidth,'OPML overflow');d.querySelector('.feed-import-options input[type=checkbox]').checked=false;d.getElementById('feed-opml-preview').requestSubmit();await wait(()=>!d.getElementById('feed-opml-preview'));
|
||||
assert(d.getElementById('feed-sources').textContent.includes('Import '+w.innerWidth),'Imported source missing');d.getElementById('feed-opml').click();await wait(()=>d.getElementById('feed-opml-file'));choose();await wait(()=>d.querySelectorAll('#feed-opml-preview tbody tr').length===3);assert(d.querySelector('#feed-opml-preview tbody input').disabled,'Duplicate after import');
|
||||
assert(d.documentElement.scrollWidth<=w.innerWidth,'OPML preview overflow');document.body.dataset.vimReady='1';
|
||||
assert(d.documentElement.scrollWidth<=w.innerWidth,'OPML preview overflow');
|
||||
[...d.querySelectorAll('.feed-source>button:first-child')].find(b=>b.textContent==='FreshRSS fixture').click();await wait(()=>d.querySelectorAll('.feed-subscriptions button').length===4&&d.querySelectorAll('.feed-row').length===2);
|
||||
const category=d.getElementById('feed-category');category.value='user/alice/label/Tech';category.dispatchEvent(new w.Event('change'));await wait(()=>d.querySelectorAll('.feed-row').length===1&&d.querySelector('.feed-row').textContent.includes('Catalog first'));
|
||||
assert(d.querySelectorAll('.feed-subscriptions button').length===2,'Category navigation');
|
||||
d.getElementById('feed-category').value='';d.getElementById('feed-category').dispatchEvent(new w.Event('change'));await wait(()=>d.querySelectorAll('.feed-row').length===2);
|
||||
d.querySelector('.feed-subscriptions button[data-feed="feed/2"]').click();await wait(()=>d.querySelectorAll('.feed-row').length===1&&d.querySelector('.feed-row').textContent.includes('Catalog second'));
|
||||
d.querySelector('.feed-row').click();await wait(()=>d.querySelector('#feed-reader .feed-body'));assert(d.querySelectorAll('#feed-reader .feed-actions button:disabled').length===2,'FreshRSS not read-only');
|
||||
if(w.innerWidth<=600)d.querySelector('#feed-toolbar button').click();
|
||||
assert(d.documentElement.scrollWidth<=w.innerWidth,'Catalog overflow');document.body.dataset.vimReady='1';
|
||||
}catch(e){const out=document.createElement('pre');out.id='error';out.textContent=e.stack;document.body.append(out);}
|
||||
})();
|
||||
Reference in new issue
Block a user