feat(feeds): inherit FreshRSS folders in cached subscription tree
Build and publish container images / images (push) Successful in 10m33s
Build and publish container images / images (push) Successful in 10m33s
This commit is contained in:
1 parent
1057be6d2b
commit
bf9a9c2847
14 files changed
+408
-55
No files matched your search
@@ -51,12 +51,12 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
switch {
|
||||
case key == "catalog" && r.Method == "GET":
|
||||
catalog, e := s.store.FeedCatalog(r.Context(), domain, actor, r.URL.Query().Get("source"))
|
||||
catalog, e := s.store.FeedCatalogSnapshot(r.Context(), domain, actor, r.URL.Query().Get("source"))
|
||||
if e != nil {
|
||||
taskError(w, e)
|
||||
return
|
||||
}
|
||||
writeEventJSON(w, 200, map[string]any{"subscriptions": catalog})
|
||||
writeEventJSON(w, 200, catalog)
|
||||
return
|
||||
case key == "note" && r.Method == "POST":
|
||||
var input struct {
|
||||
@@ -170,7 +170,7 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
items, e := s.store.FeedItems(r.Context(), domain, actor, q.Get("source"), q.Get("filter"), q.Get("q"), before, q.Get("key"), q.Get("feed"), q.Get("category"))
|
||||
items, e := s.store.FeedItems(r.Context(), domain, actor, q.Get("source"), q.Get("filter"), q.Get("q"), before, q.Get("key"), q.Get("feed"), q.Get("category"), q.Get("category_mode"))
|
||||
if e != nil {
|
||||
taskError(w, e)
|
||||
return
|
||||
|
||||
@@ -49,6 +49,7 @@ func TestFeedCatalogBackgroundSnapshotIsolation(t *testing.T) {
|
||||
}
|
||||
calls := 0
|
||||
failCatalog := false
|
||||
failFolders := false
|
||||
client := &http.Client{Transport: feedTransportFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
body := ""
|
||||
@@ -63,7 +64,7 @@ func TestFeedCatalogBackgroundSnapshotIsolation(t *testing.T) {
|
||||
t.Error("article request")
|
||||
}
|
||||
body = `{"items":[{"id":"a","title":"First","origin":{"title":"Same title","streamId":"feed/1"},"categories":["user/alice/label/Tech"]},{"id":"b","title":"Second","origin":{"title":"Same title","streamId":"feed/2"},"categories":["user/alice/label/Other"]}]}`
|
||||
if failCatalog {
|
||||
if failCatalog || failFolders {
|
||||
body = `{"items":[]}`
|
||||
}
|
||||
case strings.HasSuffix(r.URL.Path, "/subscription/list"):
|
||||
@@ -74,6 +75,11 @@ func TestFeedCatalogBackgroundSnapshotIsolation(t *testing.T) {
|
||||
return nil, errors.New("synthetic timeout")
|
||||
}
|
||||
body = testFeedCatalog
|
||||
case strings.HasSuffix(r.URL.Path, "/tag/list"):
|
||||
if failFolders {
|
||||
return nil, errors.New("synthetic folder failure")
|
||||
}
|
||||
body = `{"tags":[{"id":"user/-/label/Empty/Folder","type":"folder"},{"id":"user/-/label/Not a folder","type":"tag"}]}`
|
||||
default:
|
||||
t.Fatalf("unexpected external request %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
@@ -89,7 +95,7 @@ func TestFeedCatalogBackgroundSnapshotIsolation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
run()
|
||||
if calls != 3 {
|
||||
if calls != 4 {
|
||||
t.Fatal("fetch count", calls)
|
||||
}
|
||||
w := feedRequestTest(s, token, "GET", "catalog?source="+v.UUID, nil)
|
||||
@@ -127,9 +133,16 @@ func TestFeedCatalogBackgroundSnapshotIsolation(t *testing.T) {
|
||||
if _, e := s.store.FeedCatalog(ctx, uuid.NewString(), actor, v.UUID); e == nil {
|
||||
t.Fatal("cross-domain catalog leak")
|
||||
}
|
||||
if calls != 3 {
|
||||
if calls != 4 {
|
||||
t.Fatal("foreground external request")
|
||||
}
|
||||
failFolders = true
|
||||
run()
|
||||
kept, e := s.store.FeedCatalogSnapshot(ctx, domain, actor, v.UUID)
|
||||
if e != nil || len(kept.Folders) != 3 || len(kept.Subscriptions) != 3 {
|
||||
t.Fatal("folder failure erased snapshot", kept, e)
|
||||
}
|
||||
failFolders = false
|
||||
failCatalog = true
|
||||
run()
|
||||
catalog, e := s.store.FeedCatalog(ctx, domain, actor, v.UUID)
|
||||
|
||||
@@ -345,7 +345,7 @@ func freshRSSLogin(ctx context.Context, client *http.Client, v store.FeedSource,
|
||||
}
|
||||
return auth, nil
|
||||
}
|
||||
func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string, catalogs ...*[]store.FeedRemoteSubscription) ([]store.FeedItem, error) {
|
||||
func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string, catalogs ...*store.FeedCatalogSnapshot) ([]store.FeedItem, error) {
|
||||
base := strings.TrimRight(v.URL, "/")
|
||||
auth, e := freshRSSLogin(ctx, client, v, user, password)
|
||||
if e != nil {
|
||||
@@ -368,7 +368,23 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
*catalogs[0] = catalog
|
||||
raw, _, _, e = feedRequest(ctx, client, "GET", base+"/reader/api/0/tag/list?output=json", auth, nil, nil)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
folders, e := parseFreshRSSFolders(raw)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
folders, e = store.FeedFolders(catalog, folders)
|
||||
if e != nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
snapshot := store.FeedCatalogSnapshot{Subscriptions: catalog, Folders: folders}
|
||||
if _, _, e = snapshot.Encode(); e != nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
*catalogs[0] = snapshot
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
@@ -504,7 +520,7 @@ func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.Event
|
||||
}
|
||||
child, stop := context.WithTimeout(ctx, 8*time.Second)
|
||||
var items []store.FeedItem
|
||||
var catalog []store.FeedRemoteSubscription
|
||||
var catalog store.FeedCatalogSnapshot
|
||||
etag, modified := v.ETag, v.Modified
|
||||
unchanged := false
|
||||
if !validCalendarSourceURL(v.URL) {
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
func parseFreshRSSFolders(raw []byte) ([]store.FeedCategory, error) {
|
||||
if len(raw) > 4<<20 || !utf8.Valid(raw) {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
var response struct {
|
||||
Tags []struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
} `json:"tags"`
|
||||
}
|
||||
if json.Unmarshal(raw, &response) != nil || response.Tags == nil || len(response.Tags) > 10000 {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
folders := []store.FeedCategory{}
|
||||
seen := map[string]bool{}
|
||||
for _, tag := range response.Tags {
|
||||
// Article tags and system state tags are not subscription folders.
|
||||
if tag.Type != "folder" {
|
||||
continue
|
||||
}
|
||||
prefix, label, ok := strings.Cut(tag.ID, "/label/")
|
||||
if !ok || !strings.HasPrefix(prefix, "user/") || len(prefix) <= 5 || label == "" || !store.ValidFeedRemoteID(tag.ID) || seen[tag.ID] {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
seen[tag.ID] = true
|
||||
folders = append(folders, store.FeedCategory{ID: tag.ID, Label: feedText(label, 512)})
|
||||
}
|
||||
if _, e := store.FeedFolders(nil, folders); e != nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
return folders, nil
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
func TestFreshRSSFoldersParsing(t *testing.T) {
|
||||
folders, e := parseFreshRSSFolders([]byte(`{"tags":[{"id":"user/-/state/com.google/starred"},{"id":"user/-/label/ArticleTag","type":"tag"},{"id":"user/-/label/技術/Linux","type":"folder"}]}`))
|
||||
if e != nil || len(folders) != 1 || folders[0].Label != "技術/Linux" {
|
||||
t.Fatal(folders, e)
|
||||
}
|
||||
for _, raw := range []string{`{}`, `{"tags":null}`, `{"tags":[{"id":"invalid","type":"folder"}]}`, `{"tags":[{"id":"user/-/label/","type":"folder"}]}`, `{"tags":[{"id":"user/-/label/A","type":"folder"},{"id":"user/-/label/A","type":"folder"}]}`} {
|
||||
if _, e = parseFreshRSSFolders([]byte(raw)); e == nil {
|
||||
t.Fatal("invalid folder accepted", raw)
|
||||
}
|
||||
}
|
||||
tags := []map[string]string{}
|
||||
for i := 0; i < 1001; i++ {
|
||||
tags = append(tags, map[string]string{"id": fmt.Sprintf("user/-/label/%d", i), "type": "folder"})
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"tags": tags})
|
||||
if _, e = parseFreshRSSFolders(raw); e == nil {
|
||||
t.Fatal("unbounded folders")
|
||||
}
|
||||
if _, e = store.FeedFolders([]store.FeedRemoteSubscription{{ID: "feed/1", Categories: []store.FeedCategory{{ID: "cat", Label: "A"}}}}, []store.FeedCategory{{ID: "cat", Label: "B"}}); e == nil {
|
||||
t.Fatal("conflicting labels")
|
||||
}
|
||||
large := store.FeedCatalogSnapshot{}
|
||||
for i := 0; i < 1000; i++ {
|
||||
large.Subscriptions = append(large.Subscriptions, store.FeedRemoteSubscription{ID: fmt.Sprintf("feed/%d", i), Title: strings.Repeat("s", 512)})
|
||||
large.Folders = append(large.Folders, store.FeedCategory{ID: fmt.Sprintf("user/-/label/%d", i), Label: strings.Repeat("f", 512)})
|
||||
}
|
||||
if _, e := store.FeedCatalogJSON(large.Subscriptions); e != nil {
|
||||
t.Fatal("catalog alone should fit", e)
|
||||
}
|
||||
if _, _, e := large.Encode(); e != store.ErrTaskDAVLimit {
|
||||
t.Fatal("combined snapshot limit", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedFolderMembershipAndUncategorized(t *testing.T) {
|
||||
s, token, domain, actor, source := feedActionFixture(t)
|
||||
ctx := context.Background()
|
||||
if _, e := s.store.DB.Exec(`UPDATE feed_sources SET next_ms=0 WHERE uuid=?`, source.UUID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
sources, e := s.store.FeedSources(ctx, domain, actor)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, v := range sources {
|
||||
if v.UUID == source.UUID {
|
||||
source = v
|
||||
}
|
||||
}
|
||||
claim, e := s.store.ClaimFeed(ctx, source)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
cat := store.FeedCategory{ID: "user/-/label/New/Folder", Label: "New/Folder"}
|
||||
snapshot := store.FeedCatalogSnapshot{Subscriptions: []store.FeedRemoteSubscription{{ID: "feed/1", Title: "Moved", Categories: []store.FeedCategory{cat}}, {ID: "feed/2", Title: "Unclassified", Categories: []store.FeedCategory{}}}, Folders: []store.FeedCategory{{ID: "user/-/label/Empty", Label: "Empty"}}}
|
||||
items := []store.FeedItem{{ID: "a", Title: "Moved", RemoteFeed: "feed/1", RemoteCategories: []string{"user/-/label/Old"}}, {ID: "b", Title: "Unclassified", RemoteFeed: "feed/2", RemoteCategories: []string{"user/-/label/ArticleTag"}}, {ID: "c", Title: "Unknown subscription", RemoteFeed: "feed/unknown"}}
|
||||
if e = s.store.CompleteFeed(ctx, claim, items, "", "", "", false, snapshot); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
query string
|
||||
want string
|
||||
}{{"category=" + url.QueryEscape(cat.ID), "a"}, {"category_mode=uncategorized", "b"}, {"category=user%2F-%2Flabel%2FOld", ""}, {"category=user%2F-%2Flabel%2FEmpty", ""}} {
|
||||
w := feedRequestTest(s, token, "GET", "items?source="+source.UUID+"&"+test.query, nil)
|
||||
var data struct {
|
||||
Items []store.FeedItem `json:"items"`
|
||||
}
|
||||
json.Unmarshal(w.Body.Bytes(), &data)
|
||||
if w.Code != 200 || (test.want == "" && len(data.Items) != 0) || (test.want != "" && (len(data.Items) != 1 || data.Items[0].ID != test.want)) {
|
||||
t.Fatal(test.query, w.Code, w.Body)
|
||||
}
|
||||
}
|
||||
for _, query := range []string{"category_mode=uncategorized", "source=" + source.UUID + "&category_mode=bad", "source=" + source.UUID + "&category_mode=uncategorized&category=" + url.QueryEscape(cat.ID)} {
|
||||
if w := feedRequestTest(s, token, "GET", "items?"+query, nil); w.Code != 422 {
|
||||
t.Fatal(query, w.Code)
|
||||
}
|
||||
}
|
||||
got, e := s.store.FeedCatalogSnapshot(ctx, domain, actor, source.UUID)
|
||||
if e != nil || len(got.Folders) != 2 {
|
||||
t.Fatal(got, e)
|
||||
}
|
||||
}
|
||||
@@ -569,7 +569,7 @@ func TestFeedsChromium(t *testing.T) {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.store.CompleteFeed(context.Background(), claim, []store.FeedItem{{ID: "catalog-a", Title: "Catalog first", RemoteFeed: "feed/1", RemoteCategories: []string{"user/alice/label/Tech"}}, {ID: "catalog-b", Title: "Catalog second", RemoteFeed: "feed/2", RemoteCategories: []string{"user/alice/label/Other"}}}, "", "", "", false, catalog); e != nil {
|
||||
if e = s.store.CompleteFeed(context.Background(), claim, []store.FeedItem{{ID: "catalog-a", Title: "Catalog first", RemoteFeed: "feed/1", RemoteCategories: []string{"user/alice/label/Tech"}}, {ID: "catalog-b", Title: "Catalog second", RemoteFeed: "feed/2", RemoteCategories: []string{"user/alice/label/Other"}}}, "", "", "", false, store.FeedCatalogSnapshot{Subscriptions: catalog, Folders: []store.FeedCategory{{ID: "user/-/label/Empty/Folder", Label: "Empty/Folder"}}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
s.store.BackgroundJobsDisabled = true
|
||||
|
||||
Vendored
+13
-5
@@ -4,6 +4,7 @@
|
||||
const frame=document.createElement('iframe');frame.style='display:block;width:100%;height:100vh;border:0';document.body.append(frame);
|
||||
try{
|
||||
await new Promise(resolve=>{frame.onload=resolve;frame.src='/v/feeds';});const d=frame.contentDocument,w=frame.contentWindow;
|
||||
for(const key of Object.keys(w.localStorage))if(key.startsWith('pageswiki.feed-folders.v1:'))w.localStorage.removeItem(key);
|
||||
await wait(()=>d.querySelectorAll('.feed-row').length===100);
|
||||
assert(d.documentElement.scrollWidth<=w.innerWidth,'Viewport overflow');
|
||||
assert(d.title.startsWith('RSS ·'),'RSS title');assert(d.getElementById('feed-status').textContent==='','Unrequested status copy');assert(w.getComputedStyle(d.getElementById('feed-status')).display==='none','Empty status takes space');
|
||||
@@ -27,10 +28,15 @@
|
||||
assert(d.documentElement.scrollWidth<=w.innerWidth,'OPML overflow');d.querySelector('.feed-import-options input[type=checkbox]').checked=false;d.getElementById('feed-opml-preview').requestSubmit();await wait(()=>!d.getElementById('feed-opml-preview'));
|
||||
assert(d.getElementById('feed-sources').textContent.includes('Import '+w.innerWidth),'Imported source missing');d.getElementById('feed-opml').click();await wait(()=>d.getElementById('feed-opml-file'));choose();await wait(()=>d.querySelectorAll('#feed-opml-preview tbody tr').length===3);assert(d.querySelector('#feed-opml-preview tbody input').disabled,'Duplicate after import');
|
||||
assert(d.documentElement.scrollWidth<=w.innerWidth,'OPML preview overflow');
|
||||
[...d.querySelectorAll('.feed-source>button:first-child')].find(b=>b.textContent==='FreshRSS fixture').click();await wait(()=>d.querySelectorAll('.feed-subscriptions button').length===4&&d.querySelectorAll('.feed-row').length===2);
|
||||
const category=d.getElementById('feed-category');category.value='user/alice/label/Tech';category.dispatchEvent(new w.Event('change'));await wait(()=>d.querySelectorAll('.feed-row').length===1&&d.querySelector('.feed-row').textContent.includes('Catalog first'));
|
||||
assert(d.querySelectorAll('.feed-subscriptions button').length===2,'Category navigation');
|
||||
d.getElementById('feed-category').value='';d.getElementById('feed-category').dispatchEvent(new w.Event('change'));await wait(()=>d.querySelectorAll('.feed-row').length===2);
|
||||
const freshButton=()=>[...d.querySelectorAll('.feed-source>button:first-child')].find(b=>b.textContent==='FreshRSS fixture');freshButton().click();await wait(()=>d.querySelectorAll('.feed-folder').length===4&&d.querySelectorAll('.feed-row').length===2);
|
||||
const folder=id=>[...d.querySelectorAll('.feed-folder')].find(n=>n.dataset.category===id),tech=()=>folder('user/alice/label/Tech');
|
||||
tech().querySelector('.feed-folder-name').click();await wait(()=>d.querySelectorAll('.feed-row').length===1&&d.querySelector('.feed-row').textContent.includes('Catalog first'));
|
||||
assert(tech().querySelector('.feed-folder-toggle').getAttribute('aria-expanded')==='false','Selecting folder expands it unexpectedly');tech().querySelector('.feed-folder-toggle').click();await wait(()=>tech().querySelector('button[data-feed="feed/1"]'));
|
||||
const empty=folder('user/-/label/Empty/Folder');assert(empty.querySelector('.feed-folder-name span').textContent==='Empty/Folder','Slash label split');empty.querySelector('.feed-folder-toggle').click();assert(!empty.querySelector('button[data-feed]'),'Empty folder fabricated a feed');empty.querySelector('.feed-folder-name').click();await wait(()=>d.querySelectorAll('.feed-row').length===0);
|
||||
d.querySelector('.feed-folder[data-mode="uncategorized"] .feed-folder-name').click();await wait(()=>d.querySelectorAll('.feed-row').length===0);
|
||||
d.querySelector('.feed-catalog .feed-tree-all').click();await wait(()=>d.querySelectorAll('.feed-row').length===2);
|
||||
[...d.querySelectorAll('.feed-source>button:first-child')].find(b=>b.textContent==='Synthetic news').click();await wait(()=>!d.querySelector('.feed-catalog'));freshButton().click();await wait(()=>tech()?.querySelector('button[data-feed="feed/1"]'));assert(tech().querySelector('.feed-folder-toggle').getAttribute('aria-expanded')==='true','Folder expansion not retained');
|
||||
folder('user/alice/label/Other').querySelector('.feed-folder-toggle').click();await wait(()=>d.querySelector('.feed-subscriptions button[data-feed="feed/2"]'));
|
||||
d.querySelector('.feed-subscriptions button[data-feed="feed/2"]').click();await wait(()=>d.querySelectorAll('.feed-row').length===1&&d.querySelector('.feed-row').textContent.includes('Catalog second'));
|
||||
d.querySelector('.feed-row').click();await wait(()=>d.querySelector('#feed-reader .feed-body'));assert(d.querySelectorAll('#feed-reader .feed-actions button:disabled').length===2,'FreshRSS not read-only');
|
||||
[...d.querySelectorAll('.feed-source')].find(n=>n.textContent.includes('FreshRSS fixture')).querySelector('button:last-child').click();await wait(()=>d.querySelector('#feed-reader form'));
|
||||
@@ -52,6 +58,8 @@
|
||||
const hs=d.getElementById('feed-history-state');hs.value='all';hs.dispatchEvent(new w.Event('change'));await wait(()=>receipt(seed.missing)?.querySelector('[data-state=confirmed]'));
|
||||
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'Escape',bubbles:true}));assert(!d.querySelector('.feed-layout').classList.contains('action-history'),'History escape');
|
||||
d.getElementById('feed-history').click();await wait(()=>receipt(seed.orphan));d.getElementById('feed-history-state').value='all';d.getElementById('feed-history-state').dispatchEvent(new w.Event('change'));await wait(()=>receipt(seed.missing)?.querySelector('[data-state=confirmed]'));
|
||||
assert(d.documentElement.scrollWidth<=w.innerWidth,'History final overflow');document.body.dataset.vimReady='1';
|
||||
assert(d.documentElement.scrollWidth<=w.innerWidth,'History final overflow');
|
||||
const probe=document.createElement('iframe');probe.hidden=true;document.body.append(probe);await new Promise(resolve=>{probe.onload=resolve;probe.src='/v/feeds?source='+encodeURIComponent(source.uuid);});await wait(()=>probe.contentDocument.querySelector('.feed-folder[data-category="user/alice/label/Tech"] .feed-folder-toggle')?.getAttribute('aria-expanded')==='true');probe.remove();
|
||||
freshButton().click();await wait(()=>tech()?.querySelector('button[data-feed="feed/1"]'));if(w.innerWidth<=700){d.querySelector('#feed-toolbar button').click();await wait(()=>d.querySelector('.feed-layout').classList.contains('sources-open'));assert(w.getComputedStyle(d.getElementById('feed-sources')).display!=='none','Mobile folders hidden');}assert(d.documentElement.scrollWidth<=w.innerWidth,'Folder tree overflow');document.body.dataset.vimReady='1';
|
||||
}catch(e){const out=document.createElement('pre');out.id='error';out.textContent=e.stack;document.body.append(out);}
|
||||
})();
|
||||
Reference in new issue
Block a user