package store import ( "context" "errors" "fmt" "strings" "testing" "github.com/google/uuid" ) func feedBackendFixture(t *testing.T, backend storeSQLBackend) (*Store, string, string) { t.Helper() s, d, a := taskBackendFixture(t, backend) for range 2 { if err := s.ensureSchemaComponent("feeds"); err != nil { t.Fatal(err) } } return s, d, a } func seedFeedSource(t *testing.T, s *Store, d, a, provider string) FeedSource { t.Helper() v, err := s.SaveFeedSource(context.Background(), d, a, FeedSource{UUID: uuid.NewString(), Domain: d, Owner: a, Provider: provider, Title: "日本語 source", URL: "https://synthetic.example.test/feed", Enabled: true, Interval: 300, WriteEnabled: provider == "freshrss"}) if err != nil { t.Fatal(err) } return v } func cacheFeedItems(t *testing.T, s *Store, v FeedSource, items []FeedItem, catalog ...FeedCatalogSnapshot) { t.Helper() claim, err := s.ClaimFeed(context.Background(), v) if err != nil { t.Fatal(err) } if err = s.CompleteFeed(context.Background(), claim, items, "synthetic-etag", "", "", false, catalog...); err != nil { t.Fatal(err) } } func TestFeedsBackendLifecycle(t *testing.T) { for _, b := range []storeSQLBackend{storeSQLite, storePostgreSQL} { t.Run(fmt.Sprint(b), func(t *testing.T) { s, d, a := feedBackendFixture(t, b) ctx := context.Background() v := seedFeedSource(t, s, d, a, "local") if due, err := s.DueFeedSources(ctx); err != nil || len(due) != 1 { t.Fatal(due, err) } cacheFeedItems(t, s, v, []FeedItem{{ID: "one", Title: "日本語 100%_ literal", Body: "text", BodyHTML: "

HTML

", PublishedMS: 1800000000001}, {ID: "two", Title: "Second", PublishedMS: 1800000000000}}) items, err := s.FeedItems(ctx, d, a, v.UUID, "", "%_", -1, "") if err != nil || len(items) != 1 || items[0].ID != "one" { t.Fatal(items, err) } item, err := s.FeedItem(ctx, d, a, v.UUID, "one") if err != nil || item.BodyHTML != "

HTML

" { t.Fatal(item, err) } if err = s.SetFeedState(ctx, d, a, v.UUID, "one", true, true, 0); err != nil { t.Fatal(err) } if err = s.SetFeedState(ctx, d, a, v.UUID, "one", false, false, 0); !errors.Is(err, ErrTaskConflict) { t.Fatal(err) } if err = s.SetFeedState(ctx, d, a, v.UUID, "one", false, false, 1<<40); !errors.Is(err, ErrTaskConflict) { t.Fatal("64-bit version not treated as stale", err) } items, err = s.FeedItems(ctx, d, a, v.UUID, "starred", "", -1, "") if err != nil || len(items) != 1 || !items[0].Read || items[0].Version != 1 { t.Fatal(items, err) } items, err = s.FeedItems(ctx, d, a, v.UUID, "unread", "", 1800000000001, v.UUID+":one") if err != nil || len(items) != 1 || items[0].ID != "two" { t.Fatal(items, err) } if _, err = s.FeedItems(ctx, d, uuid.NewString(), "", "", "", -1, ""); !errors.Is(err, ErrAccessDenied) { t.Fatal(err) } v.Title = "Renamed" updated, err := s.SaveFeedSource(ctx, d, a, v) if err != nil || updated.Version != 2 { t.Fatal(updated, err) } if _, err = s.SaveFeedSource(ctx, d, a, v); !errors.Is(err, ErrTaskConflict) { t.Fatal(err) } claim, err := s.ClaimFeed(ctx, updated) if err != nil { t.Fatal(err) } if err = s.CompleteFeed(ctx, claim, nil, "", "", "network", false); err != nil { t.Fatal(err) } if item, err = s.FeedItem(ctx, d, a, v.UUID, "one"); err != nil || item.Body != "text" { t.Fatal(item, err) } if err = s.CompleteFeed(ctx, claim, nil, "", "", "", false); !errors.Is(err, ErrTaskConflict) { t.Fatal("stale worker accepted", err) } s.ReadOnly = true if err = s.FeedSourceAction(ctx, d, a, v.UUID, "delete", 2); !errors.Is(err, ErrReadOnly) { t.Fatal(err) } s.ReadOnly = false if err = s.FeedSourceAction(ctx, d, a, v.UUID, "delete", 2); err != nil { t.Fatal(err) } for _, table := range []string{"feed_sources", "feed_items", "feed_states"} { var n int if err = s.DB.QueryRow(`SELECT count(*) FROM ` + table).Scan(&n); err != nil || n != 0 { t.Fatal(table, n, err) } } }) } } func TestFeedsBackendCatalogAndRemoteReceipts(t *testing.T) { for _, b := range []storeSQLBackend{storeSQLite, storePostgreSQL} { t.Run(fmt.Sprint(b), func(t *testing.T) { s, d, a := feedBackendFixture(t, b) ctx := context.Background() v := seedFeedSource(t, s, d, a, "freshrss") catalog := FeedCatalogSnapshot{Subscriptions: []FeedRemoteSubscription{{ID: "feed/a", Title: "A", Categories: []FeedCategory{{ID: "cat/a", Label: "分類"}}}, {ID: "feed/b", Title: "B"}}, Folders: []FeedCategory{{ID: "cat/empty", Label: "Empty"}}} cacheFeedItems(t, s, v, []FeedItem{{ID: "one", Title: "First", PublishedMS: 1, RemoteFeed: "feed/a"}, {ID: "two", Title: "Second", PublishedMS: 2, RemoteFeed: "feed/b"}}, catalog) if got, err := s.FeedCatalogSnapshot(ctx, d, a, v.UUID); err != nil || len(got.Folders) != 2 || len(got.Subscriptions) != 2 { t.Fatal(got, err) } if items, err := s.FeedItems(ctx, d, a, v.UUID, "", "", -1, "", "", "cat/a"); err != nil || len(items) != 1 || items[0].ID != "one" { t.Fatal(items, err) } if items, err := s.FeedItems(ctx, d, a, v.UUID, "", "", -1, "", "", "", "uncategorized"); err != nil || len(items) != 1 || items[0].ID != "two" { t.Fatal(items, err) } if err := s.SetFeedState(ctx, d, a, v.UUID, "one", true, false, 0); !errors.Is(err, ErrAccessDenied) { t.Fatal(err) } input := FeedAction{RequestID: uuid.NewString(), Source: v.UUID, ID: "one", Field: "read", Desired: true, SourceVersion: v.Version} action, err := s.QueueFeedAction(ctx, d, a, input) if err != nil { t.Fatal(err) } if replay, err := s.QueueFeedAction(ctx, d, a, input); err != nil || replay.RequestID != action.RequestID { t.Fatal(replay, err) } if due, err := s.DueFeedActions(ctx); err != nil || len(due) != 1 { t.Fatal(due, err) } action, source, err := s.ClaimFeedAction(ctx, action) if err != nil { t.Fatal(err) } if err = s.MarkFeedActionSending(ctx, action, source); err != nil { t.Fatal(err) } if err = s.CompleteFeedAction(ctx, action, source, "confirmed", "", &FeedItem{ID: "one", Read: true, Starred: true}); err != nil { t.Fatal(err) } if got, err := s.FeedAction(ctx, d, a, v.UUID, "one"); err != nil || got.Status != "confirmed" { t.Fatal(got, err) } input.RequestID, input.ID = uuid.NewString(), "two" action, err = s.QueueFeedAction(ctx, d, a, input) if err != nil { t.Fatal(err) } action, source, err = s.ClaimFeedAction(ctx, action) if err != nil { t.Fatal(err) } if err = s.MarkFeedActionSending(ctx, action, source); err != nil { t.Fatal(err) } if _, err = s.DB.Exec(`UPDATE feed_actions SET until_ms=0 WHERE request_id=`+s.sqlParam(1), action.RequestID); err != nil { t.Fatal(err) } if due, err := s.DueFeedActions(ctx); err != nil || len(due) != 0 { t.Fatal("uncertain write retried", due, err) } if got, err := s.FeedActionRecord(ctx, d, a, action.RequestID); err != nil || got.Status != "unknown" || !got.CanCheck { t.Fatal(got, err) } v.Title = "Source changed after uncertain send" v, err = s.SaveFeedSource(ctx, d, a, v) if err != nil { t.Fatal(err) } if got, err := s.FeedActionRecord(ctx, d, a, action.RequestID); err != nil || got.Status != "unknown" || got.Error != "source_changed" { t.Fatal(got, err) } if err = s.RecheckFeedAction(ctx, d, a, action.RequestID); err != nil { t.Fatal(err) } due, err := s.DueFeedActions(ctx) if err != nil || len(due) != 1 || due[0].Status != "check_queued" { t.Fatal(due, err) } recheck, currentSource, err := s.ClaimFeedAction(ctx, due[0]) if err != nil || recheck.Status != "checking" { t.Fatal(recheck, err) } if err = s.MarkFeedActionSending(ctx, recheck, currentSource); !errors.Is(err, ErrTaskConflict) { t.Fatal("recheck became remote write", err) } if err = s.CompleteFeedAction(ctx, recheck, currentSource, "unknown", "network", nil); err != nil { t.Fatal(err) } // A source deletion preserves both terminal and uncertain receipts. if err = s.FeedSourceAction(ctx, d, a, v.UUID, "delete", v.Version); err != nil { t.Fatal(err) } history, err := s.FeedActionHistory(ctx, d, a, FeedActionQuery{Source: v.UUID, Status: "all", Before: -1}) if err != nil || len(history) != 2 || history[0].SourcePresent || history[0].CanCheck { t.Fatal(history, err) } if got, err := s.FeedActionRecord(ctx, d, a, action.RequestID); err != nil || got.Status != "unknown" || got.Error != "source_deleted" { t.Fatal(got, err) } last := history[0] if tail, err := s.FeedActionHistory(ctx, d, a, FeedActionQuery{Source: v.UUID, Status: "all", Before: last.CreatedMS, Key: last.RequestID}); err != nil || len(tail) != 1 { t.Fatal(tail, err) } v.Version = 0 if _, err = s.SaveFeedSource(ctx, d, a, v); !errors.Is(err, ErrTaskConflict) { t.Fatal("receipt rebound", err) } }) } } func TestFeedsBackendImportAndWiki(t *testing.T) { for _, b := range []storeSQLBackend{storeSQLite, storePostgreSQL} { t.Run(fmt.Sprint(b), func(t *testing.T) { s, d, a := feedBackendFixture(t, b) ctx := context.Background() input := FeedImport{RequestID: uuid.NewString(), Expected: FeedSourcesSnapshot(nil), Enabled: true, Interval: 300, Entries: []FeedSubscription{{Title: "import", Folder: "folder", URL: "https://synthetic.example.test/feed"}}} ids, err := s.ImportFeedSubscriptions(ctx, d, a, input) if err != nil || len(ids) != 1 { t.Fatal(ids, err) } if replay, err := s.ImportFeedSubscriptions(ctx, d, a, input); err != nil || len(replay) != 1 || replay[0] != ids[0] { t.Fatal(replay, err) } sources, err := s.FeedSources(ctx, d, a) if err != nil || len(sources) != 1 { t.Fatal(sources, err) } v := sources[0] cacheFeedItems(t, s, v, []FeedItem{{ID: "one", Title: "Note", PublishedMS: 1}}) request := uuid.NewString() page, err := s.CaptureFeedNote(ctx, d, a, request, v.UUID, "one", "/feed/日本語", "Note", "") if err != nil { t.Fatal(err) } if replay, err := s.CaptureFeedNote(ctx, d, a, request, v.UUID, "one", "/feed/日本語", "Note", ""); err != nil || replay != page { t.Fatal(replay, err) } if rev, err := s.GetLatestRevision(page); err != nil || !strings.Contains(rev.Body, "<script>") { t.Fatal(rev, err) } if err = s.FeedSourceAction(ctx, d, a, v.UUID, "delete", 1); err != nil { t.Fatal(err) } if _, err = s.ImportFeedSubscriptions(ctx, d, a, input); err != nil { t.Fatal(err) } if sources, err = s.FeedSources(ctx, d, a); err != nil || len(sources) != 0 { t.Fatal("retry resurrected source", sources, err) } }) } } func TestPostgreSQLFeedsConcurrencyAndRecovery(t *testing.T) { s, d, a := feedBackendFixture(t, storePostgreSQL) ctx := context.Background() v := seedFeedSource(t, s, d, a, "local") peer := &Store{DB: s.DB, sqlBackend: storePostgreSQL, DocumentRoot: s.DocumentRoot} type result struct { value FeedSource err error } results := make(chan result, 2) for _, w := range []*Store{s, peer} { go func(w *Store) { v, e := w.ClaimFeed(ctx, v); results <- result{v, e} }(w) } var claim FeedSource wins := 0 for range 2 { r := <-results if r.err == nil { wins++ claim = r.value } else if !errors.Is(r.err, ErrTaskConflict) { t.Fatal(r.err) } } if wins != 1 { t.Fatal(wins) } if _, err := s.DB.Exec(`ALTER TABLE feed_items RENAME COLUMN title TO unavailable_title`); err != nil { t.Fatal(err) } if err := s.CompleteFeed(ctx, claim, []FeedItem{{ID: "one", Title: "synthetic", PublishedMS: 1}}, "new", "", "", false); err == nil { t.Fatal("ignored cache failure") } var lease string if err := s.DB.QueryRow(`SELECT lease FROM feed_sources WHERE uuid=$1`, v.UUID).Scan(&lease); err != nil || lease != claim.Lease { t.Fatal(lease, err) } if _, err := s.DB.Exec(`ALTER TABLE feed_items RENAME COLUMN unavailable_title TO title`); err != nil { t.Fatal(err) } if err := s.CompleteFeed(ctx, claim, []FeedItem{{ID: "one", Title: "synthetic", PublishedMS: 1}}, "new", "", "", false); err != nil { t.Fatal(err) } // Limit each recovery transaction even when a large queue was interrupted. if _, err := s.DB.Exec(`INSERT INTO feed_actions(domain_uuid,owner_uuid,request_id,source_uuid,item_id,field,desired,source_version,status,created_ms) SELECT $1,$2,'expired-'||n,$3,'expired-'||n,'read',1,1,'sending',0 FROM generate_series(1,101) n`, d, a, v.UUID); err != nil { t.Fatal(err) } for _, want := range []int{100, 101} { if due, err := s.DueFeedActions(ctx); err != nil || len(due) != 0 { t.Fatal(due, err) } var got int if err := s.DB.QueryRow(`SELECT count(*) FROM feed_actions WHERE status='unknown'`).Scan(&got); err != nil || got != want { t.Fatal(got, err) } } if _, err := s.DB.Exec(`UPDATE feed_sources SET next_ms=0`); err != nil { t.Fatal(err) } for _, raw := range []string{`broken`, `{"organizer.feeds":false}`, `{"other":123}`, `{"organizer.feeds":null}`} { if _, err := s.DB.Exec(`INSERT INTO config(domain_uuid,key,value,updated_at) VALUES($1,'system_extension_states',$2,0) ON CONFLICT(domain_uuid,key) DO UPDATE SET value=excluded.value`, d, raw); err != nil { t.Fatal(err) } if due, err := s.DueFeedSources(ctx); err != nil || len(due) != 0 { t.Fatal(due, err) } if _, err := s.FeedSources(ctx, d, a); !errors.Is(err, ErrAccessDenied) { t.Fatal(err) } } if _, err := s.DB.Exec(`UPDATE config SET value='{}' WHERE domain_uuid=$1 AND key='system_extension_states'`, d); err != nil { t.Fatal(err) } if due, err := s.DueFeedSources(ctx); err != nil || len(due) != 1 { t.Fatal(due, err) } if _, err := s.DB.Exec(`UPDATE campuses SET enabled=0`); err != nil { t.Fatal(err) } if due, err := s.DueFeedSources(ctx); err != nil || len(due) != 0 { t.Fatal(due, err) } if _, err := s.FeedSources(ctx, d, a); !errors.Is(err, ErrAccessDenied) { t.Fatal(err) } if err := s.EnsureEventJobsSchema(); err != nil { t.Fatal(err) } if err := s.EnsureFeedEvent(); err != nil { t.Fatal(err) } }