369 lines
14 KiB
Go
369 lines
14 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
func feedBackendFixture(t *testing.T, backend storeSQLBackend) (*Store, string, string) {
|
|
t.Helper()
|
|
s, d, a := taskBackendFixture(t, backend)
|
|
for range 2 {
|
|
if err := s.ensureSchemaComponent("feeds"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return s, d, a
|
|
}
|
|
|
|
func seedFeedSource(t *testing.T, s *Store, d, a, provider string) FeedSource {
|
|
t.Helper()
|
|
v, err := s.SaveFeedSource(context.Background(), d, a, FeedSource{UUID: uuid.NewString(), Domain: d, Owner: a, Provider: provider, Title: "日本語 source", URL: "https://synthetic.example.test/feed", Enabled: true, Interval: 300, WriteEnabled: provider == "freshrss"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return v
|
|
}
|
|
|
|
func cacheFeedItems(t *testing.T, s *Store, v FeedSource, items []FeedItem, catalog ...FeedCatalogSnapshot) {
|
|
t.Helper()
|
|
claim, err := s.ClaimFeed(context.Background(), v)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = s.CompleteFeed(context.Background(), claim, items, "synthetic-etag", "", "", false, catalog...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestFeedsBackendLifecycle(t *testing.T) {
|
|
for _, b := range []storeSQLBackend{storeSQLite, storePostgreSQL} {
|
|
t.Run(fmt.Sprint(b), func(t *testing.T) {
|
|
s, d, a := feedBackendFixture(t, b)
|
|
ctx := context.Background()
|
|
v := seedFeedSource(t, s, d, a, "local")
|
|
if due, err := s.DueFeedSources(ctx); err != nil || len(due) != 1 {
|
|
t.Fatal(due, err)
|
|
}
|
|
cacheFeedItems(t, s, v, []FeedItem{{ID: "one", Title: "日本語 100%_ literal", Body: "text", BodyHTML: "<p>HTML</p>", PublishedMS: 1800000000001}, {ID: "two", Title: "Second", PublishedMS: 1800000000000}})
|
|
items, err := s.FeedItems(ctx, d, a, v.UUID, "", "%_", -1, "")
|
|
if err != nil || len(items) != 1 || items[0].ID != "one" {
|
|
t.Fatal(items, err)
|
|
}
|
|
item, err := s.FeedItem(ctx, d, a, v.UUID, "one")
|
|
if err != nil || item.BodyHTML != "<p>HTML</p>" {
|
|
t.Fatal(item, err)
|
|
}
|
|
if err = s.SetFeedState(ctx, d, a, v.UUID, "one", true, true, 0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = s.SetFeedState(ctx, d, a, v.UUID, "one", false, false, 0); !errors.Is(err, ErrTaskConflict) {
|
|
t.Fatal(err)
|
|
}
|
|
if err = s.SetFeedState(ctx, d, a, v.UUID, "one", false, false, 1<<40); !errors.Is(err, ErrTaskConflict) {
|
|
t.Fatal("64-bit version not treated as stale", err)
|
|
}
|
|
items, err = s.FeedItems(ctx, d, a, v.UUID, "starred", "", -1, "")
|
|
if err != nil || len(items) != 1 || !items[0].Read || items[0].Version != 1 {
|
|
t.Fatal(items, err)
|
|
}
|
|
items, err = s.FeedItems(ctx, d, a, v.UUID, "unread", "", 1800000000001, v.UUID+":one")
|
|
if err != nil || len(items) != 1 || items[0].ID != "two" {
|
|
t.Fatal(items, err)
|
|
}
|
|
if _, err = s.FeedItems(ctx, d, uuid.NewString(), "", "", "", -1, ""); !errors.Is(err, ErrAccessDenied) {
|
|
t.Fatal(err)
|
|
}
|
|
v.Title = "Renamed"
|
|
updated, err := s.SaveFeedSource(ctx, d, a, v)
|
|
if err != nil || updated.Version != 2 {
|
|
t.Fatal(updated, err)
|
|
}
|
|
if _, err = s.SaveFeedSource(ctx, d, a, v); !errors.Is(err, ErrTaskConflict) {
|
|
t.Fatal(err)
|
|
}
|
|
claim, err := s.ClaimFeed(ctx, updated)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = s.CompleteFeed(ctx, claim, nil, "", "", "network", false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if item, err = s.FeedItem(ctx, d, a, v.UUID, "one"); err != nil || item.Body != "text" {
|
|
t.Fatal(item, err)
|
|
}
|
|
if err = s.CompleteFeed(ctx, claim, nil, "", "", "", false); !errors.Is(err, ErrTaskConflict) {
|
|
t.Fatal("stale worker accepted", err)
|
|
}
|
|
s.ReadOnly = true
|
|
if err = s.FeedSourceAction(ctx, d, a, v.UUID, "delete", 2); !errors.Is(err, ErrReadOnly) {
|
|
t.Fatal(err)
|
|
}
|
|
s.ReadOnly = false
|
|
if err = s.FeedSourceAction(ctx, d, a, v.UUID, "delete", 2); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, table := range []string{"feed_sources", "feed_items", "feed_states"} {
|
|
var n int
|
|
if err = s.DB.QueryRow(`SELECT count(*) FROM ` + table).Scan(&n); err != nil || n != 0 {
|
|
t.Fatal(table, n, err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFeedsBackendCatalogAndRemoteReceipts(t *testing.T) {
|
|
for _, b := range []storeSQLBackend{storeSQLite, storePostgreSQL} {
|
|
t.Run(fmt.Sprint(b), func(t *testing.T) {
|
|
s, d, a := feedBackendFixture(t, b)
|
|
ctx := context.Background()
|
|
v := seedFeedSource(t, s, d, a, "freshrss")
|
|
catalog := FeedCatalogSnapshot{Subscriptions: []FeedRemoteSubscription{{ID: "feed/a", Title: "A", Categories: []FeedCategory{{ID: "cat/a", Label: "分類"}}}, {ID: "feed/b", Title: "B"}}, Folders: []FeedCategory{{ID: "cat/empty", Label: "Empty"}}}
|
|
cacheFeedItems(t, s, v, []FeedItem{{ID: "one", Title: "First", PublishedMS: 1, RemoteFeed: "feed/a"}, {ID: "two", Title: "Second", PublishedMS: 2, RemoteFeed: "feed/b"}}, catalog)
|
|
if got, err := s.FeedCatalogSnapshot(ctx, d, a, v.UUID); err != nil || len(got.Folders) != 2 || len(got.Subscriptions) != 2 {
|
|
t.Fatal(got, err)
|
|
}
|
|
if items, err := s.FeedItems(ctx, d, a, v.UUID, "", "", -1, "", "", "cat/a"); err != nil || len(items) != 1 || items[0].ID != "one" {
|
|
t.Fatal(items, err)
|
|
}
|
|
if items, err := s.FeedItems(ctx, d, a, v.UUID, "", "", -1, "", "", "", "uncategorized"); err != nil || len(items) != 1 || items[0].ID != "two" {
|
|
t.Fatal(items, err)
|
|
}
|
|
if err := s.SetFeedState(ctx, d, a, v.UUID, "one", true, false, 0); !errors.Is(err, ErrAccessDenied) {
|
|
t.Fatal(err)
|
|
}
|
|
input := FeedAction{RequestID: uuid.NewString(), Source: v.UUID, ID: "one", Field: "read", Desired: true, SourceVersion: v.Version}
|
|
action, err := s.QueueFeedAction(ctx, d, a, input)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if replay, err := s.QueueFeedAction(ctx, d, a, input); err != nil || replay.RequestID != action.RequestID {
|
|
t.Fatal(replay, err)
|
|
}
|
|
if due, err := s.DueFeedActions(ctx); err != nil || len(due) != 1 {
|
|
t.Fatal(due, err)
|
|
}
|
|
action, source, err := s.ClaimFeedAction(ctx, action)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = s.MarkFeedActionSending(ctx, action, source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = s.CompleteFeedAction(ctx, action, source, "confirmed", "", &FeedItem{ID: "one", Read: true, Starred: true}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, err := s.FeedAction(ctx, d, a, v.UUID, "one"); err != nil || got.Status != "confirmed" {
|
|
t.Fatal(got, err)
|
|
}
|
|
input.RequestID, input.ID = uuid.NewString(), "two"
|
|
action, err = s.QueueFeedAction(ctx, d, a, input)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
action, source, err = s.ClaimFeedAction(ctx, action)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = s.MarkFeedActionSending(ctx, action, source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = s.DB.Exec(`UPDATE feed_actions SET until_ms=0 WHERE request_id=`+s.sqlParam(1), action.RequestID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if due, err := s.DueFeedActions(ctx); err != nil || len(due) != 0 {
|
|
t.Fatal("uncertain write retried", due, err)
|
|
}
|
|
if got, err := s.FeedActionRecord(ctx, d, a, action.RequestID); err != nil || got.Status != "unknown" || !got.CanCheck {
|
|
t.Fatal(got, err)
|
|
}
|
|
v.Title = "Source changed after uncertain send"
|
|
v, err = s.SaveFeedSource(ctx, d, a, v)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, err := s.FeedActionRecord(ctx, d, a, action.RequestID); err != nil || got.Status != "unknown" || got.Error != "source_changed" {
|
|
t.Fatal(got, err)
|
|
}
|
|
if err = s.RecheckFeedAction(ctx, d, a, action.RequestID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
due, err := s.DueFeedActions(ctx)
|
|
if err != nil || len(due) != 1 || due[0].Status != "check_queued" {
|
|
t.Fatal(due, err)
|
|
}
|
|
recheck, currentSource, err := s.ClaimFeedAction(ctx, due[0])
|
|
if err != nil || recheck.Status != "checking" {
|
|
t.Fatal(recheck, err)
|
|
}
|
|
if err = s.MarkFeedActionSending(ctx, recheck, currentSource); !errors.Is(err, ErrTaskConflict) {
|
|
t.Fatal("recheck became remote write", err)
|
|
}
|
|
if err = s.CompleteFeedAction(ctx, recheck, currentSource, "unknown", "network", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A source deletion preserves both terminal and uncertain receipts.
|
|
if err = s.FeedSourceAction(ctx, d, a, v.UUID, "delete", v.Version); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
history, err := s.FeedActionHistory(ctx, d, a, FeedActionQuery{Source: v.UUID, Status: "all", Before: -1})
|
|
if err != nil || len(history) != 2 || history[0].SourcePresent || history[0].CanCheck {
|
|
t.Fatal(history, err)
|
|
}
|
|
if got, err := s.FeedActionRecord(ctx, d, a, action.RequestID); err != nil || got.Status != "unknown" || got.Error != "source_deleted" {
|
|
t.Fatal(got, err)
|
|
}
|
|
last := history[0]
|
|
if tail, err := s.FeedActionHistory(ctx, d, a, FeedActionQuery{Source: v.UUID, Status: "all", Before: last.CreatedMS, Key: last.RequestID}); err != nil || len(tail) != 1 {
|
|
t.Fatal(tail, err)
|
|
}
|
|
v.Version = 0
|
|
if _, err = s.SaveFeedSource(ctx, d, a, v); !errors.Is(err, ErrTaskConflict) {
|
|
t.Fatal("receipt rebound", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFeedsBackendImportAndWiki(t *testing.T) {
|
|
for _, b := range []storeSQLBackend{storeSQLite, storePostgreSQL} {
|
|
t.Run(fmt.Sprint(b), func(t *testing.T) {
|
|
s, d, a := feedBackendFixture(t, b)
|
|
ctx := context.Background()
|
|
input := FeedImport{RequestID: uuid.NewString(), Expected: FeedSourcesSnapshot(nil), Enabled: true, Interval: 300, Entries: []FeedSubscription{{Title: "import", Folder: "folder", URL: "https://synthetic.example.test/feed"}}}
|
|
ids, err := s.ImportFeedSubscriptions(ctx, d, a, input)
|
|
if err != nil || len(ids) != 1 {
|
|
t.Fatal(ids, err)
|
|
}
|
|
if replay, err := s.ImportFeedSubscriptions(ctx, d, a, input); err != nil || len(replay) != 1 || replay[0] != ids[0] {
|
|
t.Fatal(replay, err)
|
|
}
|
|
sources, err := s.FeedSources(ctx, d, a)
|
|
if err != nil || len(sources) != 1 {
|
|
t.Fatal(sources, err)
|
|
}
|
|
v := sources[0]
|
|
cacheFeedItems(t, s, v, []FeedItem{{ID: "one", Title: "Note", PublishedMS: 1}})
|
|
request := uuid.NewString()
|
|
page, err := s.CaptureFeedNote(ctx, d, a, request, v.UUID, "one", "/feed/日本語", "Note", "<script>inert</script>")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if replay, err := s.CaptureFeedNote(ctx, d, a, request, v.UUID, "one", "/feed/日本語", "Note", "<script>inert</script>"); err != nil || replay != page {
|
|
t.Fatal(replay, err)
|
|
}
|
|
if rev, err := s.GetLatestRevision(page); err != nil || !strings.Contains(rev.Body, "<script>") {
|
|
t.Fatal(rev, err)
|
|
}
|
|
if err = s.FeedSourceAction(ctx, d, a, v.UUID, "delete", 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = s.ImportFeedSubscriptions(ctx, d, a, input); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if sources, err = s.FeedSources(ctx, d, a); err != nil || len(sources) != 0 {
|
|
t.Fatal("retry resurrected source", sources, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLFeedsConcurrencyAndRecovery(t *testing.T) {
|
|
s, d, a := feedBackendFixture(t, storePostgreSQL)
|
|
ctx := context.Background()
|
|
v := seedFeedSource(t, s, d, a, "local")
|
|
peer := &Store{DB: s.DB, sqlBackend: storePostgreSQL, DocumentRoot: s.DocumentRoot}
|
|
type result struct {
|
|
value FeedSource
|
|
err error
|
|
}
|
|
results := make(chan result, 2)
|
|
for _, w := range []*Store{s, peer} {
|
|
go func(w *Store) { v, e := w.ClaimFeed(ctx, v); results <- result{v, e} }(w)
|
|
}
|
|
var claim FeedSource
|
|
wins := 0
|
|
for range 2 {
|
|
r := <-results
|
|
if r.err == nil {
|
|
wins++
|
|
claim = r.value
|
|
} else if !errors.Is(r.err, ErrTaskConflict) {
|
|
t.Fatal(r.err)
|
|
}
|
|
}
|
|
if wins != 1 {
|
|
t.Fatal(wins)
|
|
}
|
|
if _, err := s.DB.Exec(`ALTER TABLE feed_items RENAME COLUMN title TO unavailable_title`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.CompleteFeed(ctx, claim, []FeedItem{{ID: "one", Title: "synthetic", PublishedMS: 1}}, "new", "", "", false); err == nil {
|
|
t.Fatal("ignored cache failure")
|
|
}
|
|
var lease string
|
|
if err := s.DB.QueryRow(`SELECT lease FROM feed_sources WHERE uuid=$1`, v.UUID).Scan(&lease); err != nil || lease != claim.Lease {
|
|
t.Fatal(lease, err)
|
|
}
|
|
if _, err := s.DB.Exec(`ALTER TABLE feed_items RENAME COLUMN unavailable_title TO title`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.CompleteFeed(ctx, claim, []FeedItem{{ID: "one", Title: "synthetic", PublishedMS: 1}}, "new", "", "", false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Limit each recovery transaction even when a large queue was interrupted.
|
|
if _, err := s.DB.Exec(`INSERT INTO feed_actions(domain_uuid,owner_uuid,request_id,source_uuid,item_id,field,desired,source_version,status,created_ms) SELECT $1,$2,'expired-'||n,$3,'expired-'||n,'read',1,1,'sending',0 FROM generate_series(1,101) n`, d, a, v.UUID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []int{100, 101} {
|
|
if due, err := s.DueFeedActions(ctx); err != nil || len(due) != 0 {
|
|
t.Fatal(due, err)
|
|
}
|
|
var got int
|
|
if err := s.DB.QueryRow(`SELECT count(*) FROM feed_actions WHERE status='unknown'`).Scan(&got); err != nil || got != want {
|
|
t.Fatal(got, err)
|
|
}
|
|
}
|
|
if _, err := s.DB.Exec(`UPDATE feed_sources SET next_ms=0`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, raw := range []string{`broken`, `{"organizer.feeds":false}`, `{"other":123}`, `{"organizer.feeds":null}`} {
|
|
if _, err := s.DB.Exec(`INSERT INTO config(domain_uuid,key,value,updated_at) VALUES($1,'system_extension_states',$2,0) ON CONFLICT(domain_uuid,key) DO UPDATE SET value=excluded.value`, d, raw); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if due, err := s.DueFeedSources(ctx); err != nil || len(due) != 0 {
|
|
t.Fatal(due, err)
|
|
}
|
|
if _, err := s.FeedSources(ctx, d, a); !errors.Is(err, ErrAccessDenied) {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := s.DB.Exec(`UPDATE config SET value='{}' WHERE domain_uuid=$1 AND key='system_extension_states'`, d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if due, err := s.DueFeedSources(ctx); err != nil || len(due) != 1 {
|
|
t.Fatal(due, err)
|
|
}
|
|
if _, err := s.DB.Exec(`UPDATE campuses SET enabled=0`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if due, err := s.DueFeedSources(ctx); err != nil || len(due) != 0 {
|
|
t.Fatal(due, err)
|
|
}
|
|
if _, err := s.FeedSources(ctx, d, a); !errors.Is(err, ErrAccessDenied) {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.EnsureEventJobsSchema(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.EnsureFeedEvent(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|