Files
pages/internal/store/event_jobs_backend_test.go
T
2026-11-15 05:47:14 +09:00

384 lines
11 KiB
Go

package store
import (
"context"
"database/sql"
"errors"
"fmt"
"testing"
"time"
)
func eventBackendFixture(t *testing.T, b storeSQLBackend) *Store {
t.Helper()
s := testNamespaceStore(t, b)
for range 2 {
if err := s.EnsureEventJobsSchema(); err != nil {
t.Fatal(err)
}
}
if b == storePostgreSQL {
release, err := s.StartNativeEventWorker(context.Background())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := release(); err != nil {
t.Error(err)
}
})
}
return s
}
func TestEventBackendLifecycle(t *testing.T) {
for _, b := range []storeSQLBackend{storeSQLite, storePostgreSQL} {
t.Run(fmt.Sprint(b), func(t *testing.T) {
s := eventBackendFixture(t, b)
ctx := context.Background()
at := time.Now().Add(-time.Second)
for _, key := range []string{"one", "two"} {
if ok, err := s.EnqueueEvent("test", key, "synthetic payload", at); err != nil || !ok {
t.Fatal(ok, err)
}
}
if ok, err := s.EnqueueEvent("test", "one", "other", at); err != nil || ok {
t.Fatal(ok, err)
}
job, err := s.ClaimDueEventAt(ctx, time.Now())
if err != nil {
t.Fatal(err)
}
if b == storePostgreSQL && job.ClaimToken == "" {
t.Fatal("missing claim token")
}
if err = s.FinishClaimedEventAt(ctx, job, nil, time.Now()); err != nil {
t.Fatal(err)
}
if b == storePostgreSQL {
if err = s.FinishClaimedEventAt(ctx, job, nil, time.Now()); !errors.Is(err, ErrEventClaim) {
t.Fatal("replayed completion", err)
}
}
logs, err := s.ListEventLogs(job.UUID, 10)
if err != nil || len(logs) != 1 {
t.Fatal(logs, err)
}
if err = s.AddEventLog(job.UUID, "", "info", "another"); err != nil {
t.Fatal(err)
}
logs, err = s.ListEventLogsBefore(job.UUID, 1, 0)
if err != nil || len(logs) != 1 {
t.Fatal(logs, err)
}
if older, err := s.ListEventLogsBefore(job.UUID, 5, logs[0].ID); err != nil || len(older) != 1 {
t.Fatal(older, err)
}
trends, err := s.ListEventLogTrends(ctx, "test", 0, 100)
if err != nil || len(trends) == 0 {
t.Fatal(trends, err)
}
if _, _, err = s.EventJobPage(ctx, "history", "", 1); err != nil {
t.Fatal(err)
}
job, err = s.ClaimDueEvent()
if err != nil {
t.Fatal(err)
}
if err = s.CancelEventJob(job.UUID); err != nil {
t.Fatal(err)
}
if stop, err := s.EventCancellationState(ctx, job.UUID); err != nil || !stop {
t.Fatal(stop, err)
}
if err = s.FinishClaimedEventAt(ctx, job, errors.New("cancelled"), time.Now()); err != nil {
t.Fatal(err)
}
if err = s.RetryEventJob(job.UUID); err != nil {
t.Fatal(err)
}
job, err = s.ClaimDueEvent()
if err != nil {
t.Fatal(err)
}
if err = s.FinishClaimedEventAt(ctx, job, ErrEventSkipped, time.Now()); err != nil {
t.Fatal(err)
}
if stats, err := s.EventStats(); err != nil || stats.Succeeded != 1 || stats.Skipped != 1 {
t.Fatal(stats, err)
}
if jobs, err := s.ListEventJobs(10); err != nil || len(jobs) != 2 {
t.Fatal(jobs, err)
}
if _, err = s.ClaimDueEvent(); !errors.Is(err, sql.ErrNoRows) {
t.Fatal(err)
}
q, err := s.GetQueueSettings()
if err != nil {
t.Fatal(err)
}
q.Paused = true
if err = s.SetQueueSettings(q); err != nil {
t.Fatal(err)
}
if q, err = s.GetQueueSettings(); err != nil || !q.Paused {
t.Fatal(q, err)
}
s.ReadOnly = true
if err = s.SetQueueSettings(q); !errors.Is(err, ErrReadOnly) {
t.Fatal(err)
}
})
}
}
func TestPostgreSQLEventScheduleAtomicityAndFencing(t *testing.T) {
s := eventBackendFixture(t, storePostgreSQL)
ctx := context.Background()
at := time.Now()
peer := &Store{DB: s.DB, sqlBackend: storePostgreSQL}
if release, err := peer.StartNativeEventWorker(ctx); !errors.Is(err, ErrEventWorker) {
if release != nil {
release()
}
t.Fatal("second owner accepted", err)
}
v := EventDefinition{Name: "Synthetic interval", EventType: "test", EventKey: "scheduled", Payload: "{}", ScheduleKind: "interval", IntervalSeconds: 60, NextRunAtMS: at.Add(-time.Second).UnixMilli(), Enabled: true, MaxAttempts: 2}
if err := s.SaveEventDefinition(&v); err != nil {
t.Fatal(err)
}
if defs, err := s.ListEventDefinitions(); err != nil || len(defs) != 1 {
t.Fatal(defs, err)
}
counts := make(chan int, 2)
errs := make(chan error, 2)
for _, w := range []*Store{s, peer} {
go func(w *Store) { n, e := w.EnqueueDueDefinitions(at); counts <- n; errs <- e }(w)
}
if n := <-counts + <-counts; n != 1 {
t.Fatal(n)
}
for range 2 {
if e := <-errs; e != nil {
t.Fatal(e)
}
}
job, err := s.ClaimDueEventAt(ctx, at)
if err != nil {
t.Fatal(err)
}
if err = s.EnsureEventJobsSchema(); err != nil {
t.Fatal(err)
}
if stats, err := s.ActiveEventStats(); err != nil || stats.Running != 1 {
t.Fatal("schema stole worker", stats, err)
}
if err = s.FinishEventAt(ctx, job.UUID, nil, at); !errors.Is(err, ErrEventClaim) {
t.Fatal(err)
}
if err = s.FinishClaimedEventAt(ctx, job, errors.New("retry"), at); err != nil {
t.Fatal(err)
}
next, err := s.ClaimDueEventAt(ctx, at.Add(time.Minute))
if err != nil {
t.Fatal(err)
}
if next.ClaimToken == job.ClaimToken {
t.Fatal("reused token")
}
if err = s.FinishClaimedEventAt(ctx, job, nil, at.Add(time.Minute)); !errors.Is(err, ErrEventClaim) {
t.Fatal("stale completion accepted", err)
}
if _, err = s.DB.Exec(`ALTER TABLE event_log_buckets RENAME COLUMN count TO unavailable_count`); err != nil {
t.Fatal(err)
}
if err = s.FinishClaimedEventAt(ctx, next, nil, at.Add(time.Minute)); !errors.Is(err, ErrEventCompletionLog) {
t.Fatal(err)
}
if stats, err := s.EventStats(); err != nil || stats.Succeeded != 1 {
t.Fatal("log error retried side effect", stats, err)
}
if _, err = s.DB.Exec(`ALTER TABLE event_log_buckets RENAME COLUMN unavailable_count TO count`); err != nil {
t.Fatal(err)
}
if err = s.DeleteEventDefinition(v.UUID); err != nil {
t.Fatal(err)
}
q, err := s.GetQueueSettings()
if err != nil {
t.Fatal(err)
}
q.MaxDepth = 1
if err = s.SetQueueSettings(q); err != nil {
t.Fatal(err)
}
v.UUID = ""
v.NextRunAtMS = at.Add(-time.Second).UnixMilli()
if err = s.SaveEventDefinition(&v); err != nil {
t.Fatal(err)
}
if _, err = s.EnqueueEvent("fill", "one", "", at); err != nil {
t.Fatal(err)
}
if n, err := s.EnqueueDueDefinitions(at); !errors.Is(err, ErrEventQueueFull) || n != 0 {
t.Fatal(n, err)
}
var due int64
if err = s.DB.QueryRow(`SELECT next_run_at_ms FROM event_definitions WHERE uuid=$1`, v.UUID).Scan(&due); err != nil || due != v.NextRunAtMS {
t.Fatal("lost schedule on full queue", due, err)
}
}
func TestPostgreSQLEventRestartAndBuiltins(t *testing.T) {
s := testNamespaceStore(t, storePostgreSQL)
ctx := context.Background()
if err := s.EnsureEventJobsSchema(); err != nil {
t.Fatal(err)
}
for range 2 {
for _, f := range []func() error{s.EnsureFeedEvent, s.EnsureTaskActivityBridgeEvent, s.EnsureTaskReminderEvent, s.EnsureTaskFocusEvent, s.EnsureTaskDigestEvent} {
if err := f(); err != nil {
t.Fatal(err)
}
}
}
defs, err := s.ListEventDefinitions()
if err != nil || len(defs) != 5 {
t.Fatal(defs, err)
}
v := defs[0]
v.Enabled = false
v.IntervalSeconds = 777
if err = s.SaveEventDefinition(&v); err != nil {
t.Fatal(err)
}
if err = s.EnsureFeedEvent(); err != nil {
t.Fatal(err)
}
if err = s.EnsureTaskActivityBridgeEvent(); err != nil {
t.Fatal(err)
}
if err = s.EnsureTaskReminderEvent(); err != nil {
t.Fatal(err)
}
if err = s.EnsureTaskFocusEvent(); err != nil {
t.Fatal(err)
}
if err = s.EnsureTaskDigestEvent(); err != nil {
t.Fatal(err)
}
var enabled bool
var interval int64
if err = s.DB.QueryRow(`SELECT enabled,interval_seconds FROM event_definitions WHERE uuid=$1`, v.UUID).Scan(&enabled, &interval); err != nil || enabled || interval != 777 {
t.Fatal(enabled, interval, err)
}
release, err := s.StartNativeEventWorker(ctx)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { release() })
if _, err = s.EnqueueEvent("synthetic", "restart", "", time.Now()); err != nil {
t.Fatal(err)
}
old, err := s.ClaimDueEvent()
if err != nil {
t.Fatal(err)
}
if err = release(); err != nil {
t.Fatal(err)
}
if _, err = s.ClaimDueEvent(); !errors.Is(err, ErrEventWorker) {
t.Fatal(err)
}
peer := &Store{DB: s.DB, sqlBackend: storePostgreSQL}
release2, err := peer.StartNativeEventWorker(ctx)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { release2() })
if stats, err := peer.EventStats(); err != nil || stats.Failed != 1 || stats.Queued != 0 {
t.Fatal("ambiguous work was retried", stats, err)
}
if err = peer.RetryEventJob(old.UUID); err != nil {
t.Fatal(err)
}
current, err := peer.ClaimDueEvent()
if err != nil {
t.Fatal(err)
}
if err = s.FinishClaimedEventAt(ctx, old, nil, time.Now()); !errors.Is(err, ErrEventClaim) {
t.Fatal("pre-restart worker accepted", err)
}
if err = peer.FinishClaimedEventAt(ctx, current, nil, time.Now()); err != nil {
t.Fatal(err)
}
}
func TestPostgreSQLEventConcurrentClaims(t *testing.T) {
s := eventBackendFixture(t, storePostgreSQL)
ctx := context.Background()
settings, err := s.GetQueueSettings()
if err != nil {
t.Fatal(err)
}
settings.Concurrency = 4
if err = s.SetQueueSettings(settings); err != nil {
t.Fatal(err)
}
for i := 0; i < 12; i++ {
if _, err = s.EnqueueEvent("synthetic", fmt.Sprint(i), "", time.Now()); err != nil {
t.Fatal(err)
}
}
jobs := make(chan *EventJob, 16)
errs := make(chan error, 16)
for range 16 {
go func() { job, err := s.ClaimDueEventAt(ctx, time.Now()); jobs <- job; errs <- err }()
}
seen := map[string]bool{}
for range 16 {
job := <-jobs
if job != nil {
if seen[job.UUID] || job.ClaimToken == "" {
t.Fatal("duplicate or unfenced claim", job.UUID)
}
seen[job.UUID] = true
}
if err := <-errs; err != nil && !errors.Is(err, sql.ErrNoRows) {
t.Fatal(err)
}
}
if len(seen) != 4 {
t.Fatal("concurrency limit", len(seen))
}
if stats, err := s.ActiveEventStats(); err != nil || stats.Running != 4 || stats.Queued != 8 {
t.Fatal(stats, err)
}
}
func TestBuiltinEventDefinitionBackend(t *testing.T) {
for _, b := range []storeSQLBackend{storeSQLite, storePostgreSQL} {
t.Run(fmt.Sprint(b), func(t *testing.T) {
s := eventBackendFixture(t, b)
v := EventDefinition{UUID: "synthetic.builtin", Name: "Calendar source synchronization", EventType: "calendar.source.sweep", EventKey: "calendar.source.sweep", ScheduleKind: "interval", IntervalSeconds: 60, Enabled: true, MaxAttempts: 1}
if err := s.EnsureBuiltinEventDefinition(v); err != nil {
t.Fatal(err)
}
v.Enabled = false
v.IntervalSeconds = 123
if err := s.SaveEventDefinition(&v); err != nil {
t.Fatal(err)
}
v.Enabled = true
v.IntervalSeconds = 60
if err := s.EnsureBuiltinEventDefinition(v); err != nil {
t.Fatal(err)
}
defs, err := s.ListEventDefinitions()
if err != nil || len(defs) != 1 || defs[0].Enabled || defs[0].IntervalSeconds != 123 {
t.Fatal(defs, err)
}
})
}
}