Take the SQLite writer lock before reading job/policy facts, bound acquisition and cancellation, preserve atomic raw logs and counters, and avoid replaying successful business operations on logging failure. Tested the isolated nine-file release slice with the full Go suite, race tests and go vet.
243 lines
7.4 KiB
Go
243 lines
7.4 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"pageswiki/internal/db"
|
|
)
|
|
|
|
func eventLogFileStore(t *testing.T) (*Store, string) {
|
|
t.Helper()
|
|
path := filepath.Join(t.TempDir(), "queue.db")
|
|
database, err := db.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { database.Close() })
|
|
s := &Store{DB: database}
|
|
if err := s.EnsureEventJobsSchema(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := database.Exec(`INSERT INTO event_jobs(uuid,event_uuid,event_type,event_key,payload,status,run_at_ms,created_at_ms,updated_at_ms) VALUES('job','event','test.logs','key','{}','running',1,1,1)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return s, path
|
|
}
|
|
|
|
func TestEventLogIndependentConcurrentWriters(t *testing.T) {
|
|
s, path := eventLogFileStore(t)
|
|
const writers, each = 8, 40
|
|
var wg sync.WaitGroup
|
|
start := make(chan struct{})
|
|
errs := make(chan error, writers+1)
|
|
for n := range writers {
|
|
database, err := db.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wg.Add(1)
|
|
go func(n int) {
|
|
defer wg.Done()
|
|
defer database.Close()
|
|
<-start
|
|
worker := &Store{DB: database}
|
|
for i := range each {
|
|
if err := worker.AddEventLog("job", "event", "info", fmt.Sprintf("writer-%d-%d", n, i)); err != nil {
|
|
errs <- err
|
|
return
|
|
}
|
|
}
|
|
}(n)
|
|
}
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
<-start
|
|
for range 160 {
|
|
if _, err := s.DB.Exec(`UPDATE event_queue_settings SET max_depth=max_depth+1 WHERE id=1`); err != nil {
|
|
errs <- err
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
close(start)
|
|
wg.Wait()
|
|
close(errs)
|
|
for err := range errs {
|
|
t.Error(err)
|
|
}
|
|
var raw, distinct, aggregated int
|
|
if err := s.DB.QueryRow(`SELECT count(*),count(DISTINCT message) FROM event_logs`).Scan(&raw, &distinct); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.DB.QueryRow(`SELECT COALESCE(sum(count),0) FROM event_log_buckets WHERE event_type='test.logs'`).Scan(&aggregated); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if raw != writers*each || raw != distinct || raw != aggregated {
|
|
t.Fatalf("lost/duplicate logs: raw=%d distinct=%d aggregate=%d", raw, distinct, aggregated)
|
|
}
|
|
}
|
|
|
|
func TestEventLogCancellationAndWaitPolicyRestored(t *testing.T) {
|
|
s, path := eventLogFileStore(t)
|
|
s.DB.SetMaxOpenConns(1)
|
|
holder, err := db.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer holder.Close()
|
|
conn, err := holder.Conn(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer conn.Close()
|
|
if _, err := conn.ExecContext(context.Background(), "BEGIN IMMEDIATE"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer conn.ExecContext(context.Background(), "ROLLBACK")
|
|
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond)
|
|
defer cancel()
|
|
start := time.Now()
|
|
if err := s.AddEventLogContext(ctx, "job", "event", "info", "must not appear"); !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Fatalf("expected deadline, got %v", err)
|
|
}
|
|
if time.Since(start) > time.Second {
|
|
t.Fatal("wait ignored cancellation")
|
|
}
|
|
if _, err := conn.ExecContext(context.Background(), "ROLLBACK"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.AddEventLog("job", "event", "info", "after cancellation"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var count, wait int
|
|
if err := s.DB.QueryRow("PRAGMA busy_timeout").Scan(&wait); err != nil || wait != 5000 {
|
|
t.Fatalf("pooled timeout leaked: %d %v", wait, err)
|
|
}
|
|
if err := s.DB.QueryRow("SELECT count(*) FROM event_logs").Scan(&count); err != nil || count != 1 {
|
|
t.Fatalf("unexpected rows: %d %v", count, err)
|
|
}
|
|
}
|
|
|
|
func TestEventLogAtomicRollbackAndEmptyEventLookup(t *testing.T) {
|
|
s, _ := eventLogFileStore(t)
|
|
s.DB.SetMaxOpenConns(1)
|
|
if _, err := s.DB.Exec(`CREATE TRIGGER fail_bucket BEFORE INSERT ON event_log_buckets BEGIN SELECT RAISE(ABORT,'injected'); END`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.AddEventLog("job", "event", "info", "rollback"); err == nil {
|
|
t.Fatal("expected failure")
|
|
}
|
|
var count int
|
|
if err := s.DB.QueryRow("SELECT count(*) FROM event_logs").Scan(&count); err != nil || count != 0 {
|
|
t.Fatalf("partial write: %d %v", count, err)
|
|
}
|
|
if _, err := s.DB.Exec(`DROP TRIGGER fail_bucket; UPDATE event_jobs SET event_uuid=''`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.AddEventLog("missing-job", "", "info", "unknown"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var eventType string
|
|
if err := s.DB.QueryRow("SELECT event_type FROM event_log_buckets").Scan(&eventType); err != nil || eventType != "unknown" {
|
|
t.Fatalf("empty event ID matched unrelated job: %q %v", eventType, err)
|
|
}
|
|
s.ReadOnly = true
|
|
if err := s.AddEventLog("job", "", "info", "readonly"); !errors.Is(err, ErrReadOnly) {
|
|
t.Fatalf("read-only accepted: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestEventCompletionReportsLogFailureWithoutChangingOutcome(t *testing.T) {
|
|
for _, retry := range []bool{false, true} {
|
|
t.Run(fmt.Sprint(retry), func(t *testing.T) {
|
|
s, _ := eventLogFileStore(t)
|
|
if _, err := s.DB.Exec(`CREATE TRIGGER fail_log BEFORE INSERT ON event_logs BEGIN SELECT RAISE(ABORT,'injected'); END`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var runErr error
|
|
want := "succeeded"
|
|
if retry {
|
|
runErr, want = errors.New("synthetic execution failure"), "queued"
|
|
}
|
|
err := s.FinishEventAt(context.Background(), "job", runErr, time.Now())
|
|
if !errors.Is(err, ErrEventCompletionLog) {
|
|
t.Fatalf("log failure hidden: %v", err)
|
|
}
|
|
var status string
|
|
if err := s.DB.QueryRow("SELECT status FROM event_jobs WHERE uuid='job'").Scan(&status); err != nil || status != want {
|
|
t.Fatalf("outcome changed: %q %v", status, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEventLogDoesNotHidePolicyReadFailure(t *testing.T) {
|
|
s, _ := eventLogFileStore(t)
|
|
if _, err := s.DB.Exec(`DROP TABLE event_log_policies`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.AddEventLog("job", "event", "info", "no fallback on database failure"); err == nil || errors.Is(err, sql.ErrNoRows) {
|
|
t.Fatalf("policy failure hidden: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestEventLogJobLookupUsesIndex(t *testing.T) {
|
|
s, _ := eventLogFileStore(t)
|
|
rows, err := s.DB.Query("EXPLAIN QUERY PLAN "+eventLogJobTypeSQL, "job")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer rows.Close()
|
|
found := false
|
|
for rows.Next() {
|
|
var id, parent, unused int
|
|
var detail string
|
|
if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(detail, "SCAN event_jobs") {
|
|
t.Fatalf("history scan under writer lock: %s", detail)
|
|
}
|
|
found = found || strings.Contains(detail, "SEARCH event_jobs USING INDEX")
|
|
}
|
|
if err := rows.Err(); err != nil || !found {
|
|
t.Fatalf("indexed lookup missing: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestEventLogCancellationDuringWriteRollsBack(t *testing.T) {
|
|
s, _ := eventLogFileStore(t)
|
|
s.DB.SetMaxOpenConns(1)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
err := s.withEventLogWriter(ctx, func(conn *sql.Conn) error {
|
|
if _, err := conn.ExecContext(ctx, `INSERT INTO event_logs(job_uuid,event_uuid,level,message,created_at_ms) VALUES('job','event','info','cancelled',1)`); err != nil {
|
|
return err
|
|
}
|
|
cancel()
|
|
return nil
|
|
})
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("expected cancellation: %v", err)
|
|
}
|
|
var count, wait int
|
|
if err := s.DB.QueryRow("SELECT count(*) FROM event_logs").Scan(&count); err != nil || count != 0 {
|
|
t.Fatalf("cancelled transaction committed: %d %v", count, err)
|
|
}
|
|
if err := s.DB.QueryRow("PRAGMA busy_timeout").Scan(&wait); err != nil || wait != 5000 {
|
|
t.Fatalf("wait not restored after rollback: %d %v", wait, err)
|
|
}
|
|
if err := s.AddEventLog("job", "event", "info", "reusable"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|