Fix event log writer contention and report persistence failures safely
Take the SQLite writer lock before reading job/policy facts, bound acquisition and cancellation, preserve atomic raw logs and counters, and avoid replaying successful business operations on logging failure. Tested the isolated nine-file release slice with the full Go suite, race tests and go vet.
This commit is contained in:
1 parent
0f686aa2fb
commit
a36934e72c
9 files changed
+537
-37
No files matched your search
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/dregarnuhr"
|
||||
@@ -34,7 +36,7 @@ func (q schedulerQueue) Cancellation(ctx context.Context, id string) (bool, erro
|
||||
return q.store.EventCancellationState(ctx, id)
|
||||
}
|
||||
func (q schedulerQueue) Started(ctx context.Context, job *dregarnuhr.Job) error {
|
||||
return q.store.AddEventLog(job.UUID, job.EventUUID, "info", "execution started")
|
||||
return q.store.AddEventLogContext(ctx, job.UUID, job.EventUUID, "info", "execution started")
|
||||
}
|
||||
func (q schedulerQueue) Finish(ctx context.Context, job *dregarnuhr.Job, err error, now time.Time) error {
|
||||
return q.store.FinishEventAt(ctx, job.UUID, err, now)
|
||||
@@ -47,6 +49,13 @@ func (s *Server) eventRunner() *dregarnuhr.Runner {
|
||||
Foreground: func() bool { return s.foregroundRequests.Load() > 0 },
|
||||
Wake: s.wakeEventScheduler,
|
||||
Report: func(stage string, job *dregarnuhr.Job, err error) {
|
||||
if errors.Is(err, store.ErrEventCompletionLog) {
|
||||
stage = "finish-log"
|
||||
}
|
||||
if strings.HasSuffix(stage, "-log") {
|
||||
reportEventLogFailure(stage, job, err)
|
||||
return
|
||||
}
|
||||
if job != nil {
|
||||
log.Printf("component=%s event scheduler %s failed job=%s type=%s: %v", dregarnuhr.Component, stage, job.UUID, job.Type, err)
|
||||
} else {
|
||||
@@ -58,6 +67,30 @@ func (s *Server) eventRunner() *dregarnuhr.Runner {
|
||||
return s.schedulerRunner
|
||||
}
|
||||
|
||||
// Logging failures must be visible without repeating a successful side effect
|
||||
// (such as push delivery), or echoing payloads, SQL text or driver messages.
|
||||
func (s *Server) addEventProgressLog(ctx context.Context, job *store.EventJob, message string) {
|
||||
if err := s.store.AddEventLogContext(ctx, job.UUID, job.EventUUID, "info", message); err != nil {
|
||||
reportEventLogFailure("progress-log", job, err)
|
||||
}
|
||||
}
|
||||
|
||||
func reportEventLogFailure(stage string, job *dregarnuhr.Job, err error) {
|
||||
category := "storage_error"
|
||||
var code interface{ Code() int }
|
||||
switch {
|
||||
case errors.Is(err, context.Canceled):
|
||||
category = "cancelled"
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
category = "deadline_exceeded"
|
||||
case errors.Is(err, store.ErrReadOnly):
|
||||
category = "read_only"
|
||||
case errors.As(err, &code):
|
||||
category = fmt.Sprintf("sqlite_%d", code.Code())
|
||||
}
|
||||
log.Printf("component=%s event scheduler %s failed job=%q type=%q reason=%s", dregarnuhr.Component, stage, job.UUID, job.Type, category)
|
||||
}
|
||||
|
||||
func (s *Server) runDueEvents() {
|
||||
if s.store == nil {
|
||||
return
|
||||
|
||||
@@ -22,21 +22,21 @@ func (s *Server) registeredEventHandlers() *dregarnuhr.Registry {
|
||||
store.TaskFocusEventType: func(ctx context.Context, job *store.EventJob) error {
|
||||
processed, err := s.store.CompleteTaskFocus(ctx, time.Now(), 100)
|
||||
if err == nil && processed > 0 {
|
||||
_ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("focus timers completed=%d", processed))
|
||||
s.addEventProgressLog(ctx, job, fmt.Sprintf("focus timers completed=%d", processed))
|
||||
}
|
||||
return err
|
||||
},
|
||||
store.TaskReminderEventType: func(ctx context.Context, job *store.EventJob) error {
|
||||
checked, sent, err := s.store.DeliverTaskReminders(ctx, time.Now(), 100)
|
||||
if err == nil && checked > 0 {
|
||||
_ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("task reminders checked=%d delivered=%d", checked, sent))
|
||||
s.addEventProgressLog(ctx, job, fmt.Sprintf("task reminders checked=%d delivered=%d", checked, sent))
|
||||
}
|
||||
return err
|
||||
},
|
||||
store.TaskActivityBridgeEventType: func(ctx context.Context, job *store.EventJob) error {
|
||||
processed, published, err := s.store.PublishTaskActivity(ctx, 100)
|
||||
if err == nil && processed > 0 {
|
||||
_ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("journal entries checked=%d published=%d", processed, published))
|
||||
s.addEventProgressLog(ctx, job, fmt.Sprintf("journal entries checked=%d published=%d", processed, published))
|
||||
}
|
||||
return err
|
||||
},
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
"pageswiki/internal/dregarnuhr"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
func TestEventLoggingFailureReportedWithoutReexecuting(t *testing.T) {
|
||||
database, err := db.Open(t.TempDir() + "/queue.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
st := &store.Store{DB: database}
|
||||
if err := st.EnsureEventJobsSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A malicious/verbose driver message must never be copied to compose logs.
|
||||
if _, err := database.Exec(`CREATE TRIGGER fail_log BEFORE INSERT ON event_logs BEGIN SELECT RAISE(ABORT,'secret-payload-123'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var output bytes.Buffer
|
||||
old := log.Writer()
|
||||
log.SetOutput(&output)
|
||||
defer log.SetOutput(old)
|
||||
s := &Server{store: st, schedulerWake: make(chan struct{}, 8)}
|
||||
if err := s.registeredEventHandlers().Register("test.log-failure", func(ctx context.Context, job *dregarnuhr.Job) error {
|
||||
s.addEventProgressLog(ctx, job, "private-progress-456")
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = st.EnqueueEvent("test.log-failure", "key", "secret-body-789", time.Now().Add(-time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.eventRunner().Dispatch(context.Background())
|
||||
select {
|
||||
case <-s.schedulerWake:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("worker did not complete")
|
||||
}
|
||||
var status string
|
||||
var attempts int
|
||||
if err := database.QueryRow(`SELECT status,attempts FROM event_jobs WHERE event_type='test.log-failure'`).Scan(&status, &attempts); err != nil || status != "succeeded" || attempts != 1 {
|
||||
t.Fatalf("log failure retried business operation: %s %d %v", status, attempts, err)
|
||||
}
|
||||
for _, stage := range []string{"start-log", "progress-log", "finish-log"} {
|
||||
if !strings.Contains(output.String(), stage+" failed") {
|
||||
t.Fatalf("missing %s diagnostic: %s", stage, output.String())
|
||||
}
|
||||
}
|
||||
for _, secret := range []string{"secret-payload-123", "private-progress-456", "secret-body-789"} {
|
||||
if strings.Contains(output.String(), secret) {
|
||||
t.Fatal("diagnostic leaked content")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,7 +43,7 @@ func (s *Server) deliverNotificationPushContext(ctx context.Context, job *store.
|
||||
return err
|
||||
}
|
||||
if stats.Sent+stats.Failed > 0 {
|
||||
_ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("component=%s push sent=%d failed=%d (no payload or endpoint logged)", ordoschnellic.Component, stats.Sent, stats.Failed))
|
||||
s.addEventProgressLog(ctx, job, fmt.Sprintf("component=%s push sent=%d failed=%d (no payload or endpoint logged)", ordoschnellic.Component, stats.Sent, stats.Failed))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -197,7 +197,7 @@ func (s *Store) FinishEventAt(ctx context.Context, uuid string, runErr error, at
|
||||
run_at_ms=?, finished_at_ms=0, updated_at_ms=? WHERE uuid=?`,
|
||||
outcome.Message, outcome.RetryAt.UnixMilli(), at.UnixMilli(), uuid)
|
||||
if err == nil {
|
||||
_ = s.AddEventLog(uuid, job.EventUUID, outcome.Level, "execution failed; retry scheduled: "+outcome.Message)
|
||||
err = s.addEventCompletionLog(ctx, uuid, job.EventUUID, outcome.Level, "execution failed; retry scheduled: "+outcome.Message)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -209,7 +209,7 @@ func (s *Store) FinishEventAt(ctx context.Context, uuid string, runErr error, at
|
||||
message += ": " + outcome.Message
|
||||
}
|
||||
if err == nil {
|
||||
_ = s.AddEventLog(uuid, job.EventUUID, outcome.Level, message)
|
||||
err = s.addEventCompletionLog(ctx, uuid, job.EventUUID, outcome.Level, message)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -411,37 +411,64 @@ func (s *Store) SetQueueSettings(q QueueSettings) error {
|
||||
}
|
||||
|
||||
func (s *Store) AddEventLog(jobUUID, eventUUID, level, message string) error {
|
||||
tx, err := s.DB.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
return s.AddEventLogContext(context.Background(), jobUUID, eventUUID, level, message)
|
||||
}
|
||||
|
||||
func (s *Store) addEventCompletionLog(ctx context.Context, jobUUID, eventUUID, level, message string) error {
|
||||
if err := s.AddEventLogContext(ctx, jobUUID, eventUUID, level, message); err != nil {
|
||||
return errors.Join(ErrEventCompletionLog, err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
eventType := "unknown"
|
||||
_ = tx.QueryRow(`SELECT event_type FROM event_jobs WHERE uuid=? OR event_uuid=? ORDER BY CASE WHEN uuid=? THEN 0 ELSE 1 END LIMIT 1`, jobUUID, eventUUID, jobUUID).Scan(&eventType)
|
||||
if eventType == "" {
|
||||
eventType = "unknown"
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddEventLogContext atomically persists a raw log and its trend counter. The
|
||||
// caller's earlier deadline wins; even Background callers have a finite budget.
|
||||
func (s *Store) AddEventLogContext(ctx context.Context, jobUUID, eventUUID, level, message string) error {
|
||||
if s.ReadOnly {
|
||||
return ErrReadOnly
|
||||
}
|
||||
policy := EventLogPolicy{EventType: "*", RetentionHours: 168, AggregateAfterHours: 24, BucketSeconds: 60, MaxRows: 5000}
|
||||
var specific EventLogPolicy
|
||||
if err := scanEventLogPolicy(tx.QueryRow(`SELECT event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows FROM event_log_policies WHERE event_type=?`, eventType), &specific); err == nil {
|
||||
policy = specific
|
||||
} else {
|
||||
_ = scanEventLogPolicy(tx.QueryRow(`SELECT event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows FROM event_log_policies WHERE event_type='*'`), &policy)
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
if _, err := tx.Exec(`INSERT INTO event_logs(job_uuid,event_uuid,level,message,created_at_ms) VALUES(?,?,?,?,?)`, jobUUID, eventUUID, level, message, now); err != nil {
|
||||
return err
|
||||
}
|
||||
bucket := int64(policy.BucketSeconds)
|
||||
if bucket < 1 {
|
||||
bucket = 60
|
||||
}
|
||||
bucketStart := now - (now % (bucket * 1000))
|
||||
if _, err := tx.Exec(`INSERT INTO event_log_buckets(event_type,bucket_start_ms,level,count,first_at_ms,last_at_ms,sample_message)
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
return s.withEventLogWriter(ctx, func(tx *sql.Conn) error {
|
||||
eventType := "unknown"
|
||||
// The common path is a primary-key lookup, not OR + sorting the retained
|
||||
// job history while holding the writer lock. Keep event-only fallback.
|
||||
err := tx.QueryRowContext(ctx, eventLogJobTypeSQL, jobUUID).Scan(&eventType)
|
||||
if errors.Is(err, sql.ErrNoRows) && eventUUID != "" {
|
||||
err = tx.QueryRowContext(ctx, `SELECT event_type FROM event_jobs WHERE event_uuid=? LIMIT 1`, eventUUID).Scan(&eventType)
|
||||
}
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
if eventType == "" {
|
||||
eventType = "unknown"
|
||||
}
|
||||
policy := EventLogPolicy{EventType: "*", RetentionHours: 168, AggregateAfterHours: 24, BucketSeconds: 60, MaxRows: 5000}
|
||||
var specific EventLogPolicy
|
||||
if err := scanEventLogPolicy(tx.QueryRowContext(ctx, `SELECT event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows FROM event_log_policies WHERE event_type=?`, eventType), &specific); err == nil {
|
||||
policy = specific
|
||||
} else if errors.Is(err, sql.ErrNoRows) {
|
||||
if err := scanEventLogPolicy(tx.QueryRowContext(ctx, `SELECT event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows FROM event_log_policies WHERE event_type='*'`), &policy); err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO event_logs(job_uuid,event_uuid,level,message,created_at_ms) VALUES(?,?,?,?,?)`, jobUUID, eventUUID, level, message, now); err != nil {
|
||||
return err
|
||||
}
|
||||
bucket := int64(policy.BucketSeconds)
|
||||
if bucket < 1 {
|
||||
bucket = 60
|
||||
}
|
||||
bucketStart := now - (now % (bucket * 1000))
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO event_log_buckets(event_type,bucket_start_ms,level,count,first_at_ms,last_at_ms,sample_message)
|
||||
VALUES(?,?,?,1,?,?,?) ON CONFLICT(event_type,bucket_start_ms,level) DO UPDATE SET count=count+1,last_at_ms=excluded.last_at_ms`, eventType, bucketStart, level, now, now, message); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func scanEventLogPolicy(scanner interface{ Scan(...any) error }, policy *EventLogPolicy) error {
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrEventCompletionLog means the job state is already persisted; callers must
|
||||
// report the missing log without retrying the business operation.
|
||||
var ErrEventCompletionLog = errors.New("event state saved but completion log failed")
|
||||
|
||||
const eventLogJobTypeSQL = `SELECT event_type FROM event_jobs WHERE uuid=?`
|
||||
|
||||
// withEventLogWriter takes the WAL writer lock BEFORE reading policy/job facts.
|
||||
// Only lock acquisition is retried: neither writes nor an ambiguous COMMIT are
|
||||
// replayed. The short SQLite wait allows caller cancellation between attempts.
|
||||
func (s *Store) withEventLogWriter(ctx context.Context, write func(*sql.Conn) error) error {
|
||||
conn, err := s.DB.Conn(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
var previousWait int
|
||||
if err := conn.QueryRowContext(ctx, "PRAGMA busy_timeout").Scan(&previousWait); err != nil {
|
||||
return err
|
||||
}
|
||||
started, committed := false, false
|
||||
defer func() {
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
bad := false
|
||||
if started && !committed {
|
||||
_, err := conn.ExecContext(cleanup, "ROLLBACK")
|
||||
bad = err != nil
|
||||
}
|
||||
if !bad {
|
||||
_, err := conn.ExecContext(cleanup, fmt.Sprintf("PRAGMA busy_timeout=%d", previousWait))
|
||||
bad = err != nil
|
||||
}
|
||||
if bad {
|
||||
// Never pool a connection with an uncertain transaction or wait policy.
|
||||
_ = conn.Raw(func(any) error { return driver.ErrBadConn })
|
||||
}
|
||||
}()
|
||||
if _, err := conn.ExecContext(ctx, "PRAGMA busy_timeout=50"); err != nil {
|
||||
return err
|
||||
}
|
||||
for delay := 10 * time.Millisecond; ; {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE")
|
||||
if err == nil {
|
||||
started = true
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
// Cancellation may race successful BEGIN in the driver.
|
||||
_ = conn.Raw(func(any) error { return driver.ErrBadConn })
|
||||
return ctx.Err()
|
||||
}
|
||||
var code interface{ Code() int }
|
||||
if !errors.As(err, &code) || code.Code()&255 != 5 { // SQLITE_BUSY family only
|
||||
return err
|
||||
}
|
||||
timer := time.NewTimer(delay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
if delay < 80*time.Millisecond {
|
||||
delay *= 2
|
||||
}
|
||||
}
|
||||
if err := write(conn); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = conn.ExecContext(ctx, "COMMIT")
|
||||
committed = err == nil
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
func eventLogFileStore(t *testing.T) (*Store, string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "queue.db")
|
||||
database, err := db.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
s := &Store{DB: database}
|
||||
if err := s.EnsureEventJobsSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := database.Exec(`INSERT INTO event_jobs(uuid,event_uuid,event_type,event_key,payload,status,run_at_ms,created_at_ms,updated_at_ms) VALUES('job','event','test.logs','key','{}','running',1,1,1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s, path
|
||||
}
|
||||
|
||||
func TestEventLogIndependentConcurrentWriters(t *testing.T) {
|
||||
s, path := eventLogFileStore(t)
|
||||
const writers, each = 8, 40
|
||||
var wg sync.WaitGroup
|
||||
start := make(chan struct{})
|
||||
errs := make(chan error, writers+1)
|
||||
for n := range writers {
|
||||
database, err := db.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(n int) {
|
||||
defer wg.Done()
|
||||
defer database.Close()
|
||||
<-start
|
||||
worker := &Store{DB: database}
|
||||
for i := range each {
|
||||
if err := worker.AddEventLog("job", "event", "info", fmt.Sprintf("writer-%d-%d", n, i)); err != nil {
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}(n)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
for range 160 {
|
||||
if _, err := s.DB.Exec(`UPDATE event_queue_settings SET max_depth=max_depth+1 WHERE id=1`); err != nil {
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Error(err)
|
||||
}
|
||||
var raw, distinct, aggregated int
|
||||
if err := s.DB.QueryRow(`SELECT count(*),count(DISTINCT message) FROM event_logs`).Scan(&raw, &distinct); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.DB.QueryRow(`SELECT COALESCE(sum(count),0) FROM event_log_buckets WHERE event_type='test.logs'`).Scan(&aggregated); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if raw != writers*each || raw != distinct || raw != aggregated {
|
||||
t.Fatalf("lost/duplicate logs: raw=%d distinct=%d aggregate=%d", raw, distinct, aggregated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventLogCancellationAndWaitPolicyRestored(t *testing.T) {
|
||||
s, path := eventLogFileStore(t)
|
||||
s.DB.SetMaxOpenConns(1)
|
||||
holder, err := db.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer holder.Close()
|
||||
conn, err := holder.Conn(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if _, err := conn.ExecContext(context.Background(), "BEGIN IMMEDIATE"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.ExecContext(context.Background(), "ROLLBACK")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
if err := s.AddEventLogContext(ctx, "job", "event", "info", "must not appear"); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected deadline, got %v", err)
|
||||
}
|
||||
if time.Since(start) > time.Second {
|
||||
t.Fatal("wait ignored cancellation")
|
||||
}
|
||||
if _, err := conn.ExecContext(context.Background(), "ROLLBACK"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.AddEventLog("job", "event", "info", "after cancellation"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var count, wait int
|
||||
if err := s.DB.QueryRow("PRAGMA busy_timeout").Scan(&wait); err != nil || wait != 5000 {
|
||||
t.Fatalf("pooled timeout leaked: %d %v", wait, err)
|
||||
}
|
||||
if err := s.DB.QueryRow("SELECT count(*) FROM event_logs").Scan(&count); err != nil || count != 1 {
|
||||
t.Fatalf("unexpected rows: %d %v", count, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventLogAtomicRollbackAndEmptyEventLookup(t *testing.T) {
|
||||
s, _ := eventLogFileStore(t)
|
||||
s.DB.SetMaxOpenConns(1)
|
||||
if _, err := s.DB.Exec(`CREATE TRIGGER fail_bucket BEFORE INSERT ON event_log_buckets BEGIN SELECT RAISE(ABORT,'injected'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.AddEventLog("job", "event", "info", "rollback"); err == nil {
|
||||
t.Fatal("expected failure")
|
||||
}
|
||||
var count int
|
||||
if err := s.DB.QueryRow("SELECT count(*) FROM event_logs").Scan(&count); err != nil || count != 0 {
|
||||
t.Fatalf("partial write: %d %v", count, err)
|
||||
}
|
||||
if _, err := s.DB.Exec(`DROP TRIGGER fail_bucket; UPDATE event_jobs SET event_uuid=''`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.AddEventLog("missing-job", "", "info", "unknown"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var eventType string
|
||||
if err := s.DB.QueryRow("SELECT event_type FROM event_log_buckets").Scan(&eventType); err != nil || eventType != "unknown" {
|
||||
t.Fatalf("empty event ID matched unrelated job: %q %v", eventType, err)
|
||||
}
|
||||
s.ReadOnly = true
|
||||
if err := s.AddEventLog("job", "", "info", "readonly"); !errors.Is(err, ErrReadOnly) {
|
||||
t.Fatalf("read-only accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventCompletionReportsLogFailureWithoutChangingOutcome(t *testing.T) {
|
||||
for _, retry := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(retry), func(t *testing.T) {
|
||||
s, _ := eventLogFileStore(t)
|
||||
if _, err := s.DB.Exec(`CREATE TRIGGER fail_log BEFORE INSERT ON event_logs BEGIN SELECT RAISE(ABORT,'injected'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var runErr error
|
||||
want := "succeeded"
|
||||
if retry {
|
||||
runErr, want = errors.New("synthetic execution failure"), "queued"
|
||||
}
|
||||
err := s.FinishEventAt(context.Background(), "job", runErr, time.Now())
|
||||
if !errors.Is(err, ErrEventCompletionLog) {
|
||||
t.Fatalf("log failure hidden: %v", err)
|
||||
}
|
||||
var status string
|
||||
if err := s.DB.QueryRow("SELECT status FROM event_jobs WHERE uuid='job'").Scan(&status); err != nil || status != want {
|
||||
t.Fatalf("outcome changed: %q %v", status, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventLogDoesNotHidePolicyReadFailure(t *testing.T) {
|
||||
s, _ := eventLogFileStore(t)
|
||||
if _, err := s.DB.Exec(`DROP TABLE event_log_policies`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.AddEventLog("job", "event", "info", "no fallback on database failure"); err == nil || errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatalf("policy failure hidden: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventLogJobLookupUsesIndex(t *testing.T) {
|
||||
s, _ := eventLogFileStore(t)
|
||||
rows, err := s.DB.Query("EXPLAIN QUERY PLAN "+eventLogJobTypeSQL, "job")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
found := false
|
||||
for rows.Next() {
|
||||
var id, parent, unused int
|
||||
var detail string
|
||||
if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(detail, "SCAN event_jobs") {
|
||||
t.Fatalf("history scan under writer lock: %s", detail)
|
||||
}
|
||||
found = found || strings.Contains(detail, "SEARCH event_jobs USING INDEX")
|
||||
}
|
||||
if err := rows.Err(); err != nil || !found {
|
||||
t.Fatalf("indexed lookup missing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventLogCancellationDuringWriteRollsBack(t *testing.T) {
|
||||
s, _ := eventLogFileStore(t)
|
||||
s.DB.SetMaxOpenConns(1)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
err := s.withEventLogWriter(ctx, func(conn *sql.Conn) error {
|
||||
if _, err := conn.ExecContext(ctx, `INSERT INTO event_logs(job_uuid,event_uuid,level,message,created_at_ms) VALUES('job','event','info','cancelled',1)`); err != nil {
|
||||
return err
|
||||
}
|
||||
cancel()
|
||||
return nil
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected cancellation: %v", err)
|
||||
}
|
||||
var count, wait int
|
||||
if err := s.DB.QueryRow("SELECT count(*) FROM event_logs").Scan(&count); err != nil || count != 0 {
|
||||
t.Fatalf("cancelled transaction committed: %d %v", count, err)
|
||||
}
|
||||
if err := s.DB.QueryRow("PRAGMA busy_timeout").Scan(&wait); err != nil || wait != 5000 {
|
||||
t.Fatalf("wait not restored after rollback: %d %v", wait, err)
|
||||
}
|
||||
if err := s.AddEventLog("job", "event", "info", "reusable"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user