diff --git a/docs/design/ja/dregarnuhr.md b/docs/design/ja/dregarnuhr.md index e48ce99..7cbbbe5 100644 --- a/docs/design/ja/dregarnuhr.md +++ b/docs/design/ja/dregarnuhr.md @@ -71,9 +71,47 @@ SQLiteの単一 `UPDATE ... RETURNING` で選択とclaimを行い、同じジョ 3. 業務ハンドラーのpanicを失敗として記録・再試行対象にする。panicの内容自体は記録しない。 4. キャンセル状態のDB読込失敗を「キャンセルなし」と扱わない。業務処理を止め、失敗経路へ送る。 5. worker contextが取り消されても、独立した5秒contextで完了保存を試みる。 - 保存失敗はログへ出す。旧式の設定・ログ用DB呼出しにはcontext未対応部分が残るため、 + 保存失敗はログへ出す。旧式の設定用DB呼出しにはcontext未対応部分が残るため、 保存処理全体の厳密な5秒上限を保証するものではない。 +## v0.2.0後の優先修正:ログ保存競合 + +2026-11-01、ソース実装・合成データ検証。本番未反映。基点は `0f686aa`。 +v0.2.0本番観察で開始ログ保存のSQLITE_BUSY_SNAPSHOT(517)が発生したため、 +読取transactionをwriterへ昇格させる `AddEventLog` を修正した。 + +- `AddEventLogContext` は専用接続の `BEGIN IMMEDIATE` で書込権を先に取得し、 + イベント型/保持policyの読取、rawログ、集約bucketを同じtransaction内で扱う。 + DBスキーマ・公開API・ジョブの実行規則は変更しない。 +- 予算は最大2秒、呼出し元に短い期限があればそちらを優先する。 + 接続取得後のSQLite busy待機を50msに限定し、BEGINのBUSYだけを10〜80ms間隔で再試行する。 + 書込開始後やCOMMIT応答不明時の再送はしない。これにより内部retryの二重集計を避ける。 + OS/接続初期化/ドライバの停止までを含む厳密な実時間保証ではない。 +- 取消時も独立した最大1秒のcleanupでROLLBACKと接続のbusy_timeout復元を試みる。 + transaction状態や待機設定を確定できない接続はpoolへ戻さない。 +- 通常のイベント型取得はJob UUIDの主キー検索。履歴全走査やOR条件のsortを + writer lock内で行わない。Job不在かつEvent UUID指定時のみ従来のfallbackを使う。 + 空のEvent UUIDで無関係なジョブを拾わない。DB読取エラーを既定policyで隠さない。 +- rawログとbucketの途中失敗は両方rollback。ログ保存が成功した場合のみ両方を確定する。 + これはログ呼出し間のexactly-once保証ではなく、単一呼出し内の原子性である。 +- 完了状態を保存後にログだけ失敗した場合は `ErrEventCompletionLog` を返す。 + server adapterは `finish-log failed` として報告し、完了状態を失敗へ変えたり + 成功した業務処理を再実行したりしない。開始/進捗の失敗も別stageで報告する。 +- ログ保存失敗の診断はcomponent、Job UUID、イベント型、エラー分類/SQLite codeだけ。 + ドライバの生メッセージ、SQL、進捗本文、payload、認証情報はcomposeログへ転載しない。 + +受入試験は独立8接続の320件ログと別writer、原子rollback、取消後の再利用・待機設定復元、 +readonly、空ID誤分類、policy読取失敗、主キー利用、完了/retry状態の維持、 +開始・進捗・完了の失敗診断と秘密の非露出を対象とする。 + +全Goテスト(`go test ./... -count=1`)、全Go vet、関連store/server試験のrace検査 +(3回)が成功。通常の関連store試験は20回反復して確認した。 +テストは一時DBと合成ジョブのみ。本番DB・外部カレンダー・実通知先は使用しない。 + +長時間ロックで保存できなかったログの自動回収、完了状態とログ自体の一体commit、 +event-only fallback検索や保持処理の最適化は後続。v0.2.0タグは変更せず、 +本番反映時は新しいPATCH版として別途検証・公開する。 + ## 未実装:リース・HA・終了処理 **atomic claimは期限付きリースではない。現時点は単一writerプロセスの運用を前提とする。** @@ -88,7 +126,8 @@ exactly-onceの配送・実行は保証しない。外部副作用の冪等性 - [ ] 成功後・完了保存前の障害、lease更新停止、旧workerの遅延完了を障害注入試験する。 - [ ] アプリケーションの停止contextをschedulerに接続し、受付停止・実行中処理のdrain・期限付き終了を実装する。 - [ ] 各業務ハンドラーへcontextを伝播する。現状は独自のBackground/timeoutを使う処理も残る。 -- [ ] 古い設定・ログDB呼出しをcontext対応にする。完了保存失敗の回収方針も定義する。 +- [x] 開始/進捗/完了ログのcontext伝播、有界writer取得、ログ保存失敗の診断(ソース実装。本番未反映)。 +- [ ] 古い設定DB呼出しをcontext対応にする。完了保存失敗・欠落ログの回収方針も定義する。 コアのLoop/Runnerはキャンセルを受け付けるが、現行アプリケーションは従来どおり Background contextで起動する。これをgraceful shutdown対応済みとは扱わない。 diff --git a/docs/operations/deployments/2026-11-01-v0.2.0.md b/docs/operations/deployments/2026-11-01-v0.2.0.md index 42d1cce..7141e99 100644 --- a/docs/operations/deployments/2026-11-01-v0.2.0.md +++ b/docs/operations/deployments/2026-11-01-v0.2.0.md @@ -117,3 +117,9 @@ panic・fatal・ジョブ失敗・再起動は検出していない。 - [ ] 独立DB接続・複数writerで、ログと集約値の原子性・重複なし・取りこぼしを検証する。 - [ ] 完了ログ等の既存呼出し元で握りつぶしている保存失敗も、本文・秘密を露出しない形で可視化する。 - [ ] 修正は新しいPATCH版としてビルド・検証・公開する。v0.2.0タグやdigestは付け替えない。 + +### 後続作業(リリース済み内容とは別) + +`0f686aa` を基点に、上記ログ競合の修正をソースへ実装した。 +契約と検証範囲は [Dregarnuhrのログ保存競合修正](../../design/ja/dregarnuhr.md#v020後の優先修正ログ保存競合) を参照。 +この追記は本番修正済みを意味しない。v0.2.0のイメージ・タグは変更していない。 diff --git a/internal/server/dregarnuhr_queue.go b/internal/server/dregarnuhr_queue.go index 0faf3b3..653f65e 100644 --- a/internal/server/dregarnuhr_queue.go +++ b/internal/server/dregarnuhr_queue.go @@ -4,7 +4,9 @@ import ( "context" "database/sql" "errors" + "fmt" "log" + "strings" "time" "pageswiki/internal/dregarnuhr" @@ -34,7 +36,7 @@ func (q schedulerQueue) Cancellation(ctx context.Context, id string) (bool, erro return q.store.EventCancellationState(ctx, id) } func (q schedulerQueue) Started(ctx context.Context, job *dregarnuhr.Job) error { - return q.store.AddEventLog(job.UUID, job.EventUUID, "info", "execution started") + return q.store.AddEventLogContext(ctx, job.UUID, job.EventUUID, "info", "execution started") } func (q schedulerQueue) Finish(ctx context.Context, job *dregarnuhr.Job, err error, now time.Time) error { return q.store.FinishEventAt(ctx, job.UUID, err, now) @@ -47,6 +49,13 @@ func (s *Server) eventRunner() *dregarnuhr.Runner { Foreground: func() bool { return s.foregroundRequests.Load() > 0 }, Wake: s.wakeEventScheduler, Report: func(stage string, job *dregarnuhr.Job, err error) { + if errors.Is(err, store.ErrEventCompletionLog) { + stage = "finish-log" + } + if strings.HasSuffix(stage, "-log") { + reportEventLogFailure(stage, job, err) + return + } if job != nil { log.Printf("component=%s event scheduler %s failed job=%s type=%s: %v", dregarnuhr.Component, stage, job.UUID, job.Type, err) } else { @@ -58,6 +67,30 @@ func (s *Server) eventRunner() *dregarnuhr.Runner { return s.schedulerRunner } +// Logging failures must be visible without repeating a successful side effect +// (such as push delivery), or echoing payloads, SQL text or driver messages. +func (s *Server) addEventProgressLog(ctx context.Context, job *store.EventJob, message string) { + if err := s.store.AddEventLogContext(ctx, job.UUID, job.EventUUID, "info", message); err != nil { + reportEventLogFailure("progress-log", job, err) + } +} + +func reportEventLogFailure(stage string, job *dregarnuhr.Job, err error) { + category := "storage_error" + var code interface{ Code() int } + switch { + case errors.Is(err, context.Canceled): + category = "cancelled" + case errors.Is(err, context.DeadlineExceeded): + category = "deadline_exceeded" + case errors.Is(err, store.ErrReadOnly): + category = "read_only" + case errors.As(err, &code): + category = fmt.Sprintf("sqlite_%d", code.Code()) + } + log.Printf("component=%s event scheduler %s failed job=%q type=%q reason=%s", dregarnuhr.Component, stage, job.UUID, job.Type, category) +} + func (s *Server) runDueEvents() { if s.store == nil { return diff --git a/internal/server/event_handlers.go b/internal/server/event_handlers.go index f972c0b..c83586d 100644 --- a/internal/server/event_handlers.go +++ b/internal/server/event_handlers.go @@ -22,21 +22,21 @@ func (s *Server) registeredEventHandlers() *dregarnuhr.Registry { store.TaskFocusEventType: func(ctx context.Context, job *store.EventJob) error { processed, err := s.store.CompleteTaskFocus(ctx, time.Now(), 100) if err == nil && processed > 0 { - _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("focus timers completed=%d", processed)) + s.addEventProgressLog(ctx, job, fmt.Sprintf("focus timers completed=%d", processed)) } return err }, store.TaskReminderEventType: func(ctx context.Context, job *store.EventJob) error { checked, sent, err := s.store.DeliverTaskReminders(ctx, time.Now(), 100) if err == nil && checked > 0 { - _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("task reminders checked=%d delivered=%d", checked, sent)) + s.addEventProgressLog(ctx, job, fmt.Sprintf("task reminders checked=%d delivered=%d", checked, sent)) } return err }, store.TaskActivityBridgeEventType: func(ctx context.Context, job *store.EventJob) error { processed, published, err := s.store.PublishTaskActivity(ctx, 100) if err == nil && processed > 0 { - _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("journal entries checked=%d published=%d", processed, published)) + s.addEventProgressLog(ctx, job, fmt.Sprintf("journal entries checked=%d published=%d", processed, published)) } return err }, diff --git a/internal/server/event_log_reporting_test.go b/internal/server/event_log_reporting_test.go new file mode 100644 index 0000000..781bf69 --- /dev/null +++ b/internal/server/event_log_reporting_test.go @@ -0,0 +1,66 @@ +package server + +import ( + "bytes" + "context" + "log" + "strings" + "testing" + "time" + + "pageswiki/internal/db" + "pageswiki/internal/dregarnuhr" + "pageswiki/internal/store" +) + +func TestEventLoggingFailureReportedWithoutReexecuting(t *testing.T) { + database, err := db.Open(t.TempDir() + "/queue.db") + if err != nil { + t.Fatal(err) + } + defer database.Close() + st := &store.Store{DB: database} + if err := st.EnsureEventJobsSchema(); err != nil { + t.Fatal(err) + } + // A malicious/verbose driver message must never be copied to compose logs. + if _, err := database.Exec(`CREATE TRIGGER fail_log BEFORE INSERT ON event_logs BEGIN SELECT RAISE(ABORT,'secret-payload-123'); END`); err != nil { + t.Fatal(err) + } + var output bytes.Buffer + old := log.Writer() + log.SetOutput(&output) + defer log.SetOutput(old) + s := &Server{store: st, schedulerWake: make(chan struct{}, 8)} + if err := s.registeredEventHandlers().Register("test.log-failure", func(ctx context.Context, job *dregarnuhr.Job) error { + s.addEventProgressLog(ctx, job, "private-progress-456") + return nil + }); err != nil { + t.Fatal(err) + } + _, err = st.EnqueueEvent("test.log-failure", "key", "secret-body-789", time.Now().Add(-time.Second)) + if err != nil { + t.Fatal(err) + } + s.eventRunner().Dispatch(context.Background()) + select { + case <-s.schedulerWake: + case <-time.After(5 * time.Second): + t.Fatal("worker did not complete") + } + var status string + var attempts int + if err := database.QueryRow(`SELECT status,attempts FROM event_jobs WHERE event_type='test.log-failure'`).Scan(&status, &attempts); err != nil || status != "succeeded" || attempts != 1 { + t.Fatalf("log failure retried business operation: %s %d %v", status, attempts, err) + } + for _, stage := range []string{"start-log", "progress-log", "finish-log"} { + if !strings.Contains(output.String(), stage+" failed") { + t.Fatalf("missing %s diagnostic: %s", stage, output.String()) + } + } + for _, secret := range []string{"secret-payload-123", "private-progress-456", "secret-body-789"} { + if strings.Contains(output.String(), secret) { + t.Fatal("diagnostic leaked content") + } + } +} diff --git a/internal/server/ordoschnellic.go b/internal/server/ordoschnellic.go index d17b6c8..ee58330 100644 --- a/internal/server/ordoschnellic.go +++ b/internal/server/ordoschnellic.go @@ -43,7 +43,7 @@ func (s *Server) deliverNotificationPushContext(ctx context.Context, job *store. return err } if stats.Sent+stats.Failed > 0 { - _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("component=%s push sent=%d failed=%d (no payload or endpoint logged)", ordoschnellic.Component, stats.Sent, stats.Failed)) + s.addEventProgressLog(ctx, job, fmt.Sprintf("component=%s push sent=%d failed=%d (no payload or endpoint logged)", ordoschnellic.Component, stats.Sent, stats.Failed)) } return nil } diff --git a/internal/store/event_jobs.go b/internal/store/event_jobs.go index acd1eaf..c816953 100644 --- a/internal/store/event_jobs.go +++ b/internal/store/event_jobs.go @@ -197,7 +197,7 @@ func (s *Store) FinishEventAt(ctx context.Context, uuid string, runErr error, at run_at_ms=?, finished_at_ms=0, updated_at_ms=? WHERE uuid=?`, outcome.Message, outcome.RetryAt.UnixMilli(), at.UnixMilli(), uuid) if err == nil { - _ = s.AddEventLog(uuid, job.EventUUID, outcome.Level, "execution failed; retry scheduled: "+outcome.Message) + err = s.addEventCompletionLog(ctx, uuid, job.EventUUID, outcome.Level, "execution failed; retry scheduled: "+outcome.Message) } return err } @@ -209,7 +209,7 @@ func (s *Store) FinishEventAt(ctx context.Context, uuid string, runErr error, at message += ": " + outcome.Message } if err == nil { - _ = s.AddEventLog(uuid, job.EventUUID, outcome.Level, message) + err = s.addEventCompletionLog(ctx, uuid, job.EventUUID, outcome.Level, message) } return err } @@ -411,37 +411,64 @@ func (s *Store) SetQueueSettings(q QueueSettings) error { } func (s *Store) AddEventLog(jobUUID, eventUUID, level, message string) error { - tx, err := s.DB.Begin() - if err != nil { - return err + return s.AddEventLogContext(context.Background(), jobUUID, eventUUID, level, message) +} + +func (s *Store) addEventCompletionLog(ctx context.Context, jobUUID, eventUUID, level, message string) error { + if err := s.AddEventLogContext(ctx, jobUUID, eventUUID, level, message); err != nil { + return errors.Join(ErrEventCompletionLog, err) } - defer tx.Rollback() - eventType := "unknown" - _ = tx.QueryRow(`SELECT event_type FROM event_jobs WHERE uuid=? OR event_uuid=? ORDER BY CASE WHEN uuid=? THEN 0 ELSE 1 END LIMIT 1`, jobUUID, eventUUID, jobUUID).Scan(&eventType) - if eventType == "" { - eventType = "unknown" + return nil +} + +// AddEventLogContext atomically persists a raw log and its trend counter. The +// caller's earlier deadline wins; even Background callers have a finite budget. +func (s *Store) AddEventLogContext(ctx context.Context, jobUUID, eventUUID, level, message string) error { + if s.ReadOnly { + return ErrReadOnly } - policy := EventLogPolicy{EventType: "*", RetentionHours: 168, AggregateAfterHours: 24, BucketSeconds: 60, MaxRows: 5000} - var specific EventLogPolicy - if err := scanEventLogPolicy(tx.QueryRow(`SELECT event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows FROM event_log_policies WHERE event_type=?`, eventType), &specific); err == nil { - policy = specific - } else { - _ = scanEventLogPolicy(tx.QueryRow(`SELECT event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows FROM event_log_policies WHERE event_type='*'`), &policy) - } - now := time.Now().UnixMilli() - if _, err := tx.Exec(`INSERT INTO event_logs(job_uuid,event_uuid,level,message,created_at_ms) VALUES(?,?,?,?,?)`, jobUUID, eventUUID, level, message, now); err != nil { - return err - } - bucket := int64(policy.BucketSeconds) - if bucket < 1 { - bucket = 60 - } - bucketStart := now - (now % (bucket * 1000)) - if _, err := tx.Exec(`INSERT INTO event_log_buckets(event_type,bucket_start_ms,level,count,first_at_ms,last_at_ms,sample_message) + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + return s.withEventLogWriter(ctx, func(tx *sql.Conn) error { + eventType := "unknown" + // The common path is a primary-key lookup, not OR + sorting the retained + // job history while holding the writer lock. Keep event-only fallback. + err := tx.QueryRowContext(ctx, eventLogJobTypeSQL, jobUUID).Scan(&eventType) + if errors.Is(err, sql.ErrNoRows) && eventUUID != "" { + err = tx.QueryRowContext(ctx, `SELECT event_type FROM event_jobs WHERE event_uuid=? LIMIT 1`, eventUUID).Scan(&eventType) + } + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + if eventType == "" { + eventType = "unknown" + } + policy := EventLogPolicy{EventType: "*", RetentionHours: 168, AggregateAfterHours: 24, BucketSeconds: 60, MaxRows: 5000} + var specific EventLogPolicy + if err := scanEventLogPolicy(tx.QueryRowContext(ctx, `SELECT event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows FROM event_log_policies WHERE event_type=?`, eventType), &specific); err == nil { + policy = specific + } else if errors.Is(err, sql.ErrNoRows) { + if err := scanEventLogPolicy(tx.QueryRowContext(ctx, `SELECT event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows FROM event_log_policies WHERE event_type='*'`), &policy); err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + } else { + return err + } + now := time.Now().UnixMilli() + if _, err := tx.ExecContext(ctx, `INSERT INTO event_logs(job_uuid,event_uuid,level,message,created_at_ms) VALUES(?,?,?,?,?)`, jobUUID, eventUUID, level, message, now); err != nil { + return err + } + bucket := int64(policy.BucketSeconds) + if bucket < 1 { + bucket = 60 + } + bucketStart := now - (now % (bucket * 1000)) + if _, err := tx.ExecContext(ctx, `INSERT INTO event_log_buckets(event_type,bucket_start_ms,level,count,first_at_ms,last_at_ms,sample_message) VALUES(?,?,?,1,?,?,?) ON CONFLICT(event_type,bucket_start_ms,level) DO UPDATE SET count=count+1,last_at_ms=excluded.last_at_ms`, eventType, bucketStart, level, now, now, message); err != nil { - return err - } - return tx.Commit() + return err + } + return nil + }) } func scanEventLogPolicy(scanner interface{ Scan(...any) error }, policy *EventLogPolicy) error { diff --git a/internal/store/event_log_write.go b/internal/store/event_log_write.go new file mode 100644 index 0000000..bd61d58 --- /dev/null +++ b/internal/store/event_log_write.go @@ -0,0 +1,87 @@ +package store + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "time" +) + +// ErrEventCompletionLog means the job state is already persisted; callers must +// report the missing log without retrying the business operation. +var ErrEventCompletionLog = errors.New("event state saved but completion log failed") + +const eventLogJobTypeSQL = `SELECT event_type FROM event_jobs WHERE uuid=?` + +// withEventLogWriter takes the WAL writer lock BEFORE reading policy/job facts. +// Only lock acquisition is retried: neither writes nor an ambiguous COMMIT are +// replayed. The short SQLite wait allows caller cancellation between attempts. +func (s *Store) withEventLogWriter(ctx context.Context, write func(*sql.Conn) error) error { + conn, err := s.DB.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + var previousWait int + if err := conn.QueryRowContext(ctx, "PRAGMA busy_timeout").Scan(&previousWait); err != nil { + return err + } + started, committed := false, false + defer func() { + cleanup, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + bad := false + if started && !committed { + _, err := conn.ExecContext(cleanup, "ROLLBACK") + bad = err != nil + } + if !bad { + _, err := conn.ExecContext(cleanup, fmt.Sprintf("PRAGMA busy_timeout=%d", previousWait)) + bad = err != nil + } + if bad { + // Never pool a connection with an uncertain transaction or wait policy. + _ = conn.Raw(func(any) error { return driver.ErrBadConn }) + } + }() + if _, err := conn.ExecContext(ctx, "PRAGMA busy_timeout=50"); err != nil { + return err + } + for delay := 10 * time.Millisecond; ; { + if err := ctx.Err(); err != nil { + return err + } + _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE") + if err == nil { + started = true + break + } + if ctx.Err() != nil { + // Cancellation may race successful BEGIN in the driver. + _ = conn.Raw(func(any) error { return driver.ErrBadConn }) + return ctx.Err() + } + var code interface{ Code() int } + if !errors.As(err, &code) || code.Code()&255 != 5 { // SQLITE_BUSY family only + return err + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + if delay < 80*time.Millisecond { + delay *= 2 + } + } + if err := write(conn); err != nil { + return err + } + _, err = conn.ExecContext(ctx, "COMMIT") + committed = err == nil + return err +} diff --git a/internal/store/event_log_write_test.go b/internal/store/event_log_write_test.go new file mode 100644 index 0000000..be25cb7 --- /dev/null +++ b/internal/store/event_log_write_test.go @@ -0,0 +1,242 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "pageswiki/internal/db" +) + +func eventLogFileStore(t *testing.T) (*Store, string) { + t.Helper() + path := filepath.Join(t.TempDir(), "queue.db") + database, err := db.Open(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { database.Close() }) + s := &Store{DB: database} + if err := s.EnsureEventJobsSchema(); err != nil { + t.Fatal(err) + } + if _, err := database.Exec(`INSERT INTO event_jobs(uuid,event_uuid,event_type,event_key,payload,status,run_at_ms,created_at_ms,updated_at_ms) VALUES('job','event','test.logs','key','{}','running',1,1,1)`); err != nil { + t.Fatal(err) + } + return s, path +} + +func TestEventLogIndependentConcurrentWriters(t *testing.T) { + s, path := eventLogFileStore(t) + const writers, each = 8, 40 + var wg sync.WaitGroup + start := make(chan struct{}) + errs := make(chan error, writers+1) + for n := range writers { + database, err := db.Open(path) + if err != nil { + t.Fatal(err) + } + wg.Add(1) + go func(n int) { + defer wg.Done() + defer database.Close() + <-start + worker := &Store{DB: database} + for i := range each { + if err := worker.AddEventLog("job", "event", "info", fmt.Sprintf("writer-%d-%d", n, i)); err != nil { + errs <- err + return + } + } + }(n) + } + wg.Add(1) + go func() { + defer wg.Done() + <-start + for range 160 { + if _, err := s.DB.Exec(`UPDATE event_queue_settings SET max_depth=max_depth+1 WHERE id=1`); err != nil { + errs <- err + return + } + } + }() + close(start) + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } + var raw, distinct, aggregated int + if err := s.DB.QueryRow(`SELECT count(*),count(DISTINCT message) FROM event_logs`).Scan(&raw, &distinct); err != nil { + t.Fatal(err) + } + if err := s.DB.QueryRow(`SELECT COALESCE(sum(count),0) FROM event_log_buckets WHERE event_type='test.logs'`).Scan(&aggregated); err != nil { + t.Fatal(err) + } + if raw != writers*each || raw != distinct || raw != aggregated { + t.Fatalf("lost/duplicate logs: raw=%d distinct=%d aggregate=%d", raw, distinct, aggregated) + } +} + +func TestEventLogCancellationAndWaitPolicyRestored(t *testing.T) { + s, path := eventLogFileStore(t) + s.DB.SetMaxOpenConns(1) + holder, err := db.Open(path) + if err != nil { + t.Fatal(err) + } + defer holder.Close() + conn, err := holder.Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if _, err := conn.ExecContext(context.Background(), "BEGIN IMMEDIATE"); err != nil { + t.Fatal(err) + } + defer conn.ExecContext(context.Background(), "ROLLBACK") + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond) + defer cancel() + start := time.Now() + if err := s.AddEventLogContext(ctx, "job", "event", "info", "must not appear"); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected deadline, got %v", err) + } + if time.Since(start) > time.Second { + t.Fatal("wait ignored cancellation") + } + if _, err := conn.ExecContext(context.Background(), "ROLLBACK"); err != nil { + t.Fatal(err) + } + if err := s.AddEventLog("job", "event", "info", "after cancellation"); err != nil { + t.Fatal(err) + } + var count, wait int + if err := s.DB.QueryRow("PRAGMA busy_timeout").Scan(&wait); err != nil || wait != 5000 { + t.Fatalf("pooled timeout leaked: %d %v", wait, err) + } + if err := s.DB.QueryRow("SELECT count(*) FROM event_logs").Scan(&count); err != nil || count != 1 { + t.Fatalf("unexpected rows: %d %v", count, err) + } +} + +func TestEventLogAtomicRollbackAndEmptyEventLookup(t *testing.T) { + s, _ := eventLogFileStore(t) + s.DB.SetMaxOpenConns(1) + if _, err := s.DB.Exec(`CREATE TRIGGER fail_bucket BEFORE INSERT ON event_log_buckets BEGIN SELECT RAISE(ABORT,'injected'); END`); err != nil { + t.Fatal(err) + } + if err := s.AddEventLog("job", "event", "info", "rollback"); err == nil { + t.Fatal("expected failure") + } + var count int + if err := s.DB.QueryRow("SELECT count(*) FROM event_logs").Scan(&count); err != nil || count != 0 { + t.Fatalf("partial write: %d %v", count, err) + } + if _, err := s.DB.Exec(`DROP TRIGGER fail_bucket; UPDATE event_jobs SET event_uuid=''`); err != nil { + t.Fatal(err) + } + if err := s.AddEventLog("missing-job", "", "info", "unknown"); err != nil { + t.Fatal(err) + } + var eventType string + if err := s.DB.QueryRow("SELECT event_type FROM event_log_buckets").Scan(&eventType); err != nil || eventType != "unknown" { + t.Fatalf("empty event ID matched unrelated job: %q %v", eventType, err) + } + s.ReadOnly = true + if err := s.AddEventLog("job", "", "info", "readonly"); !errors.Is(err, ErrReadOnly) { + t.Fatalf("read-only accepted: %v", err) + } +} + +func TestEventCompletionReportsLogFailureWithoutChangingOutcome(t *testing.T) { + for _, retry := range []bool{false, true} { + t.Run(fmt.Sprint(retry), func(t *testing.T) { + s, _ := eventLogFileStore(t) + if _, err := s.DB.Exec(`CREATE TRIGGER fail_log BEFORE INSERT ON event_logs BEGIN SELECT RAISE(ABORT,'injected'); END`); err != nil { + t.Fatal(err) + } + var runErr error + want := "succeeded" + if retry { + runErr, want = errors.New("synthetic execution failure"), "queued" + } + err := s.FinishEventAt(context.Background(), "job", runErr, time.Now()) + if !errors.Is(err, ErrEventCompletionLog) { + t.Fatalf("log failure hidden: %v", err) + } + var status string + if err := s.DB.QueryRow("SELECT status FROM event_jobs WHERE uuid='job'").Scan(&status); err != nil || status != want { + t.Fatalf("outcome changed: %q %v", status, err) + } + }) + } +} + +func TestEventLogDoesNotHidePolicyReadFailure(t *testing.T) { + s, _ := eventLogFileStore(t) + if _, err := s.DB.Exec(`DROP TABLE event_log_policies`); err != nil { + t.Fatal(err) + } + if err := s.AddEventLog("job", "event", "info", "no fallback on database failure"); err == nil || errors.Is(err, sql.ErrNoRows) { + t.Fatalf("policy failure hidden: %v", err) + } +} + +func TestEventLogJobLookupUsesIndex(t *testing.T) { + s, _ := eventLogFileStore(t) + rows, err := s.DB.Query("EXPLAIN QUERY PLAN "+eventLogJobTypeSQL, "job") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + found := false + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + if strings.Contains(detail, "SCAN event_jobs") { + t.Fatalf("history scan under writer lock: %s", detail) + } + found = found || strings.Contains(detail, "SEARCH event_jobs USING INDEX") + } + if err := rows.Err(); err != nil || !found { + t.Fatalf("indexed lookup missing: %v", err) + } +} + +func TestEventLogCancellationDuringWriteRollsBack(t *testing.T) { + s, _ := eventLogFileStore(t) + s.DB.SetMaxOpenConns(1) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := s.withEventLogWriter(ctx, func(conn *sql.Conn) error { + if _, err := conn.ExecContext(ctx, `INSERT INTO event_logs(job_uuid,event_uuid,level,message,created_at_ms) VALUES('job','event','info','cancelled',1)`); err != nil { + return err + } + cancel() + return nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected cancellation: %v", err) + } + var count, wait int + if err := s.DB.QueryRow("SELECT count(*) FROM event_logs").Scan(&count); err != nil || count != 0 { + t.Fatalf("cancelled transaction committed: %d %v", count, err) + } + if err := s.DB.QueryRow("PRAGMA busy_timeout").Scan(&wait); err != nil || wait != 5000 { + t.Fatalf("wait not restored after rollback: %d %v", wait, err) + } + if err := s.AddEventLog("job", "event", "info", "reusable"); err != nil { + t.Fatal(err) + } +}