Fix event log writer contention and report persistence failures safely
Build and publish container images / images (push) Successful in 16m17s
Release container images / release (push) Successful in 39s

Take the SQLite writer lock before reading job/policy facts, bound acquisition and cancellation, preserve atomic raw logs and counters, and avoid replaying successful business operations on logging failure. Tested the isolated nine-file release slice with the full Go suite, race tests and go vet.
This commit is contained in:
Anya Tokugawa committed 2026-11-01 19:40:26 +09:00
1 parent 0f686aa2fb
commit a36934e72c
9 files changed
+537 -37

No files matched your search

+34 -1
View File
@@ -4,7 +4,9 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"log"
"strings"
"time"
"pageswiki/internal/dregarnuhr"
@@ -34,7 +36,7 @@ func (q schedulerQueue) Cancellation(ctx context.Context, id string) (bool, erro
return q.store.EventCancellationState(ctx, id)
}
func (q schedulerQueue) Started(ctx context.Context, job *dregarnuhr.Job) error {
return q.store.AddEventLog(job.UUID, job.EventUUID, "info", "execution started")
return q.store.AddEventLogContext(ctx, job.UUID, job.EventUUID, "info", "execution started")
}
func (q schedulerQueue) Finish(ctx context.Context, job *dregarnuhr.Job, err error, now time.Time) error {
return q.store.FinishEventAt(ctx, job.UUID, err, now)
@@ -47,6 +49,13 @@ func (s *Server) eventRunner() *dregarnuhr.Runner {
Foreground: func() bool { return s.foregroundRequests.Load() > 0 },
Wake: s.wakeEventScheduler,
Report: func(stage string, job *dregarnuhr.Job, err error) {
if errors.Is(err, store.ErrEventCompletionLog) {
stage = "finish-log"
}
if strings.HasSuffix(stage, "-log") {
reportEventLogFailure(stage, job, err)
return
}
if job != nil {
log.Printf("component=%s event scheduler %s failed job=%s type=%s: %v", dregarnuhr.Component, stage, job.UUID, job.Type, err)
} else {
@@ -58,6 +67,30 @@ func (s *Server) eventRunner() *dregarnuhr.Runner {
return s.schedulerRunner
}
// Logging failures must be visible without repeating a successful side effect
// (such as push delivery), or echoing payloads, SQL text or driver messages.
func (s *Server) addEventProgressLog(ctx context.Context, job *store.EventJob, message string) {
if err := s.store.AddEventLogContext(ctx, job.UUID, job.EventUUID, "info", message); err != nil {
reportEventLogFailure("progress-log", job, err)
}
}
func reportEventLogFailure(stage string, job *dregarnuhr.Job, err error) {
category := "storage_error"
var code interface{ Code() int }
switch {
case errors.Is(err, context.Canceled):
category = "cancelled"
case errors.Is(err, context.DeadlineExceeded):
category = "deadline_exceeded"
case errors.Is(err, store.ErrReadOnly):
category = "read_only"
case errors.As(err, &code):
category = fmt.Sprintf("sqlite_%d", code.Code())
}
log.Printf("component=%s event scheduler %s failed job=%q type=%q reason=%s", dregarnuhr.Component, stage, job.UUID, job.Type, category)
}
func (s *Server) runDueEvents() {
if s.store == nil {
return
+3 -3
View File
@@ -22,21 +22,21 @@ func (s *Server) registeredEventHandlers() *dregarnuhr.Registry {
store.TaskFocusEventType: func(ctx context.Context, job *store.EventJob) error {
processed, err := s.store.CompleteTaskFocus(ctx, time.Now(), 100)
if err == nil && processed > 0 {
_ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("focus timers completed=%d", processed))
s.addEventProgressLog(ctx, job, fmt.Sprintf("focus timers completed=%d", processed))
}
return err
},
store.TaskReminderEventType: func(ctx context.Context, job *store.EventJob) error {
checked, sent, err := s.store.DeliverTaskReminders(ctx, time.Now(), 100)
if err == nil && checked > 0 {
_ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("task reminders checked=%d delivered=%d", checked, sent))
s.addEventProgressLog(ctx, job, fmt.Sprintf("task reminders checked=%d delivered=%d", checked, sent))
}
return err
},
store.TaskActivityBridgeEventType: func(ctx context.Context, job *store.EventJob) error {
processed, published, err := s.store.PublishTaskActivity(ctx, 100)
if err == nil && processed > 0 {
_ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("journal entries checked=%d published=%d", processed, published))
s.addEventProgressLog(ctx, job, fmt.Sprintf("journal entries checked=%d published=%d", processed, published))
}
return err
},
@@ -0,0 +1,66 @@
package server
import (
"bytes"
"context"
"log"
"strings"
"testing"
"time"
"pageswiki/internal/db"
"pageswiki/internal/dregarnuhr"
"pageswiki/internal/store"
)
func TestEventLoggingFailureReportedWithoutReexecuting(t *testing.T) {
database, err := db.Open(t.TempDir() + "/queue.db")
if err != nil {
t.Fatal(err)
}
defer database.Close()
st := &store.Store{DB: database}
if err := st.EnsureEventJobsSchema(); err != nil {
t.Fatal(err)
}
// A malicious/verbose driver message must never be copied to compose logs.
if _, err := database.Exec(`CREATE TRIGGER fail_log BEFORE INSERT ON event_logs BEGIN SELECT RAISE(ABORT,'secret-payload-123'); END`); err != nil {
t.Fatal(err)
}
var output bytes.Buffer
old := log.Writer()
log.SetOutput(&output)
defer log.SetOutput(old)
s := &Server{store: st, schedulerWake: make(chan struct{}, 8)}
if err := s.registeredEventHandlers().Register("test.log-failure", func(ctx context.Context, job *dregarnuhr.Job) error {
s.addEventProgressLog(ctx, job, "private-progress-456")
return nil
}); err != nil {
t.Fatal(err)
}
_, err = st.EnqueueEvent("test.log-failure", "key", "secret-body-789", time.Now().Add(-time.Second))
if err != nil {
t.Fatal(err)
}
s.eventRunner().Dispatch(context.Background())
select {
case <-s.schedulerWake:
case <-time.After(5 * time.Second):
t.Fatal("worker did not complete")
}
var status string
var attempts int
if err := database.QueryRow(`SELECT status,attempts FROM event_jobs WHERE event_type='test.log-failure'`).Scan(&status, &attempts); err != nil || status != "succeeded" || attempts != 1 {
t.Fatalf("log failure retried business operation: %s %d %v", status, attempts, err)
}
for _, stage := range []string{"start-log", "progress-log", "finish-log"} {
if !strings.Contains(output.String(), stage+" failed") {
t.Fatalf("missing %s diagnostic: %s", stage, output.String())
}
}
for _, secret := range []string{"secret-payload-123", "private-progress-456", "secret-body-789"} {
if strings.Contains(output.String(), secret) {
t.Fatal("diagnostic leaked content")
}
}
}
+1 -1
View File
@@ -43,7 +43,7 @@ func (s *Server) deliverNotificationPushContext(ctx context.Context, job *store.
return err
}
if stats.Sent+stats.Failed > 0 {
_ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("component=%s push sent=%d failed=%d (no payload or endpoint logged)", ordoschnellic.Component, stats.Sent, stats.Failed))
s.addEventProgressLog(ctx, job, fmt.Sprintf("component=%s push sent=%d failed=%d (no payload or endpoint logged)", ordoschnellic.Component, stats.Sent, stats.Failed))
}
return nil
}