Fix event log writer contention and report persistence failures safely
Take the SQLite writer lock before reading job/policy facts, bound acquisition and cancellation, preserve atomic raw logs and counters, and avoid replaying successful business operations on logging failure. Tested the isolated nine-file release slice with the full Go suite, race tests and go vet.
This commit is contained in:
1 parent
0f686aa2fb
commit
a36934e72c
9 files changed
+537
-37
No files matched your search
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/dregarnuhr"
|
||||
@@ -34,7 +36,7 @@ func (q schedulerQueue) Cancellation(ctx context.Context, id string) (bool, erro
|
||||
return q.store.EventCancellationState(ctx, id)
|
||||
}
|
||||
func (q schedulerQueue) Started(ctx context.Context, job *dregarnuhr.Job) error {
|
||||
return q.store.AddEventLog(job.UUID, job.EventUUID, "info", "execution started")
|
||||
return q.store.AddEventLogContext(ctx, job.UUID, job.EventUUID, "info", "execution started")
|
||||
}
|
||||
func (q schedulerQueue) Finish(ctx context.Context, job *dregarnuhr.Job, err error, now time.Time) error {
|
||||
return q.store.FinishEventAt(ctx, job.UUID, err, now)
|
||||
@@ -47,6 +49,13 @@ func (s *Server) eventRunner() *dregarnuhr.Runner {
|
||||
Foreground: func() bool { return s.foregroundRequests.Load() > 0 },
|
||||
Wake: s.wakeEventScheduler,
|
||||
Report: func(stage string, job *dregarnuhr.Job, err error) {
|
||||
if errors.Is(err, store.ErrEventCompletionLog) {
|
||||
stage = "finish-log"
|
||||
}
|
||||
if strings.HasSuffix(stage, "-log") {
|
||||
reportEventLogFailure(stage, job, err)
|
||||
return
|
||||
}
|
||||
if job != nil {
|
||||
log.Printf("component=%s event scheduler %s failed job=%s type=%s: %v", dregarnuhr.Component, stage, job.UUID, job.Type, err)
|
||||
} else {
|
||||
@@ -58,6 +67,30 @@ func (s *Server) eventRunner() *dregarnuhr.Runner {
|
||||
return s.schedulerRunner
|
||||
}
|
||||
|
||||
// Logging failures must be visible without repeating a successful side effect
|
||||
// (such as push delivery), or echoing payloads, SQL text or driver messages.
|
||||
func (s *Server) addEventProgressLog(ctx context.Context, job *store.EventJob, message string) {
|
||||
if err := s.store.AddEventLogContext(ctx, job.UUID, job.EventUUID, "info", message); err != nil {
|
||||
reportEventLogFailure("progress-log", job, err)
|
||||
}
|
||||
}
|
||||
|
||||
func reportEventLogFailure(stage string, job *dregarnuhr.Job, err error) {
|
||||
category := "storage_error"
|
||||
var code interface{ Code() int }
|
||||
switch {
|
||||
case errors.Is(err, context.Canceled):
|
||||
category = "cancelled"
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
category = "deadline_exceeded"
|
||||
case errors.Is(err, store.ErrReadOnly):
|
||||
category = "read_only"
|
||||
case errors.As(err, &code):
|
||||
category = fmt.Sprintf("sqlite_%d", code.Code())
|
||||
}
|
||||
log.Printf("component=%s event scheduler %s failed job=%q type=%q reason=%s", dregarnuhr.Component, stage, job.UUID, job.Type, category)
|
||||
}
|
||||
|
||||
func (s *Server) runDueEvents() {
|
||||
if s.store == nil {
|
||||
return
|
||||
|
||||
@@ -22,21 +22,21 @@ func (s *Server) registeredEventHandlers() *dregarnuhr.Registry {
|
||||
store.TaskFocusEventType: func(ctx context.Context, job *store.EventJob) error {
|
||||
processed, err := s.store.CompleteTaskFocus(ctx, time.Now(), 100)
|
||||
if err == nil && processed > 0 {
|
||||
_ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("focus timers completed=%d", processed))
|
||||
s.addEventProgressLog(ctx, job, fmt.Sprintf("focus timers completed=%d", processed))
|
||||
}
|
||||
return err
|
||||
},
|
||||
store.TaskReminderEventType: func(ctx context.Context, job *store.EventJob) error {
|
||||
checked, sent, err := s.store.DeliverTaskReminders(ctx, time.Now(), 100)
|
||||
if err == nil && checked > 0 {
|
||||
_ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("task reminders checked=%d delivered=%d", checked, sent))
|
||||
s.addEventProgressLog(ctx, job, fmt.Sprintf("task reminders checked=%d delivered=%d", checked, sent))
|
||||
}
|
||||
return err
|
||||
},
|
||||
store.TaskActivityBridgeEventType: func(ctx context.Context, job *store.EventJob) error {
|
||||
processed, published, err := s.store.PublishTaskActivity(ctx, 100)
|
||||
if err == nil && processed > 0 {
|
||||
_ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("journal entries checked=%d published=%d", processed, published))
|
||||
s.addEventProgressLog(ctx, job, fmt.Sprintf("journal entries checked=%d published=%d", processed, published))
|
||||
}
|
||||
return err
|
||||
},
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
"pageswiki/internal/dregarnuhr"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
func TestEventLoggingFailureReportedWithoutReexecuting(t *testing.T) {
|
||||
database, err := db.Open(t.TempDir() + "/queue.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
st := &store.Store{DB: database}
|
||||
if err := st.EnsureEventJobsSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A malicious/verbose driver message must never be copied to compose logs.
|
||||
if _, err := database.Exec(`CREATE TRIGGER fail_log BEFORE INSERT ON event_logs BEGIN SELECT RAISE(ABORT,'secret-payload-123'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var output bytes.Buffer
|
||||
old := log.Writer()
|
||||
log.SetOutput(&output)
|
||||
defer log.SetOutput(old)
|
||||
s := &Server{store: st, schedulerWake: make(chan struct{}, 8)}
|
||||
if err := s.registeredEventHandlers().Register("test.log-failure", func(ctx context.Context, job *dregarnuhr.Job) error {
|
||||
s.addEventProgressLog(ctx, job, "private-progress-456")
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = st.EnqueueEvent("test.log-failure", "key", "secret-body-789", time.Now().Add(-time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.eventRunner().Dispatch(context.Background())
|
||||
select {
|
||||
case <-s.schedulerWake:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("worker did not complete")
|
||||
}
|
||||
var status string
|
||||
var attempts int
|
||||
if err := database.QueryRow(`SELECT status,attempts FROM event_jobs WHERE event_type='test.log-failure'`).Scan(&status, &attempts); err != nil || status != "succeeded" || attempts != 1 {
|
||||
t.Fatalf("log failure retried business operation: %s %d %v", status, attempts, err)
|
||||
}
|
||||
for _, stage := range []string{"start-log", "progress-log", "finish-log"} {
|
||||
if !strings.Contains(output.String(), stage+" failed") {
|
||||
t.Fatalf("missing %s diagnostic: %s", stage, output.String())
|
||||
}
|
||||
}
|
||||
for _, secret := range []string{"secret-payload-123", "private-progress-456", "secret-body-789"} {
|
||||
if strings.Contains(output.String(), secret) {
|
||||
t.Fatal("diagnostic leaked content")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,7 +43,7 @@ func (s *Server) deliverNotificationPushContext(ctx context.Context, job *store.
|
||||
return err
|
||||
}
|
||||
if stats.Sent+stats.Failed > 0 {
|
||||
_ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("component=%s push sent=%d failed=%d (no payload or endpoint logged)", ordoschnellic.Component, stats.Sent, stats.Failed))
|
||||
s.addEventProgressLog(ctx, job, fmt.Sprintf("component=%s push sent=%d failed=%d (no payload or endpoint logged)", ordoschnellic.Component, stats.Sent, stats.Failed))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user