Fix event log writer contention and report persistence failures safely
Take the SQLite writer lock before reading job/policy facts, bound acquisition and cancellation, preserve atomic raw logs and counters, and avoid replaying successful business operations on logging failure. Tested the isolated nine-file release slice with the full Go suite, race tests and go vet.
This commit is contained in:
1 parent
0f686aa2fb
commit
a36934e72c
9 files changed
+537
-37
No files matched your search
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/dregarnuhr"
|
||||
@@ -34,7 +36,7 @@ func (q schedulerQueue) Cancellation(ctx context.Context, id string) (bool, erro
|
||||
return q.store.EventCancellationState(ctx, id)
|
||||
}
|
||||
func (q schedulerQueue) Started(ctx context.Context, job *dregarnuhr.Job) error {
|
||||
return q.store.AddEventLog(job.UUID, job.EventUUID, "info", "execution started")
|
||||
return q.store.AddEventLogContext(ctx, job.UUID, job.EventUUID, "info", "execution started")
|
||||
}
|
||||
func (q schedulerQueue) Finish(ctx context.Context, job *dregarnuhr.Job, err error, now time.Time) error {
|
||||
return q.store.FinishEventAt(ctx, job.UUID, err, now)
|
||||
@@ -47,6 +49,13 @@ func (s *Server) eventRunner() *dregarnuhr.Runner {
|
||||
Foreground: func() bool { return s.foregroundRequests.Load() > 0 },
|
||||
Wake: s.wakeEventScheduler,
|
||||
Report: func(stage string, job *dregarnuhr.Job, err error) {
|
||||
if errors.Is(err, store.ErrEventCompletionLog) {
|
||||
stage = "finish-log"
|
||||
}
|
||||
if strings.HasSuffix(stage, "-log") {
|
||||
reportEventLogFailure(stage, job, err)
|
||||
return
|
||||
}
|
||||
if job != nil {
|
||||
log.Printf("component=%s event scheduler %s failed job=%s type=%s: %v", dregarnuhr.Component, stage, job.UUID, job.Type, err)
|
||||
} else {
|
||||
@@ -58,6 +67,30 @@ func (s *Server) eventRunner() *dregarnuhr.Runner {
|
||||
return s.schedulerRunner
|
||||
}
|
||||
|
||||
// Logging failures must be visible without repeating a successful side effect
|
||||
// (such as push delivery), or echoing payloads, SQL text or driver messages.
|
||||
func (s *Server) addEventProgressLog(ctx context.Context, job *store.EventJob, message string) {
|
||||
if err := s.store.AddEventLogContext(ctx, job.UUID, job.EventUUID, "info", message); err != nil {
|
||||
reportEventLogFailure("progress-log", job, err)
|
||||
}
|
||||
}
|
||||
|
||||
func reportEventLogFailure(stage string, job *dregarnuhr.Job, err error) {
|
||||
category := "storage_error"
|
||||
var code interface{ Code() int }
|
||||
switch {
|
||||
case errors.Is(err, context.Canceled):
|
||||
category = "cancelled"
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
category = "deadline_exceeded"
|
||||
case errors.Is(err, store.ErrReadOnly):
|
||||
category = "read_only"
|
||||
case errors.As(err, &code):
|
||||
category = fmt.Sprintf("sqlite_%d", code.Code())
|
||||
}
|
||||
log.Printf("component=%s event scheduler %s failed job=%q type=%q reason=%s", dregarnuhr.Component, stage, job.UUID, job.Type, category)
|
||||
}
|
||||
|
||||
func (s *Server) runDueEvents() {
|
||||
if s.store == nil {
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user