Fix event log writer contention and report persistence failures safely
Take the SQLite writer lock before reading job/policy facts, bound acquisition and cancellation, preserve atomic raw logs and counters, and avoid replaying successful business operations on logging failure. Tested the isolated nine-file release slice with the full Go suite, race tests and go vet.
This commit is contained in:
1 parent
0f686aa2fb
commit
a36934e72c
9 files changed
+537
-37
No files matched your search
@@ -0,0 +1,87 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrEventCompletionLog means the job state is already persisted; callers must
|
||||
// report the missing log without retrying the business operation.
|
||||
var ErrEventCompletionLog = errors.New("event state saved but completion log failed")
|
||||
|
||||
const eventLogJobTypeSQL = `SELECT event_type FROM event_jobs WHERE uuid=?`
|
||||
|
||||
// withEventLogWriter takes the WAL writer lock BEFORE reading policy/job facts.
|
||||
// Only lock acquisition is retried: neither writes nor an ambiguous COMMIT are
|
||||
// replayed. The short SQLite wait allows caller cancellation between attempts.
|
||||
func (s *Store) withEventLogWriter(ctx context.Context, write func(*sql.Conn) error) error {
|
||||
conn, err := s.DB.Conn(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
var previousWait int
|
||||
if err := conn.QueryRowContext(ctx, "PRAGMA busy_timeout").Scan(&previousWait); err != nil {
|
||||
return err
|
||||
}
|
||||
started, committed := false, false
|
||||
defer func() {
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
bad := false
|
||||
if started && !committed {
|
||||
_, err := conn.ExecContext(cleanup, "ROLLBACK")
|
||||
bad = err != nil
|
||||
}
|
||||
if !bad {
|
||||
_, err := conn.ExecContext(cleanup, fmt.Sprintf("PRAGMA busy_timeout=%d", previousWait))
|
||||
bad = err != nil
|
||||
}
|
||||
if bad {
|
||||
// Never pool a connection with an uncertain transaction or wait policy.
|
||||
_ = conn.Raw(func(any) error { return driver.ErrBadConn })
|
||||
}
|
||||
}()
|
||||
if _, err := conn.ExecContext(ctx, "PRAGMA busy_timeout=50"); err != nil {
|
||||
return err
|
||||
}
|
||||
for delay := 10 * time.Millisecond; ; {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE")
|
||||
if err == nil {
|
||||
started = true
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
// Cancellation may race successful BEGIN in the driver.
|
||||
_ = conn.Raw(func(any) error { return driver.ErrBadConn })
|
||||
return ctx.Err()
|
||||
}
|
||||
var code interface{ Code() int }
|
||||
if !errors.As(err, &code) || code.Code()&255 != 5 { // SQLITE_BUSY family only
|
||||
return err
|
||||
}
|
||||
timer := time.NewTimer(delay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
if delay < 80*time.Millisecond {
|
||||
delay *= 2
|
||||
}
|
||||
}
|
||||
if err := write(conn); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = conn.ExecContext(ctx, "COMMIT")
|
||||
committed = err == nil
|
||||
return err
|
||||
}
|
||||
Reference in new issue
Block a user