package store import ( "context" "database/sql" "database/sql/driver" "errors" "fmt" "time" "github.com/google/uuid" "pageswiki/internal/db" "pageswiki/internal/dregarnuhr" ) var ErrEventClaim = errors.New("event execution claim is stale or missing") var ErrEventWorker = errors.New("native event worker requires an exclusive live session") const eventWorkerLock = `hashtextextended(current_database()||':'||current_schema()||':pageswiki-event-worker',0)` // AcquireEventWorker preserves SQLite's existing instance-lock ownership; // PostgreSQL also needs a database-scoped owner across distinct data roots. func (s *Store) AcquireEventWorker(ctx context.Context) (func() error, error) { if s.ReadOnly || s.BackgroundJobsDisabled { return nil, ErrEventWorker } if s.sqlBackend == storeSQLite { return func() error { return nil }, nil } return s.StartNativeEventWorker(ctx) } // StartNativeEventWorker owns one dedicated connection for this single-instance // scheduler. The caller must stop/join workers before releasing it. This is not // HA failover: a lost session stops admission, and ambiguous work needs review. func (s *Store) StartNativeEventWorker(ctx context.Context) (func() error, error) { if s.ReadOnly { return nil, ErrReadOnly } if s.sqlBackend != storePostgreSQL || s.BackgroundJobsDisabled { return nil, ErrEventWorker } s.eventWorkerMu.Lock() defer s.eventWorkerMu.Unlock() if s.eventWorkerConn != nil { return nil, ErrEventWorker } ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() conn, err := s.DB.Conn(ctx) if err != nil { return nil, s.sqlError("worker owner", err) } var acquired bool err = conn.QueryRowContext(ctx, `SELECT pg_try_advisory_lock(`+eventWorkerLock+`)`).Scan(&acquired) if err != nil || !acquired { _ = conn.Raw(func(any) error { return driver.ErrBadConn }) conn.Close() if err != nil { return nil, s.sqlError("worker owner", err) } return nil, ErrEventWorker } cleanup := func() error { c, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() var unlocked bool e := conn.QueryRowContext(c, `SELECT pg_advisory_unlock(`+eventWorkerLock+`)`).Scan(&unlocked) if e != nil || !unlocked { _ = conn.Raw(func(any) error { return driver.ErrBadConn }) } closeErr := conn.Close() if e != nil { return s.sqlError("worker release", e) } return closeErr } if err = s.recoverNativeEvents(ctx); err != nil { cleanup() return nil, err } s.eventWorkerConn = conn return func() error { s.eventWorkerMu.Lock() defer s.eventWorkerMu.Unlock() if s.eventWorkerConn != conn { return nil } s.eventWorkerConn = nil return cleanup() }, nil } func (s *Store) nativeEventWorkerReady(ctx context.Context) error { s.eventWorkerMu.Lock() defer s.eventWorkerMu.Unlock() if s.eventWorkerConn == nil { return ErrEventWorker } ctx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() var one int if err := s.eventWorkerConn.QueryRowContext(ctx, `SELECT 1`).Scan(&one); err != nil { return ErrEventWorker } return nil } func (s *Store) execEventSQL(query string, args ...any) (result sql.Result, err error) { if s.ReadOnly { return nil, ErrReadOnly } if s.sqlBackend != storePostgreSQL { return s.DB.Exec(query, args...) } err = s.withNativeEventWriter(context.Background(), func(tx *sql.Tx) error { var e error result, e = tx.Exec(query, args...) return e }) return result, err } // Builtin registration must preserve an administrator's existing schedule. func (s *Store) ensureNativeBuiltinEvent(id, name, kind string, interval int64, priority, attempts int) error { now := time.Now().UnixMilli() _, err := s.execEventSQL(`INSERT INTO event_definitions(uuid,name,event_type,event_key,payload,schedule_kind,interval_seconds,next_run_at_ms,enabled,priority,max_attempts,created_at_ms,updated_at_ms) VALUES($1,$2,$3,'local','{}','interval',$4,$5,1,$6,$7,$5,$5) ON CONFLICT(uuid) DO NOTHING`, id, name, kind, interval, now, priority, attempts) return err } func (s *Store) retryNativeEvent(id string) error { err := s.withNativeEventWriter(context.Background(), func(tx *sql.Tx) error { var depth, limit int if e := tx.QueryRow(`SELECT (SELECT count(*) FROM event_jobs WHERE status IN ('queued','running')),max_depth FROM event_queue_settings WHERE id=1`).Scan(&depth, &limit); e != nil { return e } if depth >= limit { return ErrEventQueueFull } r, e := tx.Exec(`UPDATE event_jobs SET status='queued',attempts=0,cancel_requested=0,error='',run_at_ms=$1,started_at_ms=0,finished_at_ms=0,updated_at_ms=$1,claim_token='' WHERE uuid=$2 AND status IN ('failed','cancelled','skipped')`, time.Now().UnixMilli(), id) if e != nil { return e } n, e := r.RowsAffected() if e != nil { return e } if n == 0 { return sql.ErrNoRows } return nil }) if err != nil { return err } return s.AddEventLog(id, "", "info", "manually requeued") } func (s *Store) ensurePostgreSQLEventJobs(ctx context.Context) error { return db.ApplyPostgreSQLMigrations(ctx, s.DB, "store-event-jobs", []db.PostgreSQLMigration{{Version: 1, Name: "single-instance event queue and execution fencing", Statements: []string{ `CREATE TABLE event_jobs(uuid TEXT COLLATE "C" PRIMARY KEY,event_type TEXT COLLATE "C" NOT NULL,event_key TEXT COLLATE "C" NOT NULL,payload TEXT NOT NULL,status TEXT NOT NULL,attempts BIGINT NOT NULL DEFAULT 0,run_at_ms BIGINT NOT NULL,error TEXT NOT NULL DEFAULT '',created_at_ms BIGINT NOT NULL,updated_at_ms BIGINT NOT NULL,event_uuid TEXT COLLATE "C" NOT NULL DEFAULT '',priority BIGINT NOT NULL DEFAULT 0,max_attempts BIGINT NOT NULL DEFAULT 3,started_at_ms BIGINT NOT NULL DEFAULT 0,finished_at_ms BIGINT NOT NULL DEFAULT 0,cancel_requested BIGINT NOT NULL DEFAULT 0,claim_token TEXT NOT NULL DEFAULT '')`, `CREATE INDEX idx_event_jobs_identity_status ON event_jobs(event_type,event_key,status)`, `CREATE UNIQUE INDEX idx_event_jobs_active_identity ON event_jobs(event_type,event_key) WHERE status IN ('queued','running')`, `CREATE INDEX idx_event_jobs_listing ON event_jobs(created_at_ms DESC,uuid DESC)`, `CREATE INDEX idx_event_jobs_status_listing ON event_jobs(status,created_at_ms DESC,uuid DESC)`, `CREATE INDEX idx_event_jobs_history ON event_jobs(created_at_ms DESC,uuid DESC) WHERE status IN ('succeeded','skipped','failed','cancelled')`, `CREATE INDEX idx_event_jobs_due ON event_jobs(status,priority DESC,run_at_ms,created_at_ms)`, `CREATE TABLE event_definitions(uuid TEXT COLLATE "C" PRIMARY KEY,name TEXT NOT NULL,event_type TEXT NOT NULL,event_key TEXT NOT NULL,payload TEXT NOT NULL DEFAULT '',schedule_kind TEXT NOT NULL DEFAULT 'manual',interval_seconds BIGINT NOT NULL DEFAULT 0,next_run_at_ms BIGINT NOT NULL DEFAULT 0,enabled BIGINT NOT NULL DEFAULT 1,priority BIGINT NOT NULL DEFAULT 0,max_attempts BIGINT NOT NULL DEFAULT 3,created_at_ms BIGINT NOT NULL,updated_at_ms BIGINT NOT NULL)`, `CREATE INDEX idx_event_definitions_due ON event_definitions(enabled,next_run_at_ms)`, `CREATE TABLE event_logs(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,job_uuid TEXT NOT NULL,event_uuid TEXT NOT NULL DEFAULT '',level TEXT NOT NULL,message TEXT NOT NULL,created_at_ms BIGINT NOT NULL)`, `CREATE INDEX idx_event_logs_job ON event_logs(job_uuid,created_at_ms DESC)`, `CREATE INDEX idx_event_logs_job_id ON event_logs(job_uuid,id DESC)`, `CREATE TABLE event_log_policies(event_type TEXT PRIMARY KEY,retention_hours BIGINT NOT NULL DEFAULT 168,aggregate_after_hours BIGINT NOT NULL DEFAULT 24,bucket_seconds BIGINT NOT NULL DEFAULT 60,max_rows BIGINT NOT NULL DEFAULT 5000)`, `INSERT INTO event_log_policies(event_type) VALUES('*')`, `CREATE TABLE event_log_buckets(event_type TEXT NOT NULL,bucket_start_ms BIGINT NOT NULL,level TEXT NOT NULL,count BIGINT NOT NULL DEFAULT 0,first_at_ms BIGINT NOT NULL,last_at_ms BIGINT NOT NULL,sample_message TEXT NOT NULL DEFAULT '',PRIMARY KEY(event_type,bucket_start_ms,level))`, `CREATE INDEX idx_event_log_buckets_time ON event_log_buckets(event_type,bucket_start_ms DESC)`, `CREATE TABLE event_queue_settings(id BIGINT PRIMARY KEY CHECK(id=1),paused BIGINT NOT NULL DEFAULT 0,concurrency BIGINT NOT NULL DEFAULT 1,max_depth BIGINT NOT NULL DEFAULT 10000,retry_base_seconds BIGINT NOT NULL DEFAULT 30,retention_hours BIGINT NOT NULL DEFAULT 168)`, `INSERT INTO event_queue_settings(id) VALUES(1)`, }, Assertions: []string{ `SELECT ` + eventJobColumns + `,claim_token FROM event_jobs WHERE 1=0`, `SELECT uuid,name,event_type,event_key,payload,schedule_kind,interval_seconds,next_run_at_ms,enabled,priority,max_attempts,created_at_ms,updated_at_ms FROM event_definitions WHERE 1=0`, `SELECT id,job_uuid,event_uuid,level,message,created_at_ms FROM event_logs WHERE 1=0`, `SELECT event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows FROM event_log_policies WHERE 1=0`, `SELECT event_type,bucket_start_ms,level,count,first_at_ms,last_at_ms,sample_message FROM event_log_buckets WHERE 1=0`, `SELECT id,paused,concurrency,max_depth,retry_base_seconds,retention_hours FROM event_queue_settings WHERE 1=0`, }}}) } func (s *Store) withNativeEventWriter(ctx context.Context, write func(*sql.Tx) error) (err error) { defer func() { err = s.sqlError("event queue", err) }() ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() tx, err := s.DB.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() if err = s.lockTaskWriter(ctx, tx); err != nil { return err } if err = write(tx); err != nil { return err } return tx.Commit() } func (s *Store) enqueueNativeEventTx(ctx context.Context, tx *sql.Tx, eventID, kind, key, payload string, at time.Time, priority, attempts int) (bool, error) { if kind == NotificationPushEventType { return false, ErrBackendFeatureUnavailable } var duplicate bool if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM event_jobs WHERE event_type=$1 AND event_key=$2 AND status IN ('queued','running'))`, kind, key).Scan(&duplicate); err != nil { return false, err } if duplicate { return false, nil } var depth, limit int if err := tx.QueryRowContext(ctx, `SELECT (SELECT count(*) FROM event_jobs WHERE status IN ('queued','running')),max_depth FROM event_queue_settings WHERE id=1`).Scan(&depth, &limit); err != nil { return false, err } if depth >= limit { return false, ErrEventQueueFull } if attempts < 1 { attempts = 1 } now := time.Now().UnixMilli() _, err := tx.ExecContext(ctx, `INSERT INTO event_jobs(uuid,event_uuid,event_type,event_key,payload,status,priority,attempts,max_attempts,run_at_ms,created_at_ms,updated_at_ms) VALUES($1,$2,$3,$4,$5,'queued',$6,0,$7,$8,$9,$9)`, uuid.NewString(), eventID, kind, key, payload, priority, attempts, at.UnixMilli(), now) return err == nil, err } var ErrEventQueueFull = errors.New("event queue is full") func (s *Store) claimNativeEvent(ctx context.Context, at time.Time) (job *EventJob, err error) { if s.BackgroundJobsDisabled { return nil, sql.ErrNoRows } if err = s.nativeEventWorkerReady(ctx); err != nil { return nil, err } err = s.withNativeEventWriter(ctx, func(tx *sql.Tx) error { var paused bool var concurrency, running int if e := tx.QueryRowContext(ctx, `SELECT paused,concurrency,(SELECT count(*) FROM event_jobs WHERE status='running') FROM event_queue_settings WHERE id=1`).Scan(&paused, &concurrency, &running); e != nil { return e } if paused || running >= concurrency { return sql.ErrNoRows } token := uuid.NewString() j, e := scanEventJob(tx.QueryRowContext(ctx, `UPDATE event_jobs SET status='running',attempts=attempts+1,started_at_ms=$1,updated_at_ms=$1,claim_token=$2 WHERE uuid=(SELECT uuid FROM event_jobs WHERE status='queued' AND cancel_requested=0 AND run_at_ms<=$1 AND event_type<>$3 ORDER BY priority DESC,run_at_ms,created_at_ms,uuid LIMIT 1) AND status='queued' RETURNING `+eventJobColumns, at.UnixMilli(), token, NotificationPushEventType)) if e != nil { return e } j.ClaimToken = token job = j return nil }) return job, err } // UUID-only completion cannot prove ownership of a native execution attempt. func (s *Store) FinishClaimedEventAt(ctx context.Context, claim *EventJob, runErr error, at time.Time) error { if claim == nil { return ErrEventClaim } if s.sqlBackend != storePostgreSQL { return s.FinishEventAt(ctx, claim.UUID, runErr, at) } if claim.ClaimToken == "" { return ErrEventClaim } var current *EventJob var outcome dregarnuhr.Completion err := s.withNativeEventWriter(ctx, func(tx *sql.Tx) error { j, e := scanEventJob(tx.QueryRowContext(ctx, `SELECT `+eventJobColumns+` FROM event_jobs WHERE uuid=$1 AND claim_token=$2 AND status='running'`, claim.UUID, claim.ClaimToken)) if errors.Is(e, sql.ErrNoRows) { return ErrEventClaim } if e != nil { return e } current = j var base int if e = tx.QueryRowContext(ctx, `SELECT retry_base_seconds FROM event_queue_settings WHERE id=1`).Scan(&base); e != nil { return e } if j.CancelRequested && runErr == nil { runErr = context.Canceled } outcome = dregarnuhr.Complete(*j, runErr, base, at) finished, retry := at.UnixMilli(), j.RunAtMS if outcome.Status == "queued" { finished = 0 retry = outcome.RetryAt.UnixMilli() } _, e = tx.ExecContext(ctx, `UPDATE event_jobs SET status=$1,error=$2,run_at_ms=$3,finished_at_ms=$4,updated_at_ms=$5,claim_token='' WHERE uuid=$6 AND claim_token=$7`, outcome.Status, outcome.Message, retry, finished, at.UnixMilli(), claim.UUID, claim.ClaimToken) return e }) if err != nil { return err } message := outcome.Status if outcome.Message != "" { message += ": " + outcome.Message } return s.addEventCompletionLog(ctx, current.UUID, current.EventUUID, outcome.Level, message) } // Called only by a future exclusive single-instance startup owner, never by // schema initialization. Ambiguous interrupted operations are NOT auto-retried. func (s *Store) recoverNativeEvents(ctx context.Context) error { return s.withNativeEventWriter(ctx, func(tx *sql.Tx) error { _, err := tx.ExecContext(ctx, `UPDATE event_jobs SET status='failed',cancel_requested=1,error='execution interrupted; explicit retry required',finished_at_ms=$1,updated_at_ms=$1,claim_token='' WHERE status='running'`, time.Now().UnixMilli()) return err }) } func (s *Store) enqueueNativeDefinitions(ctx context.Context, at time.Time) (count int, err error) { err = s.withNativeEventWriter(ctx, func(tx *sql.Tx) error { rows, e := tx.QueryContext(ctx, `SELECT uuid,event_type,event_key,payload,priority,max_attempts,next_run_at_ms FROM event_definitions WHERE enabled=1 AND schedule_kind IN ('once','interval') AND next_run_at_ms>0 AND next_run_at_ms<=$1 AND event_type<>$2 ORDER BY next_run_at_ms,uuid LIMIT 32`, at.UnixMilli(), NotificationPushEventType) if e != nil { return e } var events []EventDefinition for rows.Next() { var v EventDefinition if e = rows.Scan(&v.UUID, &v.EventType, &v.EventKey, &v.Payload, &v.Priority, &v.MaxAttempts, &v.NextRunAtMS); e != nil { rows.Close() return e } events = append(events, v) } e = rows.Err() rows.Close() if e != nil { return e } for _, v := range events { queued, e := s.enqueueNativeEventTx(ctx, tx, v.UUID, v.EventType, v.EventKey+":"+v.UUID+":"+fmt.Sprint(v.NextRunAtMS), v.Payload, at, v.Priority, v.MaxAttempts) if e != nil { return e } if queued { count++ } if _, e = tx.ExecContext(ctx, `UPDATE event_definitions SET next_run_at_ms=CASE schedule_kind WHEN 'once' THEN 0 ELSE $1+interval_seconds*1000 END,enabled=CASE schedule_kind WHEN 'once' THEN 0 ELSE enabled END,updated_at_ms=$1 WHERE uuid=$2`, at.UnixMilli(), v.UUID); e != nil { return e } } return nil }) if err != nil { return 0, err } return count, nil } func (s *Store) runNativeDefinition(id string) (queued bool, err error) { err = s.withNativeEventWriter(context.Background(), func(tx *sql.Tx) error { var v EventDefinition if e := tx.QueryRow(`SELECT uuid,event_type,event_key,payload,priority,max_attempts FROM event_definitions WHERE uuid=$1`, id).Scan(&v.UUID, &v.EventType, &v.EventKey, &v.Payload, &v.Priority, &v.MaxAttempts); e != nil { return e } var e error queued, e = s.enqueueNativeEventTx(context.Background(), tx, v.UUID, v.EventType, v.EventKey+":"+uuid.NewString(), v.Payload, time.Now(), v.Priority, v.MaxAttempts) return e }) return queued, err }