feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s
Build and publish container images / images (push) Failing after 12m29s
This commit is contained in:
1 parent
96065d1e89
commit
0ba1dcfef0
243 files changed
+21441
-3090
No files matched your search
@@ -0,0 +1,371 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"pageswiki/internal/db"
|
||||
"pageswiki/internal/dregarnuhr"
|
||||
)
|
||||
|
||||
var ErrEventClaim = errors.New("event execution claim is stale or missing")
|
||||
var ErrEventWorker = errors.New("native event worker requires an exclusive live session")
|
||||
|
||||
const eventWorkerLock = `hashtextextended(current_database()||':'||current_schema()||':pageswiki-event-worker',0)`
|
||||
|
||||
// AcquireEventWorker preserves SQLite's existing instance-lock ownership;
|
||||
// PostgreSQL also needs a database-scoped owner across distinct data roots.
|
||||
func (s *Store) AcquireEventWorker(ctx context.Context) (func() error, error) {
|
||||
if s.ReadOnly || s.BackgroundJobsDisabled {
|
||||
return nil, ErrEventWorker
|
||||
}
|
||||
if s.sqlBackend == storeSQLite {
|
||||
return func() error { return nil }, nil
|
||||
}
|
||||
return s.StartNativeEventWorker(ctx)
|
||||
}
|
||||
|
||||
// StartNativeEventWorker owns one dedicated connection for this single-instance
|
||||
// scheduler. The caller must stop/join workers before releasing it. This is not
|
||||
// HA failover: a lost session stops admission, and ambiguous work needs review.
|
||||
func (s *Store) StartNativeEventWorker(ctx context.Context) (func() error, error) {
|
||||
if s.ReadOnly {
|
||||
return nil, ErrReadOnly
|
||||
}
|
||||
if s.sqlBackend != storePostgreSQL || s.BackgroundJobsDisabled {
|
||||
return nil, ErrEventWorker
|
||||
}
|
||||
s.eventWorkerMu.Lock()
|
||||
defer s.eventWorkerMu.Unlock()
|
||||
if s.eventWorkerConn != nil {
|
||||
return nil, ErrEventWorker
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
conn, err := s.DB.Conn(ctx)
|
||||
if err != nil {
|
||||
return nil, s.sqlError("worker owner", err)
|
||||
}
|
||||
var acquired bool
|
||||
err = conn.QueryRowContext(ctx, `SELECT pg_try_advisory_lock(`+eventWorkerLock+`)`).Scan(&acquired)
|
||||
if err != nil || !acquired {
|
||||
_ = conn.Raw(func(any) error { return driver.ErrBadConn })
|
||||
conn.Close()
|
||||
if err != nil {
|
||||
return nil, s.sqlError("worker owner", err)
|
||||
}
|
||||
return nil, ErrEventWorker
|
||||
}
|
||||
cleanup := func() error {
|
||||
c, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
var unlocked bool
|
||||
e := conn.QueryRowContext(c, `SELECT pg_advisory_unlock(`+eventWorkerLock+`)`).Scan(&unlocked)
|
||||
if e != nil || !unlocked {
|
||||
_ = conn.Raw(func(any) error { return driver.ErrBadConn })
|
||||
}
|
||||
closeErr := conn.Close()
|
||||
if e != nil {
|
||||
return s.sqlError("worker release", e)
|
||||
}
|
||||
return closeErr
|
||||
}
|
||||
if err = s.recoverNativeEvents(ctx); err != nil {
|
||||
cleanup()
|
||||
return nil, err
|
||||
}
|
||||
s.eventWorkerConn = conn
|
||||
return func() error {
|
||||
s.eventWorkerMu.Lock()
|
||||
defer s.eventWorkerMu.Unlock()
|
||||
if s.eventWorkerConn != conn {
|
||||
return nil
|
||||
}
|
||||
s.eventWorkerConn = nil
|
||||
return cleanup()
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Store) nativeEventWorkerReady(ctx context.Context) error {
|
||||
s.eventWorkerMu.Lock()
|
||||
defer s.eventWorkerMu.Unlock()
|
||||
if s.eventWorkerConn == nil {
|
||||
return ErrEventWorker
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Second)
|
||||
defer cancel()
|
||||
var one int
|
||||
if err := s.eventWorkerConn.QueryRowContext(ctx, `SELECT 1`).Scan(&one); err != nil {
|
||||
return ErrEventWorker
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) execEventSQL(query string, args ...any) (result sql.Result, err error) {
|
||||
if s.ReadOnly {
|
||||
return nil, ErrReadOnly
|
||||
}
|
||||
if s.sqlBackend != storePostgreSQL {
|
||||
return s.DB.Exec(query, args...)
|
||||
}
|
||||
err = s.withNativeEventWriter(context.Background(), func(tx *sql.Tx) error {
|
||||
var e error
|
||||
result, e = tx.Exec(query, args...)
|
||||
return e
|
||||
})
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Builtin registration must preserve an administrator's existing schedule.
|
||||
func (s *Store) ensureNativeBuiltinEvent(id, name, kind string, interval int64, priority, attempts int) error {
|
||||
now := time.Now().UnixMilli()
|
||||
_, err := s.execEventSQL(`INSERT INTO event_definitions(uuid,name,event_type,event_key,payload,schedule_kind,interval_seconds,next_run_at_ms,enabled,priority,max_attempts,created_at_ms,updated_at_ms) VALUES($1,$2,$3,'local','{}','interval',$4,$5,1,$6,$7,$5,$5) ON CONFLICT(uuid) DO NOTHING`, id, name, kind, interval, now, priority, attempts)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) retryNativeEvent(id string) error {
|
||||
err := s.withNativeEventWriter(context.Background(), func(tx *sql.Tx) error {
|
||||
var depth, limit int
|
||||
if e := tx.QueryRow(`SELECT (SELECT count(*) FROM event_jobs WHERE status IN ('queued','running')),max_depth FROM event_queue_settings WHERE id=1`).Scan(&depth, &limit); e != nil {
|
||||
return e
|
||||
}
|
||||
if depth >= limit {
|
||||
return ErrEventQueueFull
|
||||
}
|
||||
r, e := tx.Exec(`UPDATE event_jobs SET status='queued',attempts=0,cancel_requested=0,error='',run_at_ms=$1,started_at_ms=0,finished_at_ms=0,updated_at_ms=$1,claim_token='' WHERE uuid=$2 AND status IN ('failed','cancelled','skipped')`, time.Now().UnixMilli(), id)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
n, e := r.RowsAffected()
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if n == 0 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.AddEventLog(id, "", "info", "manually requeued")
|
||||
}
|
||||
|
||||
func (s *Store) ensurePostgreSQLEventJobs(ctx context.Context) error {
|
||||
return db.ApplyPostgreSQLMigrations(ctx, s.DB, "store-event-jobs", []db.PostgreSQLMigration{{Version: 1, Name: "single-instance event queue and execution fencing", Statements: []string{
|
||||
`CREATE TABLE event_jobs(uuid TEXT COLLATE "C" PRIMARY KEY,event_type TEXT COLLATE "C" NOT NULL,event_key TEXT COLLATE "C" NOT NULL,payload TEXT NOT NULL,status TEXT NOT NULL,attempts BIGINT NOT NULL DEFAULT 0,run_at_ms BIGINT NOT NULL,error TEXT NOT NULL DEFAULT '',created_at_ms BIGINT NOT NULL,updated_at_ms BIGINT NOT NULL,event_uuid TEXT COLLATE "C" NOT NULL DEFAULT '',priority BIGINT NOT NULL DEFAULT 0,max_attempts BIGINT NOT NULL DEFAULT 3,started_at_ms BIGINT NOT NULL DEFAULT 0,finished_at_ms BIGINT NOT NULL DEFAULT 0,cancel_requested BIGINT NOT NULL DEFAULT 0,claim_token TEXT NOT NULL DEFAULT '')`,
|
||||
`CREATE INDEX idx_event_jobs_identity_status ON event_jobs(event_type,event_key,status)`,
|
||||
`CREATE UNIQUE INDEX idx_event_jobs_active_identity ON event_jobs(event_type,event_key) WHERE status IN ('queued','running')`,
|
||||
`CREATE INDEX idx_event_jobs_listing ON event_jobs(created_at_ms DESC,uuid DESC)`,
|
||||
`CREATE INDEX idx_event_jobs_status_listing ON event_jobs(status,created_at_ms DESC,uuid DESC)`,
|
||||
`CREATE INDEX idx_event_jobs_history ON event_jobs(created_at_ms DESC,uuid DESC) WHERE status IN ('succeeded','skipped','failed','cancelled')`,
|
||||
`CREATE INDEX idx_event_jobs_due ON event_jobs(status,priority DESC,run_at_ms,created_at_ms)`,
|
||||
`CREATE TABLE event_definitions(uuid TEXT COLLATE "C" PRIMARY KEY,name TEXT NOT NULL,event_type TEXT NOT NULL,event_key TEXT NOT NULL,payload TEXT NOT NULL DEFAULT '',schedule_kind TEXT NOT NULL DEFAULT 'manual',interval_seconds BIGINT NOT NULL DEFAULT 0,next_run_at_ms BIGINT NOT NULL DEFAULT 0,enabled BIGINT NOT NULL DEFAULT 1,priority BIGINT NOT NULL DEFAULT 0,max_attempts BIGINT NOT NULL DEFAULT 3,created_at_ms BIGINT NOT NULL,updated_at_ms BIGINT NOT NULL)`,
|
||||
`CREATE INDEX idx_event_definitions_due ON event_definitions(enabled,next_run_at_ms)`,
|
||||
`CREATE TABLE event_logs(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,job_uuid TEXT NOT NULL,event_uuid TEXT NOT NULL DEFAULT '',level TEXT NOT NULL,message TEXT NOT NULL,created_at_ms BIGINT NOT NULL)`,
|
||||
`CREATE INDEX idx_event_logs_job ON event_logs(job_uuid,created_at_ms DESC)`,
|
||||
`CREATE INDEX idx_event_logs_job_id ON event_logs(job_uuid,id DESC)`,
|
||||
`CREATE TABLE event_log_policies(event_type TEXT PRIMARY KEY,retention_hours BIGINT NOT NULL DEFAULT 168,aggregate_after_hours BIGINT NOT NULL DEFAULT 24,bucket_seconds BIGINT NOT NULL DEFAULT 60,max_rows BIGINT NOT NULL DEFAULT 5000)`,
|
||||
`INSERT INTO event_log_policies(event_type) VALUES('*')`,
|
||||
`CREATE TABLE event_log_buckets(event_type TEXT NOT NULL,bucket_start_ms BIGINT NOT NULL,level TEXT NOT NULL,count BIGINT NOT NULL DEFAULT 0,first_at_ms BIGINT NOT NULL,last_at_ms BIGINT NOT NULL,sample_message TEXT NOT NULL DEFAULT '',PRIMARY KEY(event_type,bucket_start_ms,level))`,
|
||||
`CREATE INDEX idx_event_log_buckets_time ON event_log_buckets(event_type,bucket_start_ms DESC)`,
|
||||
`CREATE TABLE event_queue_settings(id BIGINT PRIMARY KEY CHECK(id=1),paused BIGINT NOT NULL DEFAULT 0,concurrency BIGINT NOT NULL DEFAULT 1,max_depth BIGINT NOT NULL DEFAULT 10000,retry_base_seconds BIGINT NOT NULL DEFAULT 30,retention_hours BIGINT NOT NULL DEFAULT 168)`,
|
||||
`INSERT INTO event_queue_settings(id) VALUES(1)`,
|
||||
}, Assertions: []string{
|
||||
`SELECT ` + eventJobColumns + `,claim_token FROM event_jobs WHERE 1=0`,
|
||||
`SELECT uuid,name,event_type,event_key,payload,schedule_kind,interval_seconds,next_run_at_ms,enabled,priority,max_attempts,created_at_ms,updated_at_ms FROM event_definitions WHERE 1=0`,
|
||||
`SELECT id,job_uuid,event_uuid,level,message,created_at_ms FROM event_logs WHERE 1=0`,
|
||||
`SELECT event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows FROM event_log_policies WHERE 1=0`,
|
||||
`SELECT event_type,bucket_start_ms,level,count,first_at_ms,last_at_ms,sample_message FROM event_log_buckets WHERE 1=0`,
|
||||
`SELECT id,paused,concurrency,max_depth,retry_base_seconds,retention_hours FROM event_queue_settings WHERE 1=0`,
|
||||
}}})
|
||||
}
|
||||
|
||||
func (s *Store) withNativeEventWriter(ctx context.Context, write func(*sql.Tx) error) (err error) {
|
||||
defer func() { err = s.sqlError("event queue", err) }()
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
tx, err := s.DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err = s.lockTaskWriter(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = write(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (s *Store) enqueueNativeEventTx(ctx context.Context, tx *sql.Tx, eventID, kind, key, payload string, at time.Time, priority, attempts int) (bool, error) {
|
||||
if kind == NotificationPushEventType {
|
||||
return false, ErrBackendFeatureUnavailable
|
||||
}
|
||||
var duplicate bool
|
||||
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM event_jobs WHERE event_type=$1 AND event_key=$2 AND status IN ('queued','running'))`, kind, key).Scan(&duplicate); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if duplicate {
|
||||
return false, nil
|
||||
}
|
||||
var depth, limit int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT (SELECT count(*) FROM event_jobs WHERE status IN ('queued','running')),max_depth FROM event_queue_settings WHERE id=1`).Scan(&depth, &limit); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if depth >= limit {
|
||||
return false, ErrEventQueueFull
|
||||
}
|
||||
if attempts < 1 {
|
||||
attempts = 1
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
_, err := tx.ExecContext(ctx, `INSERT INTO event_jobs(uuid,event_uuid,event_type,event_key,payload,status,priority,attempts,max_attempts,run_at_ms,created_at_ms,updated_at_ms) VALUES($1,$2,$3,$4,$5,'queued',$6,0,$7,$8,$9,$9)`, uuid.NewString(), eventID, kind, key, payload, priority, attempts, at.UnixMilli(), now)
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
var ErrEventQueueFull = errors.New("event queue is full")
|
||||
|
||||
func (s *Store) claimNativeEvent(ctx context.Context, at time.Time) (job *EventJob, err error) {
|
||||
if s.BackgroundJobsDisabled {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
if err = s.nativeEventWorkerReady(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = s.withNativeEventWriter(ctx, func(tx *sql.Tx) error {
|
||||
var paused bool
|
||||
var concurrency, running int
|
||||
if e := tx.QueryRowContext(ctx, `SELECT paused,concurrency,(SELECT count(*) FROM event_jobs WHERE status='running') FROM event_queue_settings WHERE id=1`).Scan(&paused, &concurrency, &running); e != nil {
|
||||
return e
|
||||
}
|
||||
if paused || running >= concurrency {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
token := uuid.NewString()
|
||||
j, e := scanEventJob(tx.QueryRowContext(ctx, `UPDATE event_jobs SET status='running',attempts=attempts+1,started_at_ms=$1,updated_at_ms=$1,claim_token=$2 WHERE uuid=(SELECT uuid FROM event_jobs WHERE status='queued' AND cancel_requested=0 AND run_at_ms<=$1 AND event_type<>$3 ORDER BY priority DESC,run_at_ms,created_at_ms,uuid LIMIT 1) AND status='queued' RETURNING `+eventJobColumns, at.UnixMilli(), token, NotificationPushEventType))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
j.ClaimToken = token
|
||||
job = j
|
||||
return nil
|
||||
})
|
||||
return job, err
|
||||
}
|
||||
|
||||
// UUID-only completion cannot prove ownership of a native execution attempt.
|
||||
func (s *Store) FinishClaimedEventAt(ctx context.Context, claim *EventJob, runErr error, at time.Time) error {
|
||||
if claim == nil {
|
||||
return ErrEventClaim
|
||||
}
|
||||
if s.sqlBackend != storePostgreSQL {
|
||||
return s.FinishEventAt(ctx, claim.UUID, runErr, at)
|
||||
}
|
||||
if claim.ClaimToken == "" {
|
||||
return ErrEventClaim
|
||||
}
|
||||
var current *EventJob
|
||||
var outcome dregarnuhr.Completion
|
||||
err := s.withNativeEventWriter(ctx, func(tx *sql.Tx) error {
|
||||
j, e := scanEventJob(tx.QueryRowContext(ctx, `SELECT `+eventJobColumns+` FROM event_jobs WHERE uuid=$1 AND claim_token=$2 AND status='running'`, claim.UUID, claim.ClaimToken))
|
||||
if errors.Is(e, sql.ErrNoRows) {
|
||||
return ErrEventClaim
|
||||
}
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
current = j
|
||||
var base int
|
||||
if e = tx.QueryRowContext(ctx, `SELECT retry_base_seconds FROM event_queue_settings WHERE id=1`).Scan(&base); e != nil {
|
||||
return e
|
||||
}
|
||||
if j.CancelRequested && runErr == nil {
|
||||
runErr = context.Canceled
|
||||
}
|
||||
outcome = dregarnuhr.Complete(*j, runErr, base, at)
|
||||
finished, retry := at.UnixMilli(), j.RunAtMS
|
||||
if outcome.Status == "queued" {
|
||||
finished = 0
|
||||
retry = outcome.RetryAt.UnixMilli()
|
||||
}
|
||||
_, e = tx.ExecContext(ctx, `UPDATE event_jobs SET status=$1,error=$2,run_at_ms=$3,finished_at_ms=$4,updated_at_ms=$5,claim_token='' WHERE uuid=$6 AND claim_token=$7`, outcome.Status, outcome.Message, retry, finished, at.UnixMilli(), claim.UUID, claim.ClaimToken)
|
||||
return e
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
message := outcome.Status
|
||||
if outcome.Message != "" {
|
||||
message += ": " + outcome.Message
|
||||
}
|
||||
return s.addEventCompletionLog(ctx, current.UUID, current.EventUUID, outcome.Level, message)
|
||||
}
|
||||
|
||||
// Called only by a future exclusive single-instance startup owner, never by
|
||||
// schema initialization. Ambiguous interrupted operations are NOT auto-retried.
|
||||
func (s *Store) recoverNativeEvents(ctx context.Context) error {
|
||||
return s.withNativeEventWriter(ctx, func(tx *sql.Tx) error {
|
||||
_, err := tx.ExecContext(ctx, `UPDATE event_jobs SET status='failed',cancel_requested=1,error='execution interrupted; explicit retry required',finished_at_ms=$1,updated_at_ms=$1,claim_token='' WHERE status='running'`, time.Now().UnixMilli())
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) enqueueNativeDefinitions(ctx context.Context, at time.Time) (count int, err error) {
|
||||
err = s.withNativeEventWriter(ctx, func(tx *sql.Tx) error {
|
||||
rows, e := tx.QueryContext(ctx, `SELECT uuid,event_type,event_key,payload,priority,max_attempts,next_run_at_ms FROM event_definitions WHERE enabled=1 AND schedule_kind IN ('once','interval') AND next_run_at_ms>0 AND next_run_at_ms<=$1 AND event_type<>$2 ORDER BY next_run_at_ms,uuid LIMIT 32`, at.UnixMilli(), NotificationPushEventType)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
var events []EventDefinition
|
||||
for rows.Next() {
|
||||
var v EventDefinition
|
||||
if e = rows.Scan(&v.UUID, &v.EventType, &v.EventKey, &v.Payload, &v.Priority, &v.MaxAttempts, &v.NextRunAtMS); e != nil {
|
||||
rows.Close()
|
||||
return e
|
||||
}
|
||||
events = append(events, v)
|
||||
}
|
||||
e = rows.Err()
|
||||
rows.Close()
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
for _, v := range events {
|
||||
queued, e := s.enqueueNativeEventTx(ctx, tx, v.UUID, v.EventType, v.EventKey+":"+v.UUID+":"+fmt.Sprint(v.NextRunAtMS), v.Payload, at, v.Priority, v.MaxAttempts)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if queued {
|
||||
count++
|
||||
}
|
||||
if _, e = tx.ExecContext(ctx, `UPDATE event_definitions SET next_run_at_ms=CASE schedule_kind WHEN 'once' THEN 0 ELSE $1+interval_seconds*1000 END,enabled=CASE schedule_kind WHEN 'once' THEN 0 ELSE enabled END,updated_at_ms=$1 WHERE uuid=$2`, at.UnixMilli(), v.UUID); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (s *Store) runNativeDefinition(id string) (queued bool, err error) {
|
||||
err = s.withNativeEventWriter(context.Background(), func(tx *sql.Tx) error {
|
||||
var v EventDefinition
|
||||
if e := tx.QueryRow(`SELECT uuid,event_type,event_key,payload,priority,max_attempts FROM event_definitions WHERE uuid=$1`, id).Scan(&v.UUID, &v.EventType, &v.EventKey, &v.Payload, &v.Priority, &v.MaxAttempts); e != nil {
|
||||
return e
|
||||
}
|
||||
var e error
|
||||
queued, e = s.enqueueNativeEventTx(context.Background(), tx, v.UUID, v.EventType, v.EventKey+":"+uuid.NewString(), v.Payload, time.Now(), v.Priority, v.MaxAttempts)
|
||||
return e
|
||||
})
|
||||
return queued, err
|
||||
}
|
||||
Reference in new issue
Block a user