feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s

This commit is contained in:
Anya Tokugawa committed 2026-11-15 05:47:14 +09:00
1 parent 96065d1e89
commit 0ba1dcfef0
243 files changed
+21441 -3090

No files matched your search

+371
View File
@@ -0,0 +1,371 @@
package store
import (
"context"
"database/sql"
"database/sql/driver"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"pageswiki/internal/db"
"pageswiki/internal/dregarnuhr"
)
var ErrEventClaim = errors.New("event execution claim is stale or missing")
var ErrEventWorker = errors.New("native event worker requires an exclusive live session")
const eventWorkerLock = `hashtextextended(current_database()||':'||current_schema()||':pageswiki-event-worker',0)`
// AcquireEventWorker preserves SQLite's existing instance-lock ownership;
// PostgreSQL also needs a database-scoped owner across distinct data roots.
func (s *Store) AcquireEventWorker(ctx context.Context) (func() error, error) {
if s.ReadOnly || s.BackgroundJobsDisabled {
return nil, ErrEventWorker
}
if s.sqlBackend == storeSQLite {
return func() error { return nil }, nil
}
return s.StartNativeEventWorker(ctx)
}
// StartNativeEventWorker owns one dedicated connection for this single-instance
// scheduler. The caller must stop/join workers before releasing it. This is not
// HA failover: a lost session stops admission, and ambiguous work needs review.
func (s *Store) StartNativeEventWorker(ctx context.Context) (func() error, error) {
if s.ReadOnly {
return nil, ErrReadOnly
}
if s.sqlBackend != storePostgreSQL || s.BackgroundJobsDisabled {
return nil, ErrEventWorker
}
s.eventWorkerMu.Lock()
defer s.eventWorkerMu.Unlock()
if s.eventWorkerConn != nil {
return nil, ErrEventWorker
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
conn, err := s.DB.Conn(ctx)
if err != nil {
return nil, s.sqlError("worker owner", err)
}
var acquired bool
err = conn.QueryRowContext(ctx, `SELECT pg_try_advisory_lock(`+eventWorkerLock+`)`).Scan(&acquired)
if err != nil || !acquired {
_ = conn.Raw(func(any) error { return driver.ErrBadConn })
conn.Close()
if err != nil {
return nil, s.sqlError("worker owner", err)
}
return nil, ErrEventWorker
}
cleanup := func() error {
c, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
var unlocked bool
e := conn.QueryRowContext(c, `SELECT pg_advisory_unlock(`+eventWorkerLock+`)`).Scan(&unlocked)
if e != nil || !unlocked {
_ = conn.Raw(func(any) error { return driver.ErrBadConn })
}
closeErr := conn.Close()
if e != nil {
return s.sqlError("worker release", e)
}
return closeErr
}
if err = s.recoverNativeEvents(ctx); err != nil {
cleanup()
return nil, err
}
s.eventWorkerConn = conn
return func() error {
s.eventWorkerMu.Lock()
defer s.eventWorkerMu.Unlock()
if s.eventWorkerConn != conn {
return nil
}
s.eventWorkerConn = nil
return cleanup()
}, nil
}
func (s *Store) nativeEventWorkerReady(ctx context.Context) error {
s.eventWorkerMu.Lock()
defer s.eventWorkerMu.Unlock()
if s.eventWorkerConn == nil {
return ErrEventWorker
}
ctx, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
var one int
if err := s.eventWorkerConn.QueryRowContext(ctx, `SELECT 1`).Scan(&one); err != nil {
return ErrEventWorker
}
return nil
}
func (s *Store) execEventSQL(query string, args ...any) (result sql.Result, err error) {
if s.ReadOnly {
return nil, ErrReadOnly
}
if s.sqlBackend != storePostgreSQL {
return s.DB.Exec(query, args...)
}
err = s.withNativeEventWriter(context.Background(), func(tx *sql.Tx) error {
var e error
result, e = tx.Exec(query, args...)
return e
})
return result, err
}
// Builtin registration must preserve an administrator's existing schedule.
func (s *Store) ensureNativeBuiltinEvent(id, name, kind string, interval int64, priority, attempts int) error {
now := time.Now().UnixMilli()
_, err := s.execEventSQL(`INSERT INTO event_definitions(uuid,name,event_type,event_key,payload,schedule_kind,interval_seconds,next_run_at_ms,enabled,priority,max_attempts,created_at_ms,updated_at_ms) VALUES($1,$2,$3,'local','{}','interval',$4,$5,1,$6,$7,$5,$5) ON CONFLICT(uuid) DO NOTHING`, id, name, kind, interval, now, priority, attempts)
return err
}
func (s *Store) retryNativeEvent(id string) error {
err := s.withNativeEventWriter(context.Background(), func(tx *sql.Tx) error {
var depth, limit int
if e := tx.QueryRow(`SELECT (SELECT count(*) FROM event_jobs WHERE status IN ('queued','running')),max_depth FROM event_queue_settings WHERE id=1`).Scan(&depth, &limit); e != nil {
return e
}
if depth >= limit {
return ErrEventQueueFull
}
r, e := tx.Exec(`UPDATE event_jobs SET status='queued',attempts=0,cancel_requested=0,error='',run_at_ms=$1,started_at_ms=0,finished_at_ms=0,updated_at_ms=$1,claim_token='' WHERE uuid=$2 AND status IN ('failed','cancelled','skipped')`, time.Now().UnixMilli(), id)
if e != nil {
return e
}
n, e := r.RowsAffected()
if e != nil {
return e
}
if n == 0 {
return sql.ErrNoRows
}
return nil
})
if err != nil {
return err
}
return s.AddEventLog(id, "", "info", "manually requeued")
}
func (s *Store) ensurePostgreSQLEventJobs(ctx context.Context) error {
return db.ApplyPostgreSQLMigrations(ctx, s.DB, "store-event-jobs", []db.PostgreSQLMigration{{Version: 1, Name: "single-instance event queue and execution fencing", Statements: []string{
`CREATE TABLE event_jobs(uuid TEXT COLLATE "C" PRIMARY KEY,event_type TEXT COLLATE "C" NOT NULL,event_key TEXT COLLATE "C" NOT NULL,payload TEXT NOT NULL,status TEXT NOT NULL,attempts BIGINT NOT NULL DEFAULT 0,run_at_ms BIGINT NOT NULL,error TEXT NOT NULL DEFAULT '',created_at_ms BIGINT NOT NULL,updated_at_ms BIGINT NOT NULL,event_uuid TEXT COLLATE "C" NOT NULL DEFAULT '',priority BIGINT NOT NULL DEFAULT 0,max_attempts BIGINT NOT NULL DEFAULT 3,started_at_ms BIGINT NOT NULL DEFAULT 0,finished_at_ms BIGINT NOT NULL DEFAULT 0,cancel_requested BIGINT NOT NULL DEFAULT 0,claim_token TEXT NOT NULL DEFAULT '')`,
`CREATE INDEX idx_event_jobs_identity_status ON event_jobs(event_type,event_key,status)`,
`CREATE UNIQUE INDEX idx_event_jobs_active_identity ON event_jobs(event_type,event_key) WHERE status IN ('queued','running')`,
`CREATE INDEX idx_event_jobs_listing ON event_jobs(created_at_ms DESC,uuid DESC)`,
`CREATE INDEX idx_event_jobs_status_listing ON event_jobs(status,created_at_ms DESC,uuid DESC)`,
`CREATE INDEX idx_event_jobs_history ON event_jobs(created_at_ms DESC,uuid DESC) WHERE status IN ('succeeded','skipped','failed','cancelled')`,
`CREATE INDEX idx_event_jobs_due ON event_jobs(status,priority DESC,run_at_ms,created_at_ms)`,
`CREATE TABLE event_definitions(uuid TEXT COLLATE "C" PRIMARY KEY,name TEXT NOT NULL,event_type TEXT NOT NULL,event_key TEXT NOT NULL,payload TEXT NOT NULL DEFAULT '',schedule_kind TEXT NOT NULL DEFAULT 'manual',interval_seconds BIGINT NOT NULL DEFAULT 0,next_run_at_ms BIGINT NOT NULL DEFAULT 0,enabled BIGINT NOT NULL DEFAULT 1,priority BIGINT NOT NULL DEFAULT 0,max_attempts BIGINT NOT NULL DEFAULT 3,created_at_ms BIGINT NOT NULL,updated_at_ms BIGINT NOT NULL)`,
`CREATE INDEX idx_event_definitions_due ON event_definitions(enabled,next_run_at_ms)`,
`CREATE TABLE event_logs(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,job_uuid TEXT NOT NULL,event_uuid TEXT NOT NULL DEFAULT '',level TEXT NOT NULL,message TEXT NOT NULL,created_at_ms BIGINT NOT NULL)`,
`CREATE INDEX idx_event_logs_job ON event_logs(job_uuid,created_at_ms DESC)`,
`CREATE INDEX idx_event_logs_job_id ON event_logs(job_uuid,id DESC)`,
`CREATE TABLE event_log_policies(event_type TEXT PRIMARY KEY,retention_hours BIGINT NOT NULL DEFAULT 168,aggregate_after_hours BIGINT NOT NULL DEFAULT 24,bucket_seconds BIGINT NOT NULL DEFAULT 60,max_rows BIGINT NOT NULL DEFAULT 5000)`,
`INSERT INTO event_log_policies(event_type) VALUES('*')`,
`CREATE TABLE event_log_buckets(event_type TEXT NOT NULL,bucket_start_ms BIGINT NOT NULL,level TEXT NOT NULL,count BIGINT NOT NULL DEFAULT 0,first_at_ms BIGINT NOT NULL,last_at_ms BIGINT NOT NULL,sample_message TEXT NOT NULL DEFAULT '',PRIMARY KEY(event_type,bucket_start_ms,level))`,
`CREATE INDEX idx_event_log_buckets_time ON event_log_buckets(event_type,bucket_start_ms DESC)`,
`CREATE TABLE event_queue_settings(id BIGINT PRIMARY KEY CHECK(id=1),paused BIGINT NOT NULL DEFAULT 0,concurrency BIGINT NOT NULL DEFAULT 1,max_depth BIGINT NOT NULL DEFAULT 10000,retry_base_seconds BIGINT NOT NULL DEFAULT 30,retention_hours BIGINT NOT NULL DEFAULT 168)`,
`INSERT INTO event_queue_settings(id) VALUES(1)`,
}, Assertions: []string{
`SELECT ` + eventJobColumns + `,claim_token FROM event_jobs WHERE 1=0`,
`SELECT uuid,name,event_type,event_key,payload,schedule_kind,interval_seconds,next_run_at_ms,enabled,priority,max_attempts,created_at_ms,updated_at_ms FROM event_definitions WHERE 1=0`,
`SELECT id,job_uuid,event_uuid,level,message,created_at_ms FROM event_logs WHERE 1=0`,
`SELECT event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows FROM event_log_policies WHERE 1=0`,
`SELECT event_type,bucket_start_ms,level,count,first_at_ms,last_at_ms,sample_message FROM event_log_buckets WHERE 1=0`,
`SELECT id,paused,concurrency,max_depth,retry_base_seconds,retention_hours FROM event_queue_settings WHERE 1=0`,
}}})
}
func (s *Store) withNativeEventWriter(ctx context.Context, write func(*sql.Tx) error) (err error) {
defer func() { err = s.sqlError("event queue", err) }()
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
tx, err := s.DB.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err = s.lockTaskWriter(ctx, tx); err != nil {
return err
}
if err = write(tx); err != nil {
return err
}
return tx.Commit()
}
func (s *Store) enqueueNativeEventTx(ctx context.Context, tx *sql.Tx, eventID, kind, key, payload string, at time.Time, priority, attempts int) (bool, error) {
if kind == NotificationPushEventType {
return false, ErrBackendFeatureUnavailable
}
var duplicate bool
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM event_jobs WHERE event_type=$1 AND event_key=$2 AND status IN ('queued','running'))`, kind, key).Scan(&duplicate); err != nil {
return false, err
}
if duplicate {
return false, nil
}
var depth, limit int
if err := tx.QueryRowContext(ctx, `SELECT (SELECT count(*) FROM event_jobs WHERE status IN ('queued','running')),max_depth FROM event_queue_settings WHERE id=1`).Scan(&depth, &limit); err != nil {
return false, err
}
if depth >= limit {
return false, ErrEventQueueFull
}
if attempts < 1 {
attempts = 1
}
now := time.Now().UnixMilli()
_, err := tx.ExecContext(ctx, `INSERT INTO event_jobs(uuid,event_uuid,event_type,event_key,payload,status,priority,attempts,max_attempts,run_at_ms,created_at_ms,updated_at_ms) VALUES($1,$2,$3,$4,$5,'queued',$6,0,$7,$8,$9,$9)`, uuid.NewString(), eventID, kind, key, payload, priority, attempts, at.UnixMilli(), now)
return err == nil, err
}
var ErrEventQueueFull = errors.New("event queue is full")
func (s *Store) claimNativeEvent(ctx context.Context, at time.Time) (job *EventJob, err error) {
if s.BackgroundJobsDisabled {
return nil, sql.ErrNoRows
}
if err = s.nativeEventWorkerReady(ctx); err != nil {
return nil, err
}
err = s.withNativeEventWriter(ctx, func(tx *sql.Tx) error {
var paused bool
var concurrency, running int
if e := tx.QueryRowContext(ctx, `SELECT paused,concurrency,(SELECT count(*) FROM event_jobs WHERE status='running') FROM event_queue_settings WHERE id=1`).Scan(&paused, &concurrency, &running); e != nil {
return e
}
if paused || running >= concurrency {
return sql.ErrNoRows
}
token := uuid.NewString()
j, e := scanEventJob(tx.QueryRowContext(ctx, `UPDATE event_jobs SET status='running',attempts=attempts+1,started_at_ms=$1,updated_at_ms=$1,claim_token=$2 WHERE uuid=(SELECT uuid FROM event_jobs WHERE status='queued' AND cancel_requested=0 AND run_at_ms<=$1 AND event_type<>$3 ORDER BY priority DESC,run_at_ms,created_at_ms,uuid LIMIT 1) AND status='queued' RETURNING `+eventJobColumns, at.UnixMilli(), token, NotificationPushEventType))
if e != nil {
return e
}
j.ClaimToken = token
job = j
return nil
})
return job, err
}
// UUID-only completion cannot prove ownership of a native execution attempt.
func (s *Store) FinishClaimedEventAt(ctx context.Context, claim *EventJob, runErr error, at time.Time) error {
if claim == nil {
return ErrEventClaim
}
if s.sqlBackend != storePostgreSQL {
return s.FinishEventAt(ctx, claim.UUID, runErr, at)
}
if claim.ClaimToken == "" {
return ErrEventClaim
}
var current *EventJob
var outcome dregarnuhr.Completion
err := s.withNativeEventWriter(ctx, func(tx *sql.Tx) error {
j, e := scanEventJob(tx.QueryRowContext(ctx, `SELECT `+eventJobColumns+` FROM event_jobs WHERE uuid=$1 AND claim_token=$2 AND status='running'`, claim.UUID, claim.ClaimToken))
if errors.Is(e, sql.ErrNoRows) {
return ErrEventClaim
}
if e != nil {
return e
}
current = j
var base int
if e = tx.QueryRowContext(ctx, `SELECT retry_base_seconds FROM event_queue_settings WHERE id=1`).Scan(&base); e != nil {
return e
}
if j.CancelRequested && runErr == nil {
runErr = context.Canceled
}
outcome = dregarnuhr.Complete(*j, runErr, base, at)
finished, retry := at.UnixMilli(), j.RunAtMS
if outcome.Status == "queued" {
finished = 0
retry = outcome.RetryAt.UnixMilli()
}
_, e = tx.ExecContext(ctx, `UPDATE event_jobs SET status=$1,error=$2,run_at_ms=$3,finished_at_ms=$4,updated_at_ms=$5,claim_token='' WHERE uuid=$6 AND claim_token=$7`, outcome.Status, outcome.Message, retry, finished, at.UnixMilli(), claim.UUID, claim.ClaimToken)
return e
})
if err != nil {
return err
}
message := outcome.Status
if outcome.Message != "" {
message += ": " + outcome.Message
}
return s.addEventCompletionLog(ctx, current.UUID, current.EventUUID, outcome.Level, message)
}
// Called only by a future exclusive single-instance startup owner, never by
// schema initialization. Ambiguous interrupted operations are NOT auto-retried.
func (s *Store) recoverNativeEvents(ctx context.Context) error {
return s.withNativeEventWriter(ctx, func(tx *sql.Tx) error {
_, err := tx.ExecContext(ctx, `UPDATE event_jobs SET status='failed',cancel_requested=1,error='execution interrupted; explicit retry required',finished_at_ms=$1,updated_at_ms=$1,claim_token='' WHERE status='running'`, time.Now().UnixMilli())
return err
})
}
func (s *Store) enqueueNativeDefinitions(ctx context.Context, at time.Time) (count int, err error) {
err = s.withNativeEventWriter(ctx, func(tx *sql.Tx) error {
rows, e := tx.QueryContext(ctx, `SELECT uuid,event_type,event_key,payload,priority,max_attempts,next_run_at_ms FROM event_definitions WHERE enabled=1 AND schedule_kind IN ('once','interval') AND next_run_at_ms>0 AND next_run_at_ms<=$1 AND event_type<>$2 ORDER BY next_run_at_ms,uuid LIMIT 32`, at.UnixMilli(), NotificationPushEventType)
if e != nil {
return e
}
var events []EventDefinition
for rows.Next() {
var v EventDefinition
if e = rows.Scan(&v.UUID, &v.EventType, &v.EventKey, &v.Payload, &v.Priority, &v.MaxAttempts, &v.NextRunAtMS); e != nil {
rows.Close()
return e
}
events = append(events, v)
}
e = rows.Err()
rows.Close()
if e != nil {
return e
}
for _, v := range events {
queued, e := s.enqueueNativeEventTx(ctx, tx, v.UUID, v.EventType, v.EventKey+":"+v.UUID+":"+fmt.Sprint(v.NextRunAtMS), v.Payload, at, v.Priority, v.MaxAttempts)
if e != nil {
return e
}
if queued {
count++
}
if _, e = tx.ExecContext(ctx, `UPDATE event_definitions SET next_run_at_ms=CASE schedule_kind WHEN 'once' THEN 0 ELSE $1+interval_seconds*1000 END,enabled=CASE schedule_kind WHEN 'once' THEN 0 ELSE enabled END,updated_at_ms=$1 WHERE uuid=$2`, at.UnixMilli(), v.UUID); e != nil {
return e
}
}
return nil
})
if err != nil {
return 0, err
}
return count, nil
}
func (s *Store) runNativeDefinition(id string) (queued bool, err error) {
err = s.withNativeEventWriter(context.Background(), func(tx *sql.Tx) error {
var v EventDefinition
if e := tx.QueryRow(`SELECT uuid,event_type,event_key,payload,priority,max_attempts FROM event_definitions WHERE uuid=$1`, id).Scan(&v.UUID, &v.EventType, &v.EventKey, &v.Payload, &v.Priority, &v.MaxAttempts); e != nil {
return e
}
var e error
queued, e = s.enqueueNativeEventTx(context.Background(), tx, v.UUID, v.EventType, v.EventKey+":"+uuid.NewString(), v.Payload, time.Now(), v.Priority, v.MaxAttempts)
return e
})
return queued, err
}