feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s

This commit is contained in:
Anya Tokugawa committed 2026-11-15 05:47:14 +09:00
1 parent 96065d1e89
commit 0ba1dcfef0
243 files changed
+21441 -3090

No files matched your search

+100 -17
View File
@@ -5,6 +5,7 @@ import (
"database/sql"
"encoding/json"
"errors"
"math"
"strings"
"time"
@@ -53,6 +54,14 @@ type ActivityOutboxCheckpoint struct {
var ErrActivityOutboxInput = errors.New("invalid activity outbox event")
func (s *Store) EnsureActivityOutboxSchema() error {
if s.ReadOnly {
return ErrReadOnly
}
if s.sqlBackend == storePostgreSQL {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return s.ensurePostgreSQLActivityConsumers(ctx)
}
statements := []string{
`CREATE TABLE IF NOT EXISTS activity_outbox (
uuid TEXT PRIMARY KEY,
@@ -97,6 +106,9 @@ func (s *Store) EnsureActivityOutboxSchema() error {
// source mutation. Repeated idempotency keys return the original envelope and
// never create a second Domain sequence number.
func (s *Store) AppendActivityOutboxTx(tx *sql.Tx, event ActivityOutboxEvent) (ActivityOutboxRecord, bool, error) {
if s.ReadOnly {
return ActivityOutboxRecord{}, false, ErrReadOnly
}
event.DomainUUID = strings.TrimSpace(event.DomainUUID)
event.EventType = strings.TrimSpace(event.EventType)
event.IdempotencyKey = strings.TrimSpace(event.IdempotencyKey)
@@ -106,6 +118,9 @@ func (s *Store) AppendActivityOutboxTx(tx *sql.Tx, event ActivityOutboxEvent) (A
if event.UUID == "" {
event.UUID = uuid.NewString()
}
if s.sqlBackend == storePostgreSQL {
return s.appendNativeActivityOutboxTx(tx, event)
}
var existing ActivityOutboxRecord
err := scanActivityOutbox(tx.QueryRow(`SELECT `+activityOutboxColumns+`
FROM activity_outbox WHERE domain_uuid=? AND idempotency_key=?`, event.DomainUUID, event.IdempotencyKey), &existing)
@@ -137,6 +152,11 @@ func (s *Store) AppendActivityOutboxTx(tx *sql.Tx, event ActivityOutboxEvent) (A
// intentionally construct only a subset of the schema) usable while every
// normal Store created through New has the outbox table.
func (s *Store) appendActivityOutboxIfAvailable(tx *sql.Tx, event ActivityOutboxEvent) error {
if s.sqlBackend == storePostgreSQL {
// Native schemas are mandatory: absence is an error, not a lost event.
_, _, err := s.AppendActivityOutboxTx(tx, event)
return err
}
var present int
if err := tx.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='activity_outbox'`).Scan(&present); err != nil {
return err
@@ -148,15 +168,55 @@ func (s *Store) appendActivityOutboxIfAvailable(tx *sql.Tx, event ActivityOutbox
return err
}
// Native application writers hold the namespace advisory lock before any
// mutation. This shares sequence allocation with metadata.NewApplication.
func (s *Store) appendNativeActivityOutboxTx(tx *sql.Tx, e ActivityOutboxEvent) (ActivityOutboxRecord, bool, error) {
var result ActivityOutboxRecord
err := scanActivityOutbox(tx.QueryRow(`SELECT `+activityOutboxColumns+` FROM activity_outbox WHERE domain_uuid=$1 AND idempotency_key=$2`, e.DomainUUID, e.IdempotencyKey), &result)
if err == nil {
return result, false, nil
}
if !errors.Is(err, sql.ErrNoRows) {
return result, false, err
}
var last int64
if err = tx.QueryRow(`SELECT COALESCE(MAX(last_sequence),0) FROM activity_outbox_heads WHERE domain_uuid=$1`, e.DomainUUID).Scan(&last); err != nil {
return result, false, err
}
if last == math.MaxInt64 {
return result, false, ErrActivityOutboxInput
}
now := time.Now().UnixMilli()
_, err = tx.Exec(`INSERT INTO activity_outbox(uuid,domain_uuid,sequence,event_type,actor_uuid,target_kind,target_uuid,payload,idempotency_key,next_attempt_ms,created_at_ms) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)`, e.UUID, e.DomainUUID, last+1, e.EventType, e.ActorUUID, e.TargetKind, e.TargetUUID, e.Payload, e.IdempotencyKey, now, now)
if err != nil {
return result, false, err
}
_, err = tx.Exec(`INSERT INTO activity_outbox_heads(domain_uuid,event_type,last_sequence) VALUES($1,$2,$3) ON CONFLICT(domain_uuid,event_type) DO UPDATE SET last_sequence=excluded.last_sequence`, e.DomainUUID, e.EventType, last+1)
if err != nil {
return result, false, err
}
err = scanActivityOutbox(tx.QueryRow(`SELECT `+activityOutboxColumns+` FROM activity_outbox WHERE uuid=$1`, e.UUID), &result)
return result, err == nil, err
}
func (s *Store) appendRevisionActivityOutboxIfAvailable(tx *sql.Tx, revision *Revision) error {
// Legacy unit fixtures may omit the entire outbox. Check that explicitly;
// once it exists, an unexpected source lookup error must abort the save,
// not silently commit a revision with no event.
var present int
if err := tx.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='activity_outbox'`).Scan(&present); err != nil {
return err
}
if present == 0 {
return nil
}
var domainUUID string
err := tx.QueryRow(`SELECT domain_uuid FROM paths WHERE object_uuid=? ORDER BY is_primary DESC, path LIMIT 1`, revision.ObjectUUID).Scan(&domainUUID)
if errors.Is(err, sql.ErrNoRows) {
return nil
}
if err != nil {
// Asset-only or legacy fixtures may not have a path for the object.
return nil
return err
}
payload, err := json.Marshal(map[string]any{
"revision_uuid": revision.UUID, "object_uuid": revision.ObjectUUID,
@@ -166,10 +226,11 @@ func (s *Store) appendRevisionActivityOutboxIfAvailable(tx *sql.Tx, revision *Re
if err != nil {
return err
}
return s.appendActivityOutboxIfAvailable(tx, ActivityOutboxEvent{
_, _, err = s.AppendActivityOutboxTx(tx, ActivityOutboxEvent{
DomainUUID: domainUUID, EventType: "revision.created", TargetKind: "revision", TargetUUID: revision.UUID,
Payload: string(payload), IdempotencyKey: "revision.created:" + revision.UUID,
})
return err
}
const activityOutboxColumns = `uuid,domain_uuid,sequence,event_type,actor_uuid,target_kind,target_uuid,payload,idempotency_key,status,attempts,next_attempt_ms,last_error,created_at_ms,claimed_at_ms`
@@ -185,13 +246,16 @@ func scanActivityOutbox(scanner interface{ Scan(...any) error }, out *ActivityOu
// authorization policy without making reads of the source object implicit.
// Deprecated: this is a legacy-status inspection, not consumer pending work.
// New workers must use ActivityOutboxForConsumer and their own checkpoint.
func (s *Store) PendingActivityOutbox(ctx context.Context, domainUUID string, limit int) ([]ActivityOutboxRecord, error) {
func (s *Store) PendingActivityOutbox(ctx context.Context, domainUUID string, limit int) (_ []ActivityOutboxRecord, resultErr error) {
defer func() { resultErr = s.sqlError("pending activity", resultErr) }()
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
if limit < 1 || limit > 500 {
limit = 100
}
rows, err := s.DB.QueryContext(ctx, `SELECT `+activityOutboxColumns+` FROM activity_outbox
WHERE status='pending' AND next_attempt_ms<=? AND (?='' OR domain_uuid=?)
ORDER BY domain_uuid,sequence LIMIT ?`, time.Now().UnixMilli(), domainUUID, domainUUID, limit)
WHERE status='pending' AND next_attempt_ms<=`+s.sqlParam(1)+` AND (`+s.sqlParam(2)+`='' OR domain_uuid=`+s.sqlParam(3)+`)
ORDER BY domain_uuid,sequence LIMIT `+s.sqlParam(4), time.Now().UnixMilli(), domainUUID, domainUUID, limit)
if err != nil {
return nil, err
}
@@ -210,7 +274,10 @@ func (s *Store) PendingActivityOutbox(ctx context.Context, domainUUID string, li
// ActivityOutboxAfter reads the immutable Domain sequence independently for
// each consumer. It intentionally includes every status: a consumer's
// checkpoint, rather than a shared job status, is the delivery contract.
func (s *Store) ActivityOutboxAfter(ctx context.Context, domainUUID string, afterSequence int64, limit int) ([]ActivityOutboxRecord, error) {
func (s *Store) ActivityOutboxAfter(ctx context.Context, domainUUID string, afterSequence int64, limit int) (_ []ActivityOutboxRecord, resultErr error) {
defer func() { resultErr = s.sqlError("activity history", resultErr) }()
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
if strings.TrimSpace(domainUUID) == "" || afterSequence < 0 {
return nil, ErrActivityOutboxInput
}
@@ -218,7 +285,7 @@ func (s *Store) ActivityOutboxAfter(ctx context.Context, domainUUID string, afte
limit = 100
}
rows, err := s.DB.QueryContext(ctx, `SELECT `+activityOutboxColumns+` FROM activity_outbox
WHERE domain_uuid=? AND sequence>? ORDER BY sequence LIMIT ?`, domainUUID, afterSequence, limit)
WHERE domain_uuid=`+s.sqlParam(1)+` AND sequence>`+s.sqlParam(2)+` ORDER BY sequence LIMIT `+s.sqlParam(3), domainUUID, afterSequence, limit)
if err != nil {
return nil, err
}
@@ -234,13 +301,16 @@ func (s *Store) ActivityOutboxAfter(ctx context.Context, domainUUID string, afte
return items, rows.Err()
}
func (s *Store) ActivityOutboxCheckpoint(consumerID, domainUUID string) (ActivityOutboxCheckpoint, error) {
func (s *Store) ActivityOutboxCheckpoint(consumerID, domainUUID string) (_ ActivityOutboxCheckpoint, resultErr error) {
defer func() { resultErr = s.sqlError("activity checkpoint", resultErr) }()
consumerID, domainUUID = strings.TrimSpace(consumerID), strings.TrimSpace(domainUUID)
if consumerID == "" || domainUUID == "" {
return ActivityOutboxCheckpoint{}, ErrActivityOutboxInput
}
var checkpoint ActivityOutboxCheckpoint
err := s.DB.QueryRow(`SELECT consumer_id,domain_uuid,last_sequence,updated_at_ms FROM activity_outbox_checkpoints WHERE consumer_id=? AND domain_uuid=?`, consumerID, domainUUID).
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
err := s.DB.QueryRowContext(ctx, `SELECT consumer_id,domain_uuid,last_sequence,updated_at_ms FROM activity_outbox_checkpoints WHERE consumer_id=`+s.sqlParam(1)+` AND domain_uuid=`+s.sqlParam(2), consumerID, domainUUID).
Scan(&checkpoint.ConsumerID, &checkpoint.DomainUUID, &checkpoint.LastSequence, &checkpoint.UpdatedAtMS)
if errors.Is(err, sql.ErrNoRows) {
return ActivityOutboxCheckpoint{ConsumerID: consumerID, DomainUUID: domainUUID}, nil
@@ -251,25 +321,35 @@ func (s *Store) ActivityOutboxCheckpoint(consumerID, domainUUID string) (Activit
// AdvanceActivityOutboxCheckpointTx is monotonic and safe to call after an
// idempotently completed consumer action. A stale retry can never move the
// checkpoint backwards.
func (s *Store) AdvanceActivityOutboxCheckpointTx(tx *sql.Tx, consumerID, domainUUID string, sequence int64) error {
func (s *Store) AdvanceActivityOutboxCheckpointTx(tx *sql.Tx, consumerID, domainUUID string, sequence int64) (resultErr error) {
defer func() { resultErr = s.sqlError("activity checkpoint", resultErr) }()
if s.ReadOnly {
return ErrReadOnly
}
consumerID, domainUUID = strings.TrimSpace(consumerID), strings.TrimSpace(domainUUID)
if consumerID == "" || domainUUID == "" || sequence < 0 {
return ErrActivityOutboxInput
}
now := time.Now().UnixMilli()
_, err := tx.Exec(`INSERT INTO activity_outbox_checkpoints(consumer_id,domain_uuid,last_sequence,updated_at_ms)
VALUES(?,?,?,?) ON CONFLICT(consumer_id,domain_uuid) DO UPDATE SET
last_sequence=CASE WHEN excluded.last_sequence>last_sequence THEN excluded.last_sequence ELSE last_sequence END,
updated_at_ms=CASE WHEN excluded.last_sequence>last_sequence THEN excluded.updated_at_ms ELSE updated_at_ms END`, consumerID, domainUUID, sequence, now)
VALUES(`+s.sqlValues(4)+`) ON CONFLICT(consumer_id,domain_uuid) DO UPDATE SET
last_sequence=CASE WHEN excluded.last_sequence>activity_outbox_checkpoints.last_sequence THEN excluded.last_sequence ELSE activity_outbox_checkpoints.last_sequence END,
updated_at_ms=CASE WHEN excluded.last_sequence>activity_outbox_checkpoints.last_sequence THEN excluded.updated_at_ms ELSE activity_outbox_checkpoints.updated_at_ms END`, consumerID, domainUUID, sequence, now)
return err
}
func (s *Store) AdvanceActivityOutboxCheckpoint(consumerID, domainUUID string, sequence int64) error {
tx, err := s.DB.Begin()
func (s *Store) AdvanceActivityOutboxCheckpoint(consumerID, domainUUID string, sequence int64) (err error) {
defer func() { err = s.sqlError("activity checkpoint", err) }()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
tx, err := s.DB.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err = s.lockTaskWriter(ctx, tx); err != nil {
return err
}
if err := s.AdvanceActivityOutboxCheckpointTx(tx, consumerID, domainUUID, sequence); err != nil {
return err
}
@@ -278,7 +358,10 @@ func (s *Store) AdvanceActivityOutboxCheckpoint(consumerID, domainUUID string, s
// Deprecated: use PendingActivityConsumerDomains; a shared status cannot
// represent delivery to independent subscribers.
func (s *Store) PendingActivityOutboxDomains(ctx context.Context) ([]string, error) {
func (s *Store) PendingActivityOutboxDomains(ctx context.Context) (_ []string, resultErr error) {
defer func() { resultErr = s.sqlError("pending activity domains", resultErr) }()
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
rows, err := s.DB.QueryContext(ctx, `SELECT DISTINCT domain_uuid FROM activity_outbox WHERE status='pending' ORDER BY domain_uuid LIMIT 100`)
if err != nil {
return nil, err