feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s
Build and publish container images / images (push) Failing after 12m29s
This commit is contained in:
1 parent
96065d1e89
commit
0ba1dcfef0
243 files changed
+21441
-3090
No files matched your search
@@ -5,6 +5,7 @@ import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -53,6 +54,14 @@ type ActivityOutboxCheckpoint struct {
|
||||
var ErrActivityOutboxInput = errors.New("invalid activity outbox event")
|
||||
|
||||
func (s *Store) EnsureActivityOutboxSchema() error {
|
||||
if s.ReadOnly {
|
||||
return ErrReadOnly
|
||||
}
|
||||
if s.sqlBackend == storePostgreSQL {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
return s.ensurePostgreSQLActivityConsumers(ctx)
|
||||
}
|
||||
statements := []string{
|
||||
`CREATE TABLE IF NOT EXISTS activity_outbox (
|
||||
uuid TEXT PRIMARY KEY,
|
||||
@@ -97,6 +106,9 @@ func (s *Store) EnsureActivityOutboxSchema() error {
|
||||
// source mutation. Repeated idempotency keys return the original envelope and
|
||||
// never create a second Domain sequence number.
|
||||
func (s *Store) AppendActivityOutboxTx(tx *sql.Tx, event ActivityOutboxEvent) (ActivityOutboxRecord, bool, error) {
|
||||
if s.ReadOnly {
|
||||
return ActivityOutboxRecord{}, false, ErrReadOnly
|
||||
}
|
||||
event.DomainUUID = strings.TrimSpace(event.DomainUUID)
|
||||
event.EventType = strings.TrimSpace(event.EventType)
|
||||
event.IdempotencyKey = strings.TrimSpace(event.IdempotencyKey)
|
||||
@@ -106,6 +118,9 @@ func (s *Store) AppendActivityOutboxTx(tx *sql.Tx, event ActivityOutboxEvent) (A
|
||||
if event.UUID == "" {
|
||||
event.UUID = uuid.NewString()
|
||||
}
|
||||
if s.sqlBackend == storePostgreSQL {
|
||||
return s.appendNativeActivityOutboxTx(tx, event)
|
||||
}
|
||||
var existing ActivityOutboxRecord
|
||||
err := scanActivityOutbox(tx.QueryRow(`SELECT `+activityOutboxColumns+`
|
||||
FROM activity_outbox WHERE domain_uuid=? AND idempotency_key=?`, event.DomainUUID, event.IdempotencyKey), &existing)
|
||||
@@ -137,6 +152,11 @@ func (s *Store) AppendActivityOutboxTx(tx *sql.Tx, event ActivityOutboxEvent) (A
|
||||
// intentionally construct only a subset of the schema) usable while every
|
||||
// normal Store created through New has the outbox table.
|
||||
func (s *Store) appendActivityOutboxIfAvailable(tx *sql.Tx, event ActivityOutboxEvent) error {
|
||||
if s.sqlBackend == storePostgreSQL {
|
||||
// Native schemas are mandatory: absence is an error, not a lost event.
|
||||
_, _, err := s.AppendActivityOutboxTx(tx, event)
|
||||
return err
|
||||
}
|
||||
var present int
|
||||
if err := tx.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='activity_outbox'`).Scan(&present); err != nil {
|
||||
return err
|
||||
@@ -148,15 +168,55 @@ func (s *Store) appendActivityOutboxIfAvailable(tx *sql.Tx, event ActivityOutbox
|
||||
return err
|
||||
}
|
||||
|
||||
// Native application writers hold the namespace advisory lock before any
|
||||
// mutation. This shares sequence allocation with metadata.NewApplication.
|
||||
func (s *Store) appendNativeActivityOutboxTx(tx *sql.Tx, e ActivityOutboxEvent) (ActivityOutboxRecord, bool, error) {
|
||||
var result ActivityOutboxRecord
|
||||
err := scanActivityOutbox(tx.QueryRow(`SELECT `+activityOutboxColumns+` FROM activity_outbox WHERE domain_uuid=$1 AND idempotency_key=$2`, e.DomainUUID, e.IdempotencyKey), &result)
|
||||
if err == nil {
|
||||
return result, false, nil
|
||||
}
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return result, false, err
|
||||
}
|
||||
var last int64
|
||||
if err = tx.QueryRow(`SELECT COALESCE(MAX(last_sequence),0) FROM activity_outbox_heads WHERE domain_uuid=$1`, e.DomainUUID).Scan(&last); err != nil {
|
||||
return result, false, err
|
||||
}
|
||||
if last == math.MaxInt64 {
|
||||
return result, false, ErrActivityOutboxInput
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
_, err = tx.Exec(`INSERT INTO activity_outbox(uuid,domain_uuid,sequence,event_type,actor_uuid,target_kind,target_uuid,payload,idempotency_key,next_attempt_ms,created_at_ms) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)`, e.UUID, e.DomainUUID, last+1, e.EventType, e.ActorUUID, e.TargetKind, e.TargetUUID, e.Payload, e.IdempotencyKey, now, now)
|
||||
if err != nil {
|
||||
return result, false, err
|
||||
}
|
||||
_, err = tx.Exec(`INSERT INTO activity_outbox_heads(domain_uuid,event_type,last_sequence) VALUES($1,$2,$3) ON CONFLICT(domain_uuid,event_type) DO UPDATE SET last_sequence=excluded.last_sequence`, e.DomainUUID, e.EventType, last+1)
|
||||
if err != nil {
|
||||
return result, false, err
|
||||
}
|
||||
err = scanActivityOutbox(tx.QueryRow(`SELECT `+activityOutboxColumns+` FROM activity_outbox WHERE uuid=$1`, e.UUID), &result)
|
||||
return result, err == nil, err
|
||||
}
|
||||
|
||||
func (s *Store) appendRevisionActivityOutboxIfAvailable(tx *sql.Tx, revision *Revision) error {
|
||||
// Legacy unit fixtures may omit the entire outbox. Check that explicitly;
|
||||
// once it exists, an unexpected source lookup error must abort the save,
|
||||
// not silently commit a revision with no event.
|
||||
var present int
|
||||
if err := tx.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='activity_outbox'`).Scan(&present); err != nil {
|
||||
return err
|
||||
}
|
||||
if present == 0 {
|
||||
return nil
|
||||
}
|
||||
var domainUUID string
|
||||
err := tx.QueryRow(`SELECT domain_uuid FROM paths WHERE object_uuid=? ORDER BY is_primary DESC, path LIMIT 1`, revision.ObjectUUID).Scan(&domainUUID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
// Asset-only or legacy fixtures may not have a path for the object.
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
"revision_uuid": revision.UUID, "object_uuid": revision.ObjectUUID,
|
||||
@@ -166,10 +226,11 @@ func (s *Store) appendRevisionActivityOutboxIfAvailable(tx *sql.Tx, revision *Re
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.appendActivityOutboxIfAvailable(tx, ActivityOutboxEvent{
|
||||
_, _, err = s.AppendActivityOutboxTx(tx, ActivityOutboxEvent{
|
||||
DomainUUID: domainUUID, EventType: "revision.created", TargetKind: "revision", TargetUUID: revision.UUID,
|
||||
Payload: string(payload), IdempotencyKey: "revision.created:" + revision.UUID,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
const activityOutboxColumns = `uuid,domain_uuid,sequence,event_type,actor_uuid,target_kind,target_uuid,payload,idempotency_key,status,attempts,next_attempt_ms,last_error,created_at_ms,claimed_at_ms`
|
||||
@@ -185,13 +246,16 @@ func scanActivityOutbox(scanner interface{ Scan(...any) error }, out *ActivityOu
|
||||
// authorization policy without making reads of the source object implicit.
|
||||
// Deprecated: this is a legacy-status inspection, not consumer pending work.
|
||||
// New workers must use ActivityOutboxForConsumer and their own checkpoint.
|
||||
func (s *Store) PendingActivityOutbox(ctx context.Context, domainUUID string, limit int) ([]ActivityOutboxRecord, error) {
|
||||
func (s *Store) PendingActivityOutbox(ctx context.Context, domainUUID string, limit int) (_ []ActivityOutboxRecord, resultErr error) {
|
||||
defer func() { resultErr = s.sqlError("pending activity", resultErr) }()
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
if limit < 1 || limit > 500 {
|
||||
limit = 100
|
||||
}
|
||||
rows, err := s.DB.QueryContext(ctx, `SELECT `+activityOutboxColumns+` FROM activity_outbox
|
||||
WHERE status='pending' AND next_attempt_ms<=? AND (?='' OR domain_uuid=?)
|
||||
ORDER BY domain_uuid,sequence LIMIT ?`, time.Now().UnixMilli(), domainUUID, domainUUID, limit)
|
||||
WHERE status='pending' AND next_attempt_ms<=`+s.sqlParam(1)+` AND (`+s.sqlParam(2)+`='' OR domain_uuid=`+s.sqlParam(3)+`)
|
||||
ORDER BY domain_uuid,sequence LIMIT `+s.sqlParam(4), time.Now().UnixMilli(), domainUUID, domainUUID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -210,7 +274,10 @@ func (s *Store) PendingActivityOutbox(ctx context.Context, domainUUID string, li
|
||||
// ActivityOutboxAfter reads the immutable Domain sequence independently for
|
||||
// each consumer. It intentionally includes every status: a consumer's
|
||||
// checkpoint, rather than a shared job status, is the delivery contract.
|
||||
func (s *Store) ActivityOutboxAfter(ctx context.Context, domainUUID string, afterSequence int64, limit int) ([]ActivityOutboxRecord, error) {
|
||||
func (s *Store) ActivityOutboxAfter(ctx context.Context, domainUUID string, afterSequence int64, limit int) (_ []ActivityOutboxRecord, resultErr error) {
|
||||
defer func() { resultErr = s.sqlError("activity history", resultErr) }()
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
if strings.TrimSpace(domainUUID) == "" || afterSequence < 0 {
|
||||
return nil, ErrActivityOutboxInput
|
||||
}
|
||||
@@ -218,7 +285,7 @@ func (s *Store) ActivityOutboxAfter(ctx context.Context, domainUUID string, afte
|
||||
limit = 100
|
||||
}
|
||||
rows, err := s.DB.QueryContext(ctx, `SELECT `+activityOutboxColumns+` FROM activity_outbox
|
||||
WHERE domain_uuid=? AND sequence>? ORDER BY sequence LIMIT ?`, domainUUID, afterSequence, limit)
|
||||
WHERE domain_uuid=`+s.sqlParam(1)+` AND sequence>`+s.sqlParam(2)+` ORDER BY sequence LIMIT `+s.sqlParam(3), domainUUID, afterSequence, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -234,13 +301,16 @@ func (s *Store) ActivityOutboxAfter(ctx context.Context, domainUUID string, afte
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) ActivityOutboxCheckpoint(consumerID, domainUUID string) (ActivityOutboxCheckpoint, error) {
|
||||
func (s *Store) ActivityOutboxCheckpoint(consumerID, domainUUID string) (_ ActivityOutboxCheckpoint, resultErr error) {
|
||||
defer func() { resultErr = s.sqlError("activity checkpoint", resultErr) }()
|
||||
consumerID, domainUUID = strings.TrimSpace(consumerID), strings.TrimSpace(domainUUID)
|
||||
if consumerID == "" || domainUUID == "" {
|
||||
return ActivityOutboxCheckpoint{}, ErrActivityOutboxInput
|
||||
}
|
||||
var checkpoint ActivityOutboxCheckpoint
|
||||
err := s.DB.QueryRow(`SELECT consumer_id,domain_uuid,last_sequence,updated_at_ms FROM activity_outbox_checkpoints WHERE consumer_id=? AND domain_uuid=?`, consumerID, domainUUID).
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
err := s.DB.QueryRowContext(ctx, `SELECT consumer_id,domain_uuid,last_sequence,updated_at_ms FROM activity_outbox_checkpoints WHERE consumer_id=`+s.sqlParam(1)+` AND domain_uuid=`+s.sqlParam(2), consumerID, domainUUID).
|
||||
Scan(&checkpoint.ConsumerID, &checkpoint.DomainUUID, &checkpoint.LastSequence, &checkpoint.UpdatedAtMS)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ActivityOutboxCheckpoint{ConsumerID: consumerID, DomainUUID: domainUUID}, nil
|
||||
@@ -251,25 +321,35 @@ func (s *Store) ActivityOutboxCheckpoint(consumerID, domainUUID string) (Activit
|
||||
// AdvanceActivityOutboxCheckpointTx is monotonic and safe to call after an
|
||||
// idempotently completed consumer action. A stale retry can never move the
|
||||
// checkpoint backwards.
|
||||
func (s *Store) AdvanceActivityOutboxCheckpointTx(tx *sql.Tx, consumerID, domainUUID string, sequence int64) error {
|
||||
func (s *Store) AdvanceActivityOutboxCheckpointTx(tx *sql.Tx, consumerID, domainUUID string, sequence int64) (resultErr error) {
|
||||
defer func() { resultErr = s.sqlError("activity checkpoint", resultErr) }()
|
||||
if s.ReadOnly {
|
||||
return ErrReadOnly
|
||||
}
|
||||
consumerID, domainUUID = strings.TrimSpace(consumerID), strings.TrimSpace(domainUUID)
|
||||
if consumerID == "" || domainUUID == "" || sequence < 0 {
|
||||
return ErrActivityOutboxInput
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
_, err := tx.Exec(`INSERT INTO activity_outbox_checkpoints(consumer_id,domain_uuid,last_sequence,updated_at_ms)
|
||||
VALUES(?,?,?,?) ON CONFLICT(consumer_id,domain_uuid) DO UPDATE SET
|
||||
last_sequence=CASE WHEN excluded.last_sequence>last_sequence THEN excluded.last_sequence ELSE last_sequence END,
|
||||
updated_at_ms=CASE WHEN excluded.last_sequence>last_sequence THEN excluded.updated_at_ms ELSE updated_at_ms END`, consumerID, domainUUID, sequence, now)
|
||||
VALUES(`+s.sqlValues(4)+`) ON CONFLICT(consumer_id,domain_uuid) DO UPDATE SET
|
||||
last_sequence=CASE WHEN excluded.last_sequence>activity_outbox_checkpoints.last_sequence THEN excluded.last_sequence ELSE activity_outbox_checkpoints.last_sequence END,
|
||||
updated_at_ms=CASE WHEN excluded.last_sequence>activity_outbox_checkpoints.last_sequence THEN excluded.updated_at_ms ELSE activity_outbox_checkpoints.updated_at_ms END`, consumerID, domainUUID, sequence, now)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) AdvanceActivityOutboxCheckpoint(consumerID, domainUUID string, sequence int64) error {
|
||||
tx, err := s.DB.Begin()
|
||||
func (s *Store) AdvanceActivityOutboxCheckpoint(consumerID, domainUUID string, sequence int64) (err error) {
|
||||
defer func() { err = s.sqlError("activity checkpoint", err) }()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
tx, err := s.DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err = s.lockTaskWriter(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.AdvanceActivityOutboxCheckpointTx(tx, consumerID, domainUUID, sequence); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -278,7 +358,10 @@ func (s *Store) AdvanceActivityOutboxCheckpoint(consumerID, domainUUID string, s
|
||||
|
||||
// Deprecated: use PendingActivityConsumerDomains; a shared status cannot
|
||||
// represent delivery to independent subscribers.
|
||||
func (s *Store) PendingActivityOutboxDomains(ctx context.Context) ([]string, error) {
|
||||
func (s *Store) PendingActivityOutboxDomains(ctx context.Context) (_ []string, resultErr error) {
|
||||
defer func() { resultErr = s.sqlError("pending activity domains", resultErr) }()
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
rows, err := s.DB.QueryContext(ctx, `SELECT DISTINCT domain_uuid FROM activity_outbox WHERE status='pending' ORDER BY domain_uuid LIMIT 100`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
Reference in new issue
Block a user