feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s
Build and publish container images / images (push) Failing after 12m29s
This commit is contained in:
1 parent
96065d1e89
commit
0ba1dcfef0
243 files changed
+21441
-3090
No files matched your search
@@ -0,0 +1,445 @@
|
||||
// Package metadata contains the portable metadata persistence contract. It does
|
||||
// not authorize requests, store payloads, run legacy Store initialization, or
|
||||
// constitute a PostgreSQL-capable application. Payloads must be persisted and
|
||||
// verified in Geduldh before publishing their hash here.
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"golang.org/x/text/language"
|
||||
"modernc.org/sqlite"
|
||||
)
|
||||
|
||||
type Backend string
|
||||
|
||||
const (
|
||||
SQLite Backend = "sqlite"
|
||||
PostgreSQL Backend = "postgres"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNotFound = errors.New("metadata not found")
|
||||
ErrConflict = errors.New("metadata conflict")
|
||||
ErrInvalid = errors.New("invalid metadata")
|
||||
)
|
||||
|
||||
type Repository struct {
|
||||
database *sql.DB
|
||||
backend Backend
|
||||
applicationNamespace bool
|
||||
}
|
||||
|
||||
// New never migrates a schema or creates keys. Ownership of database stays with
|
||||
// the caller. Backend must match the actual connection; no fallback is allowed.
|
||||
func New(database *sql.DB, backend Backend) (*Repository, error) {
|
||||
if database == nil || (backend != SQLite && backend != PostgreSQL) {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
return &Repository{database: database, backend: backend}, nil
|
||||
}
|
||||
|
||||
// NewApplication uses the application's namespace/administration lock before
|
||||
// the metadata Domain/Object locks. Namespace validation is repeated inside
|
||||
// the commit transaction. This does not replace request authorization, perform
|
||||
// migrations, or make an incomplete application Store ready to serve traffic.
|
||||
func NewApplication(database *sql.DB, backend Backend) (*Repository, error) {
|
||||
r, err := New(database, backend)
|
||||
if err == nil {
|
||||
r.applicationNamespace = true
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
|
||||
// p generates a placeholder, rather than rewriting arbitrary SQL (which can
|
||||
// contain JSON operators, strings or comments with literal question marks).
|
||||
func (r *Repository) p(n int) string {
|
||||
if r.backend == PostgreSQL {
|
||||
return fmt.Sprintf("$%d", n)
|
||||
}
|
||||
return "?"
|
||||
}
|
||||
|
||||
func (r *Repository) failure(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return context.Canceled
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return context.DeadlineExceeded
|
||||
}
|
||||
var pg *pgconn.PgError
|
||||
if errors.As(err, &pg) {
|
||||
if pg.Code == "23505" || pg.Code == "40001" || pg.Code == "40P01" {
|
||||
return ErrConflict
|
||||
}
|
||||
return fmt.Errorf("PostgreSQL metadata operation failed (SQLSTATE %s)", pg.Code)
|
||||
}
|
||||
var se *sqlite.Error
|
||||
if errors.As(err, &se) {
|
||||
switch se.Code() {
|
||||
case 1555, 2067:
|
||||
return ErrConflict
|
||||
}
|
||||
return fmt.Errorf("SQLite metadata operation failed (code %d)", se.Code())
|
||||
}
|
||||
// Network errors may include a credential-bearing DSN or server addresses.
|
||||
return errors.New("metadata database operation failed")
|
||||
}
|
||||
|
||||
func canonicalPath(raw string) (string, error) {
|
||||
p := strings.TrimSpace(raw)
|
||||
if !strings.HasPrefix(p, "/") {
|
||||
return "", ErrInvalid
|
||||
}
|
||||
if p != "/" {
|
||||
p = strings.TrimRight(p, "/")
|
||||
}
|
||||
if p == "" || !utf8.ValidString(p) || strings.ContainsAny(p, "\x00\r\n") {
|
||||
return "", ErrInvalid
|
||||
}
|
||||
if p != "/" {
|
||||
for _, part := range strings.Split(p[1:], "/") {
|
||||
if part == "" || part == "." || part == ".." {
|
||||
return "", ErrInvalid
|
||||
}
|
||||
}
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func validHash(hash string) bool {
|
||||
if len(hash) != 64 {
|
||||
return false
|
||||
}
|
||||
for _, c := range hash {
|
||||
if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// write locks before any read. SQLite acquires its single-writer reservation;
|
||||
// PostgreSQL locks a stable, namespaced key for the whole transaction. Path
|
||||
// writers share a domain key; revisions additionally take an object key after
|
||||
// the domain key, even when an object is bound in more than one domain.
|
||||
func (r *Repository) write(ctx context.Context, key string, fn func(*sql.Tx) error) error {
|
||||
tx, err := r.database.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if r.backend == PostgreSQL {
|
||||
if r.applicationNamespace {
|
||||
if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(1885431653,1853058931)`); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
}
|
||||
err = r.lock(ctx, tx, key)
|
||||
} else {
|
||||
_, err = tx.ExecContext(ctx, `UPDATE paths SET created_at=created_at WHERE 0`)
|
||||
}
|
||||
if err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
if r.applicationNamespace {
|
||||
if !strings.HasPrefix(key, "domain/") {
|
||||
return ErrInvalid
|
||||
}
|
||||
var local bool
|
||||
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM domains d JOIN campuses c ON c.uuid=d.campus_uuid WHERE d.uuid=`+r.p(1)+` AND d.enabled=1 AND c.enabled=1 AND d.placement='local')`, strings.TrimPrefix(key, "domain/")).Scan(&local); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
if !local {
|
||||
return ErrNotFound
|
||||
}
|
||||
}
|
||||
if err = fn(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
return r.failure(tx.Commit())
|
||||
}
|
||||
|
||||
func (r *Repository) lock(ctx context.Context, tx *sql.Tx, key string) error {
|
||||
if r.backend != PostgreSQL {
|
||||
return nil
|
||||
}
|
||||
h := sha256.Sum256([]byte("pageswiki/metadata/" + key))
|
||||
_, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock($1)`, int64(binary.BigEndian.Uint64(h[:8])))
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *Repository) CreatePage(ctx context.Context, domain, path string) (string, error) {
|
||||
p, err := canonicalPath(path)
|
||||
if err != nil || strings.TrimSpace(domain) == "" {
|
||||
return "", ErrInvalid
|
||||
}
|
||||
id := uuid.NewString()
|
||||
err = r.write(ctx, "domain/"+domain, func(tx *sql.Tx) error {
|
||||
now := time.Now().Unix()
|
||||
for _, q := range []struct {
|
||||
sql string
|
||||
args []any
|
||||
}{
|
||||
{`INSERT INTO objects(uuid,kind,created_at) VALUES (` + r.p(1) + `,'page',` + r.p(2) + `)`, []any{id, now}},
|
||||
{`INSERT INTO object_origins(object_uuid,domain_uuid,created_at) VALUES (` + r.p(1) + `,` + r.p(2) + `,` + r.p(3) + `)`, []any{id, domain, now}},
|
||||
{`INSERT INTO paths(domain_uuid,path,object_uuid,created_at,is_primary) VALUES (` + r.p(1) + `,` + r.p(2) + `,` + r.p(3) + `,` + r.p(4) + `,1)`, []any{domain, p, id, now}},
|
||||
} {
|
||||
if _, err := tx.ExecContext(ctx, q.sql, q.args...); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
}
|
||||
if r.applicationNamespace {
|
||||
return r.appendPageOutbox(ctx, tx, domain, p, id)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// CreateObject records provenance for the explicit create/save/bind API.
|
||||
// It does not grant authority over an arbitrary unbound legacy Object.
|
||||
func (r *Repository) CreateObject(ctx context.Context, domain, kind string) (string, error) {
|
||||
if domain == "" || strings.TrimSpace(kind) == "" {
|
||||
return "", ErrInvalid
|
||||
}
|
||||
id := uuid.NewString()
|
||||
err := r.write(ctx, "domain/"+domain, func(tx *sql.Tx) error {
|
||||
now := time.Now().Unix()
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO objects(uuid,kind,created_at) VALUES(`+r.p(1)+`,`+r.p(2)+`,`+r.p(3)+`)`, id, kind, now); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
_, err := tx.ExecContext(ctx, `INSERT INTO object_origins(object_uuid,domain_uuid,created_at) VALUES(`+r.p(1)+`,`+r.p(2)+`,`+r.p(3)+`)`, id, domain, now)
|
||||
return r.failure(err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (r *Repository) ResolvePath(ctx context.Context, domain, path string) (string, error) {
|
||||
var id string
|
||||
err := r.database.QueryRowContext(ctx, `SELECT object_uuid FROM paths WHERE domain_uuid=`+r.p(1)+` AND path=`+r.p(2), domain, path).Scan(&id)
|
||||
return id, r.failure(err)
|
||||
}
|
||||
|
||||
// BindPath creates an alias without replacing an existing binding. Cross-domain
|
||||
// sharing requires a separate explicit authorization/import contract.
|
||||
func (r *Repository) BindPath(ctx context.Context, domain, path, object string) error {
|
||||
p, err := canonicalPath(path)
|
||||
if err != nil || domain == "" || object == "" {
|
||||
return ErrInvalid
|
||||
}
|
||||
return r.write(ctx, "domain/"+domain, func(tx *sql.Tx) error {
|
||||
var count int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT count(*) FROM object_origins WHERE object_uuid=`+r.p(1)+` AND domain_uuid=`+r.p(2), object, domain).Scan(&count); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
if count == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
_, err := tx.ExecContext(ctx, `INSERT INTO paths(domain_uuid,path,object_uuid,created_at,is_primary) VALUES (`+r.p(1)+`,`+r.p(2)+`,`+r.p(3)+`,`+r.p(4)+`,0)`, domain, p, object, time.Now().Unix())
|
||||
return r.failure(err)
|
||||
})
|
||||
}
|
||||
|
||||
// RenameTree moves both real and virtual folders without LIKE wildcard or
|
||||
// case-folding semantics. It does not merge into occupied destinations. City,
|
||||
// ACL, activity outbox and cache invalidation must be coordinated by the future
|
||||
// application-level transaction; do not substitute this for legacy UI rename yet.
|
||||
func (r *Repository) RenameTree(ctx context.Context, domain, source, destination string) error {
|
||||
from, e1 := canonicalPath(source)
|
||||
to, e2 := canonicalPath(destination)
|
||||
if e1 != nil || e2 != nil || domain == "" || from == "/" || to == "/" || from == to || strings.HasPrefix(to, from+"/") || strings.HasPrefix(from, to+"/") {
|
||||
return ErrInvalid
|
||||
}
|
||||
return r.write(ctx, "domain/"+domain, func(tx *sql.Tx) error {
|
||||
var occupied int
|
||||
// Binary/C collation makes [prefix+'/', prefix+'0') an exact,
|
||||
// indexable descendant range, including Unicode and literal %/_ names.
|
||||
err := tx.QueryRowContext(ctx, `SELECT 1 FROM paths WHERE domain_uuid=`+r.p(1)+` AND (path=`+r.p(2)+` OR (path>=`+r.p(3)+` AND path<`+r.p(4)+`)) LIMIT 1`, domain, to, to+"/", to+"0").Scan(&occupied)
|
||||
if err == nil {
|
||||
return ErrConflict
|
||||
}
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return r.failure(err)
|
||||
}
|
||||
// A single SQL update avoids loading the subtree or issuing one query
|
||||
// per path. substr counts characters (not UTF-8 bytes) on both DBs.
|
||||
result, err := tx.ExecContext(ctx, `UPDATE paths SET path=`+r.p(1)+` || substr(path,`+r.p(2)+`) WHERE domain_uuid=`+r.p(3)+` AND (path=`+r.p(4)+` OR (path>=`+r.p(5)+` AND path<`+r.p(6)+`))`, to, len([]rune(from))+1, domain, from, from+"/", from+"0")
|
||||
if err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
count, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
if count == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
type Revision struct {
|
||||
UUID string
|
||||
ObjectUUID string
|
||||
DocumentHash string
|
||||
Language string
|
||||
ContentType string
|
||||
Filename string
|
||||
Message string
|
||||
CreatedAtMS int64
|
||||
}
|
||||
|
||||
func normalizedLanguage(raw string) (string, error) {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return "und", nil
|
||||
}
|
||||
tag, err := language.Parse(raw)
|
||||
if err != nil {
|
||||
return "", ErrInvalid
|
||||
}
|
||||
return tag.String(), nil
|
||||
}
|
||||
|
||||
const revisionColumns = `uuid,object_uuid,document_hash,lang,content_type,filename,message,created_at_ms`
|
||||
|
||||
// Application create/save/bind records creation provenance before any Path.
|
||||
// Provenance ceases to grant scope as soon as any live binding exists.
|
||||
// Expressions are owned SQL identifiers/placeholders, never request strings.
|
||||
func (r *Repository) objectDomainScope(object, domain string) string {
|
||||
scope := `SELECT domain_uuid FROM paths WHERE object_uuid=` + object
|
||||
if r.applicationNamespace {
|
||||
scope += ` UNION ALL SELECT domain_uuid FROM object_origins WHERE object_uuid=` + object + ` AND NOT EXISTS(SELECT 1 FROM paths WHERE object_uuid=` + object + `)`
|
||||
}
|
||||
return domain + ` IN (` + scope + `)`
|
||||
}
|
||||
|
||||
func scanRevision(row interface{ Scan(...any) error }) (Revision, error) {
|
||||
var v Revision
|
||||
err := row.Scan(&v.UUID, &v.ObjectUUID, &v.DocumentHash, &v.Language, &v.ContentType, &v.Filename, &v.Message, &v.CreatedAtMS)
|
||||
return v, err
|
||||
}
|
||||
|
||||
func (r *Repository) Head(ctx context.Context, domain, object, lang string) (Revision, error) {
|
||||
l, err := normalizedLanguage(lang)
|
||||
if err != nil {
|
||||
return Revision{}, err
|
||||
}
|
||||
v, err := scanRevision(r.database.QueryRowContext(ctx, `SELECT `+revisionColumns+` FROM revisions r WHERE object_uuid=`+r.p(1)+` AND lang=`+r.p(2)+` AND (`+r.objectDomainScope("r.object_uuid", r.p(3))+`) ORDER BY created_at_ms DESC,uuid DESC LIMIT 1`, object, l, domain))
|
||||
return v, r.failure(err)
|
||||
}
|
||||
|
||||
func (r *Repository) GetRevision(ctx context.Context, domain, revisionUUID string) (Revision, error) {
|
||||
v, err := scanRevision(r.database.QueryRowContext(ctx, `SELECT `+revisionColumns+` FROM revisions r WHERE uuid=`+r.p(1)+` AND (`+r.objectDomainScope("r.object_uuid", r.p(2))+`)`, revisionUUID, domain))
|
||||
return v, r.failure(err)
|
||||
}
|
||||
|
||||
// RestoreRevision appends a new revision pointing to the historical Document;
|
||||
// it never rewrites old revision metadata or bypasses the exact-language CAS.
|
||||
func (r *Repository) RestoreRevision(ctx context.Context, domain, revisionUUID, expected, message string) (Revision, error) {
|
||||
v, err := r.GetRevision(ctx, domain, revisionUUID)
|
||||
if err != nil {
|
||||
return Revision{}, err
|
||||
}
|
||||
v.Message = message
|
||||
refs, err := r.revisionReferences(ctx, domain, revisionUUID)
|
||||
if err != nil {
|
||||
return Revision{}, err
|
||||
}
|
||||
return r.appendRevision(ctx, domain, v, expected, refs, "")
|
||||
}
|
||||
|
||||
// AppendRevision uses exact-language compare-and-swap. Empty expected means
|
||||
// there is no previous revision in this language. All writes use this contract;
|
||||
// monotonic timestamps remove dependence on SQLite's hidden rowid ordering.
|
||||
func (r *Repository) AppendRevision(ctx context.Context, domain string, v Revision, expected string) (Revision, error) {
|
||||
return r.appendRevision(ctx, domain, v, expected, nil, "")
|
||||
}
|
||||
|
||||
func (r *Repository) appendRevision(ctx context.Context, domain string, v Revision, expected string, refs []string, actor string) (Revision, error) {
|
||||
return r.appendRevisionExpected(ctx, domain, v, &expected, refs, actor)
|
||||
}
|
||||
|
||||
func (r *Repository) appendRevisionExpected(ctx context.Context, domain string, v Revision, expected *string, refs []string, actor string) (Revision, error) {
|
||||
lang, err := normalizedLanguage(v.Language)
|
||||
if err != nil || !validHash(v.DocumentHash) || v.ObjectUUID == "" || strings.TrimSpace(v.ContentType) == "" {
|
||||
return Revision{}, ErrInvalid
|
||||
}
|
||||
v.UUID = uuid.NewString()
|
||||
v.Language = lang
|
||||
err = r.write(ctx, "domain/"+domain, func(tx *sql.Tx) error {
|
||||
// Always acquire Domain before Object. This also protects per-Domain
|
||||
// outbox sequence allocation for different objects and languages.
|
||||
if err := r.lock(ctx, tx, "object/"+v.ObjectUUID); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
var bound bool
|
||||
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=`+r.p(1)+` AND (`+r.objectDomainScope("o.uuid", r.p(2))+`))`, v.ObjectUUID, domain).Scan(&bound); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
if !bound {
|
||||
return ErrNotFound
|
||||
}
|
||||
var current string
|
||||
var previous int64
|
||||
err := tx.QueryRowContext(ctx, `SELECT uuid,created_at_ms FROM revisions WHERE object_uuid=`+r.p(1)+` AND lang=`+r.p(2)+` ORDER BY created_at_ms DESC,uuid DESC LIMIT 1`, v.ObjectUUID, lang).Scan(¤t, &previous)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return r.failure(err)
|
||||
}
|
||||
if expected != nil && current != *expected {
|
||||
return ErrConflict
|
||||
}
|
||||
if previous == math.MaxInt64 {
|
||||
return ErrInvalid
|
||||
}
|
||||
v.CreatedAtMS = time.Now().UnixMilli()
|
||||
if v.CreatedAtMS <= previous {
|
||||
v.CreatedAtMS = previous + 1
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO revisions(uuid,object_uuid,document_hash,lang,content_type,filename,message,created_at_ms,created_at,body) VALUES (`+r.p(1)+`,`+r.p(2)+`,`+r.p(3)+`,`+r.p(4)+`,`+r.p(5)+`,`+r.p(6)+`,`+r.p(7)+`,`+r.p(8)+`,`+r.p(9)+`,'')`, v.UUID, v.ObjectUUID, v.DocumentHash, v.Language, v.ContentType, v.Filename, v.Message, v.CreatedAtMS, v.CreatedAtMS/1000)
|
||||
if err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, hash := range refs {
|
||||
if !validHash(hash) {
|
||||
return ErrInvalid
|
||||
}
|
||||
if seen[hash] {
|
||||
continue
|
||||
}
|
||||
seen[hash] = true
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO revision_document_refs(revision_uuid,document_hash) VALUES (`+r.p(1)+`,`+r.p(2)+`)`, v.UUID, hash); err != nil {
|
||||
return r.failure(err)
|
||||
}
|
||||
}
|
||||
return r.appendRevisionOutbox(ctx, tx, domain, actor, v)
|
||||
})
|
||||
if err != nil {
|
||||
return Revision{}, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
Reference in new issue
Block a user