feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s

This commit is contained in:
Anya Tokugawa committed 2026-11-15 05:47:14 +09:00
1 parent 96065d1e89
commit 0ba1dcfef0
243 files changed
+21441 -3090

No files matched your search

+88
View File
@@ -0,0 +1,88 @@
package metadata
import (
"context"
"database/sql"
"errors"
"strings"
"time"
"github.com/google/uuid"
)
type Asset struct {
DomainUUID string
Address string
ObjectUUID string
HeadDocumentHash string
ContentType string
Filename string
SizeBytes int64
CreatedAtMS int64
UpdatedAtMS int64
}
const assetColumns = `domain_uuid,address,object_uuid,head_document_hash,content_type,filename,size_bytes,created_at_ms,updated_at_ms`
func scanAsset(row interface{ Scan(...any) error }) (Asset, error) {
var a Asset
err := row.Scan(&a.DomainUUID, &a.Address, &a.ObjectUUID, &a.HeadDocumentHash, &a.ContentType, &a.Filename, &a.SizeBytes, &a.CreatedAtMS, &a.UpdatedAtMS)
return a, err
}
func (r *Repository) GetAsset(ctx context.Context, domain, address string) (Asset, error) {
a, err := scanAsset(r.database.QueryRowContext(ctx, `SELECT `+assetColumns+` FROM assets WHERE domain_uuid=`+r.p(1)+` AND address=`+r.p(2), domain, address))
return a, r.failure(err)
}
// PutAsset publishes an already-durable Document hash and appends its audit
// history in the same transaction. Empty expected is create-only; overwrites
// compare the previous hash and keep the asset's object UUID stable.
func (r *Repository) PutAsset(ctx context.Context, a Asset, expected, actor string) (Asset, error) {
a.Address = strings.TrimSpace(a.Address)
parts := strings.SplitN(a.Address, ":", 3)
if a.DomainUUID == "" || len(parts) != 3 || parts[0] != "asset" || strings.TrimSpace(parts[1]) == "" || strings.TrimSpace(parts[2]) == "" || strings.ContainsAny(a.Address, "\x00\r\n") || !validHash(a.HeadDocumentHash) || strings.TrimSpace(a.ContentType) == "" || a.SizeBytes < 0 {
return Asset{}, ErrInvalid
}
err := r.write(ctx, "domain/"+a.DomainUUID, func(tx *sql.Tx) error {
old, err := scanAsset(tx.QueryRowContext(ctx, `SELECT `+assetColumns+` FROM assets WHERE domain_uuid=`+r.p(1)+` AND address=`+r.p(2), a.DomainUUID, a.Address))
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return r.failure(err)
}
if old.HeadDocumentHash != expected {
return ErrConflict
}
if r.applicationNamespace && err == nil && old.HeadDocumentHash == a.HeadDocumentHash && old.ContentType == a.ContentType && old.Filename == a.Filename && old.SizeBytes == a.SizeBytes {
a = old
return nil
}
a.UpdatedAtMS = time.Now().UnixMilli()
if err == nil {
a.ObjectUUID = old.ObjectUUID
a.CreatedAtMS = old.CreatedAtMS
if a.UpdatedAtMS <= old.UpdatedAtMS {
a.UpdatedAtMS = old.UpdatedAtMS + 1
}
_, err = tx.ExecContext(ctx, `UPDATE assets SET head_document_hash=`+r.p(1)+`,content_type=`+r.p(2)+`,filename=`+r.p(3)+`,size_bytes=`+r.p(4)+`,updated_at_ms=`+r.p(5)+` WHERE domain_uuid=`+r.p(6)+` AND address=`+r.p(7), a.HeadDocumentHash, a.ContentType, a.Filename, a.SizeBytes, a.UpdatedAtMS, a.DomainUUID, a.Address)
} else {
a.ObjectUUID = uuid.NewString()
a.CreatedAtMS = a.UpdatedAtMS
if _, err = tx.ExecContext(ctx, `INSERT INTO objects(uuid,kind,created_at) VALUES (`+r.p(1)+`,'asset',`+r.p(2)+`)`, a.ObjectUUID, a.CreatedAtMS/1000); err != nil {
return r.failure(err)
}
if _, err = tx.ExecContext(ctx, `INSERT INTO object_origins(object_uuid,domain_uuid,created_at) VALUES (`+r.p(1)+`,`+r.p(2)+`,`+r.p(3)+`)`, a.ObjectUUID, a.DomainUUID, a.CreatedAtMS/1000); err != nil {
return r.failure(err)
}
_, err = tx.ExecContext(ctx, `INSERT INTO assets (`+assetColumns+`) VALUES (`+r.p(1)+`,`+r.p(2)+`,`+r.p(3)+`,`+r.p(4)+`,`+r.p(5)+`,`+r.p(6)+`,`+r.p(7)+`,`+r.p(8)+`,`+r.p(9)+`)`, a.DomainUUID, a.Address, a.ObjectUUID, a.HeadDocumentHash, a.ContentType, a.Filename, a.SizeBytes, a.CreatedAtMS, a.UpdatedAtMS)
}
if err != nil {
return r.failure(err)
}
_, err = tx.ExecContext(ctx, `INSERT INTO asset_head_history(domain_uuid,address,action,document_hash,content_type,filename,size_bytes,actor_uuid,created_at_ms) VALUES (`+r.p(1)+`,`+r.p(2)+`,'put',`+r.p(3)+`,`+r.p(4)+`,`+r.p(5)+`,`+r.p(6)+`,`+r.p(7)+`,`+r.p(8)+`)`, a.DomainUUID, a.Address, a.HeadDocumentHash, a.ContentType, a.Filename, a.SizeBytes, actor, a.UpdatedAtMS)
return r.failure(err)
})
if err != nil {
return Asset{}, err
}
return a, nil
}
+115
View File
@@ -0,0 +1,115 @@
package metadata
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"mime"
"pageswiki/internal/documentrefs"
"pageswiki/internal/geduldh"
)
type SavedDocument struct {
Revision Revision
Payload []byte
}
// RestoreDocument preserves the historical immutable embedded references; it
// must not parse Asset provenance and rebind it to today's mutable Asset HEAD.
func (r *Repository) RestoreDocument(ctx context.Context, documents geduldh.Store, domain, revision, expected, message, actor string) (Revision, error) {
if documents == nil {
return Revision{}, ErrInvalid
}
v, err := r.GetRevision(ctx, domain, revision)
if err != nil {
return Revision{}, err
}
refs, err := r.revisionReferences(ctx, domain, revision)
if err != nil {
return Revision{}, err
}
for _, hash := range append([]string{v.DocumentHash}, refs...) {
if err := documents.Verify(ctx, hash); err != nil {
return Revision{}, fmt.Errorf("verify restored Document: %w", err)
}
}
v.Message = message
return r.appendRevision(ctx, domain, v, expected, refs, actor)
}
// SaveDocument freezes Domain-local Asset HEADs, verifies embedded payloads,
// persists the immutable Document, then atomically publishes Revision, refs and
// outbox. No SQL transaction/lock is held during filesystem or remote I/O.
// A failed metadata commit may leave an unreferenced immutable blob; it must
// never delete that blob (another writer may already reference the same hash).
// Concurrent GC needs a separate pin protocol before this pipeline is enabled
// in a deployment with background deletion. This is not an authorization API.
func (r *Repository) SaveDocument(ctx context.Context, documents geduldh.Store, domain string, v Revision, payload []byte, expected, actor string) (SavedDocument, error) {
return r.saveDocument(ctx, documents, domain, v, payload, &expected, actor)
}
// SaveDocumentUnconditionally preserves append-without-CAS semantics for import
// and legacy clients. Concurrent saves still append immutable revisions with
// monotonic timestamps; it must not be used to silently retry a failed CAS.
func (r *Repository) SaveDocumentUnconditionally(ctx context.Context, documents geduldh.Store, domain string, v Revision, payload []byte, actor string) (SavedDocument, error) {
return r.saveDocument(ctx, documents, domain, v, payload, nil, actor)
}
func (r *Repository) saveDocument(ctx context.Context, documents geduldh.Store, domain string, v Revision, payload []byte, expected *string, actor string) (SavedDocument, error) {
if documents == nil {
return SavedDocument{}, ErrInvalid
}
lang, err := normalizedLanguage(v.Language)
if err != nil {
return SavedDocument{}, err
}
v.Language = lang
if _, _, err := mime.ParseMediaType(v.ContentType); err != nil {
return SavedDocument{}, ErrInvalid
}
var bound bool
if err := r.database.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=`+r.p(1)+` AND (`+r.objectDomainScope("o.uuid", r.p(2))+`))`, v.ObjectUUID, domain).Scan(&bound); err != nil {
return SavedDocument{}, r.failure(err)
}
if !bound {
return SavedDocument{}, ErrNotFound
}
// Fail stale saves before expensive payload work; repeat CAS in the commit
// transaction because a different writer may commit after this preflight.
head, err := r.Head(ctx, domain, v.ObjectUUID, v.Language)
if err != nil && !errors.Is(err, ErrNotFound) {
return SavedDocument{}, err
}
if expected != nil && head.UUID != *expected {
return SavedDocument{}, ErrConflict
}
frozen, refs, err := documentrefs.Freeze(ctx, payload, v.ContentType, func(ctx context.Context, address string) (string, error) {
a, err := r.GetAsset(ctx, domain, address)
if errors.Is(err, ErrNotFound) {
return "", documentrefs.ErrUnavailable
}
return a.HeadDocumentHash, err
})
if err != nil {
return SavedDocument{}, err
}
for _, hash := range refs {
if err := documents.Verify(ctx, hash); err != nil {
return SavedDocument{}, fmt.Errorf("verify embedded Document: %w", err)
}
}
sum := sha256.Sum256(frozen)
v.DocumentHash = hex.EncodeToString(sum[:])
if err := documents.PutIfAbsent(ctx, v.DocumentHash, int64(len(frozen)), bytes.NewReader(frozen)); err != nil {
return SavedDocument{}, fmt.Errorf("persist Document: %w", err)
}
created, err := r.appendRevisionExpected(ctx, domain, v, expected, refs, actor)
if err != nil {
return SavedDocument{}, err
}
return SavedDocument{Revision: created, Payload: frozen}, nil
}
+257
View File
@@ -0,0 +1,257 @@
package metadata
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"strings"
"sync"
"testing"
"time"
"pageswiki/internal/documentrefs"
"pageswiki/internal/geduldh"
)
type faultDocuments struct {
geduldh.Store
putErr error
verifyErr error
afterPut func()
}
func (f faultDocuments) PutIfAbsent(ctx context.Context, h string, n int64, reader io.Reader) error {
if f.putErr != nil {
return f.putErr
}
if err := f.Store.PutIfAbsent(ctx, h, n, reader); err != nil {
return err
}
if f.afterPut != nil {
f.afterPut()
}
return nil
}
func (f faultDocuments) Verify(ctx context.Context, h string) error {
if f.verifyErr != nil {
return f.verifyErr
}
return f.Store.Verify(ctx, h)
}
func documentFixture(t *testing.T, r *Repository) (geduldh.Store, string, Revision) {
t.Helper()
ctx := context.Background()
documents, err := geduldh.NewLocal(t.TempDir())
if err != nil {
t.Fatal(err)
}
content := []byte("synthetic image")
h := hash(string(content))
if err := documents.PutIfAbsent(ctx, h, int64(len(content)), bytes.NewReader(content)); err != nil {
t.Fatal(err)
}
if _, err := r.PutAsset(ctx, Asset{DomainUUID: "test", Address: "asset:image:cover", HeadDocumentHash: h, ContentType: "image/png", SizeBytes: int64(len(content))}, "", "actor"); err != nil {
t.Fatal(err)
}
object, err := r.CreatePage(ctx, "test", "/guide")
if err != nil {
t.Fatal(err)
}
return documents, h, Revision{ObjectUUID: object, Language: "ja", ContentType: "text/html; charset=utf-8", Message: "synthetic save"}
}
func TestDocumentSaveReferencesOutboxAndRestore(t *testing.T) {
for _, backend := range []Backend{SQLite, PostgreSQL} {
t.Run(string(backend), func(t *testing.T) {
r := testRepository(t, backend)
ctx := context.Background()
docs, assetHash, v := documentFixture(t, r)
body := []byte(`<h1>Guide</h1><img src="asset:image:cover"><img src="asset:image:cover">`)
if _, err := r.PutAsset(ctx, Asset{DomainUUID: "private-domain", Address: "asset:image:private", HeadDocumentHash: assetHash, ContentType: "image/png", SizeBytes: 15}, "", "actor"); err != nil {
t.Fatal(err)
}
if _, err := r.SaveDocument(ctx, docs, "test", v, []byte(`<img src="asset:image:private">`), "", "actor"); !errors.Is(err, documentrefs.ErrUnavailable) {
t.Fatal("foreign Asset resolved", err)
}
first, err := r.SaveDocument(ctx, docs, "test", v, body, "", "actor")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(first.Payload), "/_/obj/sha256:"+assetHash) {
t.Fatal("mutable URL not frozen")
}
if err := docs.Verify(ctx, first.Revision.DocumentHash); err != nil {
t.Fatal("metadata published without durable Document", err)
}
refs, err := r.revisionReferences(ctx, "test", first.Revision.UUID)
if err != nil || len(refs) != 1 || refs[0] != assetHash {
t.Fatal("embedded reference", err, refs)
}
var eventDomain, target, payload, actor string
var sequence int64
if err := r.database.QueryRowContext(ctx, `SELECT domain_uuid,target_uuid,payload,actor_uuid,sequence FROM activity_outbox`).Scan(&eventDomain, &target, &payload, &actor, &sequence); err != nil {
t.Fatal(err)
}
var envelope map[string]string
if err := json.Unmarshal([]byte(payload), &envelope); err != nil || eventDomain != "test" || actor != "actor" || sequence != 1 || target != first.Revision.UUID || envelope["document_hash"] != first.Revision.DocumentHash {
t.Fatal("outbox envelope mismatch", err)
}
newContent := []byte("next synthetic image")
newHash := hash(string(newContent))
if err := docs.PutIfAbsent(ctx, newHash, int64(len(newContent)), bytes.NewReader(newContent)); err != nil {
t.Fatal(err)
}
if _, err := r.PutAsset(ctx, Asset{DomainUUID: "test", Address: "asset:image:cover", HeadDocumentHash: newHash, ContentType: "image/png", SizeBytes: int64(len(newContent))}, assetHash, "actor"); err != nil {
t.Fatal(err)
}
second, err := r.SaveDocument(ctx, docs, "test", v, first.Payload, first.Revision.UUID, "actor")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(second.Payload), newHash) {
t.Fatal("provenance not refreshed")
}
restored, err := r.RestoreRevision(ctx, "test", first.Revision.UUID, second.Revision.UUID, "restore")
if err != nil {
t.Fatal(err)
}
refs, err = r.revisionReferences(ctx, "test", restored.UUID)
if err != nil || len(refs) != 1 || refs[0] != assetHash || restored.DocumentHash != first.Revision.DocumentHash {
t.Fatal("restore followed new Asset HEAD instead of historical refs", err)
}
var count int
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM activity_outbox`).Scan(&count); err != nil || count != 3 {
t.Fatal("save/restore event count", count, err)
}
})
}
}
func TestDocumentSaveFailureIsAtomic(t *testing.T) {
for _, backend := range []Backend{SQLite, PostgreSQL} {
for _, stage := range []string{"references", "outbox"} {
t.Run(string(backend)+"/"+stage, func(t *testing.T) {
r := testRepository(t, backend)
ctx := context.Background()
docs, _, v := documentFixture(t, r)
body := []byte(`<img src="asset:image:cover">`)
first, err := r.SaveDocument(ctx, docs, "test", v, body, "", "actor")
if err != nil {
t.Fatal(err)
}
ddl := `CREATE UNIQUE INDEX synthetic_save_failure ON revision_document_refs(document_hash)`
if stage == "outbox" {
ddl = `CREATE UNIQUE INDEX synthetic_save_failure ON activity_outbox(domain_uuid)`
}
if _, err := r.database.ExecContext(ctx, ddl); err != nil {
t.Fatal(err)
}
if _, err := r.SaveDocument(ctx, docs, "test", v, append([]byte(`<p>change</p>`), body...), first.Revision.UUID, "actor"); !errors.Is(err, ErrConflict) {
t.Fatal("injected error ignored", err)
}
for _, table := range []string{"revisions", "revision_document_refs", "activity_outbox"} {
var n int
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM `+table).Scan(&n); err != nil || n != 1 {
t.Fatal("partial commit", table, n, err)
}
}
var last int64
if err := r.database.QueryRowContext(ctx, `SELECT last_sequence FROM activity_outbox_heads WHERE domain_uuid='test'`).Scan(&last); err != nil || last != 1 {
t.Fatal("sequence advanced on failure", err)
}
if _, err := r.database.ExecContext(ctx, `DROP INDEX synthetic_save_failure`); err != nil {
t.Fatal(err)
}
if _, err := r.SaveDocument(ctx, docs, "test", v, body, first.Revision.UUID, "actor"); err != nil {
t.Fatal("retry after rollback", err)
}
})
}
}
}
func TestDocumentSavePersistenceFailureAndConcurrentEdit(t *testing.T) {
for _, backend := range []Backend{SQLite, PostgreSQL} {
t.Run(string(backend), func(t *testing.T) {
r := testRepository(t, backend)
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
docs, _, v := documentFixture(t, r)
body := []byte(`<img src="asset:image:cover">`)
for _, failing := range []faultDocuments{{Store: docs, putErr: io.ErrClosedPipe}, {Store: docs, verifyErr: geduldh.ErrHashMismatch}} {
if _, err := r.SaveDocument(ctx, failing, "test", v, body, "", "actor"); err == nil {
t.Fatal("Document failure ignored")
}
var n int
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM revisions`).Scan(&n); err != nil || n != 0 {
t.Fatal("failed persistence published revision", err)
}
}
first, err := r.SaveDocument(ctx, docs, "test", v, body, "", "actor")
if err != nil {
t.Fatal(err)
}
// The callback writes using a different transaction during Document I/O.
// It would deadlock if SaveDocument held SQL locks over the payload write.
race := faultDocuments{Store: docs, afterPut: func() {
peer, _ := New(r.database, backend)
competing := v
competing.DocumentHash = first.Revision.DocumentHash
if _, err := peer.AppendRevision(ctx, "test", competing, first.Revision.UUID); err != nil {
t.Fatal("I/O held metadata lock", err)
}
}}
if _, err := r.SaveDocument(ctx, race, "test", v, []byte("<p>stale</p>"), first.Revision.UUID, "actor"); !errors.Is(err, ErrConflict) {
t.Fatal("CAS not repeated after payload I/O", err)
}
var n int
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM activity_outbox`).Scan(&n); err != nil || n != 2 {
t.Fatal("conflicting write emitted event", err)
}
})
}
}
func TestOutboxSequenceForConcurrentObjects(t *testing.T) {
for _, backend := range []Backend{SQLite, PostgreSQL} {
t.Run(string(backend), func(t *testing.T) {
r := testRepository(t, backend)
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
objects := make([]string, 6)
for i := range objects {
var err error
objects[i], err = r.CreatePage(ctx, "test", "/"+strings.Repeat("a", i+1))
if err != nil {
t.Fatal(err)
}
}
var wg sync.WaitGroup
errs := make(chan error, len(objects))
for _, id := range objects {
wg.Add(1)
go func(id string) {
defer wg.Done()
peer, _ := New(r.database, backend)
_, err := peer.AppendRevision(ctx, "test", Revision{ObjectUUID: id, DocumentHash: hash(id), ContentType: "text/html"}, "")
errs <- err
}(id)
}
wg.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
var count int
var min, max int64
if err := r.database.QueryRowContext(ctx, `SELECT count(*),min(sequence),max(sequence) FROM activity_outbox`).Scan(&count, &min, &max); err != nil || count != 6 || min != 1 || max != 6 {
t.Fatal("invalid concurrent sequence", count, min, max, err)
}
})
}
}
+244
View File
@@ -0,0 +1,244 @@
package metadata
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"sort"
"strings"
)
// A core-compatible schema is a rehearsal scope, not an application readiness
// claim. New application tables and security triggers always block core-only
// copying; unknown data must never be silently dropped as a "cache".
type PortabilityReport struct {
Scope string `json:"scope"`
ApplicationReady bool `json:"application_ready"`
CoreCompatible bool `json:"core_compatible"`
Tables []TablePortability `json:"tables"`
Blockers []string `json:"blockers"`
}
type TablePortability struct {
Name string `json:"name"`
Component string `json:"component"`
State string `json:"state"`
}
type tableSpec struct {
name string
columns []string
keys []string
identity bool
}
// Explicit columns and primary-key order define an unambiguous streaming
// fingerprint and an FK-safe copy order. Migration ledgers stay backend-local.
var coreTables = []tableSpec{
{"objects", strings.Split("uuid,kind,created_at", ","), []string{"uuid"}, false},
{"object_origins", strings.Split("object_uuid,domain_uuid,created_at", ","), []string{"object_uuid"}, false},
{"paths", strings.Split("domain_uuid,path,object_uuid,created_at,is_primary", ","), []string{"domain_uuid", "path"}, false},
{"revisions", strings.Split(revisionColumns+",created_at,body", ","), []string{"uuid"}, false},
{"assets", strings.Split(assetColumns, ","), []string{"domain_uuid", "address"}, false},
{"asset_head_history", strings.Split("id,domain_uuid,address,action,document_hash,content_type,filename,size_bytes,actor_uuid,created_at_ms", ","), []string{"id"}, true},
{"revision_document_refs", []string{"revision_uuid", "document_hash"}, []string{"revision_uuid", "document_hash"}, false},
{"activity_outbox", strings.Split("uuid,domain_uuid,sequence,event_type,actor_uuid,target_kind,target_uuid,payload,idempotency_key,status,attempts,next_attempt_ms,last_error,created_at_ms,claimed_at_ms", ","), []string{"uuid"}, false},
{"activity_outbox_heads", []string{"domain_uuid", "event_type", "last_sequence"}, []string{"domain_uuid", "event_type"}, false},
}
func componentForTable(name string) string {
for _, prefix := range []string{"auth", "principal", "group", "membership", "task", "calendar", "event", "notification", "activity", "federation", "domain", "campus", "config", "import", "resource", "adapter", "user", "profile", "contact", "feed", "periodical", "search", "locality", "city"} {
if strings.HasPrefix(name, prefix) {
return prefix
}
}
if name == "zones" || name == "spots" || strings.HasPrefix(name, "spot_") || strings.HasPrefix(name, "street_") || name == "topology_events" {
return "city"
}
return "unclassified"
}
func (r *Repository) InspectPortability(ctx context.Context) (PortabilityReport, error) {
tx, err := r.database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead})
if err != nil {
return PortabilityReport{}, r.failure(err)
}
defer tx.Rollback()
return r.inspectPortability(ctx, tx)
}
func (r *Repository) inspectPortability(ctx context.Context, tx *sql.Tx) (PortabilityReport, error) {
report := PortabilityReport{Scope: "portable-core-rehearsal-only", Tables: []TablePortability{}, Blockers: []string{}}
known := map[string]tableSpec{}
for _, s := range coreTables {
known[s.name] = s
}
query := `SELECT name,type FROM sqlite_master WHERE type IN ('table','view','trigger') AND substr(name,1,7)!='sqlite_' ORDER BY name`
if r.backend == PostgreSQL {
query = `SELECT c.relname,CASE WHEN c.relkind IN ('r','p') THEN 'table' ELSE 'view' END FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=current_schema() AND c.relkind IN ('r','p','v','m','f')
UNION ALL SELECT t.tgname,'trigger' FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=current_schema() AND NOT t.tgisinternal
UNION ALL SELECT p.proname,'routine' FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace WHERE n.nspname=current_schema()
UNION ALL SELECT r.rulename,'rule' FROM pg_rewrite r JOIN pg_class c ON c.oid=r.ev_class JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=current_schema() AND r.rulename!='_RETURN'
UNION ALL SELECT c.relname,'sequence' FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=current_schema() AND c.relkind='S' AND NOT EXISTS(SELECT 1 FROM pg_depend d JOIN pg_class t ON t.oid=d.refobjid JOIN pg_namespace tn ON tn.oid=t.relnamespace WHERE d.objid=c.oid AND d.classid='pg_class'::regclass AND d.refclassid='pg_class'::regclass AND tn.nspname=current_schema() AND t.relname='asset_head_history' AND d.deptype='i') ORDER BY 1`
}
rows, err := tx.QueryContext(ctx, query)
if err != nil {
return report, r.failure(err)
}
types := map[string]string{}
for rows.Next() {
var name, kind string
if err := rows.Scan(&name, &kind); err != nil {
rows.Close()
return report, r.failure(err)
}
if kind != "table" {
report.Blockers = append(report.Blockers, fmt.Sprintf("unported %s: %s", kind, name))
continue
}
types[name] = kind
}
err = rows.Err()
rows.Close()
if err != nil {
return report, r.failure(err)
}
names := make([]string, 0, len(types))
for name := range types {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
kind := types[name]
if kind != "table" {
report.Blockers = append(report.Blockers, fmt.Sprintf("unported %s: %s", kind, name))
continue
}
if name == "schema_migration_steps" {
continue
}
item := TablePortability{Name: name, Component: componentForTable(name), State: "unported"}
if _, ok := known[name]; ok {
item.Component = "core-metadata"
item.State = "shape-pending"
} else {
report.Blockers = append(report.Blockers, "unported table: "+name)
}
report.Tables = append(report.Tables, item)
}
for _, spec := range coreTables {
if types[spec.name] != "table" {
report.Blockers = append(report.Blockers, "missing core table: "+spec.name)
}
}
if types["schema_migration_steps"] != "table" {
report.Blockers = append(report.Blockers, "missing portable migration ledger")
} else {
if err := r.inspectCoreHistory(ctx, tx, &report); err != nil {
return report, err
}
}
for i := range report.Tables {
item := &report.Tables[i]
spec, ok := known[item.Name]
if !ok {
continue
}
columns, err := r.tableColumns(ctx, tx, item.Name)
if err != nil {
return report, err
}
actual := map[string]bool{}
for _, c := range columns {
actual[c] = true
}
matches := len(actual) == len(spec.columns)
for _, c := range spec.columns {
matches = matches && actual[c]
}
if !matches {
item.State = "schema-drift"
report.Blockers = append(report.Blockers, "column mismatch: "+item.Name)
} else {
item.State = "core-shape-compatible"
}
}
if err := r.inspectCatalog(ctx, tx, &report); err != nil {
return report, err
}
sort.Strings(report.Blockers)
report.CoreCompatible = len(report.Blockers) == 0
// This core-only report does not certify full-application readiness. Use
// the Store application importer for the production migration inventory.
// ApplicationReady intentionally remains false here because non-core
// repositories, canonical files and worker state are outside this check.
return report, nil
}
func (r *Repository) tableColumns(ctx context.Context, tx *sql.Tx, table string) ([]string, error) {
query := `SELECT name FROM pragma_table_info(` + r.p(1) + `) ORDER BY cid`
if r.backend == PostgreSQL {
query = `SELECT column_name FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1 ORDER BY ordinal_position`
}
rows, err := tx.QueryContext(ctx, query, table)
if err != nil {
return nil, r.failure(err)
}
defer rows.Close()
var out []string
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return nil, r.failure(err)
}
out = append(out, name)
}
return out, r.failure(rows.Err())
}
func (r *Repository) inspectCoreHistory(ctx context.Context, tx *sql.Tx, report *PortabilityReport) error {
sqliteSteps, pgSteps := r.migrationDefinitions()
checksums := map[int]string{}
names := map[int]string{}
add := func(version int, name string, step any) {
data, _ := json.Marshal(step)
sum := sha256.Sum256(data)
checksums[version] = hex.EncodeToString(sum[:])
names[version] = name
}
for _, s := range sqliteSteps {
add(s.Version, s.Name, s)
}
for _, s := range pgSteps {
add(s.Version, s.Name, s)
}
rows, err := tx.QueryContext(ctx, `SELECT component,version,name,checksum FROM schema_migration_steps ORDER BY component,version`)
if err != nil {
return r.failure(err)
}
defer rows.Close()
applied := 0
for rows.Next() {
var component, name, checksum string
var version int
if err := rows.Scan(&component, &version, &name, &checksum); err != nil {
return r.failure(err)
}
if component != "core-metadata" {
report.Blockers = append(report.Blockers, "unported migration component: "+component)
continue
}
if version != applied+1 || names[version] != name || checksums[version] != checksum {
report.Blockers = append(report.Blockers, "portable migration history mismatch")
}
applied++
}
if err := rows.Err(); err != nil {
return r.failure(err)
}
if applied != len(checksums) {
report.Blockers = append(report.Blockers, "portable migrations are not current")
}
return nil
}
+230
View File
@@ -0,0 +1,230 @@
package metadata
import (
"context"
"database/sql"
"fmt"
"strings"
)
// Only the owned portable-core schema is certified here. Existing application
// tables are NOT adopted by translating arbitrary SQLite DDL to PostgreSQL.
func (r *Repository) inspectCatalog(ctx context.Context, tx *sql.Tx, report *PortabilityReport) error {
if r.backend == SQLite {
return r.inspectSQLiteCatalog(ctx, tx, report)
}
return r.inspectPostgreSQLCatalog(ctx, tx, report)
}
func (r *Repository) inspectSQLiteCatalog(ctx context.Context, tx *sql.Tx, report *PortabilityReport) error {
steps, _ := r.migrationDefinitions()
expected := map[string]string{}
for _, step := range steps {
for _, statement := range step.Statements {
fields := strings.Fields(statement)
if len(fields) > 2 && fields[0] == "CREATE" && (fields[1] == "TABLE" || fields[1] == "INDEX") {
name := strings.SplitN(fields[2], "(", 2)[0]
expected[name] = strings.TrimSpace(statement)
}
}
}
rows, err := tx.QueryContext(ctx, `SELECT name,type,sql FROM sqlite_master WHERE type IN ('table','index') AND sql IS NOT NULL AND name!='schema_migration_steps' AND substr(name,1,7)!='sqlite_' ORDER BY name`)
if err != nil {
return r.failure(err)
}
defer rows.Close()
seen := map[string]bool{}
for rows.Next() {
var name, kind, ddl string
if err := rows.Scan(&name, &kind, &ddl); err != nil {
return r.failure(err)
}
if wanted, ok := expected[name]; ok {
seen[name] = true
if wanted != strings.TrimSpace(ddl) {
report.Blockers = append(report.Blockers, "catalog mismatch: "+name)
}
} else if kind == "index" {
report.Blockers = append(report.Blockers, "unported index: "+name)
}
}
if err := rows.Err(); err != nil {
return r.failure(err)
}
for name := range expected {
if !seen[name] {
report.Blockers = append(report.Blockers, "missing catalog object: "+name)
}
}
return nil
}
func numericColumn(column string) bool {
switch column {
case "created_at", "created_at_ms", "updated_at_ms", "is_primary", "size_bytes", "id", "sequence", "attempts", "next_attempt_ms", "claimed_at_ms", "last_sequence":
return true
}
return false
}
func (r *Repository) inspectPostgreSQLCatalog(ctx context.Context, tx *sql.Tx, report *PortabilityReport) error {
var ledgerPolicy bool
if err := tx.QueryRowContext(ctx, `SELECT COALESCE(bool_or(c.relrowsecurity OR c.relforcerowsecurity OR EXISTS(SELECT 1 FROM pg_policy p WHERE p.polrelid=c.oid)),false) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=current_schema() AND c.relname='schema_migration_steps'`).Scan(&ledgerPolicy); err != nil {
return r.failure(err)
}
if ledgerPolicy {
report.Blockers = append(report.Blockers, "unported policy: schema_migration_steps")
}
defaults := map[string]string{
"paths.is_primary": "0", "revisions.filename": "''::text", "revisions.message": "''::text", "revisions.body": "''::text", "assets.filename": "''::text",
"activity_outbox.actor_uuid": "''::text", "activity_outbox.target_kind": "''::text", "activity_outbox.target_uuid": "''::text", "activity_outbox.payload": "''::text", "activity_outbox.status": "'pending'::text", "activity_outbox.attempts": "0", "activity_outbox.last_error": "''::text", "activity_outbox.claimed_at_ms": "0",
}
constraints := map[string][]string{
"objects": {"PRIMARY KEY (uuid)"},
"object_origins": {"PRIMARY KEY (object_uuid)", "FOREIGN KEY (object_uuid) REFERENCES objects(uuid)"},
"paths": {"PRIMARY KEY (domain_uuid, path)", "FOREIGN KEY (object_uuid) REFERENCES objects(uuid)"},
"revisions": {"PRIMARY KEY (uuid)", "FOREIGN KEY (object_uuid) REFERENCES objects(uuid)"},
"assets": {"PRIMARY KEY (domain_uuid, address)", "UNIQUE (object_uuid)", "FOREIGN KEY (object_uuid) REFERENCES objects(uuid)", "CHECK ((size_bytes >= 0))"},
"asset_head_history": {"PRIMARY KEY (id)", "CHECK ((size_bytes >= 0))"},
"revision_document_refs": {"PRIMARY KEY (revision_uuid, document_hash)", "FOREIGN KEY (revision_uuid) REFERENCES revisions(uuid)"},
"activity_outbox": {"PRIMARY KEY (uuid)", "UNIQUE (domain_uuid, sequence)", "UNIQUE (domain_uuid, idempotency_key)"},
"activity_outbox_heads": {"PRIMARY KEY (domain_uuid, event_type)"},
}
identityTablePresent := false
for _, spec := range coreTables {
var ordinary bool
if err := tx.QueryRowContext(ctx, `SELECT c.relkind='r' AND c.relpersistence='p' AND NOT c.relrowsecurity AND NOT c.relforcerowsecurity AND NOT EXISTS(SELECT 1 FROM pg_policy p WHERE p.polrelid=c.oid) AND NOT EXISTS(SELECT 1 FROM pg_inherits x WHERE x.inhrelid=c.oid OR x.inhparent=c.oid) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=current_schema() AND c.relname=$1`, spec.name).Scan(&ordinary); err != nil {
if err == sql.ErrNoRows {
report.Blockers = append(report.Blockers, "missing table: "+spec.name)
continue
}
return r.failure(err)
}
if !ordinary {
report.Blockers = append(report.Blockers, "unported policy or table mode: "+spec.name)
}
if spec.identity {
identityTablePresent = true
}
rows, err := tx.QueryContext(ctx, `SELECT a.attname,format_type(a.atttypid,a.atttypmod),a.attnotnull,COALESCE(pg_get_expr(d.adbin,d.adrelid),''),a.attidentity,a.attgenerated,COALESCE(col.collname,'') FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid JOIN pg_namespace n ON n.oid=c.relnamespace LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum LEFT JOIN pg_collation col ON col.oid=a.attcollation WHERE n.nspname=current_schema() AND c.relname=$1 AND a.attnum>0 AND NOT a.attisdropped`, spec.name)
if err != nil {
return r.failure(err)
}
for rows.Next() {
var name, kind, def, identity, generated, collation string
var required bool
if err := rows.Scan(&name, &kind, &required, &def, &identity, &generated, &collation); err != nil {
rows.Close()
return r.failure(err)
}
wantType := "text"
if numericColumn(name) {
wantType = "bigint"
}
if name == "is_primary" || name == "attempts" {
wantType = "integer"
}
wantIdentity := ""
if spec.identity && name == "id" {
wantIdentity = "a"
}
wantCollation := ""
if wantType == "text" {
wantCollation = "default"
}
if spec.name == "paths" && name == "path" || spec.name == "assets" && name == "address" {
wantCollation = "C"
}
if kind != wantType || !required || def != defaults[spec.name+"."+name] || identity != wantIdentity || generated != "" || collation != wantCollation {
report.Blockers = append(report.Blockers, fmt.Sprintf("column contract mismatch: %s.%s", spec.name, name))
}
}
err = rows.Err()
rows.Close()
if err != nil {
return r.failure(err)
}
rows, err = tx.QueryContext(ctx, `SELECT pg_get_constraintdef(c.oid),c.convalidated,c.condeferrable,c.condeferred FROM pg_constraint c JOIN pg_class t ON t.oid=c.conrelid JOIN pg_namespace n ON n.oid=t.relnamespace WHERE n.nspname=current_schema() AND t.relname=$1`, spec.name)
if err != nil {
return r.failure(err)
}
wanted := map[string]bool{}
for _, s := range constraints[spec.name] {
wanted[s] = true
}
actual := map[string]bool{}
matches := true
constraintCount := 0
for rows.Next() {
constraintCount++
var def string
var valid, deferable, deferred bool
if err := rows.Scan(&def, &valid, &deferable, &deferred); err != nil {
rows.Close()
return r.failure(err)
}
actual[def] = true
matches = matches && valid && !deferable && !deferred && wanted[def]
}
err = rows.Err()
rows.Close()
if err != nil {
return r.failure(err)
}
if !matches || len(actual) != len(wanted) || constraintCount != len(wanted) {
report.Blockers = append(report.Blockers, "constraint contract mismatch: "+spec.name)
}
}
var schema string
if err := tx.QueryRowContext(ctx, `SELECT quote_ident(current_schema())`).Scan(&schema); err != nil {
return r.failure(err)
}
indexes := map[string]string{
"idx_paths_domain_object": "paths USING btree (domain_uuid, object_uuid)",
"idx_revisions_metadata_head": "revisions USING btree (object_uuid, lang, created_at_ms DESC, uuid DESC)",
"idx_asset_metadata_history": "asset_head_history USING btree (domain_uuid, address, id)",
"idx_metadata_document_refs": "revision_document_refs USING btree (document_hash, revision_uuid)",
"idx_metadata_outbox_pending": "activity_outbox USING btree (status, next_attempt_ms, domain_uuid, sequence)",
}
rows, err := tx.QueryContext(ctx, `SELECT i.relname,pg_get_indexdef(i.oid),x.indisvalid,x.indisready FROM pg_index x JOIN pg_class i ON i.oid=x.indexrelid JOIN pg_class t ON t.oid=x.indrelid JOIN pg_namespace n ON n.oid=t.relnamespace WHERE n.nspname=current_schema() AND t.relname!='schema_migration_steps' AND NOT EXISTS(SELECT 1 FROM pg_constraint con WHERE con.conindid=i.oid)`)
if err != nil {
return r.failure(err)
}
defer rows.Close()
seen := map[string]bool{}
for rows.Next() {
var name, def string
var valid, ready bool
if err := rows.Scan(&name, &def, &valid, &ready); err != nil {
return r.failure(err)
}
suffix, ok := indexes[name]
if !ok {
report.Blockers = append(report.Blockers, "unported index: "+name)
continue
}
seen[name] = true
if def != "CREATE INDEX "+name+" ON "+schema+"."+suffix || !valid || !ready {
report.Blockers = append(report.Blockers, "index contract mismatch: "+name)
}
}
if err := rows.Err(); err != nil {
return r.failure(err)
}
for name := range indexes {
if !seen[name] {
report.Blockers = append(report.Blockers, "missing index: "+name)
}
}
var sequenceMatches bool
if identityTablePresent {
err = tx.QueryRowContext(ctx, `SELECT seqstart=1 AND seqincrement=1 AND seqmin=1 AND seqmax=9223372036854775807 AND seqcache=1 AND NOT seqcycle FROM pg_sequence WHERE seqrelid=pg_get_serial_sequence('asset_head_history','id')::regclass`).Scan(&sequenceMatches)
}
if err != nil && err != sql.ErrNoRows {
return r.failure(err)
}
if !sequenceMatches {
report.Blockers = append(report.Blockers, "identity sequence contract mismatch: asset_head_history.id")
}
return nil
}
@@ -0,0 +1,69 @@
package metadata
import (
"context"
"testing"
)
func TestPortabilityRejectsIndexDrift(t *testing.T) {
for _, backend := range []Backend{SQLite, PostgreSQL} {
for _, change := range []string{"missing", "wrong", "extra"} {
t.Run(string(backend)+"/"+change, func(t *testing.T) {
r := testRepository(t, backend)
ctx := context.Background()
if change != "extra" {
if _, err := r.database.ExecContext(ctx, `DROP INDEX idx_paths_domain_object`); err != nil {
t.Fatal(err)
}
}
if change == "wrong" {
if _, err := r.database.ExecContext(ctx, `CREATE INDEX idx_paths_domain_object ON paths(object_uuid)`); err != nil {
t.Fatal(err)
}
}
if change == "extra" {
if _, err := r.database.ExecContext(ctx, `CREATE UNIQUE INDEX unported_unique_rule ON paths(object_uuid)`); err != nil {
t.Fatal(err)
}
}
report, err := r.InspectPortability(ctx)
if err != nil || report.CoreCompatible {
t.Fatal("index drift accepted", err, report)
}
})
}
}
}
func TestPostgreSQLPortabilityRejectsContractDrift(t *testing.T) {
changes := map[string]string{
"type": `ALTER TABLE assets ALTER COLUMN size_bytes TYPE INTEGER`,
"default": `ALTER TABLE revisions ALTER COLUMN message SET DEFAULT 'different'`,
"nullable": `ALTER TABLE revisions ALTER COLUMN content_type DROP NOT NULL`,
"foreign-key": `ALTER TABLE paths DROP CONSTRAINT paths_object_uuid_fkey`,
"collation": `ALTER TABLE paths ALTER COLUMN path TYPE TEXT COLLATE "default"`,
"identity-mode": `ALTER TABLE asset_head_history ALTER COLUMN id SET GENERATED BY DEFAULT`,
"identity-drop": `ALTER TABLE asset_head_history ALTER COLUMN id DROP IDENTITY`,
"missing-table": `DROP TABLE asset_head_history`,
"identity-step": `ALTER TABLE asset_head_history ALTER COLUMN id SET INCREMENT BY 7`,
"rls": `ALTER TABLE objects ENABLE ROW LEVEL SECURITY`,
"ledger-rls": `ALTER TABLE schema_migration_steps ENABLE ROW LEVEL SECURITY`,
"routine": `CREATE FUNCTION unported_routine() RETURNS INTEGER LANGUAGE SQL AS 'SELECT 1'`,
"sequence": `CREATE SEQUENCE unported_counter`,
"rule": `CREATE RULE unported_rule AS ON DELETE TO paths DO INSTEAD NOTHING`,
"duplicate-fk": `ALTER TABLE paths ADD CONSTRAINT duplicate_fk FOREIGN KEY (object_uuid) REFERENCES objects(uuid)`,
}
for name, q := range changes {
t.Run(name, func(t *testing.T) {
r := testRepository(t, PostgreSQL)
ctx := context.Background()
if _, err := r.database.ExecContext(ctx, q); err != nil {
t.Fatal(err)
}
report, err := r.InspectPortability(ctx)
if err != nil || report.CoreCompatible {
t.Fatal("contract drift accepted", err, report)
}
})
}
}
+471
View File
@@ -0,0 +1,471 @@
package metadata
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
hashpkg "hash"
"math"
"os"
"strconv"
"strings"
"unicode/utf8"
"pageswiki/internal/geduldh"
)
var (
ErrMigrationBlocked = errors.New("metadata migration blocked; inspect portability report")
ErrDestinationNotEmpty = errors.New("metadata migration destination is not empty")
ErrIntegrity = errors.New("metadata integrity check failed")
)
type TableFingerprint struct {
Table string `json:"table"`
Rows int64 `json:"rows"`
SHA256 string `json:"sha256"`
}
type CoreSnapshot struct {
Scope string `json:"scope"`
Tables []TableFingerprint `json:"tables"`
IdentityNext int64 `json:"asset_history_identity_next"`
SHA256 string `json:"sha256"`
}
type RehearsalOptions struct {
// Caller asserts that all writers and GC are stopped for both instances.
// A SQL snapshot alone is not a cross-store or online migration protocol.
Offline bool
Documents geduldh.Store
}
// CloneCoreForRehearsal copies only a compatible portable core into an already
// initialized, empty staging schema. It is deliberately not a production
// migration entry point. Unknown tables/triggers/views/columns/ledger entries
// block copying rather than silently losing application data or permissions.
// Canonical payloads are verified in the caller's retained Document store;
// calendar files, secrets, application configuration and blobs are NOT copied.
func CloneCoreForRehearsal(ctx context.Context, source, destination *Repository, options RehearsalOptions) (CoreSnapshot, error) {
if !options.Offline || options.Documents == nil || source == nil || destination == nil || source.database == destination.database {
return CoreSnapshot{}, ErrMigrationBlocked
}
if same, err := sameDatabase(ctx, source, destination); err != nil {
return CoreSnapshot{}, err
} else if same {
return CoreSnapshot{}, ErrMigrationBlocked
}
src, err := source.database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead})
if err != nil {
return CoreSnapshot{}, source.failure(err)
}
defer src.Rollback()
report, err := source.inspectPortability(ctx, src)
if err != nil {
return CoreSnapshot{}, err
}
if !report.CoreCompatible {
return CoreSnapshot{}, ErrMigrationBlocked
}
if err := source.verifyCoreRelations(ctx, src); err != nil {
return CoreSnapshot{}, err
}
// Reject an occupied/incompatible destination before verifying a large
// Document store. Recheck while holding destination locks before copying.
preflight, err := destination.database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead})
if err != nil {
return CoreSnapshot{}, destination.failure(err)
}
err = destination.checkEmpty(ctx, preflight)
_ = preflight.Rollback()
if err != nil {
return CoreSnapshot{}, err
}
if err := source.verifyDocuments(ctx, src, options.Documents); err != nil {
return CoreSnapshot{}, err
}
var result CoreSnapshot
err = destination.write(ctx, "migration/portable-core-rehearsal", func(dst *sql.Tx) error {
if destination.backend == PostgreSQL {
// Lock all tables, not just a cooperative advisory key: an accidental
// concurrent writer must not race the empty-destination check.
names := []string{`"schema_migration_steps"`}
for _, s := range coreTables {
names = append(names, quote(s.name))
}
if _, err := dst.ExecContext(ctx, `LOCK TABLE `+strings.Join(names, ",")+` IN ACCESS EXCLUSIVE MODE`); err != nil {
return destination.failure(err)
}
}
if err := destination.checkEmpty(ctx, dst); err != nil {
return err
}
var expected []TableFingerprint
for _, spec := range coreTables {
fingerprint, err := source.streamTable(ctx, src, spec, func(values []any) error {
marks := make([]string, len(values))
for i := range values {
marks[i] = destination.p(i + 1)
}
q := `INSERT INTO ` + quote(spec.name) + ` (` + quoted(spec.columns) + `)`
if spec.identity && destination.backend == PostgreSQL {
q += ` OVERRIDING SYSTEM VALUE`
}
_, err := dst.ExecContext(ctx, q+` VALUES (`+strings.Join(marks, ",")+`)`, values...)
return destination.failure(err)
})
if err != nil {
return err
}
expected = append(expected, fingerprint)
}
if err := destination.verifyCoreRelations(ctx, dst); err != nil {
return err
}
next, err := source.identityNext(ctx, src)
if err != nil {
return err
}
if err := destination.restoreIdentity(ctx, dst, next); err != nil {
return err
}
actual, err := destination.fingerprint(ctx, dst)
if err != nil {
return err
}
want := snapshot(expected, next)
if want.SHA256 != actual.SHA256 {
return ErrIntegrity
}
result = actual
return nil
})
if err != nil {
return CoreSnapshot{}, err
}
return result, nil
}
// FingerprintCore reads a single snapshot and never mutates the database. It
// identifies metadata content, not a backup of canonical payloads or secrets.
func (r *Repository) FingerprintCore(ctx context.Context) (CoreSnapshot, error) {
tx, err := r.database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead})
if err != nil {
return CoreSnapshot{}, r.failure(err)
}
defer tx.Rollback()
report, err := r.inspectPortability(ctx, tx)
if err != nil {
return CoreSnapshot{}, err
}
if !report.CoreCompatible {
return CoreSnapshot{}, ErrMigrationBlocked
}
if err := r.verifyCoreRelations(ctx, tx); err != nil {
return CoreSnapshot{}, err
}
return r.fingerprint(ctx, tx)
}
func (r *Repository) fingerprint(ctx context.Context, tx *sql.Tx) (CoreSnapshot, error) {
var tables []TableFingerprint
for _, spec := range coreTables {
f, err := r.streamTable(ctx, tx, spec, nil)
if err != nil {
return CoreSnapshot{}, err
}
tables = append(tables, f)
}
next, err := r.identityNext(ctx, tx)
if err != nil {
return CoreSnapshot{}, err
}
return snapshot(tables, next), nil
}
func snapshot(tables []TableFingerprint, next int64) CoreSnapshot {
data, _ := json.Marshal(struct {
Tables []TableFingerprint
IdentityNext int64
}{tables, next})
sum := sha256.Sum256(data)
return CoreSnapshot{Scope: "portable-core-rehearsal-only", Tables: tables, IdentityNext: next, SHA256: hex.EncodeToString(sum[:])}
}
func quote(s string) string { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` }
func quoted(names []string) string {
parts := make([]string, len(names))
for i, n := range names {
parts[i] = quote(n)
}
return strings.Join(parts, ",")
}
func frame(h hashpkg.Hash, data []byte) {
var size [8]byte
binary.BigEndian.PutUint64(size[:], uint64(len(data)))
h.Write(size[:])
h.Write(data)
}
func (r *Repository) streamTable(ctx context.Context, tx *sql.Tx, spec tableSpec, consume func([]any) error) (TableFingerprint, error) {
order := make([]string, len(spec.keys))
for i, k := range spec.keys {
order[i] = quote(k)
if !spec.identity {
if r.backend == PostgreSQL {
order[i] += ` COLLATE "C"`
} else {
order[i] += ` COLLATE BINARY`
}
}
}
rows, err := tx.QueryContext(ctx, `SELECT `+quoted(spec.columns)+` FROM `+quote(spec.name)+` ORDER BY `+strings.Join(order, ","))
if err != nil {
return TableFingerprint{}, r.failure(err)
}
defer rows.Close()
h := sha256.New()
frame(h, []byte(spec.name))
data, _ := json.Marshal(spec.columns)
frame(h, data)
var count int64
for rows.Next() {
if err := ctx.Err(); err != nil {
return TableFingerprint{}, err
}
values := make([]any, len(spec.columns))
pointers := make([]any, len(values))
for i := range values {
pointers[i] = &values[i]
}
if err := rows.Scan(pointers...); err != nil {
return TableFingerprint{}, r.failure(err)
}
for i, v := range values {
numeric := numericColumn(spec.columns[i])
switch value := v.(type) {
case string:
if numeric {
return TableFingerprint{}, fmt.Errorf("%w: noninteger in %s.%s", ErrIntegrity, spec.name, spec.columns[i])
}
if !utf8.ValidString(value) || strings.ContainsRune(value, 0) {
return TableFingerprint{}, fmt.Errorf("%w: nonportable text in %s.%s", ErrIntegrity, spec.name, spec.columns[i])
}
if (spec.columns[i] == "document_hash" || spec.columns[i] == "head_document_hash") && !validHash(value) {
return TableFingerprint{}, fmt.Errorf("%w: invalid hash in %s", ErrIntegrity, spec.name)
}
case int64:
if !numeric {
return TableFingerprint{}, fmt.Errorf("%w: nontext in %s.%s", ErrIntegrity, spec.name, spec.columns[i])
}
default:
return TableFingerprint{}, fmt.Errorf("%w: unsupported value type in %s.%s", ErrIntegrity, spec.name, spec.columns[i])
}
}
data, err := json.Marshal(values)
if err != nil {
return TableFingerprint{}, ErrIntegrity
}
frame(h, data)
if consume != nil {
if err := consume(values); err != nil {
return TableFingerprint{}, err
}
}
count++
}
if err := rows.Err(); err != nil {
return TableFingerprint{}, r.failure(err)
}
return TableFingerprint{Table: spec.name, Rows: count, SHA256: hex.EncodeToString(h.Sum(nil))}, nil
}
func (r *Repository) verifyCoreRelations(ctx context.Context, tx *sql.Tx) error {
checks := []struct{ name, query string }{
{"object origin", `SELECT 1 FROM objects o WHERE NOT EXISTS(SELECT 1 FROM object_origins x WHERE x.object_uuid=o.uuid) LIMIT 1`},
{"origin object", `SELECT 1 FROM object_origins x WHERE NOT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=x.object_uuid) LIMIT 1`},
{"path object", `SELECT 1 FROM paths p WHERE NOT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=p.object_uuid) LIMIT 1`},
{"revision object", `SELECT 1 FROM revisions r WHERE NOT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=r.object_uuid) LIMIT 1`},
{"asset object", `SELECT 1 FROM assets a WHERE NOT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=a.object_uuid) LIMIT 1`},
{"revision reference", `SELECT 1 FROM revision_document_refs x WHERE NOT EXISTS(SELECT 1 FROM revisions r WHERE r.uuid=x.revision_uuid) LIMIT 1`},
{"outbox watermark", `SELECT 1 FROM activity_outbox o WHERE o.sequence<1 OR NOT EXISTS(SELECT 1 FROM activity_outbox_heads h WHERE h.domain_uuid=o.domain_uuid AND h.event_type=o.event_type AND h.last_sequence>=o.sequence) LIMIT 1`},
{"revision event", `SELECT 1 FROM activity_outbox o WHERE o.event_type='revision.created' AND NOT EXISTS(SELECT 1 FROM revisions r WHERE r.uuid=o.target_uuid) LIMIT 1`},
{"asset history id", `SELECT 1 FROM asset_head_history WHERE id<1 LIMIT 1`},
}
for _, check := range checks {
var found int
err := tx.QueryRowContext(ctx, check.query).Scan(&found)
if err == nil {
return fmt.Errorf("%w: %s", ErrIntegrity, check.name)
}
if !errors.Is(err, sql.ErrNoRows) {
return r.failure(err)
}
}
return nil
}
func (r *Repository) verifyDocuments(ctx context.Context, tx *sql.Tx, documents geduldh.Store) error {
rows, err := tx.QueryContext(ctx, `SELECT document_hash FROM revisions UNION SELECT head_document_hash FROM assets UNION SELECT document_hash FROM asset_head_history UNION SELECT document_hash FROM revision_document_refs ORDER BY 1`)
if err != nil {
return r.failure(err)
}
defer rows.Close()
for rows.Next() {
var hash string
if err := rows.Scan(&hash); err != nil {
return r.failure(err)
}
if !validHash(hash) {
return ErrIntegrity
}
if err := documents.Verify(ctx, hash); err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
return fmt.Errorf("%w: Document verification failed", ErrIntegrity)
}
}
return r.failure(rows.Err())
}
func (r *Repository) checkEmpty(ctx context.Context, tx *sql.Tx) error {
report, err := r.inspectPortability(ctx, tx)
if err != nil {
return err
}
if !report.CoreCompatible {
return ErrMigrationBlocked
}
for _, s := range coreTables {
var occupied bool
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM `+quote(s.name)+`)`).Scan(&occupied); err != nil {
return r.failure(err)
}
if occupied {
return ErrDestinationNotEmpty
}
}
return nil
}
func (r *Repository) identityNext(ctx context.Context, tx *sql.Tx) (int64, error) {
var max sql.NullInt64
if err := tx.QueryRowContext(ctx, `SELECT max(id) FROM asset_head_history`).Scan(&max); err != nil {
return 0, r.failure(err)
}
if max.Int64 < 0 || max.Int64 == math.MaxInt64 {
return 0, ErrIntegrity
}
next := max.Int64 + 1
if r.backend == SQLite {
var seq sql.NullInt64
if err := tx.QueryRowContext(ctx, `SELECT max(seq) FROM sqlite_sequence WHERE name='asset_head_history'`).Scan(&seq); err != nil {
return 0, r.failure(err)
}
if seq.Int64 < 0 || seq.Int64 == math.MaxInt64 {
return 0, ErrIntegrity
}
if seq.Int64+1 > next {
next = seq.Int64 + 1
}
} else {
var schema, name string
if err := tx.QueryRowContext(ctx, `SELECT n.nspname,c.relname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE c.oid=pg_get_serial_sequence('asset_head_history','id')::regclass`).Scan(&schema, &name); err != nil {
return 0, r.failure(err)
}
var last int64
var called bool
if err := tx.QueryRowContext(ctx, `SELECT last_value,is_called FROM `+quote(schema)+`.`+quote(name)).Scan(&last, &called); err != nil {
return 0, r.failure(err)
}
if last < 1 || last == math.MaxInt64 {
return 0, ErrIntegrity
}
if called {
last++
}
if last > next {
next = last
}
}
return next, nil
}
func (r *Repository) restoreIdentity(ctx context.Context, tx *sql.Tx, next int64) error {
if next < 1 {
return ErrIntegrity
}
if r.backend == PostgreSQL {
// RESTART rolls back with the transaction; setval would not.
_, err := tx.ExecContext(ctx, `ALTER TABLE asset_head_history ALTER COLUMN id RESTART WITH `+strconv.FormatInt(next, 10))
return r.failure(err)
}
result, err := tx.ExecContext(ctx, `UPDATE sqlite_sequence SET seq=? WHERE name='asset_head_history'`, next-1)
if err != nil {
return r.failure(err)
}
n, err := result.RowsAffected()
if err != nil {
return r.failure(err)
}
if n == 0 {
_, err = tx.ExecContext(ctx, `INSERT INTO sqlite_sequence(name,seq) VALUES ('asset_head_history',?)`, next-1)
}
return r.failure(err)
}
func sameDatabase(ctx context.Context, a, b *Repository) (bool, error) {
if a.backend != b.backend {
return false, nil
}
if a.backend == SQLite {
path := func(r *Repository) (string, error) {
var seq int
var name, file string
err := r.database.QueryRowContext(ctx, `SELECT seq,name,file FROM pragma_database_list WHERE name='main'`).Scan(&seq, &name, &file)
if err != nil {
return "", r.failure(err)
}
if file == "" {
return "", ErrMigrationBlocked
}
return file, nil
}
x, err := path(a)
if err != nil {
return false, err
}
y, err := path(b)
if err != nil {
return false, err
}
xInfo, err := os.Stat(x)
if err != nil {
return false, ErrMigrationBlocked
}
yInfo, err := os.Stat(y)
if err != nil {
return false, ErrMigrationBlocked
}
return os.SameFile(xInfo, yInfo), nil
}
type identity struct{ database, schema, start string }
read := func(r *Repository) (identity, error) {
var i identity
err := r.database.QueryRowContext(ctx, `SELECT current_database(),current_schema(),extract(epoch from pg_postmaster_start_time())::text`).Scan(&i.database, &i.schema, &i.start)
return i, r.failure(err)
}
x, err := read(a)
if err != nil {
return false, err
}
y, err := read(b)
if err != nil {
return false, err
}
return x == y, nil
}
+321
View File
@@ -0,0 +1,321 @@
package metadata
import (
"context"
"errors"
"net/url"
"os"
"strings"
"testing"
"time"
"pageswiki/internal/db"
"pageswiki/internal/geduldh"
)
func TestPortabilityIsFailClosed(t *testing.T) {
for _, backend := range []Backend{SQLite, PostgreSQL} {
for _, change := range []string{"table", "view", "column", "history", "component"} {
t.Run(string(backend)+"/"+change, func(t *testing.T) {
r := testRepository(t, backend)
ctx := context.Background()
report, err := r.InspectPortability(ctx)
if err != nil || !report.CoreCompatible || report.ApplicationReady {
t.Fatal("fresh core readiness", err, report)
}
q := map[string]string{
"table": `CREATE TABLE auth_private_users(secret TEXT)`,
"view": `CREATE VIEW private_view AS SELECT uuid FROM objects`,
"column": `ALTER TABLE objects ADD COLUMN unexpected TEXT`,
"history": `UPDATE schema_migration_steps SET checksum='tampered' WHERE version=1`,
"component": `INSERT INTO schema_migration_steps(component,version,name,checksum,applied_at_ms) VALUES ('future',1,'other','unknown',1)`,
}[change]
if _, err := r.database.ExecContext(ctx, q); err != nil {
t.Fatal(err)
}
report, err = r.InspectPortability(ctx)
if err != nil || report.CoreCompatible || len(report.Blockers) == 0 {
t.Fatal("unsupported state accepted", err, report)
}
if _, err := r.FingerprintCore(ctx); !errors.Is(err, ErrMigrationBlocked) {
t.Fatal("unsupported state fingerprinted as portable", err)
}
})
}
}
}
func TestPortabilityDoesNotHideSameNamedTrigger(t *testing.T) {
r := testRepository(t, SQLite)
ctx := context.Background()
if _, err := r.database.ExecContext(ctx, `CREATE TRIGGER objects AFTER UPDATE ON objects BEGIN SELECT 1; END`); err != nil {
t.Fatal(err)
}
report, err := r.InspectPortability(ctx)
if err != nil || report.CoreCompatible {
t.Fatal("security trigger hidden by table name", err)
}
if !strings.Contains(strings.Join(report.Blockers, " "), "trigger: objects") {
t.Fatal("trigger not identified", report)
}
}
func seedRehearsal(t *testing.T, r *Repository) (geduldh.Store, Revision) {
t.Helper()
ctx := context.Background()
docs, _, v := documentFixture(t, r)
first, err := r.SaveDocument(ctx, docs, "test", v, []byte(`<h1>移行</h1><img src="asset:image:cover">`), "", "actor")
if err != nil {
t.Fatal(err)
}
if err := r.BindPath(ctx, "test", "/alias", v.ObjectUUID); err != nil {
t.Fatal(err)
}
for _, path := range []string{"/日本語%_", "/Case", "/case", "/é", "/e\u0301"} {
if err := r.BindPath(ctx, "test", path, v.ObjectUUID); err != nil {
t.Fatal(err)
}
}
// Values above JavaScript's safe-integer range must never pass through
// float64 on the migration path. Also exercise identity/sequence recovery.
const large = int64(9007199254741011)
if _, err := r.database.ExecContext(ctx, `UPDATE asset_head_history SET id=`+r.p(1), large); err != nil {
t.Fatal(err)
}
if _, err := r.database.ExecContext(ctx, `UPDATE activity_outbox SET sequence=`+r.p(1)+`,status='failed',attempts=3,last_error='synthetic'`, large); err != nil {
t.Fatal(err)
}
if _, err := r.database.ExecContext(ctx, `UPDATE activity_outbox_heads SET last_sequence=`+r.p(1), large); err != nil {
t.Fatal(err)
}
return docs, first.Revision
}
func TestCoreFingerprintIncludesContentAndPurgedIdentity(t *testing.T) {
source := testRepository(t, SQLite)
target := testRepository(t, SQLite)
ctx := context.Background()
docs, v := seedRehearsal(t, source)
before, err := source.FingerprintCore(ctx)
if err != nil {
t.Fatal(err)
}
if _, err := source.database.ExecContext(ctx, `UPDATE objects SET kind='changed' WHERE uuid=?`, v.ObjectUUID); err != nil {
t.Fatal(err)
}
after, err := source.FingerprintCore(ctx)
if err != nil || after.SHA256 == before.SHA256 {
t.Fatal("same row counts hid content change", err)
}
// A retained generator high-water mark must survive even if the highest
// history rows have been purged. Reusing an old history URL is not safe.
if _, err := source.database.ExecContext(ctx, `UPDATE sqlite_sequence SET seq=9007199254741999 WHERE name='asset_head_history'`); err != nil {
t.Fatal(err)
}
result, err := CloneCoreForRehearsal(ctx, source, target, RehearsalOptions{Offline: true, Documents: docs})
if err != nil || result.IdentityNext != 9007199254742000 {
t.Fatal("purged sequence lost", err, result.IdentityNext)
}
a, err := target.GetAsset(ctx, "test", "asset:image:cover")
if err != nil {
t.Fatal(err)
}
if _, err := target.PutAsset(ctx, a, a.HeadDocumentHash, "actor"); err != nil {
t.Fatal(err)
}
var id int64
if err := target.database.QueryRowContext(ctx, `SELECT max(id) FROM asset_head_history`).Scan(&id); err != nil || id != result.IdentityNext {
t.Fatal("old history ID reused", err, id)
}
if _, err := source.database.ExecContext(ctx, `UPDATE objects SET created_at='not-an-integer'`); err != nil {
t.Fatal(err)
}
if _, err := source.FingerprintCore(ctx); !errors.Is(err, ErrIntegrity) {
t.Fatal("invalid integer type accepted", err)
}
}
func TestCoreRoundTripSQLitePostgreSQLSQLite(t *testing.T) {
source := testRepository(t, SQLite)
target := testRepository(t, PostgreSQL)
restored := testRepository(t, SQLite)
ctx := context.Background()
docs, first := seedRehearsal(t, source)
before, err := source.FingerprintCore(ctx)
if err != nil {
t.Fatal(err)
}
result, err := CloneCoreForRehearsal(ctx, source, target, RehearsalOptions{Offline: true, Documents: docs})
if err != nil || result.SHA256 != before.SHA256 {
t.Fatal("SQLite to PostgreSQL", err, result)
}
got, err := target.Head(ctx, "test", first.ObjectUUID, first.Language)
if err != nil || got != first {
t.Fatal("revision identity/content changed", err)
}
asset, err := target.GetAsset(ctx, "test", "asset:image:cover")
if err != nil {
t.Fatal(err)
}
asset.Filename = "new.png"
if _, err := target.PutAsset(ctx, asset, asset.HeadDocumentHash, "actor"); err != nil {
t.Fatal("identity sequence not repaired", err)
}
var id int64
if err := target.database.QueryRowContext(ctx, `SELECT max(id) FROM asset_head_history`).Scan(&id); err != nil || id != 9007199254741012 {
t.Fatal("identity precision/restart", id, err)
}
if _, err := target.database.ExecContext(ctx, `ALTER TABLE asset_head_history ALTER COLUMN id RESTART WITH 9007199254743000`); err != nil {
t.Fatal(err)
}
next, err := target.SaveDocument(ctx, docs, "test", first, []byte(`<h1>PostgreSQLで更新</h1>`), first.UUID, "actor")
if err != nil {
t.Fatal(err)
}
targetState, err := target.FingerprintCore(ctx)
if err != nil {
t.Fatal(err)
}
back, err := CloneCoreForRehearsal(ctx, target, restored, RehearsalOptions{Offline: true, Documents: docs})
if err != nil || back.SHA256 != targetState.SHA256 {
t.Fatal("PostgreSQL to SQLite", err)
}
if got, err := restored.Head(ctx, "test", first.ObjectUUID, first.Language); err != nil || got.UUID != next.Revision.UUID {
t.Fatal("post-cutover update lost on rollback", err)
}
if got, err := source.FingerprintCore(ctx); err != nil || got.SHA256 != before.SHA256 {
t.Fatal("source was modified", err)
}
if _, err := restored.PutAsset(ctx, asset, asset.HeadDocumentHash, "actor"); err != nil {
t.Fatal("SQLite sequence not recovered", err)
}
if err := restored.database.QueryRowContext(ctx, `SELECT max(id) FROM asset_head_history`).Scan(&id); err != nil || id != 9007199254743000 {
t.Fatal("PostgreSQL retained generator state lost", id, err)
}
}
func TestCoreCloneRejectsSameDatabaseThroughDifferentPools(t *testing.T) {
for _, backend := range []Backend{SQLite, PostgreSQL} {
t.Run(string(backend), func(t *testing.T) {
r := testRepository(t, backend)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
var alias *Repository
if backend == SQLite {
var file string
if err := r.database.QueryRow(`SELECT file FROM pragma_database_list WHERE name='main'`).Scan(&file); err != nil {
t.Fatal(err)
}
database, err := db.Open(file)
if err != nil {
t.Fatal(err)
}
defer database.Close()
alias, _ = New(database, backend)
} else {
var schema string
if err := r.database.QueryRow(`SELECT current_schema()`).Scan(&schema); err != nil {
t.Fatal(err)
}
u, err := url.Parse(os.Getenv("PAGESWIKI_TEST_POSTGRES_DSN"))
if err != nil {
t.Fatal("invalid test DSN")
}
q := u.Query()
q.Set("search_path", schema)
u.RawQuery = q.Encode()
database, err := db.OpenPostgreSQL(ctx, db.PostgreSQLOptions{DSN: u.String(), AllowInsecure: true})
if err != nil {
t.Fatal(err)
}
defer database.Close()
alias, _ = New(database, backend)
}
documents, _ := geduldh.NewLocal(t.TempDir())
if _, err := CloneCoreForRehearsal(ctx, r, alias, RehearsalOptions{Offline: true, Documents: documents}); !errors.Is(err, ErrMigrationBlocked) {
t.Fatal("self-copy not rejected before locking", err)
}
})
}
}
func TestCoreCloneRejectsUnsafeConditionsAndRollsBack(t *testing.T) {
for _, backend := range []Backend{SQLite, PostgreSQL} {
t.Run(string(backend), func(t *testing.T) {
source := testRepository(t, SQLite)
target := testRepository(t, backend)
ctx := context.Background()
docs, _ := seedRehearsal(t, source)
if _, err := CloneCoreForRehearsal(ctx, source, target, RehearsalOptions{Documents: docs}); !errors.Is(err, ErrMigrationBlocked) {
t.Fatal("online migration permitted", err)
}
if _, err := CloneCoreForRehearsal(ctx, source, target, RehearsalOptions{Offline: true}); !errors.Is(err, ErrMigrationBlocked) {
t.Fatal("unverified Documents permitted", err)
}
empty, err := target.FingerprintCore(ctx)
if err != nil {
t.Fatal(err)
}
// Extra uniqueness rules must block the plan before any copying.
if _, err := target.database.ExecContext(ctx, `CREATE UNIQUE INDEX synthetic_rehearsal_failure ON paths(object_uuid)`); err != nil {
t.Fatal(err)
}
if _, err := CloneCoreForRehearsal(ctx, source, target, RehearsalOptions{Offline: true, Documents: docs}); !errors.Is(err, ErrMigrationBlocked) {
t.Fatal("unexpected unique index accepted", err)
}
if _, err := target.database.ExecContext(ctx, `DROP INDEX synthetic_rehearsal_failure`); err != nil {
t.Fatal(err)
}
if got, err := target.FingerprintCore(ctx); err != nil || got.SHA256 != empty.SHA256 {
t.Fatal("partial destination rows survived", err)
}
// SQLite can contain a BLOB despite a TEXT declaration. Detect it
// while streaming, after earlier tables have been inserted, and roll
// back every destination row instead of coercing or truncating it.
if _, err := source.database.ExecContext(ctx, `UPDATE paths SET path=CAST(path AS BLOB)`); err != nil {
t.Fatal(err)
}
if _, err := CloneCoreForRehearsal(ctx, source, target, RehearsalOptions{Offline: true, Documents: docs}); !errors.Is(err, ErrIntegrity) {
t.Fatal("nonportable row accepted", err)
}
if got, err := target.FingerprintCore(ctx); err != nil || got.SHA256 != empty.SHA256 {
t.Fatal("mid-copy failure left rows", err)
}
if _, err := source.database.ExecContext(ctx, `UPDATE paths SET path=CAST(path AS TEXT)`); err != nil {
t.Fatal(err)
}
if _, err := target.CreatePage(ctx, "test", "/do-not-overwrite"); err != nil {
t.Fatal(err)
}
occupied, err := target.FingerprintCore(ctx)
if err != nil {
t.Fatal(err)
}
if _, err := CloneCoreForRehearsal(ctx, source, target, RehearsalOptions{Offline: true, Documents: docs}); !errors.Is(err, ErrDestinationNotEmpty) {
t.Fatal("occupied destination overwritten", err)
}
if got, err := target.FingerprintCore(ctx); err != nil || got.SHA256 != occupied.SHA256 {
t.Fatal("occupied target changed", err)
}
})
}
}
func TestCoreCloneRejectsMissingDocumentsAndBrokenRelations(t *testing.T) {
r := testRepository(t, SQLite)
target := testRepository(t, SQLite)
ctx := context.Background()
docs, _ := seedRehearsal(t, r)
if _, err := r.database.ExecContext(ctx, `UPDATE paths SET object_uuid='missing'`); err != nil {
t.Fatal(err)
}
if _, err := CloneCoreForRehearsal(ctx, r, target, RehearsalOptions{Offline: true, Documents: docs}); !errors.Is(err, ErrIntegrity) {
t.Fatal("broken references copied", err)
}
r = testRepository(t, SQLite)
docs, _ = seedRehearsal(t, r)
missing, _ := geduldh.NewLocal(t.TempDir())
if _, err := CloneCoreForRehearsal(ctx, r, target, RehearsalOptions{Offline: true, Documents: missing}); !errors.Is(err, ErrIntegrity) {
t.Fatal("missing Document accepted", err)
}
}
+445
View File
@@ -0,0 +1,445 @@
// Package metadata contains the portable metadata persistence contract. It does
// not authorize requests, store payloads, run legacy Store initialization, or
// constitute a PostgreSQL-capable application. Payloads must be persisted and
// verified in Geduldh before publishing their hash here.
package metadata
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/binary"
"errors"
"fmt"
"math"
"strings"
"time"
"unicode/utf8"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgconn"
"golang.org/x/text/language"
"modernc.org/sqlite"
)
type Backend string
const (
SQLite Backend = "sqlite"
PostgreSQL Backend = "postgres"
)
var (
ErrNotFound = errors.New("metadata not found")
ErrConflict = errors.New("metadata conflict")
ErrInvalid = errors.New("invalid metadata")
)
type Repository struct {
database *sql.DB
backend Backend
applicationNamespace bool
}
// New never migrates a schema or creates keys. Ownership of database stays with
// the caller. Backend must match the actual connection; no fallback is allowed.
func New(database *sql.DB, backend Backend) (*Repository, error) {
if database == nil || (backend != SQLite && backend != PostgreSQL) {
return nil, ErrInvalid
}
return &Repository{database: database, backend: backend}, nil
}
// NewApplication uses the application's namespace/administration lock before
// the metadata Domain/Object locks. Namespace validation is repeated inside
// the commit transaction. This does not replace request authorization, perform
// migrations, or make an incomplete application Store ready to serve traffic.
func NewApplication(database *sql.DB, backend Backend) (*Repository, error) {
r, err := New(database, backend)
if err == nil {
r.applicationNamespace = true
}
return r, err
}
// p generates a placeholder, rather than rewriting arbitrary SQL (which can
// contain JSON operators, strings or comments with literal question marks).
func (r *Repository) p(n int) string {
if r.backend == PostgreSQL {
return fmt.Sprintf("$%d", n)
}
return "?"
}
func (r *Repository) failure(err error) error {
if err == nil {
return nil
}
if errors.Is(err, sql.ErrNoRows) {
return ErrNotFound
}
if errors.Is(err, context.Canceled) {
return context.Canceled
}
if errors.Is(err, context.DeadlineExceeded) {
return context.DeadlineExceeded
}
var pg *pgconn.PgError
if errors.As(err, &pg) {
if pg.Code == "23505" || pg.Code == "40001" || pg.Code == "40P01" {
return ErrConflict
}
return fmt.Errorf("PostgreSQL metadata operation failed (SQLSTATE %s)", pg.Code)
}
var se *sqlite.Error
if errors.As(err, &se) {
switch se.Code() {
case 1555, 2067:
return ErrConflict
}
return fmt.Errorf("SQLite metadata operation failed (code %d)", se.Code())
}
// Network errors may include a credential-bearing DSN or server addresses.
return errors.New("metadata database operation failed")
}
func canonicalPath(raw string) (string, error) {
p := strings.TrimSpace(raw)
if !strings.HasPrefix(p, "/") {
return "", ErrInvalid
}
if p != "/" {
p = strings.TrimRight(p, "/")
}
if p == "" || !utf8.ValidString(p) || strings.ContainsAny(p, "\x00\r\n") {
return "", ErrInvalid
}
if p != "/" {
for _, part := range strings.Split(p[1:], "/") {
if part == "" || part == "." || part == ".." {
return "", ErrInvalid
}
}
}
return p, nil
}
func validHash(hash string) bool {
if len(hash) != 64 {
return false
}
for _, c := range hash {
if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') {
return false
}
}
return true
}
// write locks before any read. SQLite acquires its single-writer reservation;
// PostgreSQL locks a stable, namespaced key for the whole transaction. Path
// writers share a domain key; revisions additionally take an object key after
// the domain key, even when an object is bound in more than one domain.
func (r *Repository) write(ctx context.Context, key string, fn func(*sql.Tx) error) error {
tx, err := r.database.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
if err != nil {
return r.failure(err)
}
defer tx.Rollback()
if r.backend == PostgreSQL {
if r.applicationNamespace {
if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(1885431653,1853058931)`); err != nil {
return r.failure(err)
}
}
err = r.lock(ctx, tx, key)
} else {
_, err = tx.ExecContext(ctx, `UPDATE paths SET created_at=created_at WHERE 0`)
}
if err != nil {
return r.failure(err)
}
if r.applicationNamespace {
if !strings.HasPrefix(key, "domain/") {
return ErrInvalid
}
var local bool
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM domains d JOIN campuses c ON c.uuid=d.campus_uuid WHERE d.uuid=`+r.p(1)+` AND d.enabled=1 AND c.enabled=1 AND d.placement='local')`, strings.TrimPrefix(key, "domain/")).Scan(&local); err != nil {
return r.failure(err)
}
if !local {
return ErrNotFound
}
}
if err = fn(tx); err != nil {
return err
}
return r.failure(tx.Commit())
}
func (r *Repository) lock(ctx context.Context, tx *sql.Tx, key string) error {
if r.backend != PostgreSQL {
return nil
}
h := sha256.Sum256([]byte("pageswiki/metadata/" + key))
_, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock($1)`, int64(binary.BigEndian.Uint64(h[:8])))
return err
}
func (r *Repository) CreatePage(ctx context.Context, domain, path string) (string, error) {
p, err := canonicalPath(path)
if err != nil || strings.TrimSpace(domain) == "" {
return "", ErrInvalid
}
id := uuid.NewString()
err = r.write(ctx, "domain/"+domain, func(tx *sql.Tx) error {
now := time.Now().Unix()
for _, q := range []struct {
sql string
args []any
}{
{`INSERT INTO objects(uuid,kind,created_at) VALUES (` + r.p(1) + `,'page',` + r.p(2) + `)`, []any{id, now}},
{`INSERT INTO object_origins(object_uuid,domain_uuid,created_at) VALUES (` + r.p(1) + `,` + r.p(2) + `,` + r.p(3) + `)`, []any{id, domain, now}},
{`INSERT INTO paths(domain_uuid,path,object_uuid,created_at,is_primary) VALUES (` + r.p(1) + `,` + r.p(2) + `,` + r.p(3) + `,` + r.p(4) + `,1)`, []any{domain, p, id, now}},
} {
if _, err := tx.ExecContext(ctx, q.sql, q.args...); err != nil {
return r.failure(err)
}
}
if r.applicationNamespace {
return r.appendPageOutbox(ctx, tx, domain, p, id)
}
return nil
})
if err != nil {
return "", err
}
return id, nil
}
// CreateObject records provenance for the explicit create/save/bind API.
// It does not grant authority over an arbitrary unbound legacy Object.
func (r *Repository) CreateObject(ctx context.Context, domain, kind string) (string, error) {
if domain == "" || strings.TrimSpace(kind) == "" {
return "", ErrInvalid
}
id := uuid.NewString()
err := r.write(ctx, "domain/"+domain, func(tx *sql.Tx) error {
now := time.Now().Unix()
if _, err := tx.ExecContext(ctx, `INSERT INTO objects(uuid,kind,created_at) VALUES(`+r.p(1)+`,`+r.p(2)+`,`+r.p(3)+`)`, id, kind, now); err != nil {
return r.failure(err)
}
_, err := tx.ExecContext(ctx, `INSERT INTO object_origins(object_uuid,domain_uuid,created_at) VALUES(`+r.p(1)+`,`+r.p(2)+`,`+r.p(3)+`)`, id, domain, now)
return r.failure(err)
})
if err != nil {
return "", err
}
return id, nil
}
func (r *Repository) ResolvePath(ctx context.Context, domain, path string) (string, error) {
var id string
err := r.database.QueryRowContext(ctx, `SELECT object_uuid FROM paths WHERE domain_uuid=`+r.p(1)+` AND path=`+r.p(2), domain, path).Scan(&id)
return id, r.failure(err)
}
// BindPath creates an alias without replacing an existing binding. Cross-domain
// sharing requires a separate explicit authorization/import contract.
func (r *Repository) BindPath(ctx context.Context, domain, path, object string) error {
p, err := canonicalPath(path)
if err != nil || domain == "" || object == "" {
return ErrInvalid
}
return r.write(ctx, "domain/"+domain, func(tx *sql.Tx) error {
var count int
if err := tx.QueryRowContext(ctx, `SELECT count(*) FROM object_origins WHERE object_uuid=`+r.p(1)+` AND domain_uuid=`+r.p(2), object, domain).Scan(&count); err != nil {
return r.failure(err)
}
if count == 0 {
return ErrNotFound
}
_, err := tx.ExecContext(ctx, `INSERT INTO paths(domain_uuid,path,object_uuid,created_at,is_primary) VALUES (`+r.p(1)+`,`+r.p(2)+`,`+r.p(3)+`,`+r.p(4)+`,0)`, domain, p, object, time.Now().Unix())
return r.failure(err)
})
}
// RenameTree moves both real and virtual folders without LIKE wildcard or
// case-folding semantics. It does not merge into occupied destinations. City,
// ACL, activity outbox and cache invalidation must be coordinated by the future
// application-level transaction; do not substitute this for legacy UI rename yet.
func (r *Repository) RenameTree(ctx context.Context, domain, source, destination string) error {
from, e1 := canonicalPath(source)
to, e2 := canonicalPath(destination)
if e1 != nil || e2 != nil || domain == "" || from == "/" || to == "/" || from == to || strings.HasPrefix(to, from+"/") || strings.HasPrefix(from, to+"/") {
return ErrInvalid
}
return r.write(ctx, "domain/"+domain, func(tx *sql.Tx) error {
var occupied int
// Binary/C collation makes [prefix+'/', prefix+'0') an exact,
// indexable descendant range, including Unicode and literal %/_ names.
err := tx.QueryRowContext(ctx, `SELECT 1 FROM paths WHERE domain_uuid=`+r.p(1)+` AND (path=`+r.p(2)+` OR (path>=`+r.p(3)+` AND path<`+r.p(4)+`)) LIMIT 1`, domain, to, to+"/", to+"0").Scan(&occupied)
if err == nil {
return ErrConflict
}
if !errors.Is(err, sql.ErrNoRows) {
return r.failure(err)
}
// A single SQL update avoids loading the subtree or issuing one query
// per path. substr counts characters (not UTF-8 bytes) on both DBs.
result, err := tx.ExecContext(ctx, `UPDATE paths SET path=`+r.p(1)+` || substr(path,`+r.p(2)+`) WHERE domain_uuid=`+r.p(3)+` AND (path=`+r.p(4)+` OR (path>=`+r.p(5)+` AND path<`+r.p(6)+`))`, to, len([]rune(from))+1, domain, from, from+"/", from+"0")
if err != nil {
return r.failure(err)
}
count, err := result.RowsAffected()
if err != nil {
return r.failure(err)
}
if count == 0 {
return ErrNotFound
}
return nil
})
}
type Revision struct {
UUID string
ObjectUUID string
DocumentHash string
Language string
ContentType string
Filename string
Message string
CreatedAtMS int64
}
func normalizedLanguage(raw string) (string, error) {
if strings.TrimSpace(raw) == "" {
return "und", nil
}
tag, err := language.Parse(raw)
if err != nil {
return "", ErrInvalid
}
return tag.String(), nil
}
const revisionColumns = `uuid,object_uuid,document_hash,lang,content_type,filename,message,created_at_ms`
// Application create/save/bind records creation provenance before any Path.
// Provenance ceases to grant scope as soon as any live binding exists.
// Expressions are owned SQL identifiers/placeholders, never request strings.
func (r *Repository) objectDomainScope(object, domain string) string {
scope := `SELECT domain_uuid FROM paths WHERE object_uuid=` + object
if r.applicationNamespace {
scope += ` UNION ALL SELECT domain_uuid FROM object_origins WHERE object_uuid=` + object + ` AND NOT EXISTS(SELECT 1 FROM paths WHERE object_uuid=` + object + `)`
}
return domain + ` IN (` + scope + `)`
}
func scanRevision(row interface{ Scan(...any) error }) (Revision, error) {
var v Revision
err := row.Scan(&v.UUID, &v.ObjectUUID, &v.DocumentHash, &v.Language, &v.ContentType, &v.Filename, &v.Message, &v.CreatedAtMS)
return v, err
}
func (r *Repository) Head(ctx context.Context, domain, object, lang string) (Revision, error) {
l, err := normalizedLanguage(lang)
if err != nil {
return Revision{}, err
}
v, err := scanRevision(r.database.QueryRowContext(ctx, `SELECT `+revisionColumns+` FROM revisions r WHERE object_uuid=`+r.p(1)+` AND lang=`+r.p(2)+` AND (`+r.objectDomainScope("r.object_uuid", r.p(3))+`) ORDER BY created_at_ms DESC,uuid DESC LIMIT 1`, object, l, domain))
return v, r.failure(err)
}
func (r *Repository) GetRevision(ctx context.Context, domain, revisionUUID string) (Revision, error) {
v, err := scanRevision(r.database.QueryRowContext(ctx, `SELECT `+revisionColumns+` FROM revisions r WHERE uuid=`+r.p(1)+` AND (`+r.objectDomainScope("r.object_uuid", r.p(2))+`)`, revisionUUID, domain))
return v, r.failure(err)
}
// RestoreRevision appends a new revision pointing to the historical Document;
// it never rewrites old revision metadata or bypasses the exact-language CAS.
func (r *Repository) RestoreRevision(ctx context.Context, domain, revisionUUID, expected, message string) (Revision, error) {
v, err := r.GetRevision(ctx, domain, revisionUUID)
if err != nil {
return Revision{}, err
}
v.Message = message
refs, err := r.revisionReferences(ctx, domain, revisionUUID)
if err != nil {
return Revision{}, err
}
return r.appendRevision(ctx, domain, v, expected, refs, "")
}
// AppendRevision uses exact-language compare-and-swap. Empty expected means
// there is no previous revision in this language. All writes use this contract;
// monotonic timestamps remove dependence on SQLite's hidden rowid ordering.
func (r *Repository) AppendRevision(ctx context.Context, domain string, v Revision, expected string) (Revision, error) {
return r.appendRevision(ctx, domain, v, expected, nil, "")
}
func (r *Repository) appendRevision(ctx context.Context, domain string, v Revision, expected string, refs []string, actor string) (Revision, error) {
return r.appendRevisionExpected(ctx, domain, v, &expected, refs, actor)
}
func (r *Repository) appendRevisionExpected(ctx context.Context, domain string, v Revision, expected *string, refs []string, actor string) (Revision, error) {
lang, err := normalizedLanguage(v.Language)
if err != nil || !validHash(v.DocumentHash) || v.ObjectUUID == "" || strings.TrimSpace(v.ContentType) == "" {
return Revision{}, ErrInvalid
}
v.UUID = uuid.NewString()
v.Language = lang
err = r.write(ctx, "domain/"+domain, func(tx *sql.Tx) error {
// Always acquire Domain before Object. This also protects per-Domain
// outbox sequence allocation for different objects and languages.
if err := r.lock(ctx, tx, "object/"+v.ObjectUUID); err != nil {
return r.failure(err)
}
var bound bool
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=`+r.p(1)+` AND (`+r.objectDomainScope("o.uuid", r.p(2))+`))`, v.ObjectUUID, domain).Scan(&bound); err != nil {
return r.failure(err)
}
if !bound {
return ErrNotFound
}
var current string
var previous int64
err := tx.QueryRowContext(ctx, `SELECT uuid,created_at_ms FROM revisions WHERE object_uuid=`+r.p(1)+` AND lang=`+r.p(2)+` ORDER BY created_at_ms DESC,uuid DESC LIMIT 1`, v.ObjectUUID, lang).Scan(&current, &previous)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return r.failure(err)
}
if expected != nil && current != *expected {
return ErrConflict
}
if previous == math.MaxInt64 {
return ErrInvalid
}
v.CreatedAtMS = time.Now().UnixMilli()
if v.CreatedAtMS <= previous {
v.CreatedAtMS = previous + 1
}
_, err = tx.ExecContext(ctx, `INSERT INTO revisions(uuid,object_uuid,document_hash,lang,content_type,filename,message,created_at_ms,created_at,body) VALUES (`+r.p(1)+`,`+r.p(2)+`,`+r.p(3)+`,`+r.p(4)+`,`+r.p(5)+`,`+r.p(6)+`,`+r.p(7)+`,`+r.p(8)+`,`+r.p(9)+`,'')`, v.UUID, v.ObjectUUID, v.DocumentHash, v.Language, v.ContentType, v.Filename, v.Message, v.CreatedAtMS, v.CreatedAtMS/1000)
if err != nil {
return r.failure(err)
}
seen := map[string]bool{}
for _, hash := range refs {
if !validHash(hash) {
return ErrInvalid
}
if seen[hash] {
continue
}
seen[hash] = true
if _, err := tx.ExecContext(ctx, `INSERT INTO revision_document_refs(revision_uuid,document_hash) VALUES (`+r.p(1)+`,`+r.p(2)+`)`, v.UUID, hash); err != nil {
return r.failure(err)
}
}
return r.appendRevisionOutbox(ctx, tx, domain, actor, v)
})
if err != nil {
return Revision{}, err
}
return v, nil
}
+362
View File
@@ -0,0 +1,362 @@
package metadata
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"errors"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/google/uuid"
"pageswiki/internal/db"
)
func testRepository(t *testing.T, backend Backend) *Repository {
t.Helper()
var database *sql.DB
var err error
if backend == SQLite {
database, err = db.Open(filepath.Join(t.TempDir(), "metadata.db"))
} else {
dsn := os.Getenv("PAGESWIKI_TEST_POSTGRES_DSN")
if dsn == "" {
t.Skip("set PAGESWIKI_TEST_POSTGRES_DSN to a disposable PostgreSQL URL")
}
u, e := url.Parse(dsn)
if e != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") {
t.Fatal("test DSN must be PostgreSQL URL")
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
root, e := db.OpenPostgreSQL(ctx, db.PostgreSQLOptions{DSN: dsn, AllowInsecure: true})
if e != nil {
t.Fatal(e)
}
schema := "pageswiki_metadata_test_" + strings.ReplaceAll(uuid.NewString(), "-", "")
if _, e = root.ExecContext(ctx, `CREATE SCHEMA "`+schema+`"`); e != nil {
root.Close()
t.Fatal("create synthetic test schema failed")
}
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if _, e := root.ExecContext(ctx, `DROP SCHEMA "`+schema+`" CASCADE`); e != nil {
t.Error("drop synthetic test schema failed")
}
root.Close()
})
query := u.Query()
query.Set("search_path", schema)
u.RawQuery = query.Encode()
database, err = db.OpenPostgreSQL(ctx, db.PostgreSQLOptions{DSN: u.String(), AllowInsecure: true})
}
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { database.Close() })
r, err := New(database, backend)
if err != nil {
t.Fatal(err)
}
if err := r.InitializeFresh(context.Background()); err != nil {
t.Fatal(err)
}
if err := r.InitializeFresh(context.Background()); err != nil {
t.Fatal("repeated initialization", err)
}
return r
}
func hash(s string) string { v := sha256.Sum256([]byte(s)); return hex.EncodeToString(v[:]) }
func TestMetadataContract(t *testing.T) {
for _, backend := range []Backend{SQLite, PostgreSQL} {
t.Run(string(backend), func(t *testing.T) {
r := testRepository(t, backend)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
mustPage := func(domain, path string) string {
t.Helper()
id, err := r.CreatePage(ctx, domain, path)
if err != nil {
t.Fatal(err)
}
return id
}
id := mustPage("domain-a", "/学区%_/通り/本")
other := mustPage("domain-b", "/学区%_/通り/本")
if id == other {
t.Fatal("object UUID reused")
}
if got, err := r.ResolvePath(ctx, "domain-a", "/学区%_/通り/本"); err != nil || got != id {
t.Fatal("path resolution", err)
}
if _, err := r.ResolvePath(ctx, "unknown", "/学区%_/通り/本"); !errors.Is(err, ErrNotFound) {
t.Fatal("domain isolation", err)
}
if _, err := r.CreatePage(ctx, "domain-a", "/学区%_/通り/本"); !errors.Is(err, ErrConflict) {
t.Fatal("duplicate path accepted", err)
}
var count int
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM objects`).Scan(&count); err != nil || count != 2 {
t.Fatal("failed create left orphan object", err, count)
}
if err := r.BindPath(ctx, "domain-a", "/alias", id); err != nil {
t.Fatal(err)
}
if err := r.BindPath(ctx, "domain-b", "/stolen", id); !errors.Is(err, ErrNotFound) {
t.Fatal("foreign alias accepted", err)
}
mustPage("domain-a", "/学区XX/通り/本")
mustPage("domain-a", "/学区%_suffix/本")
if err := r.RenameTree(ctx, "domain-a", "/学区%_", "/新しい学区"); err != nil {
t.Fatal("virtual folder rename", err)
}
if got, err := r.ResolvePath(ctx, "domain-a", "/新しい学区/通り/本"); err != nil || got != id {
t.Fatal("UUID not preserved", err)
}
for _, path := range []string{"/学区XX/通り/本", "/学区%_suffix/本"} {
if _, err := r.ResolvePath(ctx, "domain-a", path); err != nil {
t.Fatal("wildcard/boundary changed unrelated path", err)
}
}
if got, err := r.ResolvePath(ctx, "domain-b", "/学区%_/通り/本"); err != nil || got != other {
t.Fatal("rename crossed domain", err)
}
mustPage("domain-a", "/occupied/something")
if err := r.RenameTree(ctx, "domain-a", "/新しい学区", "/occupied"); !errors.Is(err, ErrConflict) {
t.Fatal("destination merged", err)
}
if _, err := r.ResolvePath(ctx, "domain-a", "/新しい学区/通り/本"); err != nil {
t.Fatal("failed move changed source", err)
}
if err := r.RenameTree(ctx, "domain-a", "/新しい学区", "/新しい学区/inside"); !errors.Is(err, ErrInvalid) {
t.Fatal("recursive destination accepted")
}
mustPage("domain-a", "/Case")
mustPage("domain-a", "/case")
real := mustPage("domain-a", "/real")
child := mustPage("domain-a", "/real/child")
if err := r.RenameTree(ctx, "domain-a", "/real", "/moved-real"); err != nil {
t.Fatal("real folder move", err)
}
for path, id := range map[string]string{"/moved-real": real, "/moved-real/child": child} {
if got, err := r.ResolvePath(ctx, "domain-a", path); err != nil || got != id {
t.Fatal("real subtree identity", err)
}
}
v := Revision{ObjectUUID: id, DocumentHash: hash("first"), Language: "ja", ContentType: "text/html", Message: "first"}
first, err := r.AppendRevision(ctx, "domain-a", v, "")
if err != nil {
t.Fatal(err)
}
if _, err := r.Head(ctx, "domain-b", id, "ja"); !errors.Is(err, ErrNotFound) {
t.Fatal("foreign revision readable", err)
}
if _, err := r.Head(ctx, "domain-a", id, "en"); !errors.Is(err, ErrNotFound) {
t.Fatal("language fallback leaked into exact head", err)
}
if _, err := r.AppendRevision(ctx, "domain-a", v, ""); !errors.Is(err, ErrConflict) {
t.Fatal("stale revision accepted", err)
}
v.Language = "en"
if _, err := r.AppendRevision(ctx, "domain-a", v, ""); err != nil {
t.Fatal("translation creation", err)
}
v.Language = "ja"
v.DocumentHash = hash("updated")
// Independent Repository objects simulate writers that cannot share a Go mutex.
wins := concurrent(t, 6, func() error {
peer, _ := New(r.database, backend)
_, err := peer.AppendRevision(ctx, "domain-a", v, first.UUID)
return err
})
if wins != 1 {
t.Fatal("CAS winners", wins)
}
head, err := r.Head(ctx, "domain-a", id, "ja")
if err != nil || head.UUID == first.UUID || head.CreatedAtMS <= first.CreatedAtMS {
t.Fatal("head ordering", err)
}
var originalHash string
if err := r.database.QueryRowContext(ctx, `SELECT document_hash FROM revisions WHERE uuid=`+r.p(1), first.UUID).Scan(&originalHash); err != nil || originalHash != first.DocumentHash {
t.Fatal("immutable revision changed", err)
}
if _, err := r.GetRevision(ctx, "domain-b", first.UUID); !errors.Is(err, ErrNotFound) {
t.Fatal("foreign revision UUID readable", err)
}
restored, err := r.RestoreRevision(ctx, "domain-a", first.UUID, head.UUID, "restore old document")
if err != nil || restored.UUID == first.UUID || restored.DocumentHash != first.DocumentHash || restored.CreatedAtMS <= head.CreatedAtMS {
t.Fatal("restore did not append immutable revision", err)
}
if _, err := r.RestoreRevision(ctx, "domain-a", first.UUID, head.UUID, "stale restore"); !errors.Is(err, ErrConflict) {
t.Fatal("restore ignored head conflict", err)
}
a := Asset{DomainUUID: "domain-a", Address: "asset:images:cover.png", HeadDocumentHash: hash("image1"), ContentType: "image/png", SizeBytes: 10}
initial, err := r.PutAsset(ctx, a, "", "actor")
if err != nil {
t.Fatal(err)
}
if _, err := r.GetAsset(ctx, "domain-b", a.Address); !errors.Is(err, ErrNotFound) {
t.Fatal("foreign asset readable", err)
}
a.HeadDocumentHash = hash("image2")
if wins := concurrent(t, 6, func() error {
peer, _ := New(r.database, backend)
_, err := peer.PutAsset(ctx, a, initial.HeadDocumentHash, "actor")
return err
}); wins != 1 {
t.Fatal("asset CAS winners", wins)
}
current, err := r.GetAsset(ctx, "domain-a", a.Address)
if err != nil || current.ObjectUUID != initial.ObjectUUID || current.UpdatedAtMS <= initial.UpdatedAtMS {
t.Fatal("asset identity/order", err)
}
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM asset_head_history`).Scan(&count); err != nil || count != 2 {
t.Fatal("history not atomic", err, count)
}
a.DomainUUID = "domain-b"
if _, err := r.PutAsset(ctx, a, "", "actor"); err != nil {
t.Fatal("asset namespace not domain-scoped", err)
}
if _, err := r.AppendRevision(ctx, "domain-a", Revision{ObjectUUID: id, DocumentHash: "bad", ContentType: "text/html"}, head.UUID); !errors.Is(err, ErrInvalid) {
t.Fatal("invalid hash accepted", err)
}
canceled, stop := context.WithCancel(ctx)
stop()
if _, err := r.CreatePage(canceled, "domain-a", "/canceled"); !errors.Is(err, context.Canceled) {
t.Fatal("cancellation lost", err)
}
})
}
}
func TestMetadataInputValidation(t *testing.T) {
for _, path := range []string{"", "relative", "//", "///", "/a//b", "/a/../b", "/a/./b", "/a\x00b", "/a\nb", "/a\xffb"} {
if _, err := canonicalPath(path); !errors.Is(err, ErrInvalid) {
t.Fatalf("invalid path accepted: %q", path)
}
}
for _, path := range []string{"/", "/日本語/📚", "/name%_", "/a/b/"} {
if _, err := canonicalPath(path); err != nil {
t.Fatalf("valid path rejected: %q", path)
}
}
if _, err := New(nil, SQLite); !errors.Is(err, ErrInvalid) {
t.Fatal("nil database accepted")
}
// Do not expose data carried by a wrapped cancellation error.
r := Repository{}
err := r.failure(errors.Join(errors.New("secret"), context.Canceled))
if !errors.Is(err, context.Canceled) || strings.Contains(err.Error(), "secret") {
t.Fatal("unsafe cancellation error")
}
}
func concurrent(t *testing.T, n int, fn func() error) int {
t.Helper()
var wg sync.WaitGroup
results := make(chan error, n)
for i := 0; i < n; i++ {
wg.Add(1)
go func() { defer wg.Done(); results <- fn() }()
}
wg.Wait()
close(results)
wins := 0
for err := range results {
if err == nil {
wins++
} else if !errors.Is(err, ErrConflict) {
t.Fatal("unexpected concurrent write error", err)
}
}
return wins
}
func TestMetadataRefusesLegacyAdoption(t *testing.T) {
for _, backend := range []Backend{SQLite, PostgreSQL} {
t.Run(string(backend), func(t *testing.T) {
r := testRepository(t, backend)
ctx := context.Background()
if _, err := r.database.ExecContext(ctx, `DELETE FROM schema_migration_steps WHERE component='core-metadata'`); err != nil {
t.Fatal(err)
}
if _, err := r.CreatePage(ctx, "test", "/preserved"); err != nil {
t.Fatal(err)
}
if err := r.InitializeFresh(ctx); err == nil {
t.Fatal("unversioned schema silently adopted")
}
if _, err := r.ResolvePath(ctx, "test", "/preserved"); err != nil {
t.Fatal("existing data changed", err)
}
})
}
}
func TestMetadataAssetHistoryFailureIsAtomic(t *testing.T) {
for _, backend := range []Backend{SQLite, PostgreSQL} {
t.Run(string(backend), func(t *testing.T) {
r := testRepository(t, backend)
ctx := context.Background()
a := Asset{DomainUUID: "test", Address: "asset:images:original.png", HeadDocumentHash: hash("original"), ContentType: "image/png", SizeBytes: 8}
original, err := r.PutAsset(ctx, a, "", "actor")
if err != nil {
t.Fatal(err)
}
// Force a history write failure after the asset HEAD write. The
// synthetic unique index is solely fault injection, not the real schema.
if _, err := r.database.ExecContext(ctx, `CREATE UNIQUE INDEX synthetic_history_failure ON asset_head_history(domain_uuid)`); err != nil {
t.Fatal(err)
}
a.HeadDocumentHash = hash("changed")
if _, err := r.PutAsset(ctx, a, original.HeadDocumentHash, "actor"); !errors.Is(err, ErrConflict) {
t.Fatal("injected history failure ignored", err)
}
got, err := r.GetAsset(ctx, "test", a.Address)
if err != nil || got.HeadDocumentHash != original.HeadDocumentHash || got.UpdatedAtMS != original.UpdatedAtMS {
t.Fatal("partial HEAD update", err)
}
a.Address = "asset:images:new.png"
if _, err := r.PutAsset(ctx, a, "", "actor"); !errors.Is(err, ErrConflict) {
t.Fatal("injected creation failure ignored", err)
}
if _, err := r.GetAsset(ctx, "test", a.Address); !errors.Is(err, ErrNotFound) {
t.Fatal("partial asset created", err)
}
for _, table := range []string{"objects", "object_origins", "assets", "asset_head_history"} {
var count int
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM `+table).Scan(&count); err != nil || count != 1 {
t.Fatal("rollback failed", table, count, err)
}
}
})
}
}
func TestMetadataDetectsMissingColumnWithoutRepair(t *testing.T) {
for _, backend := range []Backend{SQLite, PostgreSQL} {
t.Run(string(backend), func(t *testing.T) {
r := testRepository(t, backend)
ctx := context.Background()
if _, err := r.database.ExecContext(ctx, `ALTER TABLE revisions RENAME COLUMN filename TO misplaced_filename`); err != nil {
t.Fatal(err)
}
if err := r.InitializeFresh(ctx); err == nil {
t.Fatal("missing column ignored")
}
var value string
if err := r.database.QueryRowContext(ctx, `SELECT misplaced_filename FROM revisions LIMIT 1`).Scan(&value); !errors.Is(err, sql.ErrNoRows) {
t.Fatal("drift silently repaired", err)
}
})
}
}
+65
View File
@@ -0,0 +1,65 @@
package metadata
import (
"context"
"database/sql"
"encoding/json"
"math"
"time"
"github.com/google/uuid"
)
// appendRevisionOutbox requires the Domain write lock. Sequence allocation uses
// small per-type watermarks, not MAX over the ever-growing event journal.
// This writes pending events only; delivery/leases remain a separate port.
func (r *Repository) appendRevisionOutbox(ctx context.Context, tx *sql.Tx, domain, actor string, v Revision) error {
payload, err := json.Marshal(map[string]string{"revision_uuid": v.UUID, "object_uuid": v.ObjectUUID, "document_hash": v.DocumentHash, "content_type": v.ContentType, "filename": v.Filename})
if err != nil {
return ErrInvalid
}
return r.appendOutbox(ctx, tx, domain, actor, "revision.created", "revision", v.UUID, string(payload))
}
func (r *Repository) appendPageOutbox(ctx context.Context, tx *sql.Tx, domain, path, object string) error {
payload, err := json.Marshal(map[string]string{"path": path, "object_uuid": object})
if err != nil {
return ErrInvalid
}
return r.appendOutbox(ctx, tx, domain, "", "page.created", "page", object, string(payload))
}
// Caller holds the Domain lock; metadata and the pending event commit together.
func (r *Repository) appendOutbox(ctx context.Context, tx *sql.Tx, domain, actor, eventType, targetKind, targetUUID, payload string) error {
var last int64
if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(last_sequence),0) FROM activity_outbox_heads WHERE domain_uuid=`+r.p(1), domain).Scan(&last); err != nil {
return r.failure(err)
}
if last == math.MaxInt64 {
return ErrInvalid
}
now := time.Now().UnixMilli()
_, err := tx.ExecContext(ctx, `INSERT INTO activity_outbox(uuid,domain_uuid,sequence,event_type,actor_uuid,target_kind,target_uuid,payload,idempotency_key,next_attempt_ms,created_at_ms) VALUES (`+r.p(1)+`,`+r.p(2)+`,`+r.p(3)+`,`+r.p(4)+`,`+r.p(5)+`,`+r.p(6)+`,`+r.p(7)+`,`+r.p(8)+`,`+r.p(9)+`,`+r.p(10)+`,`+r.p(11)+`)`, uuid.NewString(), domain, last+1, eventType, actor, targetKind, targetUUID, payload, eventType+":"+targetUUID, now, now)
if err != nil {
return r.failure(err)
}
_, err = tx.ExecContext(ctx, `INSERT INTO activity_outbox_heads(domain_uuid,event_type,last_sequence) VALUES (`+r.p(1)+`,`+r.p(2)+`,`+r.p(3)+`) ON CONFLICT(domain_uuid,event_type) DO UPDATE SET last_sequence=excluded.last_sequence`, domain, eventType, last+1)
return r.failure(err)
}
func (r *Repository) revisionReferences(ctx context.Context, domain, revisionUUID string) ([]string, error) {
rows, err := r.database.QueryContext(ctx, `SELECT document_hash FROM revision_document_refs WHERE revision_uuid=`+r.p(1)+` AND EXISTS (SELECT 1 FROM revisions r WHERE r.uuid=`+r.p(2)+` AND (`+r.objectDomainScope("r.object_uuid", r.p(3))+`)) ORDER BY document_hash`, revisionUUID, revisionUUID, domain)
if err != nil {
return nil, r.failure(err)
}
defer rows.Close()
var hashes []string
for rows.Next() {
var h string
if err := rows.Scan(&h); err != nil {
return nil, r.failure(err)
}
hashes = append(hashes, h)
}
return hashes, r.failure(rows.Err())
}
+75
View File
@@ -0,0 +1,75 @@
package metadata
import (
"context"
"pageswiki/internal/db"
)
// InitializeFresh bootstraps an empty database/schema and is idempotent after
// its first successful migration. Existing unversioned application tables are
// deliberately rejected, not adopted or converted. Production import and
// legacy schema reconciliation require a separate, verified migration plan.
func (r *Repository) InitializeFresh(ctx context.Context) error {
sqliteSteps, postgresSteps := r.migrationDefinitions()
if r.backend == PostgreSQL {
return db.ApplyPostgreSQLMigrations(ctx, r.database, "core-metadata", postgresSteps)
}
return db.ApplySQLiteMigrations(ctx, r.database, "core-metadata", sqliteSteps)
}
// Keep the exact released definitions shared by initialization and offline
// compatibility inspection. Neither inspection nor copying runs migrations.
func (r *Repository) migrationDefinitions() ([]db.SQLMigration, []db.PostgreSQLMigration) {
identity := "INTEGER PRIMARY KEY AUTOINCREMENT"
addressType := "TEXT"
if r.backend == PostgreSQL {
identity = "BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY"
// Path/asset identity must not depend on database locale or an ICU
// nondeterministic collation chosen by the database administrator.
addressType = `TEXT COLLATE "C"`
}
statements := []string{
`CREATE TABLE objects(uuid TEXT PRIMARY KEY,kind TEXT NOT NULL,created_at BIGINT NOT NULL)`,
`CREATE TABLE object_origins(object_uuid TEXT PRIMARY KEY REFERENCES objects(uuid),domain_uuid TEXT NOT NULL,created_at BIGINT NOT NULL)`,
`CREATE TABLE paths(domain_uuid TEXT NOT NULL,path ` + addressType + ` NOT NULL,object_uuid TEXT NOT NULL REFERENCES objects(uuid),created_at BIGINT NOT NULL,is_primary INTEGER NOT NULL DEFAULT 0,PRIMARY KEY(domain_uuid,path))`,
`CREATE INDEX idx_paths_domain_object ON paths(domain_uuid,object_uuid)`,
`CREATE TABLE revisions(uuid TEXT PRIMARY KEY,object_uuid TEXT NOT NULL REFERENCES objects(uuid),document_hash TEXT NOT NULL,lang TEXT NOT NULL,content_type TEXT NOT NULL,filename TEXT NOT NULL DEFAULT '',message TEXT NOT NULL DEFAULT '',created_at_ms BIGINT NOT NULL,created_at BIGINT NOT NULL,body TEXT NOT NULL DEFAULT '')`,
`CREATE INDEX idx_revisions_metadata_head ON revisions(object_uuid,lang,created_at_ms DESC,uuid DESC)`,
`CREATE TABLE assets(domain_uuid TEXT NOT NULL,address ` + addressType + ` NOT NULL,object_uuid TEXT NOT NULL UNIQUE REFERENCES objects(uuid),head_document_hash TEXT NOT NULL,content_type TEXT NOT NULL,filename TEXT NOT NULL DEFAULT '',size_bytes BIGINT NOT NULL CHECK(size_bytes>=0),created_at_ms BIGINT NOT NULL,updated_at_ms BIGINT NOT NULL,PRIMARY KEY(domain_uuid,address))`,
`CREATE TABLE asset_head_history(id ` + identity + `,domain_uuid TEXT NOT NULL,address TEXT NOT NULL,action TEXT NOT NULL,document_hash TEXT NOT NULL,content_type TEXT NOT NULL,filename TEXT NOT NULL,size_bytes BIGINT NOT NULL CHECK(size_bytes>=0),actor_uuid TEXT NOT NULL,created_at_ms BIGINT NOT NULL)`,
`CREATE INDEX idx_asset_metadata_history ON asset_head_history(domain_uuid,address,id)`,
}
// Minimal shape checks also run on subsequent initialization. Full catalog
// assertions for types, collation, constraints and indexes are a separate
// adoption gate before this schema can replace a legacy production store.
assertions := []string{
`SELECT uuid,kind,created_at FROM objects WHERE 1=0`,
`SELECT object_uuid,domain_uuid,created_at FROM object_origins WHERE 1=0`,
`SELECT domain_uuid,path,object_uuid,created_at,is_primary FROM paths WHERE 1=0`,
`SELECT ` + revisionColumns + `,created_at,body FROM revisions WHERE 1=0`,
`SELECT ` + assetColumns + ` FROM assets WHERE 1=0`,
`SELECT id,domain_uuid,address,action,document_hash,content_type,filename,size_bytes,actor_uuid,created_at_ms FROM asset_head_history WHERE 1=0`,
}
referenceStatements := []string{
`CREATE TABLE revision_document_refs(revision_uuid TEXT NOT NULL REFERENCES revisions(uuid),document_hash TEXT NOT NULL,PRIMARY KEY(revision_uuid,document_hash))`,
`CREATE INDEX idx_metadata_document_refs ON revision_document_refs(document_hash,revision_uuid)`,
`CREATE TABLE activity_outbox(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,sequence BIGINT NOT NULL,event_type TEXT NOT NULL,actor_uuid TEXT NOT NULL DEFAULT '',target_kind TEXT NOT NULL DEFAULT '',target_uuid TEXT NOT NULL DEFAULT '',payload TEXT NOT NULL DEFAULT '',idempotency_key TEXT NOT NULL,status TEXT NOT NULL DEFAULT 'pending',attempts INTEGER NOT NULL DEFAULT 0,next_attempt_ms BIGINT NOT NULL,last_error TEXT NOT NULL DEFAULT '',created_at_ms BIGINT NOT NULL,claimed_at_ms BIGINT NOT NULL DEFAULT 0,UNIQUE(domain_uuid,sequence),UNIQUE(domain_uuid,idempotency_key))`,
`CREATE TABLE activity_outbox_heads(domain_uuid TEXT NOT NULL,event_type TEXT NOT NULL,last_sequence BIGINT NOT NULL,PRIMARY KEY(domain_uuid,event_type))`,
`CREATE INDEX idx_metadata_outbox_pending ON activity_outbox(status,next_attempt_ms,domain_uuid,sequence)`,
}
currentAssertions := append(append([]string(nil), assertions...),
`SELECT revision_uuid,document_hash FROM revision_document_refs WHERE 1=0`,
`SELECT uuid,domain_uuid,sequence,event_type,actor_uuid,target_kind,target_uuid,payload,idempotency_key,status,attempts,next_attempt_ms,last_error,created_at_ms,claimed_at_ms FROM activity_outbox WHERE 1=0`,
`SELECT domain_uuid,event_type,last_sequence FROM activity_outbox_heads WHERE 1=0`)
if r.backend == PostgreSQL {
return nil, []db.PostgreSQLMigration{
{Version: 1, Name: "initial portable core", Statements: statements, Assertions: assertions},
{Version: 2, Name: "revision references and outbox", Statements: referenceStatements, Assertions: currentAssertions},
}
}
return []db.SQLMigration{
{Version: 1, Name: "initial portable core", Statements: statements, Assertions: assertions},
{Version: 2, Name: "revision references and outbox", Statements: referenceStatements, Assertions: currentAssertions},
}, nil
}
+64
View File
@@ -0,0 +1,64 @@
package metadata
import (
"context"
"testing"
)
func TestReferenceMigrationPreservesV1AndRetriesAtomically(t *testing.T) {
for _, backend := range []Backend{SQLite, PostgreSQL} {
t.Run(string(backend), func(t *testing.T) {
r := testRepository(t, backend)
ctx := context.Background()
id, err := r.CreatePage(ctx, "test", "/existing")
if err != nil {
t.Fatal(err)
}
// Reconstruct the preceding portable schema using its unchanged v1
// ledger. None of this targets legacy or production application data.
for _, q := range []string{`DROP TABLE revision_document_refs`, `DROP TABLE activity_outbox`, `DROP TABLE activity_outbox_heads`, `DELETE FROM schema_migration_steps WHERE component='core-metadata' AND version=2`} {
if _, err := r.database.ExecContext(ctx, q); err != nil {
t.Fatal(err)
}
}
_, err = r.database.ExecContext(ctx, `INSERT INTO revisions(uuid,object_uuid,document_hash,lang,content_type,created_at_ms,created_at) VALUES ('existing-revision',`+r.p(1)+`,`+r.p(2)+`,'und','text/plain',1000,1)`, id, hash("old"))
if err != nil {
t.Fatal(err)
}
// An unrelated existing table must stop adoption. The earlier DDL in
// the same migration must roll back and permit an explicit retry.
if _, err := r.database.ExecContext(ctx, `CREATE TABLE activity_outbox(unrelated INTEGER)`); err != nil {
t.Fatal(err)
}
if err := r.InitializeFresh(ctx); err == nil {
t.Fatal("unmanaged outbox table adopted")
}
var count int
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM revision_document_refs`).Scan(&count); err == nil {
t.Fatal("partial v2 DDL remained")
}
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM schema_migration_steps WHERE component='core-metadata' AND version=2`).Scan(&count); err != nil || count != 0 {
t.Fatal("failed migration recorded", err)
}
if _, err := r.database.ExecContext(ctx, `DROP TABLE activity_outbox`); err != nil {
t.Fatal(err)
}
if err := r.InitializeFresh(ctx); err != nil {
t.Fatal("migration retry failed", err)
}
old, err := r.Head(ctx, "test", id, "und")
if err != nil || old.UUID != "existing-revision" || old.DocumentHash != hash("old") {
t.Fatal("v1 revision altered", err)
}
if got, err := r.ResolvePath(ctx, "test", "/existing"); err != nil || got != id {
t.Fatal("v1 binding altered", err)
}
if _, err := r.AppendRevision(ctx, "test", Revision{ObjectUUID: id, DocumentHash: hash("new"), ContentType: "text/plain"}, old.UUID); err != nil {
t.Fatal("post-migration save failed", err)
}
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM activity_outbox`).Scan(&count); err != nil || count != 1 {
t.Fatal("historical revisions spuriously notified", err, count)
}
})
}
}