472 lines
15 KiB
Go
472 lines
15 KiB
Go
package metadata
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
hashpkg "hash"
|
|
"math"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"unicode/utf8"
|
|
|
|
"pageswiki/internal/geduldh"
|
|
)
|
|
|
|
var (
|
|
ErrMigrationBlocked = errors.New("metadata migration blocked; inspect portability report")
|
|
ErrDestinationNotEmpty = errors.New("metadata migration destination is not empty")
|
|
ErrIntegrity = errors.New("metadata integrity check failed")
|
|
)
|
|
|
|
type TableFingerprint struct {
|
|
Table string `json:"table"`
|
|
Rows int64 `json:"rows"`
|
|
SHA256 string `json:"sha256"`
|
|
}
|
|
type CoreSnapshot struct {
|
|
Scope string `json:"scope"`
|
|
Tables []TableFingerprint `json:"tables"`
|
|
IdentityNext int64 `json:"asset_history_identity_next"`
|
|
SHA256 string `json:"sha256"`
|
|
}
|
|
type RehearsalOptions struct {
|
|
// Caller asserts that all writers and GC are stopped for both instances.
|
|
// A SQL snapshot alone is not a cross-store or online migration protocol.
|
|
Offline bool
|
|
Documents geduldh.Store
|
|
}
|
|
|
|
// CloneCoreForRehearsal copies only a compatible portable core into an already
|
|
// initialized, empty staging schema. It is deliberately not a production
|
|
// migration entry point. Unknown tables/triggers/views/columns/ledger entries
|
|
// block copying rather than silently losing application data or permissions.
|
|
// Canonical payloads are verified in the caller's retained Document store;
|
|
// calendar files, secrets, application configuration and blobs are NOT copied.
|
|
func CloneCoreForRehearsal(ctx context.Context, source, destination *Repository, options RehearsalOptions) (CoreSnapshot, error) {
|
|
if !options.Offline || options.Documents == nil || source == nil || destination == nil || source.database == destination.database {
|
|
return CoreSnapshot{}, ErrMigrationBlocked
|
|
}
|
|
if same, err := sameDatabase(ctx, source, destination); err != nil {
|
|
return CoreSnapshot{}, err
|
|
} else if same {
|
|
return CoreSnapshot{}, ErrMigrationBlocked
|
|
}
|
|
src, err := source.database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead})
|
|
if err != nil {
|
|
return CoreSnapshot{}, source.failure(err)
|
|
}
|
|
defer src.Rollback()
|
|
report, err := source.inspectPortability(ctx, src)
|
|
if err != nil {
|
|
return CoreSnapshot{}, err
|
|
}
|
|
if !report.CoreCompatible {
|
|
return CoreSnapshot{}, ErrMigrationBlocked
|
|
}
|
|
if err := source.verifyCoreRelations(ctx, src); err != nil {
|
|
return CoreSnapshot{}, err
|
|
}
|
|
// Reject an occupied/incompatible destination before verifying a large
|
|
// Document store. Recheck while holding destination locks before copying.
|
|
preflight, err := destination.database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead})
|
|
if err != nil {
|
|
return CoreSnapshot{}, destination.failure(err)
|
|
}
|
|
err = destination.checkEmpty(ctx, preflight)
|
|
_ = preflight.Rollback()
|
|
if err != nil {
|
|
return CoreSnapshot{}, err
|
|
}
|
|
if err := source.verifyDocuments(ctx, src, options.Documents); err != nil {
|
|
return CoreSnapshot{}, err
|
|
}
|
|
var result CoreSnapshot
|
|
err = destination.write(ctx, "migration/portable-core-rehearsal", func(dst *sql.Tx) error {
|
|
if destination.backend == PostgreSQL {
|
|
// Lock all tables, not just a cooperative advisory key: an accidental
|
|
// concurrent writer must not race the empty-destination check.
|
|
names := []string{`"schema_migration_steps"`}
|
|
for _, s := range coreTables {
|
|
names = append(names, quote(s.name))
|
|
}
|
|
if _, err := dst.ExecContext(ctx, `LOCK TABLE `+strings.Join(names, ",")+` IN ACCESS EXCLUSIVE MODE`); err != nil {
|
|
return destination.failure(err)
|
|
}
|
|
}
|
|
if err := destination.checkEmpty(ctx, dst); err != nil {
|
|
return err
|
|
}
|
|
var expected []TableFingerprint
|
|
for _, spec := range coreTables {
|
|
fingerprint, err := source.streamTable(ctx, src, spec, func(values []any) error {
|
|
marks := make([]string, len(values))
|
|
for i := range values {
|
|
marks[i] = destination.p(i + 1)
|
|
}
|
|
q := `INSERT INTO ` + quote(spec.name) + ` (` + quoted(spec.columns) + `)`
|
|
if spec.identity && destination.backend == PostgreSQL {
|
|
q += ` OVERRIDING SYSTEM VALUE`
|
|
}
|
|
_, err := dst.ExecContext(ctx, q+` VALUES (`+strings.Join(marks, ",")+`)`, values...)
|
|
return destination.failure(err)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
expected = append(expected, fingerprint)
|
|
}
|
|
if err := destination.verifyCoreRelations(ctx, dst); err != nil {
|
|
return err
|
|
}
|
|
next, err := source.identityNext(ctx, src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := destination.restoreIdentity(ctx, dst, next); err != nil {
|
|
return err
|
|
}
|
|
actual, err := destination.fingerprint(ctx, dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
want := snapshot(expected, next)
|
|
if want.SHA256 != actual.SHA256 {
|
|
return ErrIntegrity
|
|
}
|
|
result = actual
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return CoreSnapshot{}, err
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
// FingerprintCore reads a single snapshot and never mutates the database. It
|
|
// identifies metadata content, not a backup of canonical payloads or secrets.
|
|
func (r *Repository) FingerprintCore(ctx context.Context) (CoreSnapshot, error) {
|
|
tx, err := r.database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead})
|
|
if err != nil {
|
|
return CoreSnapshot{}, r.failure(err)
|
|
}
|
|
defer tx.Rollback()
|
|
report, err := r.inspectPortability(ctx, tx)
|
|
if err != nil {
|
|
return CoreSnapshot{}, err
|
|
}
|
|
if !report.CoreCompatible {
|
|
return CoreSnapshot{}, ErrMigrationBlocked
|
|
}
|
|
if err := r.verifyCoreRelations(ctx, tx); err != nil {
|
|
return CoreSnapshot{}, err
|
|
}
|
|
return r.fingerprint(ctx, tx)
|
|
}
|
|
|
|
func (r *Repository) fingerprint(ctx context.Context, tx *sql.Tx) (CoreSnapshot, error) {
|
|
var tables []TableFingerprint
|
|
for _, spec := range coreTables {
|
|
f, err := r.streamTable(ctx, tx, spec, nil)
|
|
if err != nil {
|
|
return CoreSnapshot{}, err
|
|
}
|
|
tables = append(tables, f)
|
|
}
|
|
next, err := r.identityNext(ctx, tx)
|
|
if err != nil {
|
|
return CoreSnapshot{}, err
|
|
}
|
|
return snapshot(tables, next), nil
|
|
}
|
|
|
|
func snapshot(tables []TableFingerprint, next int64) CoreSnapshot {
|
|
data, _ := json.Marshal(struct {
|
|
Tables []TableFingerprint
|
|
IdentityNext int64
|
|
}{tables, next})
|
|
sum := sha256.Sum256(data)
|
|
return CoreSnapshot{Scope: "portable-core-rehearsal-only", Tables: tables, IdentityNext: next, SHA256: hex.EncodeToString(sum[:])}
|
|
}
|
|
|
|
func quote(s string) string { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` }
|
|
func quoted(names []string) string {
|
|
parts := make([]string, len(names))
|
|
for i, n := range names {
|
|
parts[i] = quote(n)
|
|
}
|
|
return strings.Join(parts, ",")
|
|
}
|
|
func frame(h hashpkg.Hash, data []byte) {
|
|
var size [8]byte
|
|
binary.BigEndian.PutUint64(size[:], uint64(len(data)))
|
|
h.Write(size[:])
|
|
h.Write(data)
|
|
}
|
|
|
|
func (r *Repository) streamTable(ctx context.Context, tx *sql.Tx, spec tableSpec, consume func([]any) error) (TableFingerprint, error) {
|
|
order := make([]string, len(spec.keys))
|
|
for i, k := range spec.keys {
|
|
order[i] = quote(k)
|
|
if !spec.identity {
|
|
if r.backend == PostgreSQL {
|
|
order[i] += ` COLLATE "C"`
|
|
} else {
|
|
order[i] += ` COLLATE BINARY`
|
|
}
|
|
}
|
|
}
|
|
rows, err := tx.QueryContext(ctx, `SELECT `+quoted(spec.columns)+` FROM `+quote(spec.name)+` ORDER BY `+strings.Join(order, ","))
|
|
if err != nil {
|
|
return TableFingerprint{}, r.failure(err)
|
|
}
|
|
defer rows.Close()
|
|
h := sha256.New()
|
|
frame(h, []byte(spec.name))
|
|
data, _ := json.Marshal(spec.columns)
|
|
frame(h, data)
|
|
var count int64
|
|
for rows.Next() {
|
|
if err := ctx.Err(); err != nil {
|
|
return TableFingerprint{}, err
|
|
}
|
|
values := make([]any, len(spec.columns))
|
|
pointers := make([]any, len(values))
|
|
for i := range values {
|
|
pointers[i] = &values[i]
|
|
}
|
|
if err := rows.Scan(pointers...); err != nil {
|
|
return TableFingerprint{}, r.failure(err)
|
|
}
|
|
for i, v := range values {
|
|
numeric := numericColumn(spec.columns[i])
|
|
switch value := v.(type) {
|
|
case string:
|
|
if numeric {
|
|
return TableFingerprint{}, fmt.Errorf("%w: noninteger in %s.%s", ErrIntegrity, spec.name, spec.columns[i])
|
|
}
|
|
if !utf8.ValidString(value) || strings.ContainsRune(value, 0) {
|
|
return TableFingerprint{}, fmt.Errorf("%w: nonportable text in %s.%s", ErrIntegrity, spec.name, spec.columns[i])
|
|
}
|
|
if (spec.columns[i] == "document_hash" || spec.columns[i] == "head_document_hash") && !validHash(value) {
|
|
return TableFingerprint{}, fmt.Errorf("%w: invalid hash in %s", ErrIntegrity, spec.name)
|
|
}
|
|
case int64:
|
|
if !numeric {
|
|
return TableFingerprint{}, fmt.Errorf("%w: nontext in %s.%s", ErrIntegrity, spec.name, spec.columns[i])
|
|
}
|
|
default:
|
|
return TableFingerprint{}, fmt.Errorf("%w: unsupported value type in %s.%s", ErrIntegrity, spec.name, spec.columns[i])
|
|
}
|
|
}
|
|
data, err := json.Marshal(values)
|
|
if err != nil {
|
|
return TableFingerprint{}, ErrIntegrity
|
|
}
|
|
frame(h, data)
|
|
if consume != nil {
|
|
if err := consume(values); err != nil {
|
|
return TableFingerprint{}, err
|
|
}
|
|
}
|
|
count++
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return TableFingerprint{}, r.failure(err)
|
|
}
|
|
return TableFingerprint{Table: spec.name, Rows: count, SHA256: hex.EncodeToString(h.Sum(nil))}, nil
|
|
}
|
|
|
|
func (r *Repository) verifyCoreRelations(ctx context.Context, tx *sql.Tx) error {
|
|
checks := []struct{ name, query string }{
|
|
{"object origin", `SELECT 1 FROM objects o WHERE NOT EXISTS(SELECT 1 FROM object_origins x WHERE x.object_uuid=o.uuid) LIMIT 1`},
|
|
{"origin object", `SELECT 1 FROM object_origins x WHERE NOT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=x.object_uuid) LIMIT 1`},
|
|
{"path object", `SELECT 1 FROM paths p WHERE NOT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=p.object_uuid) LIMIT 1`},
|
|
{"revision object", `SELECT 1 FROM revisions r WHERE NOT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=r.object_uuid) LIMIT 1`},
|
|
{"asset object", `SELECT 1 FROM assets a WHERE NOT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=a.object_uuid) LIMIT 1`},
|
|
{"revision reference", `SELECT 1 FROM revision_document_refs x WHERE NOT EXISTS(SELECT 1 FROM revisions r WHERE r.uuid=x.revision_uuid) LIMIT 1`},
|
|
{"outbox watermark", `SELECT 1 FROM activity_outbox o WHERE o.sequence<1 OR NOT EXISTS(SELECT 1 FROM activity_outbox_heads h WHERE h.domain_uuid=o.domain_uuid AND h.event_type=o.event_type AND h.last_sequence>=o.sequence) LIMIT 1`},
|
|
{"revision event", `SELECT 1 FROM activity_outbox o WHERE o.event_type='revision.created' AND NOT EXISTS(SELECT 1 FROM revisions r WHERE r.uuid=o.target_uuid) LIMIT 1`},
|
|
{"asset history id", `SELECT 1 FROM asset_head_history WHERE id<1 LIMIT 1`},
|
|
}
|
|
for _, check := range checks {
|
|
var found int
|
|
err := tx.QueryRowContext(ctx, check.query).Scan(&found)
|
|
if err == nil {
|
|
return fmt.Errorf("%w: %s", ErrIntegrity, check.name)
|
|
}
|
|
if !errors.Is(err, sql.ErrNoRows) {
|
|
return r.failure(err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Repository) verifyDocuments(ctx context.Context, tx *sql.Tx, documents geduldh.Store) error {
|
|
rows, err := tx.QueryContext(ctx, `SELECT document_hash FROM revisions UNION SELECT head_document_hash FROM assets UNION SELECT document_hash FROM asset_head_history UNION SELECT document_hash FROM revision_document_refs ORDER BY 1`)
|
|
if err != nil {
|
|
return r.failure(err)
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var hash string
|
|
if err := rows.Scan(&hash); err != nil {
|
|
return r.failure(err)
|
|
}
|
|
if !validHash(hash) {
|
|
return ErrIntegrity
|
|
}
|
|
if err := documents.Verify(ctx, hash); err != nil {
|
|
if ctx.Err() != nil {
|
|
return ctx.Err()
|
|
}
|
|
return fmt.Errorf("%w: Document verification failed", ErrIntegrity)
|
|
}
|
|
}
|
|
return r.failure(rows.Err())
|
|
}
|
|
|
|
func (r *Repository) checkEmpty(ctx context.Context, tx *sql.Tx) error {
|
|
report, err := r.inspectPortability(ctx, tx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !report.CoreCompatible {
|
|
return ErrMigrationBlocked
|
|
}
|
|
for _, s := range coreTables {
|
|
var occupied bool
|
|
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM `+quote(s.name)+`)`).Scan(&occupied); err != nil {
|
|
return r.failure(err)
|
|
}
|
|
if occupied {
|
|
return ErrDestinationNotEmpty
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Repository) identityNext(ctx context.Context, tx *sql.Tx) (int64, error) {
|
|
var max sql.NullInt64
|
|
if err := tx.QueryRowContext(ctx, `SELECT max(id) FROM asset_head_history`).Scan(&max); err != nil {
|
|
return 0, r.failure(err)
|
|
}
|
|
if max.Int64 < 0 || max.Int64 == math.MaxInt64 {
|
|
return 0, ErrIntegrity
|
|
}
|
|
next := max.Int64 + 1
|
|
if r.backend == SQLite {
|
|
var seq sql.NullInt64
|
|
if err := tx.QueryRowContext(ctx, `SELECT max(seq) FROM sqlite_sequence WHERE name='asset_head_history'`).Scan(&seq); err != nil {
|
|
return 0, r.failure(err)
|
|
}
|
|
if seq.Int64 < 0 || seq.Int64 == math.MaxInt64 {
|
|
return 0, ErrIntegrity
|
|
}
|
|
if seq.Int64+1 > next {
|
|
next = seq.Int64 + 1
|
|
}
|
|
} else {
|
|
var schema, name string
|
|
if err := tx.QueryRowContext(ctx, `SELECT n.nspname,c.relname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE c.oid=pg_get_serial_sequence('asset_head_history','id')::regclass`).Scan(&schema, &name); err != nil {
|
|
return 0, r.failure(err)
|
|
}
|
|
var last int64
|
|
var called bool
|
|
if err := tx.QueryRowContext(ctx, `SELECT last_value,is_called FROM `+quote(schema)+`.`+quote(name)).Scan(&last, &called); err != nil {
|
|
return 0, r.failure(err)
|
|
}
|
|
if last < 1 || last == math.MaxInt64 {
|
|
return 0, ErrIntegrity
|
|
}
|
|
if called {
|
|
last++
|
|
}
|
|
if last > next {
|
|
next = last
|
|
}
|
|
}
|
|
return next, nil
|
|
}
|
|
|
|
func (r *Repository) restoreIdentity(ctx context.Context, tx *sql.Tx, next int64) error {
|
|
if next < 1 {
|
|
return ErrIntegrity
|
|
}
|
|
if r.backend == PostgreSQL {
|
|
// RESTART rolls back with the transaction; setval would not.
|
|
_, err := tx.ExecContext(ctx, `ALTER TABLE asset_head_history ALTER COLUMN id RESTART WITH `+strconv.FormatInt(next, 10))
|
|
return r.failure(err)
|
|
}
|
|
result, err := tx.ExecContext(ctx, `UPDATE sqlite_sequence SET seq=? WHERE name='asset_head_history'`, next-1)
|
|
if err != nil {
|
|
return r.failure(err)
|
|
}
|
|
n, err := result.RowsAffected()
|
|
if err != nil {
|
|
return r.failure(err)
|
|
}
|
|
if n == 0 {
|
|
_, err = tx.ExecContext(ctx, `INSERT INTO sqlite_sequence(name,seq) VALUES ('asset_head_history',?)`, next-1)
|
|
}
|
|
return r.failure(err)
|
|
}
|
|
|
|
func sameDatabase(ctx context.Context, a, b *Repository) (bool, error) {
|
|
if a.backend != b.backend {
|
|
return false, nil
|
|
}
|
|
if a.backend == SQLite {
|
|
path := func(r *Repository) (string, error) {
|
|
var seq int
|
|
var name, file string
|
|
err := r.database.QueryRowContext(ctx, `SELECT seq,name,file FROM pragma_database_list WHERE name='main'`).Scan(&seq, &name, &file)
|
|
if err != nil {
|
|
return "", r.failure(err)
|
|
}
|
|
if file == "" {
|
|
return "", ErrMigrationBlocked
|
|
}
|
|
return file, nil
|
|
}
|
|
x, err := path(a)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
y, err := path(b)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
xInfo, err := os.Stat(x)
|
|
if err != nil {
|
|
return false, ErrMigrationBlocked
|
|
}
|
|
yInfo, err := os.Stat(y)
|
|
if err != nil {
|
|
return false, ErrMigrationBlocked
|
|
}
|
|
return os.SameFile(xInfo, yInfo), nil
|
|
}
|
|
type identity struct{ database, schema, start string }
|
|
read := func(r *Repository) (identity, error) {
|
|
var i identity
|
|
err := r.database.QueryRowContext(ctx, `SELECT current_database(),current_schema(),extract(epoch from pg_postmaster_start_time())::text`).Scan(&i.database, &i.schema, &i.start)
|
|
return i, r.failure(err)
|
|
}
|
|
x, err := read(a)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
y, err := read(b)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return x == y, nil
|
|
}
|