package metadata import ( "context" "crypto/sha256" "database/sql" "encoding/binary" "encoding/hex" "encoding/json" "errors" "fmt" hashpkg "hash" "math" "os" "strconv" "strings" "unicode/utf8" "pageswiki/internal/geduldh" ) var ( ErrMigrationBlocked = errors.New("metadata migration blocked; inspect portability report") ErrDestinationNotEmpty = errors.New("metadata migration destination is not empty") ErrIntegrity = errors.New("metadata integrity check failed") ) type TableFingerprint struct { Table string `json:"table"` Rows int64 `json:"rows"` SHA256 string `json:"sha256"` } type CoreSnapshot struct { Scope string `json:"scope"` Tables []TableFingerprint `json:"tables"` IdentityNext int64 `json:"asset_history_identity_next"` SHA256 string `json:"sha256"` } type RehearsalOptions struct { // Caller asserts that all writers and GC are stopped for both instances. // A SQL snapshot alone is not a cross-store or online migration protocol. Offline bool Documents geduldh.Store } // CloneCoreForRehearsal copies only a compatible portable core into an already // initialized, empty staging schema. It is deliberately not a production // migration entry point. Unknown tables/triggers/views/columns/ledger entries // block copying rather than silently losing application data or permissions. // Canonical payloads are verified in the caller's retained Document store; // calendar files, secrets, application configuration and blobs are NOT copied. func CloneCoreForRehearsal(ctx context.Context, source, destination *Repository, options RehearsalOptions) (CoreSnapshot, error) { if !options.Offline || options.Documents == nil || source == nil || destination == nil || source.database == destination.database { return CoreSnapshot{}, ErrMigrationBlocked } if same, err := sameDatabase(ctx, source, destination); err != nil { return CoreSnapshot{}, err } else if same { return CoreSnapshot{}, ErrMigrationBlocked } src, err := source.database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead}) if err != nil { return CoreSnapshot{}, source.failure(err) } defer src.Rollback() report, err := source.inspectPortability(ctx, src) if err != nil { return CoreSnapshot{}, err } if !report.CoreCompatible { return CoreSnapshot{}, ErrMigrationBlocked } if err := source.verifyCoreRelations(ctx, src); err != nil { return CoreSnapshot{}, err } // Reject an occupied/incompatible destination before verifying a large // Document store. Recheck while holding destination locks before copying. preflight, err := destination.database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead}) if err != nil { return CoreSnapshot{}, destination.failure(err) } err = destination.checkEmpty(ctx, preflight) _ = preflight.Rollback() if err != nil { return CoreSnapshot{}, err } if err := source.verifyDocuments(ctx, src, options.Documents); err != nil { return CoreSnapshot{}, err } var result CoreSnapshot err = destination.write(ctx, "migration/portable-core-rehearsal", func(dst *sql.Tx) error { if destination.backend == PostgreSQL { // Lock all tables, not just a cooperative advisory key: an accidental // concurrent writer must not race the empty-destination check. names := []string{`"schema_migration_steps"`} for _, s := range coreTables { names = append(names, quote(s.name)) } if _, err := dst.ExecContext(ctx, `LOCK TABLE `+strings.Join(names, ",")+` IN ACCESS EXCLUSIVE MODE`); err != nil { return destination.failure(err) } } if err := destination.checkEmpty(ctx, dst); err != nil { return err } var expected []TableFingerprint for _, spec := range coreTables { fingerprint, err := source.streamTable(ctx, src, spec, func(values []any) error { marks := make([]string, len(values)) for i := range values { marks[i] = destination.p(i + 1) } q := `INSERT INTO ` + quote(spec.name) + ` (` + quoted(spec.columns) + `)` if spec.identity && destination.backend == PostgreSQL { q += ` OVERRIDING SYSTEM VALUE` } _, err := dst.ExecContext(ctx, q+` VALUES (`+strings.Join(marks, ",")+`)`, values...) return destination.failure(err) }) if err != nil { return err } expected = append(expected, fingerprint) } if err := destination.verifyCoreRelations(ctx, dst); err != nil { return err } next, err := source.identityNext(ctx, src) if err != nil { return err } if err := destination.restoreIdentity(ctx, dst, next); err != nil { return err } actual, err := destination.fingerprint(ctx, dst) if err != nil { return err } want := snapshot(expected, next) if want.SHA256 != actual.SHA256 { return ErrIntegrity } result = actual return nil }) if err != nil { return CoreSnapshot{}, err } return result, nil } // FingerprintCore reads a single snapshot and never mutates the database. It // identifies metadata content, not a backup of canonical payloads or secrets. func (r *Repository) FingerprintCore(ctx context.Context) (CoreSnapshot, error) { tx, err := r.database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead}) if err != nil { return CoreSnapshot{}, r.failure(err) } defer tx.Rollback() report, err := r.inspectPortability(ctx, tx) if err != nil { return CoreSnapshot{}, err } if !report.CoreCompatible { return CoreSnapshot{}, ErrMigrationBlocked } if err := r.verifyCoreRelations(ctx, tx); err != nil { return CoreSnapshot{}, err } return r.fingerprint(ctx, tx) } func (r *Repository) fingerprint(ctx context.Context, tx *sql.Tx) (CoreSnapshot, error) { var tables []TableFingerprint for _, spec := range coreTables { f, err := r.streamTable(ctx, tx, spec, nil) if err != nil { return CoreSnapshot{}, err } tables = append(tables, f) } next, err := r.identityNext(ctx, tx) if err != nil { return CoreSnapshot{}, err } return snapshot(tables, next), nil } func snapshot(tables []TableFingerprint, next int64) CoreSnapshot { data, _ := json.Marshal(struct { Tables []TableFingerprint IdentityNext int64 }{tables, next}) sum := sha256.Sum256(data) return CoreSnapshot{Scope: "portable-core-rehearsal-only", Tables: tables, IdentityNext: next, SHA256: hex.EncodeToString(sum[:])} } func quote(s string) string { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` } func quoted(names []string) string { parts := make([]string, len(names)) for i, n := range names { parts[i] = quote(n) } return strings.Join(parts, ",") } func frame(h hashpkg.Hash, data []byte) { var size [8]byte binary.BigEndian.PutUint64(size[:], uint64(len(data))) h.Write(size[:]) h.Write(data) } func (r *Repository) streamTable(ctx context.Context, tx *sql.Tx, spec tableSpec, consume func([]any) error) (TableFingerprint, error) { order := make([]string, len(spec.keys)) for i, k := range spec.keys { order[i] = quote(k) if !spec.identity { if r.backend == PostgreSQL { order[i] += ` COLLATE "C"` } else { order[i] += ` COLLATE BINARY` } } } rows, err := tx.QueryContext(ctx, `SELECT `+quoted(spec.columns)+` FROM `+quote(spec.name)+` ORDER BY `+strings.Join(order, ",")) if err != nil { return TableFingerprint{}, r.failure(err) } defer rows.Close() h := sha256.New() frame(h, []byte(spec.name)) data, _ := json.Marshal(spec.columns) frame(h, data) var count int64 for rows.Next() { if err := ctx.Err(); err != nil { return TableFingerprint{}, err } values := make([]any, len(spec.columns)) pointers := make([]any, len(values)) for i := range values { pointers[i] = &values[i] } if err := rows.Scan(pointers...); err != nil { return TableFingerprint{}, r.failure(err) } for i, v := range values { numeric := numericColumn(spec.columns[i]) switch value := v.(type) { case string: if numeric { return TableFingerprint{}, fmt.Errorf("%w: noninteger in %s.%s", ErrIntegrity, spec.name, spec.columns[i]) } if !utf8.ValidString(value) || strings.ContainsRune(value, 0) { return TableFingerprint{}, fmt.Errorf("%w: nonportable text in %s.%s", ErrIntegrity, spec.name, spec.columns[i]) } if (spec.columns[i] == "document_hash" || spec.columns[i] == "head_document_hash") && !validHash(value) { return TableFingerprint{}, fmt.Errorf("%w: invalid hash in %s", ErrIntegrity, spec.name) } case int64: if !numeric { return TableFingerprint{}, fmt.Errorf("%w: nontext in %s.%s", ErrIntegrity, spec.name, spec.columns[i]) } default: return TableFingerprint{}, fmt.Errorf("%w: unsupported value type in %s.%s", ErrIntegrity, spec.name, spec.columns[i]) } } data, err := json.Marshal(values) if err != nil { return TableFingerprint{}, ErrIntegrity } frame(h, data) if consume != nil { if err := consume(values); err != nil { return TableFingerprint{}, err } } count++ } if err := rows.Err(); err != nil { return TableFingerprint{}, r.failure(err) } return TableFingerprint{Table: spec.name, Rows: count, SHA256: hex.EncodeToString(h.Sum(nil))}, nil } func (r *Repository) verifyCoreRelations(ctx context.Context, tx *sql.Tx) error { checks := []struct{ name, query string }{ {"object origin", `SELECT 1 FROM objects o WHERE NOT EXISTS(SELECT 1 FROM object_origins x WHERE x.object_uuid=o.uuid) LIMIT 1`}, {"origin object", `SELECT 1 FROM object_origins x WHERE NOT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=x.object_uuid) LIMIT 1`}, {"path object", `SELECT 1 FROM paths p WHERE NOT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=p.object_uuid) LIMIT 1`}, {"revision object", `SELECT 1 FROM revisions r WHERE NOT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=r.object_uuid) LIMIT 1`}, {"asset object", `SELECT 1 FROM assets a WHERE NOT EXISTS(SELECT 1 FROM objects o WHERE o.uuid=a.object_uuid) LIMIT 1`}, {"revision reference", `SELECT 1 FROM revision_document_refs x WHERE NOT EXISTS(SELECT 1 FROM revisions r WHERE r.uuid=x.revision_uuid) LIMIT 1`}, {"outbox watermark", `SELECT 1 FROM activity_outbox o WHERE o.sequence<1 OR NOT EXISTS(SELECT 1 FROM activity_outbox_heads h WHERE h.domain_uuid=o.domain_uuid AND h.event_type=o.event_type AND h.last_sequence>=o.sequence) LIMIT 1`}, {"revision event", `SELECT 1 FROM activity_outbox o WHERE o.event_type='revision.created' AND NOT EXISTS(SELECT 1 FROM revisions r WHERE r.uuid=o.target_uuid) LIMIT 1`}, {"asset history id", `SELECT 1 FROM asset_head_history WHERE id<1 LIMIT 1`}, } for _, check := range checks { var found int err := tx.QueryRowContext(ctx, check.query).Scan(&found) if err == nil { return fmt.Errorf("%w: %s", ErrIntegrity, check.name) } if !errors.Is(err, sql.ErrNoRows) { return r.failure(err) } } return nil } func (r *Repository) verifyDocuments(ctx context.Context, tx *sql.Tx, documents geduldh.Store) error { rows, err := tx.QueryContext(ctx, `SELECT document_hash FROM revisions UNION SELECT head_document_hash FROM assets UNION SELECT document_hash FROM asset_head_history UNION SELECT document_hash FROM revision_document_refs ORDER BY 1`) if err != nil { return r.failure(err) } defer rows.Close() for rows.Next() { var hash string if err := rows.Scan(&hash); err != nil { return r.failure(err) } if !validHash(hash) { return ErrIntegrity } if err := documents.Verify(ctx, hash); err != nil { if ctx.Err() != nil { return ctx.Err() } return fmt.Errorf("%w: Document verification failed", ErrIntegrity) } } return r.failure(rows.Err()) } func (r *Repository) checkEmpty(ctx context.Context, tx *sql.Tx) error { report, err := r.inspectPortability(ctx, tx) if err != nil { return err } if !report.CoreCompatible { return ErrMigrationBlocked } for _, s := range coreTables { var occupied bool if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM `+quote(s.name)+`)`).Scan(&occupied); err != nil { return r.failure(err) } if occupied { return ErrDestinationNotEmpty } } return nil } func (r *Repository) identityNext(ctx context.Context, tx *sql.Tx) (int64, error) { var max sql.NullInt64 if err := tx.QueryRowContext(ctx, `SELECT max(id) FROM asset_head_history`).Scan(&max); err != nil { return 0, r.failure(err) } if max.Int64 < 0 || max.Int64 == math.MaxInt64 { return 0, ErrIntegrity } next := max.Int64 + 1 if r.backend == SQLite { var seq sql.NullInt64 if err := tx.QueryRowContext(ctx, `SELECT max(seq) FROM sqlite_sequence WHERE name='asset_head_history'`).Scan(&seq); err != nil { return 0, r.failure(err) } if seq.Int64 < 0 || seq.Int64 == math.MaxInt64 { return 0, ErrIntegrity } if seq.Int64+1 > next { next = seq.Int64 + 1 } } else { var schema, name string if err := tx.QueryRowContext(ctx, `SELECT n.nspname,c.relname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE c.oid=pg_get_serial_sequence('asset_head_history','id')::regclass`).Scan(&schema, &name); err != nil { return 0, r.failure(err) } var last int64 var called bool if err := tx.QueryRowContext(ctx, `SELECT last_value,is_called FROM `+quote(schema)+`.`+quote(name)).Scan(&last, &called); err != nil { return 0, r.failure(err) } if last < 1 || last == math.MaxInt64 { return 0, ErrIntegrity } if called { last++ } if last > next { next = last } } return next, nil } func (r *Repository) restoreIdentity(ctx context.Context, tx *sql.Tx, next int64) error { if next < 1 { return ErrIntegrity } if r.backend == PostgreSQL { // RESTART rolls back with the transaction; setval would not. _, err := tx.ExecContext(ctx, `ALTER TABLE asset_head_history ALTER COLUMN id RESTART WITH `+strconv.FormatInt(next, 10)) return r.failure(err) } result, err := tx.ExecContext(ctx, `UPDATE sqlite_sequence SET seq=? WHERE name='asset_head_history'`, next-1) if err != nil { return r.failure(err) } n, err := result.RowsAffected() if err != nil { return r.failure(err) } if n == 0 { _, err = tx.ExecContext(ctx, `INSERT INTO sqlite_sequence(name,seq) VALUES ('asset_head_history',?)`, next-1) } return r.failure(err) } func sameDatabase(ctx context.Context, a, b *Repository) (bool, error) { if a.backend != b.backend { return false, nil } if a.backend == SQLite { path := func(r *Repository) (string, error) { var seq int var name, file string err := r.database.QueryRowContext(ctx, `SELECT seq,name,file FROM pragma_database_list WHERE name='main'`).Scan(&seq, &name, &file) if err != nil { return "", r.failure(err) } if file == "" { return "", ErrMigrationBlocked } return file, nil } x, err := path(a) if err != nil { return false, err } y, err := path(b) if err != nil { return false, err } xInfo, err := os.Stat(x) if err != nil { return false, ErrMigrationBlocked } yInfo, err := os.Stat(y) if err != nil { return false, ErrMigrationBlocked } return os.SameFile(xInfo, yInfo), nil } type identity struct{ database, schema, start string } read := func(r *Repository) (identity, error) { var i identity err := r.database.QueryRowContext(ctx, `SELECT current_database(),current_schema(),extract(epoch from pg_postmaster_start_time())::text`).Scan(&i.database, &i.schema, &i.start) return i, r.failure(err) } x, err := read(a) if err != nil { return false, err } y, err := read(b) if err != nil { return false, err } return x == y, nil }