package store import ( "context" "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" "errors" "fmt" "os" "path/filepath" "sort" "strings" "time" "pageswiki/internal/calendar" "pageswiki/internal/metadata" ) type ImportedTable struct { Name string `json:"name"` Rows int64 `json:"rows"` SHA256 string `json:"sha256"` } type applicationImportRefusal struct{ reason string } func (e *applicationImportRefusal) Error() string { return e.reason } func importRefuse(reason string) error { return &applicationImportRefusal{reason} } func importRefusef(format string, args ...any) error { return importRefuse(fmt.Sprintf(format, args...)) } type ApplicationImportReport struct { Tables []ImportedTable `json:"tables"` Omitted []string `json:"omitted"` Documents int64 `json:"documents"` Fingerprint string `json:"fingerprint"` Sequences map[string]int64 `json:"sequences"` Transforms []string `json:"transforms"` } // ImportSQLiteApplication imports into an EMPTY, already prepared PostgreSQL // schema. The caller holds an exclusive data-directory lock and stops all // writers. Files stay at documentRoot; no source rows or canonical files are // changed. This is not a merge or an online migration. func ImportSQLiteApplication(ctx context.Context, source, target *sql.DB, documentRoot string) (report ApplicationImportReport, err error) { key, err := existingAuthKey(documentRoot) if err != nil { return report, err } root, err := filepath.Abs(documentRoot) if err != nil { return report, err } if info, e := os.Stat(root); e != nil || !info.IsDir() { return report, ErrDocumentStoreRequired } s := &Store{DB: target, sqlBackend: storePostgreSQL, DocumentRoot: root} defer func() { if err != nil { var refused *applicationImportRefusal if errors.As(err, &refused) { return } err = s.sqlError("application import", err) } }() from, err := source.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) if err != nil { return report, err } defer from.Rollback() tables, err := inspectApplicationImportSource(ctx, from) if err != nil { return report, err } to, err := target.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelSerializable}) if err != nil { return report, err } defer to.Rollback() if err = inspectApplicationImportTarget(ctx, to); err != nil { return report, err } if _, err = to.ExecContext(ctx, `SELECT pg_advisory_xact_lock(1885431653,1853058931)`); err != nil { return report, err } var completed bool if err = to.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM application_import_receipts)`).Scan(&completed); err != nil { return report, err } if completed { return report, importRefuse("application import already completed; overwrite is forbidden") } names := make([]string, 0, len(applicationImportColumns)) for name := range applicationImportColumns { names = append(names, name) } sort.Strings(names) // Validate all destination columns and take table locks before emptiness // checks. Only exact schema-provided seed rows may be removed. for _, name := range names { if _, err = to.ExecContext(ctx, `LOCK TABLE `+importIdent(name)+` IN ACCESS EXCLUSIVE MODE`); err != nil { return report, err } var n int64 if err = to.QueryRowContext(ctx, `SELECT count(*) FROM `+importIdent(name)).Scan(&n); err != nil { return report, err } if n > 0 { predicate, ok := importSeedPredicates[name] if !ok { return report, importRefusef("import destination is not empty: %s", name) } var valid bool if err = to.QueryRowContext(ctx, `SELECT count(*)=1 AND bool_and(`+predicate+`) FROM `+importIdent(name)).Scan(&valid); err != nil { return report, err } if !valid { return report, importRefusef("import destination seed has changed: %s", name) } } // User triggers derive state or revoke publication on normal writes. // Suspend them only inside this transaction while restoring exact state. // Foreign keys and CHECK/UNIQUE constraints stay enabled throughout. if _, err = to.ExecContext(ctx, `ALTER TABLE `+importIdent(name)+` DISABLE TRIGGER USER`); err != nil { return report, err } } for name := range importSeedPredicates { if _, err = to.ExecContext(ctx, `DELETE FROM `+importIdent(name)); err != nil { return report, err } } order, err := importTableOrder(ctx, to, names) if err != nil { return report, err } for _, name := range order { columns := applicationImportColumns[name] if !tables[name] { if !strings.HasPrefix(name, "calendar_") { return report, importRefusef("missing source table: %s", name) } continue } fingerprint, err := copyImportTable(ctx, from, to, name, strings.Split(columns, ",")) if err != nil { return report, err } report.Tables = append(report.Tables, fingerprint) } // Source-only structures are explicitly classified, not inferred by prefix. for name := range tables { if _, ok := applicationImportOmissions[name]; ok { report.Omitted = append(report.Omitted, name) } } sort.Strings(report.Omitted) report.Transforms = []string{"task DAV index keys: SQLite NUL-separated to native v1 base64url (resources and deletion tombstones)"} // Verify every retained Document reference before committing metadata. docs, err := s.documentStorage() if err != nil { return report, err } refs, err := to.QueryContext(ctx, `SELECT document_hash FROM revisions UNION SELECT head_document_hash FROM assets UNION SELECT document_hash FROM revision_document_refs UNION SELECT document_hash FROM asset_head_history`) if err != nil { return report, err } for refs.Next() { var hash string if err = refs.Scan(&hash); err != nil { refs.Close() return report, err } if err = docs.Verify(ctx, hash); err != nil { refs.Close() return report, importRefuse("canonical Document missing or hash mismatch; import rolled back") } report.Documents++ } err = refs.Err() refs.Close() if err != nil { return report, err } if err = validateImportedBindings(ctx, to); err != nil { return report, err } for _, name := range names { if _, err = to.ExecContext(ctx, `ALTER TABLE `+importIdent(name)+` ENABLE TRIGGER USER`); err != nil { return report, err } } // Sequence reset uses transactional ALTER, not non-transactional setval. report.Sequences, err = sourceImportSequences(ctx, from) if err != nil { return report, err } if err = resetImportIdentities(ctx, to, report.Sequences); err != nil { return report, err } sort.Slice(report.Tables, func(i, j int) bool { return report.Tables[i].Name < report.Tables[j].Name }) b, _ := json.Marshal(report) sum := sha256.Sum256(b) report.Fingerprint = hex.EncodeToString(sum[:]) b, _ = json.Marshal(report) ks := sha256.Sum256(key) if _, err = to.ExecContext(ctx, `INSERT INTO application_import_receipts VALUES(1,$1,$2,$3,$4,$5)`, report.Fingerprint, root, hex.EncodeToString(ks[:]), string(b), time.Now().UnixMilli()); err != nil { return report, err } if err = ctx.Err(); err != nil { return report, err } err = to.Commit() return report, err } var importSeedPredicates = map[string]string{ "organizer_contact_epoch": `id=1 AND value ~ '^[a-f0-9]{64}$'`, "organizer_activity_bridge": `id=1 AND last_sequence=0`, "navigation_generation": `id=1 AND version=1`, "event_queue_settings": `id=1 AND concurrency=1 AND max_depth=10000 AND paused=0 AND retry_base_seconds=30 AND retention_hours=168`, "event_log_policies": `event_type='*' AND retention_hours=168 AND aggregate_after_hours=24 AND bucket_seconds=60 AND max_rows=5000`, } func importIdent(s string) string { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` } func inspectApplicationImportSource(ctx context.Context, tx *sql.Tx) (map[string]bool, error) { rows, err := tx.QueryContext(ctx, `SELECT name FROM sqlite_master WHERE type='table' AND substr(name,1,7)!='sqlite_' ORDER BY name`) if err != nil { return nil, err } names := map[string]bool{} for rows.Next() { var n string if err = rows.Scan(&n); err != nil { rows.Close() return nil, err } names[n] = true } err = rows.Err() rows.Close() if err != nil { return nil, err } for name := range names { expected, ok := applicationImportColumns[name] if !ok { expected, ok = applicationImportOmissions[name] } if !ok { return nil, importRefusef("unclassified source table: %s", name) } cols, err := tx.QueryContext(ctx, `SELECT name FROM pragma_table_info(?) ORDER BY name`, name) if err != nil { return nil, err } var got []string for cols.Next() { var n string if err = cols.Scan(&n); err != nil { cols.Close() return nil, err } got = append(got, n) } err = cols.Err() cols.Close() if err != nil { return nil, err } if strings.Join(got, ",") != expected { return nil, importRefusef("source columns differ: %s", name) } } for name := range applicationImportColumns { if !names[name] && !strings.HasPrefix(name, "calendar_") { return nil, importRefusef("missing source table: %s", name) } } objects, err := tx.QueryContext(ctx, `SELECT type,name,sql FROM sqlite_master WHERE type IN ('trigger','view') ORDER BY name`) if err != nil { return nil, err } for objects.Next() { var kind, name, ddl string if err = objects.Scan(&kind, &name, &ddl); err != nil { objects.Close() return nil, err } sum := sha256.Sum256([]byte(ddl)) if applicationImportSourceObjects[name] != kind+":"+hex.EncodeToString(sum[:]) { objects.Close() return nil, importRefusef("unclassified or modified source %s: %s", kind, name) } } err = objects.Err() objects.Close() if err != nil { return nil, err } return names, nil } func inspectApplicationImportTarget(ctx context.Context, tx *sql.Tx) error { rows, err := tx.QueryContext(ctx, `SELECT table_name,column_name FROM information_schema.columns WHERE table_schema=current_schema() ORDER BY table_name,column_name`) if err != nil { return err } got := map[string][]string{} for rows.Next() { var t, c string if err = rows.Scan(&t, &c); err != nil { rows.Close() return err } got[t] = append(got[t], c) } err = rows.Err() rows.Close() if err != nil { return err } for name, cols := range got { if name == "schema_migration_steps" || name == "application_import_receipts" { continue } expected, ok := applicationImportColumns[name] if !ok { return importRefusef("unclassified destination table: %s", name) } if name == "event_jobs" { expected += ",claim_token" p := strings.Split(expected, ",") sort.Strings(p) expected = strings.Join(p, ",") } if strings.Join(cols, ",") != expected { return importRefusef("destination columns differ: %s", name) } } for name := range applicationImportColumns { if _, ok := got[name]; !ok { return importRefusef("missing destination table: %s", name) } } return nil } func importTableOrder(ctx context.Context, tx *sql.Tx, names []string) ([]string, error) { deps := map[string][]string{} rows, err := tx.QueryContext(ctx, `SELECT a.relname,b.relname FROM pg_constraint f JOIN pg_class a ON a.oid=f.conrelid JOIN pg_class b ON b.oid=f.confrelid JOIN pg_namespace n ON n.oid=a.relnamespace WHERE f.contype='f' AND n.nspname=current_schema()`) if err != nil { return nil, err } for rows.Next() { var a, b string if err = rows.Scan(&a, &b); err != nil { rows.Close() return nil, err } if a != b { deps[a] = append(deps[a], b) } } err = rows.Err() rows.Close() if err != nil { return nil, err } var order []string seen := map[string]bool{} for len(order) < len(names) { changed := false for _, name := range names { if seen[name] { continue } ready := true for _, dep := range deps[name] { ready = ready && seen[dep] } if ready { order = append(order, name) seen[name] = true changed = true } } if !changed { return nil, importRefuse("unsupported import foreign key dependency") } } return order, nil } func importSelect(name string, columns []string, pg bool) string { quoted := make([]string, len(columns)) for i, c := range columns { quoted[i] = importIdent(c) } // All columns participate, including duplicate legacy tags. SQL collation // is kept bytewise to produce the same deterministic row stream. order := make([]string, len(columns)) for i, q := range quoted { if pg { order[i] = q + ` NULLS FIRST` } else { order[i] = q + ` COLLATE BINARY NULLS FIRST` } } if name == "organizer_task_deletions" { order = []string{`"domain_uuid" COLLATE BINARY`, `"item_uuid" COLLATE BINARY`} if pg { order = []string{`"domain_uuid" COLLATE "C"`, `"item_uuid" COLLATE "C"`} } } return `SELECT ` + strings.Join(quoted, ",") + ` FROM ` + importIdent(name) + ` ORDER BY ` + strings.Join(order, ",") } func copyImportTable(ctx context.Context, from, to *sql.Tx, name string, columns []string) (ImportedTable, error) { result := ImportedTable{Name: name} hash := sha256.New() rows, err := from.QueryContext(ctx, importSelect(name, columns, false)) if err != nil { return result, err } defer rows.Close() quoted := make([]string, len(columns)) for i, c := range columns { quoted[i] = importIdent(c) } var batch [][]any bytes := 0 flush := func() error { if len(batch) == 0 { return nil } args := make([]any, 0, len(batch)*len(columns)) values := make([]string, 0, len(batch)) for _, row := range batch { p := make([]string, len(columns)) for i, v := range row { args = append(args, v) p[i] = fmt.Sprintf("$%d", len(args)) } values = append(values, "("+strings.Join(p, ",")+")") } _, err := to.ExecContext(ctx, `INSERT INTO `+importIdent(name)+` (`+strings.Join(quoted, ",")+`) OVERRIDING SYSTEM VALUE VALUES `+strings.Join(values, ","), args...) batch = nil bytes = 0 return err } for rows.Next() { v := make([]any, len(columns)) dest := make([]any, len(v)) for i := range v { dest[i] = &v[i] } if err = rows.Scan(dest...); err != nil { return result, err } if err = normalizeImportRow(name, columns, v); err != nil { return result, err } b, err := importRowBytes(v) if err != nil { return result, err } if len(b) > 64<<20 { return result, importRefuse("import row exceeds 64 MiB safety limit") } hash.Write(b) hash.Write([]byte{'\n'}) result.Rows++ batch = append(batch, v) bytes += len(b) if len(batch) >= 100 || bytes >= 2<<20 { if err = flush(); err != nil { return result, err } } } if err = rows.Err(); err != nil { return result, err } rows.Close() if err = flush(); err != nil { return result, err } result.SHA256 = hex.EncodeToString(hash.Sum(nil)) // Re-read retained values, not only row counts. Bytea/NULL/integer/text // retain their JSON type tags, so conversion or loss fails the transaction. selectSQL := importSelect(name, columns, true) // Force bytewise ordering for TEXT regardless of the database default. meta, err := to.QueryContext(ctx, `SELECT column_name FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1 AND data_type IN ('text','character varying','character')`, name) if err != nil { return result, err } textCols := map[string]bool{} for meta.Next() { var c string if err = meta.Scan(&c); err != nil { meta.Close() return result, err } textCols[c] = true } err = meta.Err() meta.Close() if err != nil { return result, err } ordering := make([]string, len(columns)) for i, c := range columns { ordering[i] = importIdent(c) if textCols[c] { ordering[i] += ` COLLATE "C"` } ordering[i] += " NULLS FIRST" } if name == "organizer_task_deletions" { ordering = []string{`"domain_uuid" COLLATE "C"`, `"item_uuid" COLLATE "C"`} } selectSQL = selectSQL[:strings.Index(selectSQL, " ORDER BY ")] + " ORDER BY " + strings.Join(ordering, ",") check, err := to.QueryContext(ctx, selectSQL) if err != nil { return result, err } defer check.Close() h := sha256.New() var count int64 for check.Next() { v := make([]any, len(columns)) dest := make([]any, len(v)) for i := range v { dest[i] = &v[i] } if err = check.Scan(dest...); err != nil { return result, err } b, err := importRowBytes(v) if err != nil { return result, err } h.Write(b) h.Write([]byte{'\n'}) count++ } if err = check.Err(); err != nil { return result, err } if count != result.Rows || hex.EncodeToString(h.Sum(nil)) != result.SHA256 { return result, importRefusef("import content fingerprint differs: %s", name) } return result, nil } func normalizeImportRow(table string, columns []string, values []any) error { for i, column := range columns { // SQLite may scan a non-NULL zero-length BLOB as []byte(nil). // pgx binds that slice as SQL NULL unless made explicitly non-nil. if b, ok := values[i].([]byte); ok && b == nil { values[i] = []byte{} } if !(table == "organizer_dav_resources" && (column == "name" || column == "uid")) && !(table == "organizer_task_deletions" && (column == "dav_name" || column == "dav_uid")) { continue } value, ok := values[i].(string) if !ok { return importRefuse("invalid task DAV index scalar") } project := "" if strings.HasPrefix(value, "\x00") { parts := strings.Split(value, "\x00") if len(parts) != 3 || parts[1] == "" || parts[2] == "" { return importRefuse("invalid legacy task DAV index key") } project, value = parts[1], parts[2] } else if strings.ContainsRune(value, 0) { return importRefuse("invalid legacy task DAV index key") } values[i] = nativeTaskDAVIndexKey(project, value) } return nil } func resetImportIdentities(ctx context.Context, tx *sql.Tx, sourceSequences map[string]int64) error { rows, err := tx.QueryContext(ctx, `SELECT table_name,column_name FROM information_schema.columns WHERE table_schema=current_schema() AND is_identity='YES'`) if err != nil { return err } var ids [][2]string for rows.Next() { var t, c string if err = rows.Scan(&t, &c); err != nil { rows.Close() return err } ids = append(ids, [2]string{t, c}) } err = rows.Err() rows.Close() if err != nil { return err } for _, id := range ids { if _, ok := applicationImportColumns[id[0]]; !ok { return importRefuse("unclassified identity") } var next int64 if err = tx.QueryRowContext(ctx, `SELECT COALESCE(max(`+importIdent(id[1])+`),0)+1 FROM `+importIdent(id[0])).Scan(&next); err != nil { return err } if next < 1 { return importRefuse("identity overflow") } if previous := sourceSequences[id[0]]; previous >= next { if previous == int64(^uint64(0)>>1) { return importRefuse("identity exhausted") } next = previous + 1 } if _, err = tx.ExecContext(ctx, `ALTER TABLE `+importIdent(id[0])+` ALTER COLUMN `+importIdent(id[1])+` RESTART WITH `+fmt.Sprint(next)); err != nil { return err } } return nil } func sourceImportSequences(ctx context.Context, tx *sql.Tx) (map[string]int64, error) { out := map[string]int64{} var exists bool if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='sqlite_sequence')`).Scan(&exists); err != nil { return nil, err } if !exists { return out, nil } rows, err := tx.QueryContext(ctx, `SELECT name,seq FROM sqlite_sequence`) if err != nil { return nil, err } defer rows.Close() for rows.Next() { var name string var seq int64 if err = rows.Scan(&name, &seq); err != nil { return nil, err } if _, ok := applicationImportColumns[name]; ok { out[name] = seq } else if _, ok := applicationImportOmissions[name]; !ok { return nil, importRefusef("unclassified sequence: %s", name) } } return out, rows.Err() } func validateImportedBindings(ctx context.Context, tx *sql.Tx) error { var bad bool err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM domains d LEFT JOIN campuses c ON c.uuid=d.campus_uuid WHERE c.uuid IS NULL) OR EXISTS(SELECT 1 FROM paths p LEFT JOIN domains d ON d.uuid=p.domain_uuid WHERE d.uuid IS NULL) OR EXISTS(SELECT 1 FROM domain_memberships m LEFT JOIN domains d ON d.uuid=m.domain_uuid LEFT JOIN principals p ON p.uuid=m.principal_uuid WHERE d.uuid IS NULL OR p.uuid IS NULL)`).Scan(&bad) if err != nil { return err } if bad { return importRefuse("imported namespace or membership integrity failed") } return nil } // PreparePostgreSQLImport creates only schema in a dedicated empty database. // It must not be used to adopt a populated or unversioned PostgreSQL schema. func PreparePostgreSQLImport(ctx context.Context, target *sql.DB, root string) error { if err := refusePopulatedImportTarget(ctx, target); err != nil { return err } s := &Store{DB: target, sqlBackend: storePostgreSQL, DocumentRoot: root} if err := s.ensurePostgreSQLNamespaces(ctx); err != nil { return err } core, err := metadata.New(target, metadata.PostgreSQL) if err != nil { return err } if err = core.InitializeFresh(ctx); err != nil { return err } if err = s.prepareNativeApplicationSchemas(ctx); err != nil { return err } _, err = calendar.NewPostgreSQL(ctx, filepath.Join(root, "calendars"), target) return err } // Preparation must not run migrations against an existing application. Failed // attempts may leave versioned schema and exact default seeds, but no user rows. func refusePopulatedImportTarget(ctx context.Context, target *sql.DB) error { rows, err := target.QueryContext(ctx, `SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_type='BASE TABLE'`) if err != nil { return err } var names []string for rows.Next() { var name string if err = rows.Scan(&name); err != nil { rows.Close() return err } names = append(names, name) } err = rows.Err() rows.Close() if err != nil { return err } for _, name := range names { if name == "schema_migration_steps" { continue } if _, ok := applicationImportColumns[name]; !ok && name != "application_import_receipts" { return importRefuse("import destination contains an unmanaged table") } var n int64 if err = target.QueryRowContext(ctx, `SELECT count(*) FROM `+importIdent(name)).Scan(&n); err != nil { return err } if n == 0 { continue } predicate, ok := importSeedPredicates[name] if !ok { return importRefusef("import destination is not empty: %s", name) } var valid bool if err = target.QueryRowContext(ctx, `SELECT count(*)=1 AND bool_and(`+predicate+`) FROM `+importIdent(name)).Scan(&valid); err != nil { return err } if !valid { return importRefusef("import destination seed has changed: %s", name) } } return nil } // JSON alone cannot distinguish a byte slice from its base64 text, nor an // integer from an integral float. Explicit tags make conversion detectable. func importRowBytes(values []any) ([]byte, error) { type cell struct { Type string Value any } row := make([]cell, len(values)) for i, value := range values { var kind string switch value.(type) { case nil: kind = "null" case string: kind = "text" case []byte: kind = "bytes" if value.([]byte) == nil { value = []byte{} } case int64: kind = "integer" case float64: kind = "float" case bool: kind = "boolean" default: return nil, importRefuse("unsupported import scalar type") } row[i] = cell{kind, value} } return json.Marshal(row) } // InspectSQLiteApplicationImport never writes or hydrates user content. func InspectSQLiteApplicationImport(ctx context.Context, source *sql.DB) ([]string, error) { tx, err := source.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) if err != nil { return nil, err } defer tx.Rollback() tables, err := inspectApplicationImportSource(ctx, tx) if err != nil { return nil, err } names := make([]string, 0, len(tables)) for name := range tables { names = append(names, name) } sort.Strings(names) return names, nil } // VerifyPostgreSQLImport gates the production selector. A successful core-only // rehearsal is insufficient. The original key and the same canonical root are // required; relocation/restore must preserve the configured mount path. func VerifyPostgreSQLImport(ctx context.Context, target *sql.DB, root string) error { key, err := existingAuthKey(root) if err != nil { return err } root, err = filepath.Abs(root) if err != nil { return err } var path, keyHash, fingerprint string var completed int64 if err = target.QueryRowContext(ctx, `SELECT document_root,key_fingerprint,source_fingerprint,completed_at_ms FROM application_import_receipts WHERE id=1`).Scan(&path, &keyHash, &fingerprint, &completed); err != nil { return importRefuse("PostgreSQL startup requires a completed full application import") } sum := sha256.Sum256(key) if root != path || keyHash != hex.EncodeToString(sum[:]) || len(fingerprint) != 64 || completed <= 0 { return importRefuse("PostgreSQL import root/key receipt mismatch") } return nil }