Refactor architecture into eight components and consolidate reliability work
Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts. Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation. Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
This commit is contained in:
1 parent
9dd425f765
commit
d5d21b7841
302 files changed
+20037
-3492
No files matched your search
+17
-1
@@ -1,9 +1,12 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
@@ -24,7 +27,20 @@ func Open(path string) (*sql.DB, error) {
|
||||
},
|
||||
}.Encode(),
|
||||
}).String()
|
||||
return sql.Open("sqlite", dsn)
|
||||
database, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// sql.Open is lazy. Establish WAL once before exposing the pool, otherwise
|
||||
// simultaneous first borrowers race the initial journal_mode transition in
|
||||
// their connection pragmas (SQLITE_BUSY before BEGIN IMMEDIATE is reached).
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := database.PingContext(ctx); err != nil {
|
||||
database.Close()
|
||||
return nil, fmt.Errorf("initialize SQLite connection: %w", err)
|
||||
}
|
||||
return database, nil
|
||||
}
|
||||
|
||||
func Init(db *sql.DB) error {
|
||||
|
||||
+87
-90
@@ -4,6 +4,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func Migrate(
|
||||
@@ -96,54 +97,21 @@ func Migrate(
|
||||
}
|
||||
}
|
||||
|
||||
alterStmts := []string{
|
||||
|
||||
`
|
||||
ALTER TABLE paths
|
||||
ADD COLUMN created_at INTEGER
|
||||
`,
|
||||
|
||||
`
|
||||
ALTER TABLE objects
|
||||
ADD COLUMN created_at INTEGER
|
||||
`,
|
||||
|
||||
`
|
||||
ALTER TABLE revisions
|
||||
ADD COLUMN created_at INTEGER
|
||||
`,
|
||||
|
||||
`
|
||||
ALTER TABLE revisions
|
||||
ADD COLUMN document_hash TEXT
|
||||
`,
|
||||
|
||||
`
|
||||
ALTER TABLE revisions
|
||||
ADD COLUMN lang TEXT NOT NULL DEFAULT 'und'
|
||||
`,
|
||||
|
||||
`
|
||||
ALTER TABLE revisions
|
||||
ADD COLUMN created_at_ms INTEGER
|
||||
`,
|
||||
`
|
||||
ALTER TABLE revisions
|
||||
ADD COLUMN content_type TEXT NOT NULL DEFAULT 'text/html; charset=utf-8'
|
||||
`,
|
||||
`
|
||||
ALTER TABLE revisions
|
||||
ADD COLUMN filename TEXT NOT NULL DEFAULT ''
|
||||
`,
|
||||
|
||||
`
|
||||
ALTER TABLE tags
|
||||
ADD COLUMN created_at INTEGER
|
||||
`,
|
||||
columns := []struct{ table, name, declaration string }{
|
||||
{"paths", "created_at", "INTEGER"},
|
||||
{"objects", "created_at", "INTEGER"},
|
||||
{"revisions", "created_at", "INTEGER"},
|
||||
{"revisions", "document_hash", "TEXT"},
|
||||
{"revisions", "lang", "TEXT NOT NULL DEFAULT 'und'"},
|
||||
{"revisions", "created_at_ms", "INTEGER"},
|
||||
{"revisions", "content_type", "TEXT NOT NULL DEFAULT 'text/html; charset=utf-8'"},
|
||||
{"revisions", "filename", "TEXT NOT NULL DEFAULT ''"},
|
||||
{"tags", "created_at", "INTEGER"},
|
||||
}
|
||||
|
||||
for _, stmt := range alterStmts {
|
||||
_, _ = db.Exec(stmt)
|
||||
for _, column := range columns {
|
||||
if err := ensureSQLiteColumn(db, column.table, column.name, column.declaration); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := migrateRevisionMetadata(db); err != nil {
|
||||
@@ -185,54 +153,47 @@ func migrateRevisionMetadata(db *sql.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
rows, err := db.Query(`
|
||||
SELECT
|
||||
uuid,
|
||||
COALESCE(body, '')
|
||||
FROM revisions
|
||||
WHERE document_hash IS NULL
|
||||
OR document_hash = ''
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
type legacyRevision struct {
|
||||
uuid string
|
||||
body string
|
||||
}
|
||||
|
||||
var revisions []legacyRevision
|
||||
|
||||
for rows.Next() {
|
||||
var revision legacyRevision
|
||||
|
||||
if err := rows.Scan(&revision.uuid, &revision.body); err != nil {
|
||||
// Fetch only identifiers in batches, then hydrate one legacy body at a
|
||||
// time. The old implementation accumulated all payloads in memory.
|
||||
lastUUID := ""
|
||||
for {
|
||||
rows, err := db.Query(`SELECT uuid FROM revisions WHERE uuid>? AND (document_hash IS NULL OR document_hash='') ORDER BY uuid LIMIT 128`, lastUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
revisions = append(revisions, revision)
|
||||
}
|
||||
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, revision := range revisions {
|
||||
sum := sha256.Sum256([]byte(revision.body))
|
||||
documentHash := hex.EncodeToString(sum[:])
|
||||
|
||||
if _, err := db.Exec(`
|
||||
UPDATE revisions
|
||||
SET document_hash = ?
|
||||
WHERE uuid = ?
|
||||
`, documentHash, revision.uuid); err != nil {
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, id := range ids {
|
||||
var body string
|
||||
if err := db.QueryRow(`SELECT COALESCE(body,'') FROM revisions WHERE uuid=?`, id).Scan(&body); err != nil {
|
||||
return err
|
||||
}
|
||||
sum := sha256.Sum256([]byte(body))
|
||||
if _, err := db.Exec(`UPDATE revisions SET document_hash=? WHERE uuid=? AND (document_hash IS NULL OR document_hash='')`, hex.EncodeToString(sum[:]), id); err != nil {
|
||||
return err
|
||||
}
|
||||
lastUUID = id
|
||||
}
|
||||
if len(ids) < 128 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if _, err = db.Exec(`
|
||||
if _, err := db.Exec(`
|
||||
UPDATE revisions
|
||||
SET
|
||||
lang = COALESCE(NULLIF(lang, ''), 'und'),
|
||||
@@ -247,7 +208,7 @@ func migrateRevisionMetadata(db *sql.DB) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = db.Exec(`
|
||||
if _, err := db.Exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_revisions_object_created
|
||||
ON revisions (
|
||||
object_uuid,
|
||||
@@ -257,9 +218,45 @@ func migrateRevisionMetadata(db *sql.DB) error {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
_, err := db.Exec(`
|
||||
INSERT OR IGNORE INTO schema_migrations (name, applied_at)
|
||||
VALUES ('revision_metadata_v1', strftime('%s','now'))
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
func sqliteColumnExists(database *sql.DB, table, column string) (bool, error) {
|
||||
// All identifiers originate in the fixed legacy bootstrap above.
|
||||
rows, err := database.Query(fmt.Sprintf("PRAGMA table_info(%q)", table))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
found := false
|
||||
for rows.Next() {
|
||||
var cid, notNull, primaryKey int
|
||||
var name, kind string
|
||||
var defaultValue any
|
||||
if err := rows.Scan(&cid, &name, &kind, ¬Null, &defaultValue, &primaryKey); err != nil {
|
||||
return false, err
|
||||
}
|
||||
found = found || name == column
|
||||
}
|
||||
return found, rows.Err()
|
||||
}
|
||||
|
||||
func ensureSQLiteColumn(database *sql.DB, table, column, declaration string) error {
|
||||
exists, err := sqliteColumnExists(database, table, column)
|
||||
if err != nil || exists {
|
||||
return err
|
||||
}
|
||||
_, err = database.Exec(fmt.Sprintf("ALTER TABLE %q ADD COLUMN %q %s", table, column, declaration))
|
||||
if err != nil {
|
||||
// Only tolerate another bootstrap adding the exact column concurrently.
|
||||
if exists, checkErr := sqliteColumnExists(database, table, column); checkErr == nil && exists {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("add column %s.%s: %w", table, column, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
type MigrationPlanStep struct {
|
||||
Version int `json:"version"`
|
||||
Name string `json:"name"`
|
||||
Statements int `json:"statements"`
|
||||
AddColumns []SQLiteColumn `json:"add_columns,omitempty"`
|
||||
}
|
||||
|
||||
type MigrationInspection struct {
|
||||
Component string `json:"component"`
|
||||
State string `json:"state"` // current, pending, blocked, unknown
|
||||
AppliedVersion int `json:"applied_version"`
|
||||
LatestVersion int `json:"latest_version"`
|
||||
Pending []MigrationPlanStep `json:"pending,omitempty"`
|
||||
ContractChecked bool `json:"contract_checked"`
|
||||
Issues []string `json:"issues,omitempty"`
|
||||
}
|
||||
|
||||
// InspectSQLiteMigrations takes a read transaction and never applies DDL/DML.
|
||||
// Pending definitions are a plan, not a successful rehearsal: data-dependent
|
||||
// DDL and assertions on not-yet-created objects require a disposable DB copy.
|
||||
func InspectSQLiteMigrations(ctx context.Context, database *sql.DB, plans map[string][]SQLMigration) ([]MigrationInspection, error) {
|
||||
tx, err := database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var ledger bool
|
||||
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_migration_steps')`).Scan(&ledger); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type record struct {
|
||||
version int
|
||||
name, checksum string
|
||||
}
|
||||
history := map[string][]record{}
|
||||
if ledger {
|
||||
rows, err := tx.QueryContext(ctx, `SELECT component,version,name,checksum FROM schema_migration_steps ORDER BY component,version`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var component string
|
||||
var r record
|
||||
if err := rows.Scan(&component, &r.version, &r.name, &r.checksum); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
history[component] = append(history[component], r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
names := make([]string, 0, len(plans)+len(history))
|
||||
for name := range plans {
|
||||
names = append(names, name)
|
||||
}
|
||||
for name := range history {
|
||||
if _, ok := plans[name]; !ok {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
result := make([]MigrationInspection, 0, len(names))
|
||||
for _, name := range names {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
steps, known := plans[name]
|
||||
item := MigrationInspection{Component: name, State: "current", LatestVersion: len(steps)}
|
||||
for i, step := range steps {
|
||||
if step.Version != i+1 || step.Name == "" || (len(step.Statements) == 0 && len(step.AddColumns) == 0 && len(step.Assertions) == 0) {
|
||||
return nil, fmt.Errorf("invalid migration definition %s/%d", name, step.Version)
|
||||
}
|
||||
}
|
||||
if !known {
|
||||
item.State = "unknown"
|
||||
item.Issues = []string{"component is not registered in this binary"}
|
||||
}
|
||||
for i, record := range history[name] {
|
||||
item.AppliedVersion = record.version
|
||||
if known && (record.version != i+1 || record.version > len(steps) || record.name != steps[record.version-1].Name || record.checksum != migrationChecksum(steps[record.version-1])) {
|
||||
item.State = "blocked"
|
||||
item.Issues = append(item.Issues, fmt.Sprintf("migration history mismatch at version %d", record.version))
|
||||
}
|
||||
}
|
||||
if item.State == "current" {
|
||||
for _, step := range steps[len(history[name]):] {
|
||||
item.Pending = append(item.Pending, MigrationPlanStep{Version: step.Version, Name: step.Name, Statements: len(step.Statements), AddColumns: step.AddColumns})
|
||||
}
|
||||
if len(item.Pending) > 0 {
|
||||
item.State = "pending"
|
||||
}
|
||||
if len(steps) > 0 && len(steps[len(steps)-1].Assertions) > 0 {
|
||||
item.ContractChecked = true
|
||||
for i, assertion := range steps[len(steps)-1].Assertions {
|
||||
rows, err := tx.QueryContext(ctx, assertion)
|
||||
if err != nil {
|
||||
item.Issues = append(item.Issues, fmt.Sprintf("assertion %d: %v", i+1, err))
|
||||
continue
|
||||
}
|
||||
violated := rows.Next()
|
||||
err = rows.Err()
|
||||
closeErr := rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
if violated {
|
||||
item.Issues = append(item.Issues, fmt.Sprintf("assertion %d failed", i+1))
|
||||
}
|
||||
}
|
||||
if len(item.Issues) > 0 && item.State == "current" {
|
||||
item.State = "blocked"
|
||||
}
|
||||
}
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, tx.Commit()
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInspectMigrationsReadOnlyLifecycle(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "inspect.db")
|
||||
writer, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer writer.Close()
|
||||
if err := writer.Ping(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reader, err := OpenReadOnly(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample(id INTEGER PRIMARY KEY, value TEXT)`, `INSERT INTO sample VALUES(1,'keep')`}, Assertions: []string{`SELECT id,value FROM sample LIMIT 0`}}}
|
||||
plans := map[string][]SQLMigration{"sample": steps}
|
||||
inspect := func(state string, issues bool) {
|
||||
t.Helper()
|
||||
got, err := InspectSQLiteMigrations(context.Background(), reader, plans)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].State != state || (len(got[0].Issues) > 0) != issues || !got[0].ContractChecked {
|
||||
t.Fatalf("report: %+v", got)
|
||||
}
|
||||
}
|
||||
inspect("pending", true)
|
||||
var count int
|
||||
if err := writer.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name IN ('sample','schema_migration_steps')`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatalf("inspection created tables: %d %v", count, err)
|
||||
}
|
||||
if _, err := reader.Exec(`CREATE TABLE forbidden(id INTEGER)`); err == nil {
|
||||
t.Fatal("connection is writable")
|
||||
}
|
||||
// Keep the writer open: the reader must see committed WAL contents too.
|
||||
if err := ApplySQLiteMigrations(context.Background(), writer, "sample", steps); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspect("current", false)
|
||||
if _, err := writer.Exec(`ALTER TABLE sample RENAME COLUMN value TO lost`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspect("blocked", true)
|
||||
var value string
|
||||
if err := writer.QueryRow(`SELECT lost FROM sample WHERE id=1`).Scan(&value); err != nil || value != "keep" {
|
||||
t.Fatalf("inspection changed data: %q %v", value, err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := InspectSQLiteMigrations(ctx, reader, plans); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancel: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectMigrationsRejectHistoryMismatch(t *testing.T) {
|
||||
for _, tc := range []struct{ name, mutation string }{
|
||||
{"checksum", `UPDATE schema_migration_steps SET checksum='changed'`},
|
||||
{"name", `UPDATE schema_migration_steps SET name='changed'`},
|
||||
{"zero", `UPDATE schema_migration_steps SET version=0`},
|
||||
{"negative", `UPDATE schema_migration_steps SET version=-1`},
|
||||
{"newer", `UPDATE schema_migration_steps SET version=2`},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
database, err := Open(filepath.Join(t.TempDir(), "inspect.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample(id INTEGER)`}}}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "sample", steps); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := database.Exec(tc.mutation); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := InspectSQLiteMigrations(context.Background(), database, map[string][]SQLMigration{"sample": steps})
|
||||
if err != nil || len(got) != 1 || got[0].State != "blocked" {
|
||||
t.Fatalf("report=%+v err=%v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectMigrationsUnknownAndPendingPlan(t *testing.T) {
|
||||
database, err := Open(filepath.Join(t.TempDir(), "inspect.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample(id INTEGER)`}}}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "sample", steps); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := InspectSQLiteMigrations(context.Background(), database, nil)
|
||||
if err != nil || len(got) != 1 || got[0].State != "unknown" {
|
||||
t.Fatalf("unknown=%+v %v", got, err)
|
||||
}
|
||||
steps = append(steps, SQLMigration{Version: 2, Name: "add_value", AddColumns: []SQLiteColumn{{Table: "sample", Name: "value", Declaration: "TEXT NOT NULL DEFAULT ''"}}, Assertions: []string{`SELECT value FROM sample LIMIT 0`}})
|
||||
got, err = InspectSQLiteMigrations(context.Background(), database, map[string][]SQLMigration{"sample": steps})
|
||||
if err != nil || len(got) != 1 {
|
||||
t.Fatalf("pending=%+v %v", got, err)
|
||||
}
|
||||
item := got[0]
|
||||
if item.State != "pending" || item.AppliedVersion != 1 || item.LatestVersion != 2 || len(item.Pending) != 1 || len(item.Pending[0].AddColumns) != 1 || len(item.Issues) == 0 {
|
||||
t.Fatalf("pending=%+v", item)
|
||||
}
|
||||
var count int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM pragma_table_info('sample') WHERE name='value'`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatalf("pending DDL applied: %d %v", count, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SQLMigration is append-only once released. Edit a new version, never an
|
||||
// applied statement. This runner is SQLite-specific; other DBs need their own
|
||||
// locking/DDL implementations rather than a driver-name substitution.
|
||||
type SQLMigration struct {
|
||||
Version int
|
||||
Name string
|
||||
Statements []string
|
||||
// Omit empty new fields to preserve checksums of previously released steps.
|
||||
AddColumns []SQLiteColumn `json:",omitempty"`
|
||||
// Each assertion must execute successfully and return no rows. The latest
|
||||
// version defines the whole current contract; older assertions are superseded
|
||||
// to permit deliberate column removal in a later, immutable migration.
|
||||
Assertions []string `json:",omitempty"`
|
||||
}
|
||||
|
||||
type SQLiteColumn struct {
|
||||
Table string
|
||||
Name string
|
||||
Declaration string
|
||||
}
|
||||
|
||||
func migrationChecksum(step SQLMigration) string {
|
||||
encoded, _ := json.Marshal(step)
|
||||
sum := sha256.Sum256(encoded)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// ApplySQLiteMigrations serializes processes on a dedicated connection with
|
||||
// BEGIN IMMEDIATE. DDL, data statements and ledger updates commit together.
|
||||
// The legacy schema_migrations table remains untouched for old installations.
|
||||
func ApplySQLiteMigrations(ctx context.Context, database *sql.DB, component string, steps []SQLMigration) error {
|
||||
if component == "" {
|
||||
return fmt.Errorf("migration component is required")
|
||||
}
|
||||
for i, step := range steps {
|
||||
if step.Version != i+1 || step.Name == "" || (len(step.Statements) == 0 && len(step.AddColumns) == 0 && len(step.Assertions) == 0) {
|
||||
return fmt.Errorf("invalid migration sequence for %s", component)
|
||||
}
|
||||
}
|
||||
conn, err := database.Conn(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("migration %s acquire connection: %w", component, err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil {
|
||||
return fmt.Errorf("migration %s begin immediate: %w", component, err)
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if committed {
|
||||
return
|
||||
}
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if _, err := conn.ExecContext(cleanup, "ROLLBACK"); err != nil {
|
||||
// Never return a connection with uncertain transaction state to
|
||||
// the pool. A failed/ambiguous COMMIT also takes this path.
|
||||
_ = conn.Raw(func(any) error { return driver.ErrBadConn })
|
||||
}
|
||||
}()
|
||||
if _, err := conn.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_migration_steps (
|
||||
component TEXT NOT NULL, version INTEGER NOT NULL, name TEXT NOT NULL,
|
||||
checksum TEXT NOT NULL, applied_at_ms INTEGER NOT NULL,
|
||||
PRIMARY KEY(component, version))`); err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := conn.QueryContext(ctx, `SELECT version,name,checksum FROM schema_migration_steps WHERE component=? ORDER BY version`, component)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
applied := 0
|
||||
for rows.Next() {
|
||||
var version int
|
||||
var name, checksum string
|
||||
if err := rows.Scan(&version, &name, &checksum); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if version != applied+1 || version > len(steps) || steps[version-1].Name != name || migrationChecksum(steps[version-1]) != checksum {
|
||||
rows.Close()
|
||||
return fmt.Errorf("migration history mismatch: component=%s version=%d (changed, missing or newer migration)", component, version)
|
||||
}
|
||||
applied++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, step := range steps[applied:] {
|
||||
for i, statement := range step.Statements {
|
||||
if _, err := conn.ExecContext(ctx, statement); err != nil {
|
||||
return fmt.Errorf("migration %s/%d %s statement %d: %w", component, step.Version, step.Name, i+1, err)
|
||||
}
|
||||
}
|
||||
for _, column := range step.AddColumns {
|
||||
if err := addSQLiteColumn(ctx, conn, column); err != nil {
|
||||
return fmt.Errorf("migration %s/%d column %s.%s: %w", component, step.Version, column.Table, column.Name, err)
|
||||
}
|
||||
}
|
||||
if _, err := conn.ExecContext(ctx, `INSERT INTO schema_migration_steps(component,version,name,checksum,applied_at_ms) VALUES(?,?,?,?,?)`, component, step.Version, step.Name, migrationChecksum(step), time.Now().UnixMilli()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(steps) > 0 {
|
||||
step := steps[len(steps)-1]
|
||||
for i, assertion := range step.Assertions {
|
||||
rows, err := conn.QueryContext(ctx, assertion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("schema contract %s/%d assertion %d: %w", component, step.Version, i+1, err)
|
||||
}
|
||||
violated := rows.Next()
|
||||
err = rows.Err()
|
||||
closeErr := rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if violated {
|
||||
return fmt.Errorf("schema contract %s/%d assertion %d failed", component, step.Version, i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
_, err = conn.ExecContext(ctx, "COMMIT")
|
||||
committed = err == nil
|
||||
return err
|
||||
}
|
||||
|
||||
func addSQLiteColumn(ctx context.Context, conn *sql.Conn, column SQLiteColumn) error {
|
||||
// Identifiers are trusted migration definitions, but validate before SQL
|
||||
// interpolation so accidental punctuation cannot change the statement.
|
||||
for _, identifier := range []string{column.Table, column.Name} {
|
||||
if identifier == "" || strings.IndexFunc(identifier, func(r rune) bool {
|
||||
return !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_')
|
||||
}) >= 0 {
|
||||
return fmt.Errorf("invalid schema identifier")
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(column.Declaration) == "" {
|
||||
return fmt.Errorf("column declaration is required")
|
||||
}
|
||||
rows, err := conn.QueryContext(ctx, `SELECT name FROM pragma_table_info(?)`, column.Table)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found, tableExists := false, false
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
found, tableExists = found || name == column.Name, true
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !tableExists {
|
||||
return fmt.Errorf("table %s is missing", column.Table)
|
||||
}
|
||||
if found {
|
||||
return nil
|
||||
}
|
||||
_, err = conn.ExecContext(ctx, fmt.Sprintf("ALTER TABLE %q ADD COLUMN %q %s", column.Table, column.Name, column.Declaration))
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSQLMigrationLedgerConcurrencyAndChecksum(t *testing.T) {
|
||||
database, err := Open(filepath.Join(t.TempDir(), "migration.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample (id INTEGER PRIMARY KEY)`, `INSERT INTO sample VALUES (1)`}}}
|
||||
var wg sync.WaitGroup
|
||||
errors := make(chan error, 6)
|
||||
for i := 0; i < 6; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
errors <- ApplySQLiteMigrations(context.Background(), database, "test", steps)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errors)
|
||||
for err := range errors {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var count int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM sample`).Scan(&count); err != nil || count != 1 {
|
||||
t.Fatalf("count=%d err=%v", count, err)
|
||||
}
|
||||
changed := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample (id TEXT)`}}}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", changed); err == nil || !strings.Contains(err.Error(), "history mismatch") {
|
||||
t.Fatalf("changed migration accepted: %v", err)
|
||||
}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", nil); err == nil {
|
||||
t.Fatal("downgrade accepted")
|
||||
}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "other", []SQLMigration{{Version: 2, Name: "gap", Statements: []string{`SELECT 1`}}}); err == nil {
|
||||
t.Fatal("version gap accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLMigrationFailureRollsBackDDLAndLedger(t *testing.T) {
|
||||
database, err := Open(filepath.Join(t.TempDir(), "migration.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
steps := []SQLMigration{{Version: 1, Name: "broken", Statements: []string{`CREATE TABLE partial (id INTEGER)`, `NOT VALID SQL`}}}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err == nil {
|
||||
t.Fatal("DDL failure ignored")
|
||||
}
|
||||
var count int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name IN ('partial','schema_migration_steps')`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatalf("partial migration remains count=%d err=%v", count, err)
|
||||
}
|
||||
steps[0].Statements = steps[0].Statements[:1]
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil {
|
||||
t.Fatal("retry after rollback", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if err := ApplySQLiteMigrations(ctx, database, "test", steps); err == nil {
|
||||
t.Fatal("cancel ignored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyRevisionHashMigrationSpansBoundedBatches(t *testing.T) {
|
||||
database, err := Open(filepath.Join(t.TempDir(), "legacy.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if _, err := database.Exec(`CREATE TABLE revisions (uuid TEXT PRIMARY KEY,object_uuid TEXT,body TEXT,message TEXT)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 260; i++ {
|
||||
if _, err := database.Exec(`INSERT INTO revisions(uuid,body) VALUES(?,?)`, fmt.Sprintf("revision-%04d", i), "body"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var count int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM revisions WHERE length(document_hash)=64 AND lang='und' AND created_at_ms=0`).Scan(&count); err != nil || count != 260 {
|
||||
t.Fatalf("migrated=%d err=%v", count, err)
|
||||
}
|
||||
if err := ensureSQLiteColumn(database, "does_not_exist", "created_at", "INTEGER"); err == nil {
|
||||
t.Fatal("ALTER failure swallowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationChecksumPreservesLegacyFormat(t *testing.T) {
|
||||
step := SQLMigration{Version: 1, Name: "legacy", Statements: []string{"SELECT 1"}}
|
||||
sum := sha256.Sum256([]byte(`{"Version":1,"Name":"legacy","Statements":["SELECT 1"]}`))
|
||||
if got := migrationChecksum(step); got != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("legacy checksum changed: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationConditionalColumnsAndAssertionsAreAtomic(t *testing.T) {
|
||||
database, err := Open(filepath.Join(t.TempDir(), "schema.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if _, err := database.Exec(`CREATE TABLE sample(id INTEGER PRIMARY KEY); INSERT INTO sample VALUES(1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := []SQLMigration{{Version: 1, Name: "adopt", AddColumns: []SQLiteColumn{{Table: "sample", Name: "value", Declaration: "TEXT NOT NULL DEFAULT ''"}}, Assertions: []string{`SELECT missing FROM sample LIMIT 0`}}}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err == nil {
|
||||
t.Fatal("invalid schema accepted")
|
||||
}
|
||||
var count int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM pragma_table_info('sample') WHERE name='value'`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatalf("ALTER was not rolled back: %d %v", count, err)
|
||||
}
|
||||
steps[0].Assertions = []string{`SELECT id,value FROM sample LIMIT 0`}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil {
|
||||
t.Fatal("repeated apply", err)
|
||||
}
|
||||
var value string
|
||||
if err := database.QueryRow(`SELECT value FROM sample WHERE id=1`).Scan(&value); err != nil || value != "" {
|
||||
t.Fatalf("legacy row lost: %q %v", value, err)
|
||||
}
|
||||
steps = append(steps, SQLMigration{Version: 2, Name: "remove_value", Statements: []string{`ALTER TABLE sample DROP COLUMN value`}, Assertions: []string{`SELECT id FROM sample LIMIT 0`}})
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil {
|
||||
t.Fatal("new contract must supersede old contract", err)
|
||||
}
|
||||
if _, err := database.Exec(`DROP TABLE sample`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err == nil {
|
||||
t.Fatal("recorded history masked missing table")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// OpenReadOnly deliberately does not migrate, set journal_mode, or create a DB.
|
||||
func OpenReadOnly(path string) (*sql.DB, error) {
|
||||
path, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: url.Values{"mode": {"ro"}, "_pragma": {"query_only(1)", "busy_timeout(5000)"}}.Encode()}).String()
|
||||
database, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = database.Ping(); err != nil {
|
||||
database.Close()
|
||||
return nil, err
|
||||
}
|
||||
return database, nil
|
||||
}
|
||||
Reference in new issue
Block a user