feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s

This commit is contained in:
Anya Tokugawa committed 2026-11-15 05:47:14 +09:00
1 parent 96065d1e89
commit 0ba1dcfef0
243 files changed
+21441 -3090

No files matched your search

+7
View File
@@ -16,6 +16,13 @@ import (
// All role changes are operator-only and offline. Never expose this as a
// domain-admin API: a data directory contains multiple campuses and domains.
func runDR(args []string, out io.Writer) error {
backend, err := configuredDatabaseBackend()
if err != nil {
return err
}
if backend != "sqlite" {
return errors.New("DR commands require the SQLite backend; use PostgreSQL backup/restore for native instances")
}
const usage = "pageswiki dr {init|status|snapshot|verify|restore|promote|resume-workers} [flags]\nUse COMMAND --help. Snapshot requires all writers stopped; promotion requires external fencing."
if len(args) == 0 || args[0] == "--help" {
_, err := fmt.Fprintln(out, usage)
+25 -5
View File
@@ -1,6 +1,7 @@
package main
import (
"context"
"errors"
"log"
"net/url"
@@ -14,18 +15,30 @@ import (
const managedCampusReconcileInterval = 5 * time.Second
func startManagedCampusReconciler(dataDir string, st *store.Store) {
func startManagedCampusReconciler(ctx context.Context, dataDir string, st *store.Store) <-chan struct{} {
done := make(chan struct{})
instanceFQDN := strings.TrimSpace(os.Getenv("PAGESWIKI_INSTANCE_FQDN"))
if instanceFQDN == "" {
return
if instanceFQDN == "" || st.ReadOnly || st.BackgroundJobsDisabled {
close(done)
return done
}
go func() {
defer close(done)
ticker := time.NewTicker(managedCampusReconcileInterval)
defer ticker.Stop()
for range ticker.C {
reconcileManagedCampuses(dataDir, st, instanceFQDN)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if ctx.Err() != nil {
return
}
reconcileManagedCampusesContext(ctx, dataDir, st, instanceFQDN)
}
}
}()
return done
}
func reconcileManagedCampuses(
@@ -33,12 +46,19 @@ func reconcileManagedCampuses(
st *store.Store,
instanceFQDN string,
) {
reconcileManagedCampusesContext(context.Background(), dataDir, st, instanceFQDN)
}
func reconcileManagedCampusesContext(ctx context.Context, dataDir string, st *store.Store, instanceFQDN string) {
campuses, err := st.ListCampuses()
if err != nil {
log.Printf("managed Campus reconciliation: %v", err)
return
}
for _, campus := range campuses {
if ctx.Err() != nil {
return
}
if campus.AuthorityMode != "managed" || !campus.Enabled {
continue
}
+23 -56
View File
@@ -1,18 +1,26 @@
package main
import (
"context"
"errors"
"log"
"os"
"os/signal"
"path/filepath"
"syscall"
"pageswiki/internal/db"
"pageswiki/internal/dr"
"pageswiki/internal/server"
"pageswiki/internal/store"
)
func main() {
if len(os.Args) > 1 && os.Args[1] == "migrate-postgres" {
if err := runPostgreSQLMigration(os.Args[2:], os.Stdout); err != nil {
log.Fatal(err)
}
return
}
if len(os.Args) > 1 && os.Args[1] == "schema" {
if err := runSchema(os.Args[2:], os.Stdout); err != nil {
log.Print(err)
@@ -29,6 +37,8 @@ func main() {
}
return
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
enrollment, err := parseExternalCampusEnrollment(os.Args[1:])
if err != nil {
log.Fatal(err)
@@ -64,6 +74,13 @@ func main() {
if err = dr.RequireRole(node, os.Getenv("PAGESWIKI_DR_EXPECT_ROLE")); err != nil {
log.Fatal(err)
}
backend, err := configuredDatabaseBackend()
if err != nil {
log.Fatal(err)
}
if backend == "postgresql" && node != nil {
log.Fatal("PostgreSQL currently supports standalone instances only; DR startup refused")
}
if node != nil && node.Role == "standby" {
if len(os.Args) > 1 {
log.Fatal("standby does not accept enrollment/startup mutation flags")
@@ -79,73 +96,23 @@ func main() {
}
st.StandbyReadUntil = node.ReadUntil()
log.Printf("DR standby: migrations, federation reconciliation and background workers disabled; snapshot=%s", node.SnapshotID)
if err := server.New(st).Run(":3000"); err != nil {
if err := serveInstance(ctx, dataDir, st, ":3000"); err != nil {
log.Fatal(err)
}
return
}
dbPath :=
filepath.Join(
dataDir,
"pages.db",
)
log.Printf(
"pages.db = %s",
dbPath,
)
database, err :=
db.Open(dbPath)
if err != nil {
log.Fatal(err)
}
err = db.Migrate(database)
if err != nil {
log.Fatal(err)
}
documentRoot, err :=
store.MigrateLegacyDocumentRoot(
dataDir,
)
if err != nil {
log.Fatal(err)
}
st :=
store.NewWithDocumentRoot(
database,
documentRoot,
)
err = st.EnsureConfigTable()
if err != nil {
log.Fatal(err)
}
err = st.EnsurePathLinkColumns()
log.Printf("database backend = %s", backend)
database, st, err := openApplicationStore(ctx, dataDir)
if err != nil {
log.Fatal(err)
}
defer database.Close()
if err := enrollExternalCampus(dataDir, st, enrollment); err != nil {
log.Fatal(err)
}
st.BackgroundJobsDisabled = node != nil && !node.WorkersEnabled
if !st.BackgroundJobsDisabled {
startManagedCampusReconciler(dataDir, st)
}
srv := server.New(st)
err = srv.Run(":3000")
err = serveInstance(ctx, dataDir, st, ":3000")
if err != nil {
log.Fatal(err)
+135
View File
@@ -0,0 +1,135 @@
package main
import (
"context"
"database/sql"
"encoding/json"
"errors"
"flag"
"io"
"os"
"path/filepath"
"strings"
"time"
"pageswiki/internal/db"
"pageswiki/internal/dr"
"pageswiki/internal/store"
)
func runPostgreSQLMigration(args []string, out io.Writer) error {
f := flag.NewFlagSet("migrate-postgres", flag.ContinueOnError)
f.SetOutput(out)
defaultDir := os.Getenv("PAGESWIKI_DATA_DIR")
if defaultDir == "" {
defaultDir = "./data"
}
dir := f.String("data-dir", defaultDir, "stopped instance data directory; existing files are retained in place")
apply := f.Bool("apply", false, "copy into an empty PostgreSQL schema after source preflight")
insecure := f.Bool("allow-insecure", false, "allow unverified PostgreSQL TLS for isolated testing only")
timeout := f.Duration("timeout", 30*time.Minute, "migration timeout")
if err := f.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return nil
}
return err
}
if f.NArg() != 0 || *timeout <= 0 {
return errors.New("invalid migration arguments")
}
if info, err := os.Stat(*dir); err != nil || !info.IsDir() {
return errors.New("existing data directory required")
}
lock, err := dr.Lock(*dir, true)
if err != nil {
return err
}
defer lock.Close()
if node, err := dr.Load(*dir); err != nil {
return err
} else if node != nil {
return errors.New("PostgreSQL migration does not support a configured DR instance")
}
source, err := db.OpenReadOnly(filepath.Join(*dir, "pages.db"))
if err != nil {
return err
}
defer source.Close()
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel()
tables, err := store.InspectSQLiteApplicationImport(ctx, source)
if err != nil {
return err
}
if !*apply {
return json.NewEncoder(out).Encode(map[string]any{"mode": "preflight", "tables": tables, "copied": false})
}
target, err := db.OpenPostgreSQL(ctx, db.PostgreSQLOptions{DSN: os.Getenv("PAGESWIKI_POSTGRES_DSN"), AllowInsecure: *insecure})
if err != nil {
return err
}
defer target.Close()
root := filepath.Join(*dir, "documents")
if err = store.PreparePostgreSQLImport(ctx, target, root); err != nil {
return err
}
report, err := store.ImportSQLiteApplication(ctx, source, target, root)
if err != nil {
return err
}
return json.NewEncoder(out).Encode(report)
}
func configuredDatabaseBackend() (string, error) {
b := strings.ToLower(strings.TrimSpace(os.Getenv("PAGESWIKI_DB_BACKEND")))
switch b {
case "", "sqlite":
return "sqlite", nil
case "postgresql":
return b, nil
default:
return "", errors.New("PAGESWIKI_DB_BACKEND must be sqlite or postgresql")
}
}
func openApplicationStore(ctx context.Context, dataDir string) (database *sql.DB, st *store.Store, err error) {
backend, err := configuredDatabaseBackend()
if err != nil {
return nil, nil, err
}
defer func() {
if err != nil && database != nil {
database.Close()
database = nil
}
}()
if backend == "postgresql" {
database, err = db.OpenPostgreSQL(ctx, db.PostgreSQLOptions{DSN: os.Getenv("PAGESWIKI_POSTGRES_DSN")})
if err != nil {
return nil, nil, err
}
root := filepath.Join(dataDir, "documents")
if err = store.VerifyPostgreSQLImport(ctx, database, root); err != nil {
return database, nil, err
}
st, err = store.NewPostgreSQLWithDocumentRoot(ctx, database, root)
return database, st, err
}
database, err = db.Open(filepath.Join(dataDir, "pages.db"))
if err != nil {
return nil, nil, err
}
if err = db.Migrate(database); err != nil {
return database, nil, err
}
root, err := store.MigrateLegacyDocumentRoot(dataDir)
if err != nil {
return database, nil, err
}
st = store.NewWithDocumentRoot(database, root)
if err = st.EnsureConfigTable(); err != nil {
return database, nil, err
}
err = st.EnsurePathLinkColumns()
return database, st, err
}
+168
View File
@@ -0,0 +1,168 @@
package main
import (
"bytes"
"context"
"encoding/json"
"github.com/google/uuid"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"time"
"pageswiki/internal/db"
"pageswiki/internal/dr"
"pageswiki/internal/server"
"pageswiki/internal/store"
)
func TestPostgreSQLMigrationAndTLSStartup(t *testing.T) {
dsn := os.Getenv("PAGESWIKI_TEST_POSTGRES_TLS_DSN")
if dsn == "" {
t.Skip("set a disposable PostgreSQL verify-full DSN")
}
ctx := context.Background()
admin, err := db.OpenPostgreSQL(ctx, db.PostgreSQLOptions{DSN: dsn})
if err != nil {
t.Fatal(err)
}
schema := "pageswiki_cli_test_" + strings.ReplaceAll(uuid.NewString(), "-", "")
if _, err = admin.Exec(`CREATE SCHEMA "` + schema + `"`); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _, _ = admin.Exec(`DROP SCHEMA "` + schema + `" CASCADE`); admin.Close() })
u, err := url.Parse(dsn)
if err != nil {
t.Fatal(err)
}
q := u.Query()
q.Set("search_path", schema)
u.RawQuery = q.Encode()
t.Setenv("PAGESWIKI_POSTGRES_DSN", u.String())
t.Setenv("PAGESWIKI_AUTH_KEY", "")
dir := t.TempDir()
database, err := db.Open(filepath.Join(dir, "pages.db"))
if err != nil {
t.Fatal(err)
}
if err = db.Migrate(database); err != nil {
t.Fatal(err)
}
s := store.NewWithDocumentRoot(database, filepath.Join(dir, "documents"))
if err = s.EnsureConfigTable(); err != nil {
t.Fatal(err)
}
domain, err := s.GetDefaultDomain()
if err != nil {
t.Fatal(err)
}
owner, err := s.BootstrapOwner(domain.UUID, "owner", "synthetic cli password", "Owner", "")
if err != nil {
t.Fatal(err)
}
token, err := s.CreateAuthSession(domain.UUID, owner.UUID)
if err != nil {
t.Fatal(err)
}
if err = database.Close(); err != nil {
t.Fatal(err)
}
var out bytes.Buffer
if err = runPostgreSQLMigration([]string{"--data-dir", dir, "--apply"}, &out); err != nil {
t.Fatal(err)
}
t.Setenv("PAGESWIKI_DB_BACKEND", "postgresql")
t.Chdir("../..")
for range 2 {
database, native, err := openApplicationStore(ctx, dir)
if err != nil {
t.Fatal(err)
}
if got, err := native.PrincipalForSession(domain.UUID, token); err != nil || got.UUID != owner.UUID {
database.Close()
t.Fatal("session lost", err)
}
srv, err := server.NewChecked(native)
if err != nil {
database.Close()
t.Fatal(err)
}
closeCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
err = srv.Close(closeCtx)
cancel()
database.Close()
if err != nil {
t.Fatal(err)
}
}
}
func TestPostgreSQLMigrationPreflight(t *testing.T) {
dir := t.TempDir()
database, err := db.Open(filepath.Join(dir, "pages.db"))
if err != nil {
t.Fatal(err)
}
if err = db.Migrate(database); err != nil {
t.Fatal(err)
}
s := store.NewWithDocumentRoot(database, filepath.Join(dir, "documents"))
if err = s.EnsureConfigTable(); err != nil {
t.Fatal(err)
}
if err = database.Close(); err != nil {
t.Fatal(err)
}
// No target is needed for the schema-only preflight.
t.Setenv("PAGESWIKI_POSTGRES_DSN", "")
var out bytes.Buffer
if err = runPostgreSQLMigration([]string{"--data-dir", dir}, &out); err != nil {
t.Fatal(err)
}
var result struct {
Mode string
Copied bool
Tables []string
}
if err = json.Unmarshal(out.Bytes(), &result); err != nil || result.Mode != "preflight" || result.Copied || len(result.Tables) < 130 {
t.Fatal(result, err)
}
lock, err := dr.Lock(dir, false)
if err != nil {
t.Fatal(err)
}
defer lock.Close()
if err = runPostgreSQLMigration([]string{"--data-dir", dir}, &out); err == nil {
t.Fatal("active instance maintenance accepted")
}
}
func TestPostgreSQLBackendRefusesFallback(t *testing.T) {
dir := t.TempDir()
for _, backend := range []string{"unsupported", "postgresql"} {
t.Setenv("PAGESWIKI_DB_BACKEND", backend)
t.Setenv("PAGESWIKI_POSTGRES_DSN", "")
if _, _, err := openApplicationStore(context.Background(), dir); err == nil {
t.Fatal("invalid configuration accepted")
}
if _, err := os.Stat(filepath.Join(dir, "pages.db")); !os.IsNotExist(err) {
t.Fatal("SQLite fallback created")
}
if err := runSchema([]string{"status", "--data-dir", dir}, &bytes.Buffer{}); err == nil {
t.Fatal("legacy inspection accepted")
}
if err := runDR([]string{"init", "--data-dir", dir}, &bytes.Buffer{}); err == nil {
t.Fatal("legacy DR accepted")
}
}
}
func TestPostgreSQLMigrationArguments(t *testing.T) {
for _, args := range [][]string{{"--timeout", "0"}, {"unexpected"}, {"--data-dir", filepath.Join(t.TempDir(), "missing")}} {
if err := runPostgreSQLMigration(args, &bytes.Buffer{}); err == nil {
t.Fatal("invalid arguments accepted", args)
}
}
}
+51 -2
View File
@@ -12,19 +12,28 @@ import (
"time"
"pageswiki/internal/db"
"pageswiki/internal/metadata"
"pageswiki/internal/store"
)
var errSchemaPending = errors.New("schema migrations are pending; no changes were applied")
var errSchemaBlocked = errors.New("schema history or contract is incompatible; no changes were applied")
var errPostgreSQLNotReady = errors.New("core-only inspection cannot certify application migration; use migrate-postgres; no changes were applied")
func runSchema(args []string, out io.Writer) error {
const usage = "pageswiki schema {status|check} [--db FILE | --data-dir DIR] [--json] [--timeout 30s]\nRead-only inspection of managed components; legacy migrations and pending DDL are not rehearsed.\ncheck exits 2 for pending migrations, 1 for incompatible state, 0 when managed components are current."
backend, err := configuredDatabaseBackend()
if err != nil {
return err
}
if backend != "sqlite" {
return errors.New("legacy schema inspection requires SQLite; PostgreSQL startup validates native migrations and import receipt")
}
const usage = "pageswiki schema {status|check|portability} [--db FILE | --data-dir DIR] [--json] [--timeout 30s]\nRead-only inspection; no migration is performed.\ncheck exits 2 for pending migrations, 1 for incompatible state, 0 when managed components are current.\nportability reports PostgreSQL migration blockers and exits 1 while full application support is incomplete."
if len(args) == 0 || args[0] == "--help" {
_, err := fmt.Fprintln(out, usage)
return err
}
if args[0] != "status" && args[0] != "check" {
if args[0] != "status" && args[0] != "check" && args[0] != "portability" {
return errors.New(usage)
}
f := flag.NewFlagSet("schema "+args[0], flag.ContinueOnError)
@@ -65,6 +74,46 @@ func runSchema(args []string, out io.Writer) error {
defer database.Close()
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel()
if args[0] == "portability" {
repository, err := metadata.New(database, metadata.SQLite)
if err != nil {
return err
}
report, err := repository.InspectPortability(ctx)
if err != nil {
return err
}
if *jsonOutput {
encoder := json.NewEncoder(out)
encoder.SetIndent("", " ")
if err := encoder.Encode(report); err != nil {
return err
}
} else {
if _, err := fmt.Fprintf(out, "PostgreSQL application ready: %t\nPortable core rehearsal compatible: %t\nScope: %s\n", report.ApplicationReady, report.CoreCompatible, report.Scope); err != nil {
return err
}
for _, table := range report.Tables {
if _, err := fmt.Fprintf(out, "%-28s %-20s %s\n", table.Name, table.Component, table.State); err != nil {
return err
}
}
for _, blocker := range report.Blockers {
if _, err := fmt.Fprintln(out, "blocked:", blocker); err != nil {
return err
}
}
if !report.ApplicationReady {
if _, err := fmt.Fprintln(out, "This legacy diagnostic covers core rehearsal only. Use migrate-postgres for full application preflight and transfer."); err != nil {
return err
}
}
}
if !report.ApplicationReady {
return errPostgreSQLNotReady
}
return nil
}
report, err := store.InspectSchema(ctx, database)
if err != nil {
return err
+51
View File
@@ -9,9 +9,60 @@ import (
"testing"
"pageswiki/internal/db"
"pageswiki/internal/metadata"
"pageswiki/internal/store"
)
func TestPortabilityCommandIsReadOnlyAndNeverClaimsApplicationReady(t *testing.T) {
dir := t.TempDir()
database, err := db.Open(filepath.Join(dir, "pages.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
if err := db.Migrate(database); err != nil {
t.Fatal(err)
}
st := store.NewWithDocumentRoot(database, filepath.Join(dir, "documents"))
if _, err := st.CreatePageAtPath("/preserved"); err != nil {
t.Fatal(err)
}
if _, err := database.Exec(`CREATE TABLE private_extension_data(secret TEXT); INSERT INTO private_extension_data VALUES ('DO_NOT_PRINT_PRIVATE_VALUE')`); err != nil {
t.Fatal(err)
}
var out bytes.Buffer
if err := runSchema([]string{"portability", "--db", filepath.Join(dir, "pages.db"), "--json"}, &out); !errors.Is(err, errPostgreSQLNotReady) {
t.Fatal("unsafe readiness result", err)
}
var report metadata.PortabilityReport
if err := json.Unmarshal(out.Bytes(), &report); err != nil {
t.Fatal(err)
}
if report.ApplicationReady || report.CoreCompatible || len(report.Blockers) == 0 {
t.Fatal("legacy application marked portable", report)
}
if bytes.Contains(out.Bytes(), []byte("DO_NOT_PRINT_PRIVATE_VALUE")) {
t.Fatal("row values leaked")
}
if _, err := st.ResolvePath("/preserved"); err != nil {
t.Fatal("inspection changed existing data", err)
}
var count int
if err := database.QueryRow(`SELECT count(*) FROM schema_migration_steps WHERE component='core-metadata'`).Scan(&count); err != nil || count != 0 {
t.Fatal("inspection migrated database", err)
}
}
func TestPortabilityCommandMissingDatabaseDoesNotCreateFiles(t *testing.T) {
dir := filepath.Join(t.TempDir(), "not-created")
if err := runSchema([]string{"portability", "--data-dir", dir}, &bytes.Buffer{}); err == nil {
t.Fatal("nonexistent DB accepted")
}
if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) {
t.Fatal("directory created", err)
}
}
func TestSchemaCommandMissingDatabaseDoesNotInitialize(t *testing.T) {
dir := filepath.Join(t.TempDir(), "missing")
var out bytes.Buffer
+35
View File
@@ -0,0 +1,35 @@
package main
import (
"context"
"pageswiki/internal/server"
"pageswiki/internal/store"
)
// Let an in-flight federation reconciliation finish before releasing the
// event-worker owner. Each remote request retains its existing network timeout;
// cancellation prevents starting another campus or reconciliation pass.
func serveInstance(ctx context.Context, dataDir string, st *store.Store, addr string) error {
srv, err := server.NewChecked(st)
if err != nil {
return err
}
reconcileCtx, stopReconciler := context.WithCancel(ctx)
reconcilerDone := startManagedCampusReconciler(reconcileCtx, dataDir, st)
serveCtx, stopServer := context.WithCancel(context.Background())
finished := make(chan struct{})
go func() {
select {
case <-ctx.Done():
<-reconcilerDone
stopServer()
case <-finished:
}
}()
err = srv.RunContext(serveCtx, addr)
stopReconciler()
<-reconcilerDone
close(finished)
stopServer()
return err
}
+50
View File
@@ -0,0 +1,50 @@
package main
import (
"context"
"pageswiki/internal/db"
"pageswiki/internal/store"
"testing"
"time"
)
func TestServeInstanceCancelled(t *testing.T) {
t.Chdir("../..")
t.Setenv("PAGESWIKI_INSTANCE_FQDN", "")
root := t.TempDir()
database, err := db.Open(root + "/pages.db")
if err != nil {
t.Fatal(err)
}
defer database.Close()
if err = db.Migrate(database); err != nil {
t.Fatal(err)
}
st := store.NewWithDocumentRoot(database, root+"/documents")
if err = st.EnsureConfigTable(); err != nil {
t.Fatal(err)
}
st.BackgroundJobsDisabled = true
ctx, cancel := context.WithCancel(context.Background())
cancel()
if err = serveInstance(ctx, root, st, "127.0.0.1:0"); err != nil {
t.Fatal(err)
}
if err = database.Ping(); err != nil {
t.Fatal("server closed caller-owned DB", err)
}
}
func TestManagedReconcilerCancellationBeforeFirstPass(t *testing.T) {
t.Setenv("PAGESWIKI_INSTANCE_FQDN", "synthetic.invalid")
ctx, cancel := context.WithCancel(context.Background())
cancel()
// No DB: cancellation must finish without attempting a repository read or
// accessing an external campus. This test does not make a network request.
done := startManagedCampusReconciler(ctx, t.TempDir(), &store.Store{})
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("reconciler did not stop")
}
}