feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s

This commit is contained in:
Anya Tokugawa committed 2026-11-15 05:47:14 +09:00
1 parent 96065d1e89
commit 0ba1dcfef0
243 files changed
+21441 -3090

No files matched your search

+108
View File
@@ -0,0 +1,108 @@
package db
import (
"context"
"database/sql"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/stdlib"
)
// PostgreSQLOptions is an instance-level secret-bearing configuration. Do not
// log it. Application startup additionally requires a verified full-import
// receipt; opening a connection alone does not authorize backend cutover.
type PostgreSQLOptions struct {
DSN string
MaxOpenConns int
MaxIdleConns int
ConnectTimeout time.Duration
// AllowInsecure permits plaintext or unverified TLS for isolated development
// only. Production defaults require verified TLS and no plaintext fallback.
AllowInsecure bool
}
// PostgreSQLError deliberately excludes server messages, SQL, connection
// strings and constraint details, all of which can contain credentials or data.
// SQLState remains available for retry/conflict classification.
type PostgreSQLError struct {
Operation string
SQLState string
}
func (e *PostgreSQLError) Error() string {
if e.SQLState != "" {
return fmt.Sprintf("PostgreSQL %s failed (SQLSTATE %s)", e.Operation, e.SQLState)
}
return "PostgreSQL " + e.Operation + " failed"
}
func postgresError(operation string, err error) error {
if errors.Is(err, context.Canceled) {
return context.Canceled
}
if errors.Is(err, context.DeadlineExceeded) {
return context.DeadlineExceeded
}
var pgErr *pgconn.PgError
result := &PostgreSQLError{Operation: operation}
if errors.As(err, &pgErr) {
result.SQLState = pgErr.Code
}
return result
}
func OpenPostgreSQL(ctx context.Context, options PostgreSQLOptions) (*sql.DB, error) {
if options.DSN == "" {
return nil, fmt.Errorf("PostgreSQL DSN is required")
}
if options.MaxOpenConns < 0 || options.MaxIdleConns < 0 || options.ConnectTimeout < 0 {
return nil, fmt.Errorf("invalid PostgreSQL pool or timeout configuration")
}
config, err := pgx.ParseConfig(options.DSN)
if err != nil {
return nil, postgresError("configuration", err)
}
if !options.AllowInsecure {
if config.TLSConfig == nil || config.TLSConfig.InsecureSkipVerify {
return nil, fmt.Errorf("PostgreSQL requires verified TLS; use sslmode=verify-full")
}
for _, fallback := range config.Fallbacks {
if fallback.TLSConfig == nil || fallback.TLSConfig.InsecureSkipVerify {
return nil, fmt.Errorf("PostgreSQL insecure connection fallback is not allowed")
}
}
}
timeout := options.ConnectTimeout
if timeout == 0 {
timeout = 10 * time.Second
}
config.ConnectTimeout = timeout
config.RuntimeParams["application_name"] = "pageswiki"
database := stdlib.OpenDB(*config)
maxOpen := options.MaxOpenConns
if maxOpen == 0 {
maxOpen = 10
}
maxIdle := options.MaxIdleConns
if maxIdle == 0 {
maxIdle = 2
}
if maxIdle > maxOpen {
maxIdle = maxOpen
}
database.SetMaxOpenConns(maxOpen)
database.SetMaxIdleConns(maxIdle)
database.SetConnMaxLifetime(30 * time.Minute)
database.SetConnMaxIdleTime(5 * time.Minute)
pingCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
if err := database.PingContext(pingCtx); err != nil {
_ = database.Close()
return nil, postgresError("connect", err)
}
return database, nil
}
+120
View File
@@ -0,0 +1,120 @@
package db
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
"time"
)
// PostgreSQLMigration contains native PostgreSQL SQL ($1 parameters, BIGINT
// timestamps). SQLite migration definitions must not be translated implicitly.
// Definitions are append-only after release. Assertions return no rows on success.
type PostgreSQLMigration struct {
Version int
Name string
Statements []string
Assertions []string `json:",omitempty"`
}
func postgresMigrationChecksum(step PostgreSQLMigration) string {
encoded, _ := json.Marshal(step)
sum := sha256.Sum256(encoded)
return hex.EncodeToString(sum[:])
}
// ApplyPostgreSQLMigrations uses one transaction-scoped advisory lock for the
// migration namespace, including ledger creation. Component-specific locks
// would race concurrent first creation of the shared ledger. READ COMMITTED
// ensures a waiter sees the previous owner's committed migration history.
// Callers must use the same search_path and lock convention for a database.
func ApplyPostgreSQLMigrations(ctx context.Context, database *sql.DB, component string, steps []PostgreSQLMigration) error {
if strings.TrimSpace(component) == "" {
return fmt.Errorf("migration component is required")
}
for i, step := range steps {
if step.Version != i+1 || strings.TrimSpace(step.Name) == "" || len(step.Statements)+len(step.Assertions) == 0 {
return fmt.Errorf("invalid PostgreSQL migration sequence for %s", component)
}
}
tx, err := database.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
if err != nil {
return postgresError("migration begin", err)
}
defer tx.Rollback()
// Stable application-specific key, independent of Go hashes and versions.
if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(1885431653, 1835624306)`); err != nil {
return postgresError("migration lock", err)
}
if _, err := tx.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_migration_steps (
component TEXT NOT NULL, version INTEGER NOT NULL, name TEXT NOT NULL,
checksum TEXT NOT NULL, applied_at_ms BIGINT NOT NULL,
PRIMARY KEY(component,version))`); err != nil {
return postgresError("migration ledger", err)
}
rows, err := tx.QueryContext(ctx, `SELECT version,name,checksum FROM schema_migration_steps WHERE component=$1 ORDER BY version`, component)
if err != nil {
return postgresError("migration history", err)
}
applied := 0
for rows.Next() {
var version int
var name, checksum string
if err := rows.Scan(&version, &name, &checksum); err != nil {
rows.Close()
return postgresError("migration history", err)
}
if version != applied+1 || version > len(steps) || steps[version-1].Name != name || postgresMigrationChecksum(steps[version-1]) != checksum {
rows.Close()
return fmt.Errorf("PostgreSQL migration history mismatch: component=%s version=%d", component, version)
}
applied++
}
err = rows.Err()
closeErr := rows.Close()
if err != nil {
return postgresError("migration history", err)
}
if closeErr != nil {
return postgresError("migration history close", closeErr)
}
for _, step := range steps[applied:] {
for i, statement := range step.Statements {
if _, err := tx.ExecContext(ctx, statement); err != nil {
return postgresError(fmt.Sprintf("migration %s/%d statement %d", component, step.Version, i+1), err)
}
}
if _, err := tx.ExecContext(ctx, `INSERT INTO schema_migration_steps(component,version,name,checksum,applied_at_ms) VALUES($1,$2,$3,$4,$5)`, component, step.Version, step.Name, postgresMigrationChecksum(step), time.Now().UnixMilli()); err != nil {
return postgresError("migration record", err)
}
}
if len(steps) > 0 {
step := steps[len(steps)-1]
for i, assertion := range step.Assertions {
rows, err := tx.QueryContext(ctx, assertion)
if err != nil {
return postgresError("schema assertion", err)
}
violated := rows.Next()
err = rows.Err()
closeErr := rows.Close()
if err != nil {
return postgresError("schema assertion", err)
}
if closeErr != nil {
return postgresError("schema assertion close", closeErr)
}
if violated {
return fmt.Errorf("PostgreSQL schema contract %s/%d assertion %d failed", component, step.Version, i+1)
}
}
}
if err := tx.Commit(); err != nil {
return postgresError("migration commit", err)
}
return nil
}
+194
View File
@@ -0,0 +1,194 @@
package db
import (
"context"
"database/sql"
"errors"
"fmt"
"net/url"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgconn"
)
func TestPostgreSQLConfigurationAndRedaction(t *testing.T) {
for _, options := range []PostgreSQLOptions{
{}, {DSN: "postgres://test:secret@localhost/db", MaxOpenConns: -1},
{DSN: "postgres://test:secret@localhost/db?sslmode=disable"},
{DSN: "postgres://test:secret@localhost/db?sslmode=require"},
{DSN: "postgres://test:secret@localhost/db?sslmode=prefer"},
{DSN: "postgres://test:secret@localhost/db?sslmode=invalid"},
} {
database, err := OpenPostgreSQL(context.Background(), options)
if database != nil || err == nil {
t.Fatal("invalid configuration accepted")
}
if strings.Contains(err.Error(), "secret") {
t.Fatal("password disclosed")
}
}
err := postgresError("write", &pgconn.PgError{Code: "23505", Message: "secret", Detail: "private row"})
var safe *PostgreSQLError
if !errors.As(err, &safe) || safe.SQLState != "23505" || strings.Contains(err.Error(), "secret") {
t.Fatal("unsafe error classification")
}
if !errors.Is(postgresError("read", context.Canceled), context.Canceled) {
t.Fatal("cancellation lost")
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
database, err := OpenPostgreSQL(ctx, PostgreSQLOptions{DSN: "postgres://test:secret@127.0.0.1:1/db?sslmode=disable", AllowInsecure: true})
if database != nil || !errors.Is(err, context.Canceled) {
t.Fatal("connection cancellation lost")
}
}
// These tests create and drop only a random schema. Use a disposable database
// and URL DSN, never a production connection. No fixtures contain real events.
func postgresTestDatabase(t *testing.T) *sql.DB {
t.Helper()
dsn := os.Getenv("PAGESWIKI_TEST_POSTGRES_DSN")
if dsn == "" {
t.Skip("set PAGESWIKI_TEST_POSTGRES_DSN to a disposable PostgreSQL URL")
}
u, err := url.Parse(dsn)
if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") {
t.Fatal("test DSN must be a PostgreSQL URL")
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
root, err := OpenPostgreSQL(ctx, PostgreSQLOptions{DSN: dsn, AllowInsecure: true})
if err != nil {
t.Fatal(err)
}
schema := "pageswiki_test_" + strings.ReplaceAll(uuid.NewString(), "-", "")
if _, err := root.ExecContext(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil {
root.Close()
t.Fatal(postgresError("test schema", err))
}
t.Cleanup(func() {
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if _, err := root.ExecContext(cleanup, `DROP SCHEMA "`+schema+`" CASCADE`); err != nil {
t.Error(postgresError("test cleanup", err))
}
root.Close()
})
query := u.Query()
query.Set("search_path", schema)
u.RawQuery = query.Encode()
database, err := OpenPostgreSQL(ctx, PostgreSQLOptions{DSN: u.String(), AllowInsecure: true, MaxOpenConns: 6})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { database.Close() })
return database
}
func TestPostgreSQLMigrationConcurrencyAndHistory(t *testing.T) {
database := postgresTestDatabase(t)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
steps := []PostgreSQLMigration{{Version: 1, Name: "initial", Statements: []string{
`CREATE TABLE sample (uuid TEXT PRIMARY KEY, created_at_ms BIGINT NOT NULL)`,
`INSERT INTO sample VALUES ('synthetic', 1800000000000)`,
}, Assertions: []string{`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM sample WHERE created_at_ms=1800000000000)`}}}
var wg sync.WaitGroup
errs := make(chan error, 6)
for i := 0; i < 6; i++ {
wg.Add(1)
go func() { defer wg.Done(); errs <- ApplyPostgreSQLMigrations(ctx, database, "test", steps) }()
}
wg.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
var count int
if err := database.QueryRowContext(ctx, `SELECT count(*) FROM sample`).Scan(&count); err != nil || count != 1 {
t.Fatalf("rows=%d error=%v", count, err)
}
if err := ApplyPostgreSQLMigrations(ctx, database, "test", nil); err == nil {
t.Fatal("downgrade accepted")
}
changed := append([]PostgreSQLMigration(nil), steps...)
changed[0].Name = "changed"
if err := ApplyPostgreSQLMigrations(ctx, database, "test", changed); err == nil {
t.Fatal("changed history accepted")
}
changed = append([]PostgreSQLMigration(nil), steps...)
changed[0].Statements = []string{`SELECT 1`}
if err := ApplyPostgreSQLMigrations(ctx, database, "test", changed); err == nil {
t.Fatal("changed SQL checksum accepted")
}
if err := ApplyPostgreSQLMigrations(ctx, database, "other", []PostgreSQLMigration{{Version: 2, Name: "gap", Statements: []string{`SELECT 1`}}}); err == nil {
t.Fatal("version gap accepted")
}
// Assertions are checked again even when no migrations remain to apply.
if _, err := database.ExecContext(ctx, `DELETE FROM sample`); err != nil {
t.Fatal(err)
}
if err := ApplyPostgreSQLMigrations(ctx, database, "test", steps); err == nil {
t.Fatal("schema assertion skipped")
}
}
func TestPostgreSQLMigrationRollbackAndLockCancellation(t *testing.T) {
database := postgresTestDatabase(t)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
broken := []PostgreSQLMigration{{Version: 1, Name: "broken", Statements: []string{`CREATE TABLE partial (id INTEGER)`, `NOT VALID SQL`}}}
if err := ApplyPostgreSQLMigrations(ctx, database, "test", broken); err == nil {
t.Fatal("SQL error ignored")
}
var name sql.NullString
if err := database.QueryRowContext(ctx, `SELECT to_regclass('partial')::text`).Scan(&name); err != nil || name.Valid {
t.Fatal("DDL not rolled back", err)
}
if err := database.QueryRowContext(ctx, `SELECT to_regclass('schema_migration_steps')::text`).Scan(&name); err != nil || name.Valid {
t.Fatal("ledger not rolled back", err)
}
broken[0].Statements = broken[0].Statements[:1]
if err := ApplyPostgreSQLMigrations(ctx, database, "test", broken); err != nil {
t.Fatal("retry failed", err)
}
assertionFailure := append(append([]PostgreSQLMigration(nil), broken...), PostgreSQLMigration{
Version: 2, Name: "assertion failure", Statements: []string{`CREATE TABLE should_rollback (id INTEGER)`}, Assertions: []string{`SELECT 1`},
})
if err := ApplyPostgreSQLMigrations(ctx, database, "test", assertionFailure); err == nil {
t.Fatal("assertion failure ignored")
}
if err := database.QueryRowContext(ctx, `SELECT to_regclass('should_rollback')::text`).Scan(&name); err != nil || name.Valid {
t.Fatal("assertion DDL not rolled back", err)
}
var version int
if err := database.QueryRowContext(ctx, `SELECT max(version) FROM schema_migration_steps WHERE component=$1`, "test").Scan(&version); err != nil || version != 1 {
t.Fatal("assertion ledger not rolled back", err)
}
tx, err := database.BeginTx(ctx, nil)
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
if _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(1885431653, 1835624306)`); err != nil {
t.Fatal(err)
}
waitCtx, stop := context.WithTimeout(ctx, 100*time.Millisecond)
defer stop()
if err := ApplyPostgreSQLMigrations(waitCtx, database, "test", broken); err == nil {
t.Fatal("blocked migration ignored deadline")
}
if err := tx.Rollback(); err != nil {
t.Fatal(err)
}
if err := ApplyPostgreSQLMigrations(ctx, database, "test", broken); err != nil {
t.Fatal(fmt.Sprintf("lock not released: %v", err))
}
}