feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s
Build and publish container images / images (push) Failing after 12m29s
This commit is contained in:
1 parent
96065d1e89
commit
0ba1dcfef0
243 files changed
+21441
-3090
No files matched your search
@@ -0,0 +1,391 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
"pageswiki/internal/metadata"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// A partial Store is constructed ONLY inside package integration tests. There
|
||||
// is intentionally no public constructor or serving mode that can bypass the
|
||||
// remaining authentication, policy, task, federation and scheduler ports.
|
||||
func testNamespaceStore(t *testing.T, backend storeSQLBackend) *Store {
|
||||
t.Helper()
|
||||
var database *sql.DB
|
||||
if backend == storeSQLite {
|
||||
var err error
|
||||
database, err = db.Open(filepath.Join(t.TempDir(), "pages.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Migrate(database); err != nil {
|
||||
database.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
st := NewWithDocumentRoot(database, filepath.Join(t.TempDir(), "documents"))
|
||||
if err := st.EnsureConfigTable(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return st
|
||||
}
|
||||
dsn := os.Getenv("PAGESWIKI_TEST_POSTGRES_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("set PAGESWIKI_TEST_POSTGRES_DSN to a disposable PostgreSQL URL")
|
||||
}
|
||||
u, err := url.Parse(dsn)
|
||||
if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") {
|
||||
t.Fatal("invalid test PostgreSQL URL")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
root, err := db.OpenPostgreSQL(ctx, db.PostgreSQLOptions{DSN: dsn, AllowInsecure: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
schema := "pageswiki_namespace_test_" + strings.ReplaceAll(uuid.NewString(), "-", "")
|
||||
if _, err := root.ExecContext(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil {
|
||||
root.Close()
|
||||
t.Fatal("create namespace test schema failed")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := root.ExecContext(ctx, `DROP SCHEMA "`+schema+`" CASCADE`); err != nil {
|
||||
t.Error("drop namespace test schema failed")
|
||||
}
|
||||
root.Close()
|
||||
})
|
||||
q := u.Query()
|
||||
q.Set("search_path", schema)
|
||||
u.RawQuery = q.Encode()
|
||||
database, err = db.OpenPostgreSQL(ctx, db.PostgreSQLOptions{DSN: u.String(), AllowInsecure: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
st := &Store{DB: database, sqlBackend: backend}
|
||||
if err := st.EnsureDomainsSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.EnsureCampusesSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.EnsureConfigTable(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
core, _ := metadata.New(database, metadata.PostgreSQL)
|
||||
if err := core.InitializeFresh(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
func seedNamespaceDomain(t *testing.T, st *Store, campus, name, host string) *Domain {
|
||||
t.Helper()
|
||||
id := uuid.NewString()
|
||||
_, err := st.DB.Exec(`INSERT INTO domains(uuid,name,title,domain,enabled,created_at,campus_uuid) VALUES (`+st.sqlValues(7)+`)`, id, name, "Title "+name, host, 1, time.Now().Unix(), campus)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := st.GetDomainByUUID(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func TestStoreNamespaceBackendContract(t *testing.T) {
|
||||
for _, backend := range []storeSQLBackend{storeSQLite, storePostgreSQL} {
|
||||
t.Run(fmt.Sprint(backend), func(t *testing.T) {
|
||||
st := testNamespaceStore(t, backend)
|
||||
campus, err := st.GetDefaultCampus()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
domain, err := st.GetDefaultDomain()
|
||||
if err != nil || domain.CampusUUID != campus.UUID {
|
||||
t.Fatal("default association", err)
|
||||
}
|
||||
if err := st.UpdateCampus(campus.ID, "default", "キャンパス", "ja-JP", "standalone", "pages.example.test", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
updated, err := st.GetCampusByUUID(campus.UUID)
|
||||
if err != nil || updated.Title != "キャンパス" || updated.DefaultLang != "ja-JP" {
|
||||
t.Fatal("Campus update", err)
|
||||
}
|
||||
if err := st.CreateCampus("OTHER", "Other", "en", "standalone", "other.example.test", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
campuses, err := st.ListCampuses()
|
||||
if err != nil || len(campuses) != 2 {
|
||||
t.Fatal("Campus list", err)
|
||||
}
|
||||
var other Campus
|
||||
for _, c := range campuses {
|
||||
if c.UUID != campus.UUID {
|
||||
other = c
|
||||
}
|
||||
}
|
||||
short := seedNamespaceDomain(t, st, campus.UUID, "short", "www")
|
||||
absolute := seedNamespaceDomain(t, st, other.UUID, "absolute", "www.example.test")
|
||||
for host, want := range map[string]string{"PAGES.EXAMPLE.TEST:443": domain.UUID, "www.pages.example.test": short.UUID, "www.example.test": absolute.UUID} {
|
||||
got, err := st.GetDomainByHost(host)
|
||||
if err != nil || got.UUID != want {
|
||||
t.Fatal("host resolution", host, err)
|
||||
}
|
||||
}
|
||||
if _, err := st.GetDomainByHost("unknown.example.test"); !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatal("unknown host became default", err)
|
||||
}
|
||||
if got, err := st.GetDomainEffectiveHost(short.UUID); err != nil || got != "www.pages.example.test" {
|
||||
t.Fatal("relative host expansion", err)
|
||||
}
|
||||
if err := st.UpdateCampus(campus.ID, "default", "wrong", "en", "standalone", "example.test", true); err == nil {
|
||||
t.Fatal("effective host collision accepted")
|
||||
}
|
||||
if got, err := st.GetCampusByUUID(campus.UUID); err != nil || got.FQDN != "pages.example.test" || got.Title != "キャンパス" {
|
||||
t.Fatal("failed validation changed Campus", err)
|
||||
}
|
||||
if err := st.UpdateCampus(campus.ID, "default", "wrong", "en", "standalone", campus.FQDN, false); err == nil {
|
||||
t.Fatal("referenced Campus disabled")
|
||||
}
|
||||
if err := st.SetConfig("locale", "ja"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value, err := st.GetConfig("locale"); err != nil || value != "ja" {
|
||||
t.Fatal("default config", err)
|
||||
}
|
||||
value := `日本語 ? ' " {{children}}` + "\n"
|
||||
if err := st.SetConfigInDomain(short.UUID, "same-key", value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.SetConfigInDomain(absolute.UUID, "same-key", "different"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := st.GetConfigInDomain(short.UUID, "same-key"); err != nil || got != value {
|
||||
t.Fatal("config values changed", err)
|
||||
}
|
||||
if got, err := st.GetConfigInDomain(absolute.UUID, "same-key"); err != nil || got != "different" {
|
||||
t.Fatal("config crossed Domain", err)
|
||||
}
|
||||
if got, err := st.GetConfigInDomain(short.UUID, "missing"); err != nil || got != "" {
|
||||
t.Fatal("missing config contract changed", err)
|
||||
}
|
||||
var timestamp int64
|
||||
if err := st.DB.QueryRow(`SELECT updated_at FROM config WHERE domain_uuid=`+st.sqlParam(1)+` AND key='same-key'`, short.UUID).Scan(×tamp); err != nil || timestamp < 1000000000 || timestamp > time.Now().Unix()+1 {
|
||||
t.Fatal("config time unit changed", timestamp, err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if err := st.SetConfigInDomainContext(ctx, short.UUID, "canceled", "secret"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal("configuration cancellation lost", err)
|
||||
}
|
||||
if _, err := st.GetConfigInDomainContext(ctx, short.UUID, "same-key"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal("read cancellation lost", err)
|
||||
}
|
||||
for _, read := range []func() error{
|
||||
func() error { _, err := st.GetDomainByHostContext(ctx, "pages.example.test"); return err },
|
||||
func() error { _, err := st.GetDomainByUUIDContext(ctx, domain.UUID); return err },
|
||||
func() error { _, err := st.GetDefaultDomainContext(ctx); return err },
|
||||
func() error { _, err := st.ListDomainsContext(ctx); return err },
|
||||
func() error { _, err := st.GetDefaultCampusContext(ctx); return err },
|
||||
func() error { _, err := st.GetCampusByUUIDContext(ctx, campus.UUID); return err },
|
||||
func() error { _, err := st.ListCampusesContext(ctx); return err },
|
||||
} {
|
||||
if err := read(); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal("namespace read lost context", err)
|
||||
}
|
||||
}
|
||||
metaBackend := metadata.SQLite
|
||||
if backend == storePostgreSQL {
|
||||
metaBackend = metadata.PostgreSQL
|
||||
}
|
||||
repo, _ := metadata.New(st.DB, metaBackend)
|
||||
object, err := repo.CreatePage(context.Background(), short.UUID, "/日本語%_/本")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := st.ResolvePathInDomain(short.UUID, "日本語%_/本/"); err != nil || got != object {
|
||||
t.Fatal("Store path lookup not using backend", err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`UPDATE domains SET placement='remote' WHERE uuid=`+st.sqlParam(1), absolute.UUID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.ResolvePathInDomain(absolute.UUID, "/anything"); err == nil {
|
||||
t.Fatal("remote namespace served locally")
|
||||
}
|
||||
if domains, err := st.ListDomains(); err != nil || len(domains) != 3 {
|
||||
t.Fatal("Domain list", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgreSQLNamespaceBootstrapAndSchemaDrift(t *testing.T) {
|
||||
st := testNamespaceStore(t, storePostgreSQL)
|
||||
if _, err := st.DB.Exec(`DELETE FROM domains; DELETE FROM campuses`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
results := make(chan error, 6)
|
||||
for i := 0; i < 6; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
peer := &Store{DB: st.DB, sqlBackend: storePostgreSQL}
|
||||
if err := peer.EnsureDomainsSchema(); err != nil {
|
||||
results <- err
|
||||
return
|
||||
}
|
||||
results <- peer.EnsureCampusesSchema()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
for err := range results {
|
||||
if err != nil {
|
||||
t.Fatal("concurrent bootstrap", err)
|
||||
}
|
||||
}
|
||||
campuses, err := st.ListCampuses()
|
||||
if err != nil || len(campuses) != 1 {
|
||||
t.Fatal("duplicate default Campus", err)
|
||||
}
|
||||
domains, err := st.ListDomains()
|
||||
if err != nil || len(domains) != 1 || domains[0].CampusUUID != campuses[0].UUID {
|
||||
t.Fatal("default Domain association", err)
|
||||
}
|
||||
if err := st.SetConfig("preserved", "data"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`ALTER TABLE config RENAME COLUMN updated_at TO misplaced_timestamp`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.EnsureConfigTable(); err == nil {
|
||||
t.Fatal("missing column silently accepted")
|
||||
}
|
||||
if got, err := st.GetConfig("preserved"); err != nil || got != "data" {
|
||||
t.Fatal("schema failure changed data", err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`ALTER TABLE config RENAME COLUMN misplaced_timestamp TO updated_at`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.EnsureConfigTable(); err != nil {
|
||||
t.Fatal("schema verification not reusable", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCampusConcurrentFQDNCreation(t *testing.T) {
|
||||
for _, backend := range []storeSQLBackend{storeSQLite, storePostgreSQL} {
|
||||
t.Run(fmt.Sprint(backend), func(t *testing.T) {
|
||||
st := testNamespaceStore(t, backend)
|
||||
var wg sync.WaitGroup
|
||||
results := make(chan error, 6)
|
||||
for i := 0; i < 6; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
peer := &Store{DB: st.DB, sqlBackend: backend}
|
||||
results <- peer.CreateCampus(fmt.Sprintf("parallel-%d", i), "Concurrent", "en", "standalone", "same.example.test", true)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
wins := 0
|
||||
for err := range results {
|
||||
if err == nil {
|
||||
wins++
|
||||
} else if !strings.Contains(err.Error(), "FQDN already exists") {
|
||||
t.Fatal("unexpected concurrent error", err)
|
||||
}
|
||||
}
|
||||
if wins != 1 {
|
||||
t.Fatal("duplicate FQDN committed", wins)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgreSQLNamespaceMissingDependenciesAndSecretRedaction(t *testing.T) {
|
||||
st := testNamespaceStore(t, storePostgreSQL)
|
||||
d, _ := st.GetDefaultDomain()
|
||||
c, _ := st.GetDefaultCampus()
|
||||
for _, fn := range []func() error{
|
||||
func() error { return st.CreateDomain(c.UUID, "blocked", "Blocked", "blocked", "local", "", "", true) },
|
||||
func() error {
|
||||
return st.UpdateDomain(d.ID, c.UUID, "blocked", "Blocked", "blocked", "local", "", "", true)
|
||||
},
|
||||
func() error { return st.DeleteDomain(d.ID) },
|
||||
func() error { return st.DeleteCampus(c.ID) },
|
||||
} {
|
||||
if err := fn(); err == nil {
|
||||
t.Fatal("missing auth dependency or last namespace deletion accepted")
|
||||
}
|
||||
}
|
||||
if count, err := st.CountDomains(); err != nil || count != 1 {
|
||||
t.Fatal("failed operation changed Domains", err)
|
||||
}
|
||||
unchanged, err := st.GetDomainByUUID(d.UUID)
|
||||
if err != nil || unchanged.Name != d.Name {
|
||||
t.Fatal("partial namespace update", unchanged, err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`ALTER TABLE config ADD CONSTRAINT synthetic_secret_failure CHECK(value != 'DO_NOT_LEAK_PRIVATE_VALUE')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = st.SetConfigInDomain(d.UUID, "sensitive", "DO_NOT_LEAK_PRIVATE_VALUE")
|
||||
var safe *db.PostgreSQLError
|
||||
if err == nil || !errors.As(err, &safe) || safe.SQLState != "23514" || strings.Contains(err.Error(), "DO_NOT_LEAK_PRIVATE_VALUE") {
|
||||
t.Fatal("unsafe configuration error", err)
|
||||
}
|
||||
encoded, _ := json.Marshal(safe)
|
||||
if strings.Contains(string(encoded), "DO_NOT_LEAK_PRIVATE_VALUE") {
|
||||
t.Fatal("private detail retained")
|
||||
}
|
||||
if err := st.sqlError("decode namespace", errors.New("driver error with DO_NOT_LEAK_PRIVATE_VALUE")); strings.Contains(err.Error(), "DO_NOT_LEAK_PRIVATE_VALUE") {
|
||||
t.Fatal("untyped driver error leaked")
|
||||
}
|
||||
validation := namespaceValidationError("campus FQDN already exists")
|
||||
if err := st.sqlError("validate namespace", validation); !errors.Is(err, validation) {
|
||||
t.Fatal("safe form validation was hidden", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgreSQLNamespaceLockDeadline(t *testing.T) {
|
||||
st := testNamespaceStore(t, storePostgreSQL)
|
||||
holder, err := st.DB.Begin()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer holder.Rollback()
|
||||
if _, err := holder.Exec(`SELECT pg_advisory_xact_lock(1885431653,1853058931)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
called := false
|
||||
err = st.namespaceWrite(ctx, func(context.Context, *sql.Tx) error { called = true; return nil })
|
||||
if called || !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatal("namespace lock ignored deadline", err)
|
||||
}
|
||||
if err := holder.Rollback(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.CreateCampus("after-timeout", "After timeout", "en", "standalone", "after.example.test", true); err != nil {
|
||||
t.Fatal("canceled connection not recovered", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user