feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s
Build and publish container images / images (push) Failing after 12m29s
This commit is contained in:
1 parent
96065d1e89
commit
0ba1dcfef0
243 files changed
+21441
-3090
No files matched your search
@@ -0,0 +1,348 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
func TestApplicationImportRoundTrip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
source := testNamespaceStore(t, storeSQLite)
|
||||
if _, err := source.NewCalendarRepository(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
domain, err := source.GetDefaultDomain()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
owner, err := source.BootstrapOwner(domain.UUID, "owner", "synthetic import password", "日本語 owner", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = source.SetConfigInDomain(domain.UUID, "authorization_mode", "enforcing"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, err := source.CreatePageAtPathInDomain(domain.UUID, "/日本語/roundtrip")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rev, err := source.SaveRevisionContent(obj, []byte("<h1>Imported page</h1>"), "commit message", "en", "text/html", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = source.AddTag(obj, "日本語 tag"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
token, err := source.CreateAuthSession(domain.UUID, owner.UUID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = source.DB.Exec(`UPDATE principals SET avatar=? WHERE uuid=?`, []byte{0, 1, 2, 255}, owner.UUID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, err := source.SaveTaskDAV(ctx, domain.UUID, owner.UUID, TaskRecord{UUID: uuid.NewString(), Kind: "task", Title: "移行タスク", Status: "todo", Bucket: "inbox"}, 0, &TaskDAVData{Name: "import.ics", UID: "import-task", Data: "synthetic VTODO"}, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
contact, err := source.SaveContact(ctx, domain.UUID, owner.UUID, Contact{UUID: uuid.NewString(), VCard: syntheticContact("import-contact")}, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
project, err := source.SaveTask(ctx, domain.UUID, owner.UUID, TaskRecord{UUID: uuid.NewString(), Kind: "project", Title: "移行 Project", Status: "todo", Bucket: "inbox"}, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectTask, err := source.SaveTaskDAV(ctx, domain.UUID, owner.UUID, TaskRecord{UUID: uuid.NewString(), Kind: "task", Title: "Project task", Status: "todo", Bucket: "inbox", ProjectUUID: project.UUID}, 0, &TaskDAVData{Name: "import.ics", UID: "import-task", Data: "synthetic VTODO"}, "", project.UUID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deleted, err := source.SaveTask(ctx, domain.UUID, owner.UUID, TaskRecord{UUID: uuid.NewString(), Kind: "task", Title: "Deleted task", Status: "todo", Bucket: "inbox", ProjectUUID: project.UUID}, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = source.DeleteTask(ctx, domain.UUID, owner.UUID, deleted.UUID, deleted.Version); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshot, err := source.ContactDAVSnapshot(ctx, domain.UUID, owner.UUID, "", 200)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Deleted highest sequence must not be reused after transfer.
|
||||
if _, err = source.DB.Exec(`UPDATE sqlite_sequence SET seq=100000 WHERE name='organizer_contact_changes'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := testNamespaceStore(t, storePostgreSQL)
|
||||
// Only synthetic bootstrap rows in this test's private schema are removed.
|
||||
if _, err = target.DB.Exec(`DELETE FROM domains;DELETE FROM campuses;DELETE FROM config`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = PreparePostgreSQLImport(ctx, target.DB, source.DocumentRoot); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = VerifyPostgreSQLImport(ctx, target.DB, source.DocumentRoot); err == nil {
|
||||
t.Fatal("unimported target accepted")
|
||||
}
|
||||
// Force a late failure after row copying, without altering canonical files.
|
||||
if _, err = source.DB.Exec(`INSERT INTO tags(object_uuid,tag,created_at) VALUES(?,?,?)`, obj, "rollback", "invalid bigint"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = ImportSQLiteApplication(ctx, source.DB, target.DB, source.DocumentRoot); err == nil {
|
||||
t.Fatal("invalid row accepted")
|
||||
}
|
||||
var count int
|
||||
if err = target.DB.QueryRow(`SELECT count(*) FROM domains`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatal("partial import", count, err)
|
||||
}
|
||||
if err = target.DB.QueryRow(`SELECT count(*) FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=current_schema() AND NOT t.tgisinternal AND t.tgenabled='D'`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatal("disabled triggers retained", count, err)
|
||||
}
|
||||
if _, err = source.DB.Exec(`DELETE FROM tags WHERE tag='rollback'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = PreparePostgreSQLImport(ctx, target.DB, source.DocumentRoot); err != nil {
|
||||
t.Fatal("retry preparation", err)
|
||||
}
|
||||
report, err := ImportSQLiteApplication(ctx, source.DB, target.DB, source.DocumentRoot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Documents != 1 || len(report.Tables) < 130 || report.Fingerprint == "" {
|
||||
t.Fatal(report)
|
||||
}
|
||||
if err = VerifyPostgreSQLImport(ctx, target.DB, source.DocumentRoot); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st, err := NewPostgreSQLWithDocumentRoot(ctx, target.DB, source.DocumentRoot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := st.GetRevision(rev.UUID)
|
||||
if err != nil || got.Body != rev.Body {
|
||||
t.Fatal(got, err)
|
||||
}
|
||||
p, err := st.PrincipalForSession(domain.UUID, token)
|
||||
if err != nil || p.UUID != owner.UUID {
|
||||
t.Fatal(p, err)
|
||||
}
|
||||
var avatar []byte
|
||||
if err = target.DB.QueryRow(`SELECT avatar FROM principals WHERE uuid=$1`, owner.UUID).Scan(&avatar); err != nil || !bytes.Equal(avatar, []byte{0, 1, 2, 255}) {
|
||||
t.Fatal(avatar, err)
|
||||
}
|
||||
tags, err := st.ListTags(obj)
|
||||
if err != nil || len(tags) != 1 || tags[0] != "日本語 tag" {
|
||||
t.Fatal(tags, err)
|
||||
}
|
||||
if got, err := st.TaskDAVByName(ctx, domain.UUID, owner.UUID, "import.ics"); err != nil || got.UUID != task.UUID || got.Title != task.Title {
|
||||
t.Fatal(got, err)
|
||||
}
|
||||
if got, err := st.TaskDAVByName(ctx, domain.UUID, owner.UUID, "import.ics", project.UUID); err != nil || got.UUID != projectTask.UUID {
|
||||
t.Fatal("project DAV identity", got, err)
|
||||
}
|
||||
if err = st.DeleteTask(ctx, domain.UUID, owner.UUID, deleted.UUID, deleted.Version); err != nil {
|
||||
t.Fatal("delete idempotency", err)
|
||||
}
|
||||
if got, err := st.Contact(ctx, domain.UUID, owner.UUID, contact.UUID); err != nil || got.VCard != contact.VCard {
|
||||
t.Fatal(got, err)
|
||||
}
|
||||
if got, err := st.ContactDAVSnapshot(ctx, domain.UUID, owner.UUID, snapshot.Token, 200); err != nil || len(got.Items) != 0 {
|
||||
t.Fatal("sync continuity", got, err)
|
||||
}
|
||||
contact.Name = "After import"
|
||||
if _, err = st.SaveContact(ctx, domain.UUID, owner.UUID, contact, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sequence int64
|
||||
if err = target.DB.QueryRow(`SELECT max(sequence) FROM organizer_contact_changes`).Scan(&sequence); err != nil || sequence <= 100000 {
|
||||
t.Fatal("sequence reused", sequence, err)
|
||||
}
|
||||
if err = PreparePostgreSQLImport(ctx, target.DB, source.DocumentRoot); err == nil {
|
||||
t.Fatal("populated target preparation accepted")
|
||||
}
|
||||
if os.Getenv("PAGESWIKI_TEST_POSTGRES_CONTAINER") != "" {
|
||||
restored := restoreImportTestSchema(t, target)
|
||||
if err = VerifyPostgreSQLImport(ctx, restored, source.DocumentRoot); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := NewPostgreSQLWithDocumentRoot(ctx, restored, source.DocumentRoot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := r.GetRevision(rev.UUID); err != nil || got.Body != rev.Body {
|
||||
t.Fatal("restored revision", err)
|
||||
}
|
||||
if got, err := r.PrincipalForSession(domain.UUID, token); err != nil || got.UUID != owner.UUID {
|
||||
t.Fatal("restored session", err)
|
||||
}
|
||||
if got, err := r.TaskDAVByName(ctx, domain.UUID, owner.UUID, "import.ics", project.UUID); err != nil || got.UUID != projectTask.UUID {
|
||||
t.Fatal("restored task DAV", err)
|
||||
}
|
||||
if _, err = r.ContactDAVSnapshot(ctx, domain.UUID, owner.UUID, snapshot.Token, 200); err != nil {
|
||||
t.Fatal("restored contact sync", err)
|
||||
}
|
||||
t.Log("schema-scoped pg_dump/pg_restore and canonical-file reuse verified")
|
||||
}
|
||||
if _, err = ImportSQLiteApplication(ctx, source.DB, target.DB, source.DocumentRoot); err == nil {
|
||||
t.Fatal("overwrite accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// Opt-in restore rehearsal: only the random schema created by this test is
|
||||
// dumped and restored, inside the explicitly selected disposable container.
|
||||
func restoreImportTestSchema(t *testing.T, source *Store) *sql.DB {
|
||||
t.Helper()
|
||||
container := os.Getenv("PAGESWIKI_TEST_POSTGRES_CONTAINER")
|
||||
u, err := url.Parse(os.Getenv("PAGESWIKI_TEST_POSTGRES_DSN"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var schema string
|
||||
if err = source.DB.QueryRow(`SELECT current_schema()`).Scan(&schema); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(schema, "pageswiki_namespace_test_") {
|
||||
t.Fatal("restore requires a private test schema")
|
||||
}
|
||||
database := strings.TrimPrefix(u.Path, "/")
|
||||
dump, err := exec.Command("docker", "exec", container, "pg_dump", "-U", u.User.Username(), "-d", database, "-n", schema, "--no-owner", "--no-acl", "-Fc").Output()
|
||||
if err != nil {
|
||||
t.Fatal("synthetic schema pg_dump failed")
|
||||
}
|
||||
old := schema + "_old"
|
||||
if _, err = source.DB.Exec(`ALTER SCHEMA ` + importIdent(schema) + ` RENAME TO ` + importIdent(old)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = source.DB.Exec(`DROP SCHEMA IF EXISTS ` + importIdent(old) + ` CASCADE`) })
|
||||
cmd := exec.Command("docker", "exec", "-i", container, "pg_restore", "-U", u.User.Username(), "-d", database, "--exit-on-error", "--no-owner", "--no-acl")
|
||||
cmd.Stdin = bytes.NewReader(dump)
|
||||
if err = cmd.Run(); err != nil {
|
||||
t.Fatal("synthetic schema pg_restore failed")
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("search_path", schema)
|
||||
u.RawQuery = q.Encode()
|
||||
restored, err := db.OpenPostgreSQL(context.Background(), db.PostgreSQLOptions{DSN: u.String(), AllowInsecure: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { restored.Close() })
|
||||
return restored
|
||||
}
|
||||
|
||||
func TestApplicationImportScalarFingerprint(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
for _, v := range []any{nil, "", []byte{}, "YQ==", []byte("a"), int64(1), float64(1), true} {
|
||||
b, err := importRowBytes([]any{v})
|
||||
if err != nil || seen[string(b)] {
|
||||
t.Fatal("scalar collision", err)
|
||||
}
|
||||
seen[string(b)] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationImportRejectsSourceDrift(t *testing.T) {
|
||||
for _, ddl := range []string{`CREATE TABLE unexpected(secret TEXT)`, `ALTER TABLE tags ADD COLUMN unexpected TEXT`, `CREATE TRIGGER unexpected AFTER INSERT ON tags BEGIN SELECT 1; END`} {
|
||||
t.Run(strings.Fields(ddl)[1], func(t *testing.T) {
|
||||
s := testNamespaceStore(t, storeSQLite)
|
||||
if _, err := InspectSQLiteApplicationImport(context.Background(), s.DB); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.DB.Exec(ddl); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := InspectSQLiteApplicationImport(context.Background(), s.DB); err == nil {
|
||||
t.Fatal("schema drift accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Opt-in schema inventory contains table/column metadata only, never user rows.
|
||||
func TestApplicationImportCatalogInventory(t *testing.T) {
|
||||
if os.Getenv("PAGESWIKI_IMPORT_CATALOG") != "1" {
|
||||
t.Skip("schema inventory")
|
||||
}
|
||||
ctx := context.Background()
|
||||
for _, backend := range []storeSQLBackend{storeSQLite, storePostgreSQL} {
|
||||
s := testNamespaceStore(t, backend)
|
||||
if backend == storePostgreSQL {
|
||||
root := filepath.Join(t.TempDir(), "documents")
|
||||
if err := os.Mkdir(root, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PAGESWIKI_AUTH_KEY", "synthetic catalog key")
|
||||
var err error
|
||||
s, err = NewPostgreSQLWithDocumentRoot(ctx, s.DB, root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := s.NewCalendarRepository(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
query := `SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name`
|
||||
if backend == storePostgreSQL {
|
||||
query = `SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_type='BASE TABLE' ORDER BY table_name`
|
||||
}
|
||||
rows, err := s.DB.Query(query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var tables []string
|
||||
for rows.Next() {
|
||||
var n string
|
||||
if err = rows.Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tables = append(tables, n)
|
||||
}
|
||||
rows.Close()
|
||||
if backend == storeSQLite {
|
||||
objects, err := s.DB.Query(`SELECT type,name,sql FROM sqlite_master WHERE type IN ('trigger','view') ORDER BY name`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for objects.Next() {
|
||||
var kind, name, ddl string
|
||||
if err = objects.Scan(&kind, &name, &ddl); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("SOURCEOBJECT %s %s %x", kind, name, sha256.Sum256([]byte(ddl)))
|
||||
}
|
||||
objects.Close()
|
||||
}
|
||||
for _, table := range tables {
|
||||
var cols *sql.Rows
|
||||
if backend == storeSQLite {
|
||||
cols, err = s.DB.Query(`SELECT name,type FROM pragma_table_info(?) ORDER BY name`, table)
|
||||
} else {
|
||||
cols, err = s.DB.Query(`SELECT column_name,data_type FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1 ORDER BY column_name`, table)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
columns := map[string]string{}
|
||||
for cols.Next() {
|
||||
var name, kind string
|
||||
if err = cols.Scan(&name, &kind); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
columns[name] = kind
|
||||
}
|
||||
cols.Close()
|
||||
b, _ := json.Marshal(columns)
|
||||
t.Logf("CATALOG %d %s %s", backend, table, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user