Files
pages/internal/dr/snapshot.go
T
Anya Tokugawa d5d21b7841
Build and publish container images / images (push) Successful in 9m34s
Release container images / release (push) Successful in 36s
Refactor architecture into eight components and consolidate reliability work
Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts.

Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation.

Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
2026-11-01 15:23:31 +09:00

580 lines
16 KiB
Go

package dr
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/google/uuid"
"golang.org/x/sys/unix"
"modernc.org/sqlite"
"pageswiki/internal/db"
)
type File struct {
Path string `json:"path"`
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
}
type Manifest struct {
Version int `json:"version"`
ID string `json:"id"`
ClusterID string `json:"cluster_id"`
PrimaryNodeID string `json:"primary_node_id"`
Epoch uint64 `json:"epoch"`
CapturedAt time.Time `json:"captured_at"`
CompletedAt time.Time `json:"completed_at"`
Scope string `json:"scope"`
Files []File `json:"files"`
}
// Include canonical stores, auth/signing keys and managed-campus enrollment.
// Caches, temporary uploads, editor workspaces and node-local DR state are NOT
// recovery authorities. Extra mounted storage/environment secrets are external.
func canonical(path string) bool {
if !filepath.IsLocal(path) || filepath.ToSlash(path) != path || strings.Contains(path, "\\") || path != filepath.Clean(path) {
return false
}
root := strings.Split(path, "/")[0]
switch root {
case "documents", "themes", "federation":
return true
case "pages.db", "auth.key", "instance-ed25519.key":
return path == root
}
return false
}
func fileHash(path string) (File, error) {
f, err := os.Open(path)
if err != nil {
return File{}, err
}
defer f.Close()
i, err := f.Stat()
if err != nil || !i.Mode().IsRegular() {
return File{}, errors.New("snapshot entry is not a regular file")
}
h := sha256.New()
n, err := io.CopyBuffer(h, f, make([]byte, 256<<10))
return File{Size: n, SHA256: hex.EncodeToString(h.Sum(nil))}, err
}
func copyFile(source, target string) error {
i, err := os.Lstat(source)
if err != nil {
return err
}
if !i.Mode().IsRegular() {
return fmt.Errorf("refusing nonregular source: %s", source)
}
in, err := os.Open(source)
if err != nil {
return err
}
defer in.Close()
opened, err := in.Stat()
if err != nil || !os.SameFile(i, opened) {
return errors.New("source changed during snapshot")
}
if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil {
return err
}
out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return err
}
_, err = io.CopyBuffer(out, in, make([]byte, 256<<10))
if err == nil {
err = out.Sync()
}
closeErr := out.Close()
if err == nil {
err = closeErr
}
return err
}
// Backup API preserves rowids (unlike VACUUM INTO) and includes committed WAL.
func backupDB(ctx context.Context, source, target string) error {
database, err := db.OpenReadOnly(source)
if err != nil {
return err
}
defer database.Close()
conn, err := database.Conn(ctx)
if err != nil {
return err
}
defer conn.Close()
err = conn.Raw(func(raw any) error {
b, err := raw.(interface {
NewBackup(string) (*sqlite.Backup, error)
}).NewBackup((&url.URL{Scheme: "file", Path: target}).String())
if err != nil {
return err
}
for {
if err = ctx.Err(); err != nil {
b.Finish()
return err
}
more, e := b.Step(256)
if e != nil {
b.Finish()
return e
}
if !more {
return b.Finish()
}
}
})
if err != nil {
return err
}
// Do not leave a WAL-mode snapshot that creates sidecars during verification.
dst, err := sql.Open("sqlite", (&url.URL{Scheme: "file", Path: target, RawQuery: "mode=rw"}).String())
if err != nil {
return err
}
_, err = dst.ExecContext(ctx, "PRAGMA journal_mode=DELETE")
closeErr := dst.Close()
if err != nil {
return err
}
if closeErr != nil {
return closeErr
}
// Finish leaves a standalone DB; fsync before publishing the generation.
f, err := os.OpenFile(target, os.O_RDWR, 0600)
if err != nil {
return err
}
defer f.Close()
if err = f.Chmod(0600); err != nil {
return err
}
return f.Sync()
}
// Snapshot requires every writer stopped, including older binaries and CLIs.
// The lock enforces this for upgraded PagesWiki processes, not arbitrary tools.
func Snapshot(ctx context.Context, dataDir, target string) (*Manifest, string, error) {
source, err := filepath.EvalSymlinks(dataDir)
if err != nil {
return nil, "", err
}
source, err = filepath.Abs(source)
if err != nil {
return nil, "", err
}
target, err = filepath.Abs(target)
if err != nil {
return nil, "", err
}
parent, err := filepath.EvalSymlinks(filepath.Dir(target))
if err != nil {
return nil, "", err
}
target = filepath.Join(parent, filepath.Base(target))
if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) {
return nil, "", errors.New("snapshot destination must be outside the data directory")
}
l, err := Lock(source, true)
if err != nil {
return nil, "", err
}
defer l.Close()
c, err := Load(source)
if err != nil {
return nil, "", err
}
if c == nil || c.Role != "primary" {
return nil, "", errors.New("initialize primary DR configuration before snapshot")
}
if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
return nil, "", errors.New("snapshot destination must not exist")
}
stage, err := os.MkdirTemp(parent, ".dr-snapshot-")
if err != nil {
return nil, "", err
}
defer os.RemoveAll(stage)
m := &Manifest{Version: 1, ID: uuid.NewString(), ClusterID: c.ClusterID, PrimaryNodeID: c.NodeID, Epoch: c.Epoch, CapturedAt: time.Now().UTC(), Scope: "instance-offline-v1"}
root := filepath.Join(stage, "data")
if err = os.Mkdir(root, 0700); err != nil {
return nil, "", err
}
info, err := os.Lstat(filepath.Join(source, "pages.db"))
if err != nil {
return nil, "", err
}
if !info.Mode().IsRegular() {
return nil, "", errors.New("snapshot database must be a regular file, not a symlink")
}
if err = checkAuthKey(source); err != nil {
return nil, "", err
}
if err = backupDB(ctx, filepath.Join(source, "pages.db"), filepath.Join(root, "pages.db")); err != nil {
return nil, "", err
}
for _, name := range []string{"documents", "themes", "federation", "auth.key", "instance-ed25519.key"} {
base := filepath.Join(source, name)
if _, e := os.Lstat(base); errors.Is(e, os.ErrNotExist) {
continue
} else if e != nil {
return nil, "", e
}
err = filepath.WalkDir(base, func(path string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if err := ctx.Err(); err != nil {
return err
}
if entry.Type()&os.ModeSymlink != 0 {
return fmt.Errorf("snapshot refuses symlink: %s", path)
}
rel, _ := filepath.Rel(source, path)
if entry.IsDir() {
return os.MkdirAll(filepath.Join(root, rel), 0700)
}
return copyFile(path, filepath.Join(root, rel))
})
if err != nil {
return nil, "", err
}
}
err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, e error) error {
if e != nil {
return e
}
if entry.IsDir() {
return syncDir(path)
}
if len(m.Files) >= 100000 {
return errors.New("snapshot exceeds 100000 files; partitioned snapshots require a later format")
}
f, e := fileHash(path)
if e != nil {
return e
}
f.Path, _ = filepath.Rel(root, path)
m.Files = append(m.Files, f)
return nil
})
if err != nil {
return nil, "", err
}
m.CompletedAt = time.Now().UTC()
encoded, err := json.MarshalIndent(m, "", " ")
if err != nil {
return nil, "", err
}
if err = os.WriteFile(filepath.Join(stage, "manifest.json"), encoded, 0600); err != nil {
return nil, "", err
}
manifestFile, err := os.OpenFile(filepath.Join(stage, "manifest.json"), os.O_RDWR, 0600)
if err != nil {
return nil, "", err
}
err = manifestFile.Sync()
manifestFile.Close()
if err != nil {
return nil, "", err
}
hash := sha256.Sum256(encoded)
if _, err = Verify(ctx, stage, hex.EncodeToString(hash[:])); err != nil {
return nil, "", err
}
if err = syncDir(stage); err != nil {
return nil, "", err
}
if err = publish(stage, target); err != nil {
return nil, "", err
}
return m, hex.EncodeToString(hash[:]), syncDir(parent)
}
func Verify(ctx context.Context, root, expectedHash string) (*Manifest, error) {
if len(expectedHash) != 64 {
return nil, errors.New("expected manifest SHA-256 is required from a trusted channel")
}
root, err := filepath.Abs(root)
if err != nil {
return nil, err
}
info, err := os.Lstat(filepath.Join(root, "manifest.json"))
if err != nil {
return nil, err
}
if !info.Mode().IsRegular() || info.Size() > 64<<20 {
return nil, errors.New("invalid or oversized snapshot manifest")
}
encoded, err := os.ReadFile(filepath.Join(root, "manifest.json"))
if err != nil {
return nil, err
}
sum := sha256.Sum256(encoded)
if hex.EncodeToString(sum[:]) != expectedHash {
return nil, errors.New("manifest SHA-256 mismatch")
}
var m Manifest
if err = json.Unmarshal(encoded, &m); err != nil {
return nil, err
}
if m.Version != 1 || m.Scope != "instance-offline-v1" || m.Epoch == 0 || len(m.Files) == 0 || len(m.Files) > 100000 || m.CapturedAt.IsZero() || m.CompletedAt.Before(m.CapturedAt) || m.CompletedAt.After(time.Now().Add(time.Minute)) {
return nil, errors.New("invalid snapshot manifest")
}
for _, id := range []string{m.ID, m.ClusterID, m.PrimaryNodeID} {
u, e := uuid.Parse(id)
if e != nil || u == uuid.Nil || u.String() != id {
return nil, errors.New("invalid snapshot identity")
}
}
seen := map[string]bool{}
hashes := map[string]string{}
for _, item := range m.Files {
if err = ctx.Err(); err != nil {
return nil, err
}
if !canonical(item.Path) || seen[item.Path] {
return nil, errors.New("invalid/duplicate snapshot path")
}
seen[item.Path] = true
hashes[item.Path] = item.SHA256
path := filepath.Join(root, "data", item.Path)
// Refuse symlinks in every component, including parent directories.
for p := path; p != filepath.Dir(root) && p != "."; p = filepath.Dir(p) {
info, e := os.Lstat(p)
if e != nil {
return nil, e
}
if info.Mode()&os.ModeSymlink != 0 {
return nil, errors.New("snapshot contains symlink")
}
if p == root || filepath.Dir(p) == p {
break
}
}
actual, e := fileHash(path)
if e != nil {
return nil, e
}
if actual.Size != item.Size || actual.SHA256 != item.SHA256 {
return nil, fmt.Errorf("snapshot payload mismatch: %s", item.Path)
}
if strings.HasPrefix(item.Path, "documents/sha256/") && filepath.Base(item.Path) != actual.SHA256 {
return nil, errors.New("CAS filename does not match payload hash")
}
}
if !seen["pages.db"] {
return nil, errors.New("snapshot database missing")
}
if err = verifyCalendarFiles(root, hashes); err != nil {
return nil, err
}
err = filepath.WalkDir(filepath.Join(root, "data"), func(path string, entry fs.DirEntry, e error) error {
if e != nil {
return e
}
if entry.IsDir() {
return nil
}
rel, _ := filepath.Rel(filepath.Join(root, "data"), path)
if !seen[rel] {
return errors.New("unlisted snapshot payload")
}
return nil
})
if err != nil {
return nil, err
}
database, err := db.OpenReadOnly(filepath.Join(root, "data", "pages.db"))
if err != nil {
return nil, err
}
defer database.Close()
var integrity string
if err = database.QueryRowContext(ctx, "PRAGMA quick_check").Scan(&integrity); err != nil {
return nil, err
}
if integrity != "ok" {
return nil, errors.New("snapshot database integrity check failed")
}
if err = verifyReferences(ctx, database, seen); err != nil {
return nil, err
}
return &m, nil
}
// Restore never merges into or overwrites a running/existing data directory.
// It publishes a new generation with a fresh physical node ID, always standby.
func Restore(ctx context.Context, snapshot, expectedHash, target string) (*Config, error) {
m, err := Verify(ctx, snapshot, expectedHash)
if err != nil {
return nil, err
}
target, err = filepath.Abs(target)
if err != nil {
return nil, err
}
parent, err := filepath.EvalSymlinks(filepath.Dir(target))
if err != nil {
return nil, err
}
target = filepath.Join(parent, filepath.Base(target))
source, err := filepath.EvalSymlinks(snapshot)
if err != nil {
return nil, err
}
source, err = filepath.Abs(source)
if err != nil {
return nil, err
}
if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) {
return nil, errors.New("restore destination must be outside the snapshot")
}
if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
return nil, errors.New("restore target must not exist; use a new generation")
}
stage, err := os.MkdirTemp(filepath.Dir(target), ".dr-restore-")
if err != nil {
return nil, err
}
defer os.RemoveAll(stage)
for _, item := range m.Files {
if err = ctx.Err(); err != nil {
return nil, err
}
if err = copyFile(filepath.Join(snapshot, "data", item.Path), filepath.Join(stage, item.Path)); err != nil {
return nil, err
}
actual, e := fileHash(filepath.Join(stage, item.Path))
if e != nil {
return nil, e
}
if actual.Size != item.Size || actual.SHA256 != item.SHA256 {
return nil, errors.New("snapshot changed during restore")
}
}
c := &Config{Version: 1, ClusterID: m.ClusterID, NodeID: uuid.NewString(), PrimaryNodeID: m.PrimaryNodeID, Role: "standby", Epoch: m.Epoch, SnapshotID: m.ID, CapturedAt: m.CapturedAt, MaxReadAgeSeconds: 3600}
if err = Save(stage, *c); err != nil {
return nil, err
}
if err = os.MkdirAll(filepath.Join(stage, "documents", "sha256"), 0700); err != nil {
return nil, err
}
// Create the lock before a read-only data mount is applied.
l, err := Lock(stage, true)
if err != nil {
return nil, err
}
l.Close()
err = filepath.WalkDir(stage, func(path string, entry fs.DirEntry, e error) error {
if e != nil {
return e
}
if entry.IsDir() {
return syncDir(path)
}
return nil
})
if err != nil {
return nil, err
}
if err = publish(stage, target); err != nil {
return nil, err
}
return c, syncDir(filepath.Dir(target))
}
// Linux deployment: atomically publish without replacing even an empty directory.
func publish(stage, target string) error {
return unix.Renameat2(unix.AT_FDCWD, stage, unix.AT_FDCWD, target, unix.RENAME_NOREPLACE)
}
func verifyReferences(ctx context.Context, database *sql.DB, seen map[string]bool) error {
// Only authoritative references, not derived projection/preview caches. Some
// installations predate optional modules, hence the explicit table check.
for _, ref := range [][2]string{{"revisions", "document_hash"}, {"assets", "head_document_hash"}, {"asset_head_history", "document_hash"}, {"organizer_items", "document_hash"}, {"organizer_revisions", "document_hash"}} {
var exists bool
if err := database.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?)", ref[0]).Scan(&exists); err != nil {
return err
}
if !exists {
continue
}
rows, err := database.QueryContext(ctx, "SELECT DISTINCT "+ref[1]+" FROM "+ref[0]+" WHERE "+ref[1]+" IS NOT NULL AND "+ref[1]+"<>''")
if err != nil {
return err
}
for rows.Next() {
var hash string
if err = rows.Scan(&hash); err != nil {
rows.Close()
return err
}
if len(hash) != 64 || !seen["documents/sha256/"+hash[:2]+"/"+hash] {
rows.Close()
return fmt.Errorf("snapshot lacks canonical payload referenced by %s", ref[0])
}
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
}
return nil
}
// Verify file-based calendar authority without parsing event bodies or making
// external calendar requests. Derived calendar indexes may be repaired only
// after promotion, never by a standby read request.
func verifyCalendarFiles(root string, hashes map[string]string) error {
for path := range hashes {
if !strings.HasPrefix(path, "documents/calendars/") || filepath.Base(path) != "calendar.json" {
continue
}
filename := filepath.Join(root, "data", path)
info, err := os.Stat(filename)
if err != nil {
return err
}
if info.Size() > 16<<20 {
return errors.New("calendar metadata exceeds snapshot verification limit")
}
encoded, err := os.ReadFile(filename)
if err != nil {
return err
}
var metadata struct {
Months map[string]string `json:"months"`
}
if err = json.Unmarshal(encoded, &metadata); err != nil {
return errors.New("invalid calendar metadata in snapshot")
}
for month, hash := range metadata.Months {
if _, err = time.Parse("2006-01", month); err != nil {
return errors.New("invalid calendar month in snapshot")
}
if len(hash) != 64 || hashes[filepath.ToSlash(filepath.Join(filepath.Dir(path), month+"."+hash+".ics"))] != hash {
return errors.New("snapshot lacks a valid calendar monthly generation")
}
}
}
return nil
}