Files
pages/internal/dr/snapshot.go
T
Anya Tokugawa b4792bb175
Build and publish container images / images (push) Successful in 16m37s
Add task capture, work reports, daily digests and private contacts
2026-11-01 22:28:16 +09:00

580 lines
17 KiB
Go

package dr
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/google/uuid"
"golang.org/x/sys/unix"
"modernc.org/sqlite"
"pageswiki/internal/db"
)
type File struct {
Path string `json:"path"`
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
}
type Manifest struct {
Version int `json:"version"`
ID string `json:"id"`
ClusterID string `json:"cluster_id"`
PrimaryNodeID string `json:"primary_node_id"`
Epoch uint64 `json:"epoch"`
CapturedAt time.Time `json:"captured_at"`
CompletedAt time.Time `json:"completed_at"`
Scope string `json:"scope"`
Files []File `json:"files"`
}
// Include canonical stores, auth/signing keys and managed-campus enrollment.
// Caches, temporary uploads, editor workspaces and node-local DR state are NOT
// recovery authorities. Extra mounted storage/environment secrets are external.
func canonical(path string) bool {
if !filepath.IsLocal(path) || filepath.ToSlash(path) != path || strings.Contains(path, "\\") || path != filepath.Clean(path) {
return false
}
root := strings.Split(path, "/")[0]
switch root {
case "documents", "themes", "federation":
return true
case "pages.db", "auth.key", "instance-ed25519.key":
return path == root
}
return false
}
func fileHash(path string) (File, error) {
f, err := os.Open(path)
if err != nil {
return File{}, err
}
defer f.Close()
i, err := f.Stat()
if err != nil || !i.Mode().IsRegular() {
return File{}, errors.New("snapshot entry is not a regular file")
}
h := sha256.New()
n, err := io.CopyBuffer(h, f, make([]byte, 256<<10))
return File{Size: n, SHA256: hex.EncodeToString(h.Sum(nil))}, err
}
func copyFile(source, target string) error {
i, err := os.Lstat(source)
if err != nil {
return err
}
if !i.Mode().IsRegular() {
return fmt.Errorf("refusing nonregular source: %s", source)
}
in, err := os.Open(source)
if err != nil {
return err
}
defer in.Close()
opened, err := in.Stat()
if err != nil || !os.SameFile(i, opened) {
return errors.New("source changed during snapshot")
}
if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil {
return err
}
out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return err
}
_, err = io.CopyBuffer(out, in, make([]byte, 256<<10))
if err == nil {
err = out.Sync()
}
closeErr := out.Close()
if err == nil {
err = closeErr
}
return err
}
// Backup API preserves rowids (unlike VACUUM INTO) and includes committed WAL.
func backupDB(ctx context.Context, source, target string) error {
database, err := db.OpenReadOnly(source)
if err != nil {
return err
}
defer database.Close()
conn, err := database.Conn(ctx)
if err != nil {
return err
}
defer conn.Close()
err = conn.Raw(func(raw any) error {
b, err := raw.(interface {
NewBackup(string) (*sqlite.Backup, error)
}).NewBackup((&url.URL{Scheme: "file", Path: target}).String())
if err != nil {
return err
}
for {
if err = ctx.Err(); err != nil {
b.Finish()
return err
}
more, e := b.Step(256)
if e != nil {
b.Finish()
return e
}
if !more {
return b.Finish()
}
}
})
if err != nil {
return err
}
// Do not leave a WAL-mode snapshot that creates sidecars during verification.
dst, err := sql.Open("sqlite", (&url.URL{Scheme: "file", Path: target, RawQuery: "mode=rw"}).String())
if err != nil {
return err
}
_, err = dst.ExecContext(ctx, "PRAGMA journal_mode=DELETE")
closeErr := dst.Close()
if err != nil {
return err
}
if closeErr != nil {
return closeErr
}
// Finish leaves a standalone DB; fsync before publishing the generation.
f, err := os.OpenFile(target, os.O_RDWR, 0600)
if err != nil {
return err
}
defer f.Close()
if err = f.Chmod(0600); err != nil {
return err
}
return f.Sync()
}
// Snapshot requires every writer stopped, including older binaries and CLIs.
// The lock enforces this for upgraded PagesWiki processes, not arbitrary tools.
func Snapshot(ctx context.Context, dataDir, target string) (*Manifest, string, error) {
source, err := filepath.EvalSymlinks(dataDir)
if err != nil {
return nil, "", err
}
source, err = filepath.Abs(source)
if err != nil {
return nil, "", err
}
target, err = filepath.Abs(target)
if err != nil {
return nil, "", err
}
parent, err := filepath.EvalSymlinks(filepath.Dir(target))
if err != nil {
return nil, "", err
}
target = filepath.Join(parent, filepath.Base(target))
if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) {
return nil, "", errors.New("snapshot destination must be outside the data directory")
}
l, err := Lock(source, true)
if err != nil {
return nil, "", err
}
defer l.Close()
c, err := Load(source)
if err != nil {
return nil, "", err
}
if c == nil || c.Role != "primary" {
return nil, "", errors.New("initialize primary DR configuration before snapshot")
}
if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
return nil, "", errors.New("snapshot destination must not exist")
}
stage, err := os.MkdirTemp(parent, ".dr-snapshot-")
if err != nil {
return nil, "", err
}
defer os.RemoveAll(stage)
m := &Manifest{Version: 1, ID: uuid.NewString(), ClusterID: c.ClusterID, PrimaryNodeID: c.NodeID, Epoch: c.Epoch, CapturedAt: time.Now().UTC(), Scope: "instance-offline-v1"}
root := filepath.Join(stage, "data")
if err = os.Mkdir(root, 0700); err != nil {
return nil, "", err
}
info, err := os.Lstat(filepath.Join(source, "pages.db"))
if err != nil {
return nil, "", err
}
if !info.Mode().IsRegular() {
return nil, "", errors.New("snapshot database must be a regular file, not a symlink")
}
if err = checkAuthKey(source); err != nil {
return nil, "", err
}
if err = backupDB(ctx, filepath.Join(source, "pages.db"), filepath.Join(root, "pages.db")); err != nil {
return nil, "", err
}
for _, name := range []string{"documents", "themes", "federation", "auth.key", "instance-ed25519.key"} {
base := filepath.Join(source, name)
if _, e := os.Lstat(base); errors.Is(e, os.ErrNotExist) {
continue
} else if e != nil {
return nil, "", e
}
err = filepath.WalkDir(base, func(path string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if err := ctx.Err(); err != nil {
return err
}
if entry.Type()&os.ModeSymlink != 0 {
return fmt.Errorf("snapshot refuses symlink: %s", path)
}
rel, _ := filepath.Rel(source, path)
if entry.IsDir() {
return os.MkdirAll(filepath.Join(root, rel), 0700)
}
return copyFile(path, filepath.Join(root, rel))
})
if err != nil {
return nil, "", err
}
}
err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, e error) error {
if e != nil {
return e
}
if entry.IsDir() {
return syncDir(path)
}
if len(m.Files) >= 100000 {
return errors.New("snapshot exceeds 100000 files; partitioned snapshots require a later format")
}
f, e := fileHash(path)
if e != nil {
return e
}
f.Path, _ = filepath.Rel(root, path)
m.Files = append(m.Files, f)
return nil
})
if err != nil {
return nil, "", err
}
m.CompletedAt = time.Now().UTC()
encoded, err := json.MarshalIndent(m, "", " ")
if err != nil {
return nil, "", err
}
if err = os.WriteFile(filepath.Join(stage, "manifest.json"), encoded, 0600); err != nil {
return nil, "", err
}
manifestFile, err := os.OpenFile(filepath.Join(stage, "manifest.json"), os.O_RDWR, 0600)
if err != nil {
return nil, "", err
}
err = manifestFile.Sync()
manifestFile.Close()
if err != nil {
return nil, "", err
}
hash := sha256.Sum256(encoded)
if _, err = Verify(ctx, stage, hex.EncodeToString(hash[:])); err != nil {
return nil, "", err
}
if err = syncDir(stage); err != nil {
return nil, "", err
}
if err = publish(stage, target); err != nil {
return nil, "", err
}
return m, hex.EncodeToString(hash[:]), syncDir(parent)
}
func Verify(ctx context.Context, root, expectedHash string) (*Manifest, error) {
if len(expectedHash) != 64 {
return nil, errors.New("expected manifest SHA-256 is required from a trusted channel")
}
root, err := filepath.Abs(root)
if err != nil {
return nil, err
}
info, err := os.Lstat(filepath.Join(root, "manifest.json"))
if err != nil {
return nil, err
}
if !info.Mode().IsRegular() || info.Size() > 64<<20 {
return nil, errors.New("invalid or oversized snapshot manifest")
}
encoded, err := os.ReadFile(filepath.Join(root, "manifest.json"))
if err != nil {
return nil, err
}
sum := sha256.Sum256(encoded)
if hex.EncodeToString(sum[:]) != expectedHash {
return nil, errors.New("manifest SHA-256 mismatch")
}
var m Manifest
if err = json.Unmarshal(encoded, &m); err != nil {
return nil, err
}
if m.Version != 1 || m.Scope != "instance-offline-v1" || m.Epoch == 0 || len(m.Files) == 0 || len(m.Files) > 100000 || m.CapturedAt.IsZero() || m.CompletedAt.Before(m.CapturedAt) || m.CompletedAt.After(time.Now().Add(time.Minute)) {
return nil, errors.New("invalid snapshot manifest")
}
for _, id := range []string{m.ID, m.ClusterID, m.PrimaryNodeID} {
u, e := uuid.Parse(id)
if e != nil || u == uuid.Nil || u.String() != id {
return nil, errors.New("invalid snapshot identity")
}
}
seen := map[string]bool{}
hashes := map[string]string{}
for _, item := range m.Files {
if err = ctx.Err(); err != nil {
return nil, err
}
if !canonical(item.Path) || seen[item.Path] {
return nil, errors.New("invalid/duplicate snapshot path")
}
seen[item.Path] = true
hashes[item.Path] = item.SHA256
path := filepath.Join(root, "data", item.Path)
// Refuse symlinks in every component, including parent directories.
for p := path; p != filepath.Dir(root) && p != "."; p = filepath.Dir(p) {
info, e := os.Lstat(p)
if e != nil {
return nil, e
}
if info.Mode()&os.ModeSymlink != 0 {
return nil, errors.New("snapshot contains symlink")
}
if p == root || filepath.Dir(p) == p {
break
}
}
actual, e := fileHash(path)
if e != nil {
return nil, e
}
if actual.Size != item.Size || actual.SHA256 != item.SHA256 {
return nil, fmt.Errorf("snapshot payload mismatch: %s", item.Path)
}
if strings.HasPrefix(item.Path, "documents/sha256/") && filepath.Base(item.Path) != actual.SHA256 {
return nil, errors.New("CAS filename does not match payload hash")
}
}
if !seen["pages.db"] {
return nil, errors.New("snapshot database missing")
}
if err = verifyCalendarFiles(root, hashes); err != nil {
return nil, err
}
err = filepath.WalkDir(filepath.Join(root, "data"), func(path string, entry fs.DirEntry, e error) error {
if e != nil {
return e
}
if entry.IsDir() {
return nil
}
rel, _ := filepath.Rel(filepath.Join(root, "data"), path)
if !seen[rel] {
return errors.New("unlisted snapshot payload")
}
return nil
})
if err != nil {
return nil, err
}
database, err := db.OpenReadOnly(filepath.Join(root, "data", "pages.db"))
if err != nil {
return nil, err
}
defer database.Close()
var integrity string
if err = database.QueryRowContext(ctx, "PRAGMA quick_check").Scan(&integrity); err != nil {
return nil, err
}
if integrity != "ok" {
return nil, errors.New("snapshot database integrity check failed")
}
if err = verifyReferences(ctx, database, seen); err != nil {
return nil, err
}
return &m, nil
}
// Restore never merges into or overwrites a running/existing data directory.
// It publishes a new generation with a fresh physical node ID, always standby.
func Restore(ctx context.Context, snapshot, expectedHash, target string) (*Config, error) {
m, err := Verify(ctx, snapshot, expectedHash)
if err != nil {
return nil, err
}
target, err = filepath.Abs(target)
if err != nil {
return nil, err
}
parent, err := filepath.EvalSymlinks(filepath.Dir(target))
if err != nil {
return nil, err
}
target = filepath.Join(parent, filepath.Base(target))
source, err := filepath.EvalSymlinks(snapshot)
if err != nil {
return nil, err
}
source, err = filepath.Abs(source)
if err != nil {
return nil, err
}
if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) {
return nil, errors.New("restore destination must be outside the snapshot")
}
if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
return nil, errors.New("restore target must not exist; use a new generation")
}
stage, err := os.MkdirTemp(filepath.Dir(target), ".dr-restore-")
if err != nil {
return nil, err
}
defer os.RemoveAll(stage)
for _, item := range m.Files {
if err = ctx.Err(); err != nil {
return nil, err
}
if err = copyFile(filepath.Join(snapshot, "data", item.Path), filepath.Join(stage, item.Path)); err != nil {
return nil, err
}
actual, e := fileHash(filepath.Join(stage, item.Path))
if e != nil {
return nil, e
}
if actual.Size != item.Size || actual.SHA256 != item.SHA256 {
return nil, errors.New("snapshot changed during restore")
}
}
c := &Config{Version: 1, ClusterID: m.ClusterID, NodeID: uuid.NewString(), PrimaryNodeID: m.PrimaryNodeID, Role: "standby", Epoch: m.Epoch, SnapshotID: m.ID, CapturedAt: m.CapturedAt, MaxReadAgeSeconds: 3600}
if err = Save(stage, *c); err != nil {
return nil, err
}
if err = os.MkdirAll(filepath.Join(stage, "documents", "sha256"), 0700); err != nil {
return nil, err
}
// Create the lock before a read-only data mount is applied.
l, err := Lock(stage, true)
if err != nil {
return nil, err
}
l.Close()
err = filepath.WalkDir(stage, func(path string, entry fs.DirEntry, e error) error {
if e != nil {
return e
}
if entry.IsDir() {
return syncDir(path)
}
return nil
})
if err != nil {
return nil, err
}
if err = publish(stage, target); err != nil {
return nil, err
}
return c, syncDir(filepath.Dir(target))
}
// Linux deployment: atomically publish without replacing even an empty directory.
func publish(stage, target string) error {
return unix.Renameat2(unix.AT_FDCWD, stage, unix.AT_FDCWD, target, unix.RENAME_NOREPLACE)
}
func verifyReferences(ctx context.Context, database *sql.DB, seen map[string]bool) error {
// Only authoritative references, not derived projection/preview caches. Some
// installations predate optional modules, hence the explicit table check.
for _, ref := range [][2]string{{"revisions", "document_hash"}, {"assets", "head_document_hash"}, {"asset_head_history", "document_hash"}, {"organizer_items", "document_hash"}, {"organizer_revisions", "document_hash"}, {"organizer_contacts", "hash"}, {"organizer_contact_revisions", "hash"}} {
var exists bool
if err := database.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?)", ref[0]).Scan(&exists); err != nil {
return err
}
if !exists {
continue
}
rows, err := database.QueryContext(ctx, "SELECT DISTINCT "+ref[1]+" FROM "+ref[0]+" WHERE "+ref[1]+" IS NOT NULL AND "+ref[1]+"<>''")
if err != nil {
return err
}
for rows.Next() {
var hash string
if err = rows.Scan(&hash); err != nil {
rows.Close()
return err
}
if len(hash) != 64 || !seen["documents/sha256/"+hash[:2]+"/"+hash] {
rows.Close()
return fmt.Errorf("snapshot lacks canonical payload referenced by %s", ref[0])
}
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
}
return nil
}
// Verify file-based calendar authority without parsing event bodies or making
// external calendar requests. Derived calendar indexes may be repaired only
// after promotion, never by a standby read request.
func verifyCalendarFiles(root string, hashes map[string]string) error {
for path := range hashes {
if !strings.HasPrefix(path, "documents/calendars/") || filepath.Base(path) != "calendar.json" {
continue
}
filename := filepath.Join(root, "data", path)
info, err := os.Stat(filename)
if err != nil {
return err
}
if info.Size() > 16<<20 {
return errors.New("calendar metadata exceeds snapshot verification limit")
}
encoded, err := os.ReadFile(filename)
if err != nil {
return err
}
var metadata struct {
Months map[string]string `json:"months"`
}
if err = json.Unmarshal(encoded, &metadata); err != nil {
return errors.New("invalid calendar metadata in snapshot")
}
for month, hash := range metadata.Months {
if _, err = time.Parse("2006-01", month); err != nil {
return errors.New("invalid calendar month in snapshot")
}
if len(hash) != 64 || hashes[filepath.ToSlash(filepath.Join(filepath.Dir(path), month+"."+hash+".ics"))] != hash {
return errors.New("snapshot lacks a valid calendar monthly generation")
}
}
}
return nil
}