Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts. Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation. Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
580 lines
16 KiB
Go
580 lines
16 KiB
Go
package dr
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"golang.org/x/sys/unix"
|
|
"modernc.org/sqlite"
|
|
"pageswiki/internal/db"
|
|
)
|
|
|
|
type File struct {
|
|
Path string `json:"path"`
|
|
Size int64 `json:"size"`
|
|
SHA256 string `json:"sha256"`
|
|
}
|
|
type Manifest struct {
|
|
Version int `json:"version"`
|
|
ID string `json:"id"`
|
|
ClusterID string `json:"cluster_id"`
|
|
PrimaryNodeID string `json:"primary_node_id"`
|
|
Epoch uint64 `json:"epoch"`
|
|
CapturedAt time.Time `json:"captured_at"`
|
|
CompletedAt time.Time `json:"completed_at"`
|
|
Scope string `json:"scope"`
|
|
Files []File `json:"files"`
|
|
}
|
|
|
|
// Include canonical stores, auth/signing keys and managed-campus enrollment.
|
|
// Caches, temporary uploads, editor workspaces and node-local DR state are NOT
|
|
// recovery authorities. Extra mounted storage/environment secrets are external.
|
|
func canonical(path string) bool {
|
|
if !filepath.IsLocal(path) || filepath.ToSlash(path) != path || strings.Contains(path, "\\") || path != filepath.Clean(path) {
|
|
return false
|
|
}
|
|
root := strings.Split(path, "/")[0]
|
|
switch root {
|
|
case "documents", "themes", "federation":
|
|
return true
|
|
case "pages.db", "auth.key", "instance-ed25519.key":
|
|
return path == root
|
|
}
|
|
return false
|
|
}
|
|
|
|
func fileHash(path string) (File, error) {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return File{}, err
|
|
}
|
|
defer f.Close()
|
|
i, err := f.Stat()
|
|
if err != nil || !i.Mode().IsRegular() {
|
|
return File{}, errors.New("snapshot entry is not a regular file")
|
|
}
|
|
h := sha256.New()
|
|
n, err := io.CopyBuffer(h, f, make([]byte, 256<<10))
|
|
return File{Size: n, SHA256: hex.EncodeToString(h.Sum(nil))}, err
|
|
}
|
|
|
|
func copyFile(source, target string) error {
|
|
i, err := os.Lstat(source)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !i.Mode().IsRegular() {
|
|
return fmt.Errorf("refusing nonregular source: %s", source)
|
|
}
|
|
in, err := os.Open(source)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer in.Close()
|
|
opened, err := in.Stat()
|
|
if err != nil || !os.SameFile(i, opened) {
|
|
return errors.New("source changed during snapshot")
|
|
}
|
|
if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil {
|
|
return err
|
|
}
|
|
out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = io.CopyBuffer(out, in, make([]byte, 256<<10))
|
|
if err == nil {
|
|
err = out.Sync()
|
|
}
|
|
closeErr := out.Close()
|
|
if err == nil {
|
|
err = closeErr
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Backup API preserves rowids (unlike VACUUM INTO) and includes committed WAL.
|
|
func backupDB(ctx context.Context, source, target string) error {
|
|
database, err := db.OpenReadOnly(source)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer database.Close()
|
|
conn, err := database.Conn(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer conn.Close()
|
|
err = conn.Raw(func(raw any) error {
|
|
b, err := raw.(interface {
|
|
NewBackup(string) (*sqlite.Backup, error)
|
|
}).NewBackup((&url.URL{Scheme: "file", Path: target}).String())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for {
|
|
if err = ctx.Err(); err != nil {
|
|
b.Finish()
|
|
return err
|
|
}
|
|
more, e := b.Step(256)
|
|
if e != nil {
|
|
b.Finish()
|
|
return e
|
|
}
|
|
if !more {
|
|
return b.Finish()
|
|
}
|
|
}
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Do not leave a WAL-mode snapshot that creates sidecars during verification.
|
|
dst, err := sql.Open("sqlite", (&url.URL{Scheme: "file", Path: target, RawQuery: "mode=rw"}).String())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = dst.ExecContext(ctx, "PRAGMA journal_mode=DELETE")
|
|
closeErr := dst.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if closeErr != nil {
|
|
return closeErr
|
|
}
|
|
// Finish leaves a standalone DB; fsync before publishing the generation.
|
|
f, err := os.OpenFile(target, os.O_RDWR, 0600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
if err = f.Chmod(0600); err != nil {
|
|
return err
|
|
}
|
|
return f.Sync()
|
|
}
|
|
|
|
// Snapshot requires every writer stopped, including older binaries and CLIs.
|
|
// The lock enforces this for upgraded PagesWiki processes, not arbitrary tools.
|
|
func Snapshot(ctx context.Context, dataDir, target string) (*Manifest, string, error) {
|
|
source, err := filepath.EvalSymlinks(dataDir)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
source, err = filepath.Abs(source)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
target, err = filepath.Abs(target)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
parent, err := filepath.EvalSymlinks(filepath.Dir(target))
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
target = filepath.Join(parent, filepath.Base(target))
|
|
if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) {
|
|
return nil, "", errors.New("snapshot destination must be outside the data directory")
|
|
}
|
|
l, err := Lock(source, true)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
defer l.Close()
|
|
c, err := Load(source)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
if c == nil || c.Role != "primary" {
|
|
return nil, "", errors.New("initialize primary DR configuration before snapshot")
|
|
}
|
|
if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
|
|
return nil, "", errors.New("snapshot destination must not exist")
|
|
}
|
|
stage, err := os.MkdirTemp(parent, ".dr-snapshot-")
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
defer os.RemoveAll(stage)
|
|
m := &Manifest{Version: 1, ID: uuid.NewString(), ClusterID: c.ClusterID, PrimaryNodeID: c.NodeID, Epoch: c.Epoch, CapturedAt: time.Now().UTC(), Scope: "instance-offline-v1"}
|
|
root := filepath.Join(stage, "data")
|
|
if err = os.Mkdir(root, 0700); err != nil {
|
|
return nil, "", err
|
|
}
|
|
info, err := os.Lstat(filepath.Join(source, "pages.db"))
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
if !info.Mode().IsRegular() {
|
|
return nil, "", errors.New("snapshot database must be a regular file, not a symlink")
|
|
}
|
|
if err = checkAuthKey(source); err != nil {
|
|
return nil, "", err
|
|
}
|
|
if err = backupDB(ctx, filepath.Join(source, "pages.db"), filepath.Join(root, "pages.db")); err != nil {
|
|
return nil, "", err
|
|
}
|
|
for _, name := range []string{"documents", "themes", "federation", "auth.key", "instance-ed25519.key"} {
|
|
base := filepath.Join(source, name)
|
|
if _, e := os.Lstat(base); errors.Is(e, os.ErrNotExist) {
|
|
continue
|
|
} else if e != nil {
|
|
return nil, "", e
|
|
}
|
|
err = filepath.WalkDir(base, func(path string, entry fs.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
return walkErr
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
if entry.Type()&os.ModeSymlink != 0 {
|
|
return fmt.Errorf("snapshot refuses symlink: %s", path)
|
|
}
|
|
rel, _ := filepath.Rel(source, path)
|
|
if entry.IsDir() {
|
|
return os.MkdirAll(filepath.Join(root, rel), 0700)
|
|
}
|
|
return copyFile(path, filepath.Join(root, rel))
|
|
})
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
}
|
|
err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, e error) error {
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if entry.IsDir() {
|
|
return syncDir(path)
|
|
}
|
|
if len(m.Files) >= 100000 {
|
|
return errors.New("snapshot exceeds 100000 files; partitioned snapshots require a later format")
|
|
}
|
|
f, e := fileHash(path)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
f.Path, _ = filepath.Rel(root, path)
|
|
m.Files = append(m.Files, f)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
m.CompletedAt = time.Now().UTC()
|
|
encoded, err := json.MarshalIndent(m, "", " ")
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
if err = os.WriteFile(filepath.Join(stage, "manifest.json"), encoded, 0600); err != nil {
|
|
return nil, "", err
|
|
}
|
|
manifestFile, err := os.OpenFile(filepath.Join(stage, "manifest.json"), os.O_RDWR, 0600)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
err = manifestFile.Sync()
|
|
manifestFile.Close()
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
hash := sha256.Sum256(encoded)
|
|
if _, err = Verify(ctx, stage, hex.EncodeToString(hash[:])); err != nil {
|
|
return nil, "", err
|
|
}
|
|
if err = syncDir(stage); err != nil {
|
|
return nil, "", err
|
|
}
|
|
if err = publish(stage, target); err != nil {
|
|
return nil, "", err
|
|
}
|
|
return m, hex.EncodeToString(hash[:]), syncDir(parent)
|
|
}
|
|
|
|
func Verify(ctx context.Context, root, expectedHash string) (*Manifest, error) {
|
|
if len(expectedHash) != 64 {
|
|
return nil, errors.New("expected manifest SHA-256 is required from a trusted channel")
|
|
}
|
|
root, err := filepath.Abs(root)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
info, err := os.Lstat(filepath.Join(root, "manifest.json"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !info.Mode().IsRegular() || info.Size() > 64<<20 {
|
|
return nil, errors.New("invalid or oversized snapshot manifest")
|
|
}
|
|
encoded, err := os.ReadFile(filepath.Join(root, "manifest.json"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sum := sha256.Sum256(encoded)
|
|
if hex.EncodeToString(sum[:]) != expectedHash {
|
|
return nil, errors.New("manifest SHA-256 mismatch")
|
|
}
|
|
var m Manifest
|
|
if err = json.Unmarshal(encoded, &m); err != nil {
|
|
return nil, err
|
|
}
|
|
if m.Version != 1 || m.Scope != "instance-offline-v1" || m.Epoch == 0 || len(m.Files) == 0 || len(m.Files) > 100000 || m.CapturedAt.IsZero() || m.CompletedAt.Before(m.CapturedAt) || m.CompletedAt.After(time.Now().Add(time.Minute)) {
|
|
return nil, errors.New("invalid snapshot manifest")
|
|
}
|
|
for _, id := range []string{m.ID, m.ClusterID, m.PrimaryNodeID} {
|
|
u, e := uuid.Parse(id)
|
|
if e != nil || u == uuid.Nil || u.String() != id {
|
|
return nil, errors.New("invalid snapshot identity")
|
|
}
|
|
}
|
|
seen := map[string]bool{}
|
|
hashes := map[string]string{}
|
|
for _, item := range m.Files {
|
|
if err = ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if !canonical(item.Path) || seen[item.Path] {
|
|
return nil, errors.New("invalid/duplicate snapshot path")
|
|
}
|
|
seen[item.Path] = true
|
|
hashes[item.Path] = item.SHA256
|
|
path := filepath.Join(root, "data", item.Path)
|
|
// Refuse symlinks in every component, including parent directories.
|
|
for p := path; p != filepath.Dir(root) && p != "."; p = filepath.Dir(p) {
|
|
info, e := os.Lstat(p)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 {
|
|
return nil, errors.New("snapshot contains symlink")
|
|
}
|
|
if p == root || filepath.Dir(p) == p {
|
|
break
|
|
}
|
|
}
|
|
actual, e := fileHash(path)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if actual.Size != item.Size || actual.SHA256 != item.SHA256 {
|
|
return nil, fmt.Errorf("snapshot payload mismatch: %s", item.Path)
|
|
}
|
|
if strings.HasPrefix(item.Path, "documents/sha256/") && filepath.Base(item.Path) != actual.SHA256 {
|
|
return nil, errors.New("CAS filename does not match payload hash")
|
|
}
|
|
}
|
|
if !seen["pages.db"] {
|
|
return nil, errors.New("snapshot database missing")
|
|
}
|
|
if err = verifyCalendarFiles(root, hashes); err != nil {
|
|
return nil, err
|
|
}
|
|
err = filepath.WalkDir(filepath.Join(root, "data"), func(path string, entry fs.DirEntry, e error) error {
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if entry.IsDir() {
|
|
return nil
|
|
}
|
|
rel, _ := filepath.Rel(filepath.Join(root, "data"), path)
|
|
if !seen[rel] {
|
|
return errors.New("unlisted snapshot payload")
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
database, err := db.OpenReadOnly(filepath.Join(root, "data", "pages.db"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer database.Close()
|
|
var integrity string
|
|
if err = database.QueryRowContext(ctx, "PRAGMA quick_check").Scan(&integrity); err != nil {
|
|
return nil, err
|
|
}
|
|
if integrity != "ok" {
|
|
return nil, errors.New("snapshot database integrity check failed")
|
|
}
|
|
if err = verifyReferences(ctx, database, seen); err != nil {
|
|
return nil, err
|
|
}
|
|
return &m, nil
|
|
}
|
|
|
|
// Restore never merges into or overwrites a running/existing data directory.
|
|
// It publishes a new generation with a fresh physical node ID, always standby.
|
|
func Restore(ctx context.Context, snapshot, expectedHash, target string) (*Config, error) {
|
|
m, err := Verify(ctx, snapshot, expectedHash)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
target, err = filepath.Abs(target)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
parent, err := filepath.EvalSymlinks(filepath.Dir(target))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
target = filepath.Join(parent, filepath.Base(target))
|
|
source, err := filepath.EvalSymlinks(snapshot)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
source, err = filepath.Abs(source)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) {
|
|
return nil, errors.New("restore destination must be outside the snapshot")
|
|
}
|
|
if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
|
|
return nil, errors.New("restore target must not exist; use a new generation")
|
|
}
|
|
stage, err := os.MkdirTemp(filepath.Dir(target), ".dr-restore-")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer os.RemoveAll(stage)
|
|
for _, item := range m.Files {
|
|
if err = ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if err = copyFile(filepath.Join(snapshot, "data", item.Path), filepath.Join(stage, item.Path)); err != nil {
|
|
return nil, err
|
|
}
|
|
actual, e := fileHash(filepath.Join(stage, item.Path))
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if actual.Size != item.Size || actual.SHA256 != item.SHA256 {
|
|
return nil, errors.New("snapshot changed during restore")
|
|
}
|
|
}
|
|
c := &Config{Version: 1, ClusterID: m.ClusterID, NodeID: uuid.NewString(), PrimaryNodeID: m.PrimaryNodeID, Role: "standby", Epoch: m.Epoch, SnapshotID: m.ID, CapturedAt: m.CapturedAt, MaxReadAgeSeconds: 3600}
|
|
if err = Save(stage, *c); err != nil {
|
|
return nil, err
|
|
}
|
|
if err = os.MkdirAll(filepath.Join(stage, "documents", "sha256"), 0700); err != nil {
|
|
return nil, err
|
|
}
|
|
// Create the lock before a read-only data mount is applied.
|
|
l, err := Lock(stage, true)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
l.Close()
|
|
err = filepath.WalkDir(stage, func(path string, entry fs.DirEntry, e error) error {
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if entry.IsDir() {
|
|
return syncDir(path)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err = publish(stage, target); err != nil {
|
|
return nil, err
|
|
}
|
|
return c, syncDir(filepath.Dir(target))
|
|
}
|
|
|
|
// Linux deployment: atomically publish without replacing even an empty directory.
|
|
func publish(stage, target string) error {
|
|
return unix.Renameat2(unix.AT_FDCWD, stage, unix.AT_FDCWD, target, unix.RENAME_NOREPLACE)
|
|
}
|
|
|
|
func verifyReferences(ctx context.Context, database *sql.DB, seen map[string]bool) error {
|
|
// Only authoritative references, not derived projection/preview caches. Some
|
|
// installations predate optional modules, hence the explicit table check.
|
|
for _, ref := range [][2]string{{"revisions", "document_hash"}, {"assets", "head_document_hash"}, {"asset_head_history", "document_hash"}, {"organizer_items", "document_hash"}, {"organizer_revisions", "document_hash"}} {
|
|
var exists bool
|
|
if err := database.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?)", ref[0]).Scan(&exists); err != nil {
|
|
return err
|
|
}
|
|
if !exists {
|
|
continue
|
|
}
|
|
rows, err := database.QueryContext(ctx, "SELECT DISTINCT "+ref[1]+" FROM "+ref[0]+" WHERE "+ref[1]+" IS NOT NULL AND "+ref[1]+"<>''")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for rows.Next() {
|
|
var hash string
|
|
if err = rows.Scan(&hash); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
if len(hash) != 64 || !seen["documents/sha256/"+hash[:2]+"/"+hash] {
|
|
rows.Close()
|
|
return fmt.Errorf("snapshot lacks canonical payload referenced by %s", ref[0])
|
|
}
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Verify file-based calendar authority without parsing event bodies or making
|
|
// external calendar requests. Derived calendar indexes may be repaired only
|
|
// after promotion, never by a standby read request.
|
|
func verifyCalendarFiles(root string, hashes map[string]string) error {
|
|
for path := range hashes {
|
|
if !strings.HasPrefix(path, "documents/calendars/") || filepath.Base(path) != "calendar.json" {
|
|
continue
|
|
}
|
|
filename := filepath.Join(root, "data", path)
|
|
info, err := os.Stat(filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.Size() > 16<<20 {
|
|
return errors.New("calendar metadata exceeds snapshot verification limit")
|
|
}
|
|
encoded, err := os.ReadFile(filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var metadata struct {
|
|
Months map[string]string `json:"months"`
|
|
}
|
|
if err = json.Unmarshal(encoded, &metadata); err != nil {
|
|
return errors.New("invalid calendar metadata in snapshot")
|
|
}
|
|
for month, hash := range metadata.Months {
|
|
if _, err = time.Parse("2006-01", month); err != nil {
|
|
return errors.New("invalid calendar month in snapshot")
|
|
}
|
|
if len(hash) != 64 || hashes[filepath.ToSlash(filepath.Join(filepath.Dir(path), month+"."+hash+".ics"))] != hash {
|
|
return errors.New("snapshot lacks a valid calendar monthly generation")
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|