package dr import ( "context" "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" "errors" "fmt" "io" "io/fs" "net/url" "os" "path/filepath" "strings" "time" "github.com/google/uuid" "golang.org/x/sys/unix" "modernc.org/sqlite" "pageswiki/internal/db" ) type File struct { Path string `json:"path"` Size int64 `json:"size"` SHA256 string `json:"sha256"` } type Manifest struct { Version int `json:"version"` ID string `json:"id"` ClusterID string `json:"cluster_id"` PrimaryNodeID string `json:"primary_node_id"` Epoch uint64 `json:"epoch"` CapturedAt time.Time `json:"captured_at"` CompletedAt time.Time `json:"completed_at"` Scope string `json:"scope"` Files []File `json:"files"` } // Include canonical stores, auth/signing keys and managed-campus enrollment. // Caches, temporary uploads, editor workspaces and node-local DR state are NOT // recovery authorities. Extra mounted storage/environment secrets are external. func canonical(path string) bool { if !filepath.IsLocal(path) || filepath.ToSlash(path) != path || strings.Contains(path, "\\") || path != filepath.Clean(path) { return false } root := strings.Split(path, "/")[0] switch root { case "documents", "themes", "federation": return true case "pages.db", "auth.key", "instance-ed25519.key": return path == root } return false } func fileHash(path string) (File, error) { f, err := os.Open(path) if err != nil { return File{}, err } defer f.Close() i, err := f.Stat() if err != nil || !i.Mode().IsRegular() { return File{}, errors.New("snapshot entry is not a regular file") } h := sha256.New() n, err := io.CopyBuffer(h, f, make([]byte, 256<<10)) return File{Size: n, SHA256: hex.EncodeToString(h.Sum(nil))}, err } func copyFile(source, target string) error { i, err := os.Lstat(source) if err != nil { return err } if !i.Mode().IsRegular() { return fmt.Errorf("refusing nonregular source: %s", source) } in, err := os.Open(source) if err != nil { return err } defer in.Close() opened, err := in.Stat() if err != nil || !os.SameFile(i, opened) { return errors.New("source changed during snapshot") } if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil { return err } out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) if err != nil { return err } _, err = io.CopyBuffer(out, in, make([]byte, 256<<10)) if err == nil { err = out.Sync() } closeErr := out.Close() if err == nil { err = closeErr } return err } // Backup API preserves rowids (unlike VACUUM INTO) and includes committed WAL. func backupDB(ctx context.Context, source, target string) error { database, err := db.OpenReadOnly(source) if err != nil { return err } defer database.Close() conn, err := database.Conn(ctx) if err != nil { return err } defer conn.Close() err = conn.Raw(func(raw any) error { b, err := raw.(interface { NewBackup(string) (*sqlite.Backup, error) }).NewBackup((&url.URL{Scheme: "file", Path: target}).String()) if err != nil { return err } for { if err = ctx.Err(); err != nil { b.Finish() return err } more, e := b.Step(256) if e != nil { b.Finish() return e } if !more { return b.Finish() } } }) if err != nil { return err } // Do not leave a WAL-mode snapshot that creates sidecars during verification. dst, err := sql.Open("sqlite", (&url.URL{Scheme: "file", Path: target, RawQuery: "mode=rw"}).String()) if err != nil { return err } _, err = dst.ExecContext(ctx, "PRAGMA journal_mode=DELETE") closeErr := dst.Close() if err != nil { return err } if closeErr != nil { return closeErr } // Finish leaves a standalone DB; fsync before publishing the generation. f, err := os.OpenFile(target, os.O_RDWR, 0600) if err != nil { return err } defer f.Close() if err = f.Chmod(0600); err != nil { return err } return f.Sync() } // Snapshot requires every writer stopped, including older binaries and CLIs. // The lock enforces this for upgraded PagesWiki processes, not arbitrary tools. func Snapshot(ctx context.Context, dataDir, target string) (*Manifest, string, error) { source, err := filepath.EvalSymlinks(dataDir) if err != nil { return nil, "", err } source, err = filepath.Abs(source) if err != nil { return nil, "", err } target, err = filepath.Abs(target) if err != nil { return nil, "", err } parent, err := filepath.EvalSymlinks(filepath.Dir(target)) if err != nil { return nil, "", err } target = filepath.Join(parent, filepath.Base(target)) if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) { return nil, "", errors.New("snapshot destination must be outside the data directory") } l, err := Lock(source, true) if err != nil { return nil, "", err } defer l.Close() c, err := Load(source) if err != nil { return nil, "", err } if c == nil || c.Role != "primary" { return nil, "", errors.New("initialize primary DR configuration before snapshot") } if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) { return nil, "", errors.New("snapshot destination must not exist") } stage, err := os.MkdirTemp(parent, ".dr-snapshot-") if err != nil { return nil, "", err } defer os.RemoveAll(stage) m := &Manifest{Version: 1, ID: uuid.NewString(), ClusterID: c.ClusterID, PrimaryNodeID: c.NodeID, Epoch: c.Epoch, CapturedAt: time.Now().UTC(), Scope: "instance-offline-v1"} root := filepath.Join(stage, "data") if err = os.Mkdir(root, 0700); err != nil { return nil, "", err } info, err := os.Lstat(filepath.Join(source, "pages.db")) if err != nil { return nil, "", err } if !info.Mode().IsRegular() { return nil, "", errors.New("snapshot database must be a regular file, not a symlink") } if err = checkAuthKey(source); err != nil { return nil, "", err } if err = backupDB(ctx, filepath.Join(source, "pages.db"), filepath.Join(root, "pages.db")); err != nil { return nil, "", err } for _, name := range []string{"documents", "themes", "federation", "auth.key", "instance-ed25519.key"} { base := filepath.Join(source, name) if _, e := os.Lstat(base); errors.Is(e, os.ErrNotExist) { continue } else if e != nil { return nil, "", e } err = filepath.WalkDir(base, func(path string, entry fs.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if err := ctx.Err(); err != nil { return err } if entry.Type()&os.ModeSymlink != 0 { return fmt.Errorf("snapshot refuses symlink: %s", path) } rel, _ := filepath.Rel(source, path) if entry.IsDir() { return os.MkdirAll(filepath.Join(root, rel), 0700) } return copyFile(path, filepath.Join(root, rel)) }) if err != nil { return nil, "", err } } err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, e error) error { if e != nil { return e } if entry.IsDir() { return syncDir(path) } if len(m.Files) >= 100000 { return errors.New("snapshot exceeds 100000 files; partitioned snapshots require a later format") } f, e := fileHash(path) if e != nil { return e } f.Path, _ = filepath.Rel(root, path) m.Files = append(m.Files, f) return nil }) if err != nil { return nil, "", err } m.CompletedAt = time.Now().UTC() encoded, err := json.MarshalIndent(m, "", " ") if err != nil { return nil, "", err } if err = os.WriteFile(filepath.Join(stage, "manifest.json"), encoded, 0600); err != nil { return nil, "", err } manifestFile, err := os.OpenFile(filepath.Join(stage, "manifest.json"), os.O_RDWR, 0600) if err != nil { return nil, "", err } err = manifestFile.Sync() manifestFile.Close() if err != nil { return nil, "", err } hash := sha256.Sum256(encoded) if _, err = Verify(ctx, stage, hex.EncodeToString(hash[:])); err != nil { return nil, "", err } if err = syncDir(stage); err != nil { return nil, "", err } if err = publish(stage, target); err != nil { return nil, "", err } return m, hex.EncodeToString(hash[:]), syncDir(parent) } func Verify(ctx context.Context, root, expectedHash string) (*Manifest, error) { if len(expectedHash) != 64 { return nil, errors.New("expected manifest SHA-256 is required from a trusted channel") } root, err := filepath.Abs(root) if err != nil { return nil, err } info, err := os.Lstat(filepath.Join(root, "manifest.json")) if err != nil { return nil, err } if !info.Mode().IsRegular() || info.Size() > 64<<20 { return nil, errors.New("invalid or oversized snapshot manifest") } encoded, err := os.ReadFile(filepath.Join(root, "manifest.json")) if err != nil { return nil, err } sum := sha256.Sum256(encoded) if hex.EncodeToString(sum[:]) != expectedHash { return nil, errors.New("manifest SHA-256 mismatch") } var m Manifest if err = json.Unmarshal(encoded, &m); err != nil { return nil, err } if m.Version != 1 || m.Scope != "instance-offline-v1" || m.Epoch == 0 || len(m.Files) == 0 || len(m.Files) > 100000 || m.CapturedAt.IsZero() || m.CompletedAt.Before(m.CapturedAt) || m.CompletedAt.After(time.Now().Add(time.Minute)) { return nil, errors.New("invalid snapshot manifest") } for _, id := range []string{m.ID, m.ClusterID, m.PrimaryNodeID} { u, e := uuid.Parse(id) if e != nil || u == uuid.Nil || u.String() != id { return nil, errors.New("invalid snapshot identity") } } seen := map[string]bool{} hashes := map[string]string{} for _, item := range m.Files { if err = ctx.Err(); err != nil { return nil, err } if !canonical(item.Path) || seen[item.Path] { return nil, errors.New("invalid/duplicate snapshot path") } seen[item.Path] = true hashes[item.Path] = item.SHA256 path := filepath.Join(root, "data", item.Path) // Refuse symlinks in every component, including parent directories. for p := path; p != filepath.Dir(root) && p != "."; p = filepath.Dir(p) { info, e := os.Lstat(p) if e != nil { return nil, e } if info.Mode()&os.ModeSymlink != 0 { return nil, errors.New("snapshot contains symlink") } if p == root || filepath.Dir(p) == p { break } } actual, e := fileHash(path) if e != nil { return nil, e } if actual.Size != item.Size || actual.SHA256 != item.SHA256 { return nil, fmt.Errorf("snapshot payload mismatch: %s", item.Path) } if strings.HasPrefix(item.Path, "documents/sha256/") && filepath.Base(item.Path) != actual.SHA256 { return nil, errors.New("CAS filename does not match payload hash") } } if !seen["pages.db"] { return nil, errors.New("snapshot database missing") } if err = verifyCalendarFiles(root, hashes); err != nil { return nil, err } err = filepath.WalkDir(filepath.Join(root, "data"), func(path string, entry fs.DirEntry, e error) error { if e != nil { return e } if entry.IsDir() { return nil } rel, _ := filepath.Rel(filepath.Join(root, "data"), path) if !seen[rel] { return errors.New("unlisted snapshot payload") } return nil }) if err != nil { return nil, err } database, err := db.OpenReadOnly(filepath.Join(root, "data", "pages.db")) if err != nil { return nil, err } defer database.Close() var integrity string if err = database.QueryRowContext(ctx, "PRAGMA quick_check").Scan(&integrity); err != nil { return nil, err } if integrity != "ok" { return nil, errors.New("snapshot database integrity check failed") } if err = verifyReferences(ctx, database, seen); err != nil { return nil, err } return &m, nil } // Restore never merges into or overwrites a running/existing data directory. // It publishes a new generation with a fresh physical node ID, always standby. func Restore(ctx context.Context, snapshot, expectedHash, target string) (*Config, error) { m, err := Verify(ctx, snapshot, expectedHash) if err != nil { return nil, err } target, err = filepath.Abs(target) if err != nil { return nil, err } parent, err := filepath.EvalSymlinks(filepath.Dir(target)) if err != nil { return nil, err } target = filepath.Join(parent, filepath.Base(target)) source, err := filepath.EvalSymlinks(snapshot) if err != nil { return nil, err } source, err = filepath.Abs(source) if err != nil { return nil, err } if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) { return nil, errors.New("restore destination must be outside the snapshot") } if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) { return nil, errors.New("restore target must not exist; use a new generation") } stage, err := os.MkdirTemp(filepath.Dir(target), ".dr-restore-") if err != nil { return nil, err } defer os.RemoveAll(stage) for _, item := range m.Files { if err = ctx.Err(); err != nil { return nil, err } if err = copyFile(filepath.Join(snapshot, "data", item.Path), filepath.Join(stage, item.Path)); err != nil { return nil, err } actual, e := fileHash(filepath.Join(stage, item.Path)) if e != nil { return nil, e } if actual.Size != item.Size || actual.SHA256 != item.SHA256 { return nil, errors.New("snapshot changed during restore") } } c := &Config{Version: 1, ClusterID: m.ClusterID, NodeID: uuid.NewString(), PrimaryNodeID: m.PrimaryNodeID, Role: "standby", Epoch: m.Epoch, SnapshotID: m.ID, CapturedAt: m.CapturedAt, MaxReadAgeSeconds: 3600} if err = Save(stage, *c); err != nil { return nil, err } if err = os.MkdirAll(filepath.Join(stage, "documents", "sha256"), 0700); err != nil { return nil, err } // Create the lock before a read-only data mount is applied. l, err := Lock(stage, true) if err != nil { return nil, err } l.Close() err = filepath.WalkDir(stage, func(path string, entry fs.DirEntry, e error) error { if e != nil { return e } if entry.IsDir() { return syncDir(path) } return nil }) if err != nil { return nil, err } if err = publish(stage, target); err != nil { return nil, err } return c, syncDir(filepath.Dir(target)) } // Linux deployment: atomically publish without replacing even an empty directory. func publish(stage, target string) error { return unix.Renameat2(unix.AT_FDCWD, stage, unix.AT_FDCWD, target, unix.RENAME_NOREPLACE) } func verifyReferences(ctx context.Context, database *sql.DB, seen map[string]bool) error { // Only authoritative references, not derived projection/preview caches. Some // installations predate optional modules, hence the explicit table check. for _, ref := range [][2]string{{"revisions", "document_hash"}, {"assets", "head_document_hash"}, {"asset_head_history", "document_hash"}, {"organizer_items", "document_hash"}, {"organizer_revisions", "document_hash"}} { var exists bool if err := database.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?)", ref[0]).Scan(&exists); err != nil { return err } if !exists { continue } rows, err := database.QueryContext(ctx, "SELECT DISTINCT "+ref[1]+" FROM "+ref[0]+" WHERE "+ref[1]+" IS NOT NULL AND "+ref[1]+"<>''") if err != nil { return err } for rows.Next() { var hash string if err = rows.Scan(&hash); err != nil { rows.Close() return err } if len(hash) != 64 || !seen["documents/sha256/"+hash[:2]+"/"+hash] { rows.Close() return fmt.Errorf("snapshot lacks canonical payload referenced by %s", ref[0]) } } err = rows.Err() rows.Close() if err != nil { return err } } return nil } // Verify file-based calendar authority without parsing event bodies or making // external calendar requests. Derived calendar indexes may be repaired only // after promotion, never by a standby read request. func verifyCalendarFiles(root string, hashes map[string]string) error { for path := range hashes { if !strings.HasPrefix(path, "documents/calendars/") || filepath.Base(path) != "calendar.json" { continue } filename := filepath.Join(root, "data", path) info, err := os.Stat(filename) if err != nil { return err } if info.Size() > 16<<20 { return errors.New("calendar metadata exceeds snapshot verification limit") } encoded, err := os.ReadFile(filename) if err != nil { return err } var metadata struct { Months map[string]string `json:"months"` } if err = json.Unmarshal(encoded, &metadata); err != nil { return errors.New("invalid calendar metadata in snapshot") } for month, hash := range metadata.Months { if _, err = time.Parse("2006-01", month); err != nil { return errors.New("invalid calendar month in snapshot") } if len(hash) != 64 || hashes[filepath.ToSlash(filepath.Join(filepath.Dir(path), month+"."+hash+".ics"))] != hash { return errors.New("snapshot lacks a valid calendar monthly generation") } } } return nil }