Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts. Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation. Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
274 lines
8.2 KiB
Go
274 lines
8.2 KiB
Go
// Package dr implements offline, operator-fenced recovery, not leader election.
|
|
package dr
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
const ConfigFile = "dr-node.json"
|
|
const LockFile = ".pageswiki-instance.lock"
|
|
|
|
// RequireRole is an optional deployment guard against mounting an empty/wrong
|
|
// data volume on a DR host and accidentally starting a new writable instance.
|
|
func RequireRole(c *Config, expected string) error {
|
|
if expected == "" {
|
|
return nil
|
|
}
|
|
if expected != "primary" && expected != "standby" {
|
|
return errors.New("PAGESWIKI_DR_EXPECT_ROLE must be primary or standby")
|
|
}
|
|
if c == nil || c.Role != expected {
|
|
return errors.New("DR role does not match PAGESWIKI_DR_EXPECT_ROLE; refusing startup")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type Promotion struct {
|
|
At time.Time `json:"at"`
|
|
PreviousNode string `json:"previous_node"`
|
|
FenceReference string `json:"fence_reference"`
|
|
SnapshotID string `json:"snapshot_id"`
|
|
Epoch uint64 `json:"epoch"`
|
|
}
|
|
|
|
// Config is node-local and MUST NOT be overwritten by a primary's snapshot.
|
|
// Epoch is an audit generation only; it is NOT a distributed fencing token.
|
|
type Config struct {
|
|
Version int `json:"version"`
|
|
ClusterID string `json:"cluster_id"`
|
|
NodeID string `json:"node_id"`
|
|
PrimaryNodeID string `json:"primary_node_id"`
|
|
Role string `json:"role"`
|
|
Epoch uint64 `json:"epoch"`
|
|
SnapshotID string `json:"snapshot_id,omitempty"`
|
|
CapturedAt time.Time `json:"captured_at,omitempty"`
|
|
MaxReadAgeSeconds int64 `json:"max_read_age_seconds"`
|
|
WorkersEnabled bool `json:"workers_enabled"`
|
|
WorkersResumedAt time.Time `json:"workers_resumed_at,omitempty"`
|
|
History []Promotion `json:"promotion_history,omitempty"`
|
|
}
|
|
|
|
func (c Config) Validate() error {
|
|
for _, id := range []string{c.ClusterID, c.NodeID, c.PrimaryNodeID} {
|
|
u, err := uuid.Parse(id)
|
|
if err != nil || u == uuid.Nil || u.String() != id {
|
|
return errors.New("DR identity must be a canonical nonzero UUID")
|
|
}
|
|
}
|
|
if c.Version != 1 || c.Epoch == 0 || c.MaxReadAgeSeconds < 1 || c.MaxReadAgeSeconds > 86400 {
|
|
return errors.New("invalid DR version, epoch or max_read_age_seconds (1..86400)")
|
|
}
|
|
if c.Role != "primary" && c.Role != "standby" {
|
|
return errors.New("DR role must be primary or standby")
|
|
}
|
|
if (c.Role == "primary") != (c.PrimaryNodeID == c.NodeID) {
|
|
return errors.New("DR role and primary identity disagree")
|
|
}
|
|
if c.Role == "standby" && (c.SnapshotID == "" || c.CapturedAt.IsZero()) {
|
|
return errors.New("standby requires a verified snapshot")
|
|
}
|
|
if c.Role == "standby" && c.WorkersEnabled {
|
|
return errors.New("standby cannot enable background workers")
|
|
}
|
|
if c.SnapshotID != "" {
|
|
u, err := uuid.Parse(c.SnapshotID)
|
|
if err != nil || u == uuid.Nil || u.String() != c.SnapshotID {
|
|
return errors.New("invalid snapshot identity")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c Config) Readable(now time.Time) bool {
|
|
return c.Role == "standby" && !c.CapturedAt.After(now.Add(time.Minute)) && now.Sub(c.CapturedAt) <= time.Duration(c.MaxReadAgeSeconds)*time.Second
|
|
}
|
|
|
|
func (c Config) ReadUntil() time.Time {
|
|
if c.CapturedAt.After(time.Now().Add(time.Minute)) {
|
|
return time.Time{}
|
|
}
|
|
return c.CapturedAt.Add(time.Duration(c.MaxReadAgeSeconds) * time.Second)
|
|
}
|
|
|
|
func Load(dataDir string) (*Config, error) {
|
|
f, err := os.Open(filepath.Join(dataDir, ConfigFile))
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer f.Close()
|
|
info, err := f.Stat()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !info.Mode().IsRegular() || info.Size() > 1<<20 {
|
|
return nil, errors.New("invalid or oversized DR configuration")
|
|
}
|
|
var c Config
|
|
d := json.NewDecoder(io.LimitReader(f, 1<<20))
|
|
d.DisallowUnknownFields()
|
|
if err = d.Decode(&c); err != nil {
|
|
return nil, err
|
|
}
|
|
if err = d.Decode(new(any)); err != io.EOF {
|
|
return nil, errors.New("trailing DR configuration data")
|
|
}
|
|
return &c, c.Validate()
|
|
}
|
|
|
|
func Save(dataDir string, c Config) error {
|
|
if err := c.Validate(); err != nil {
|
|
return err
|
|
}
|
|
b, err := json.MarshalIndent(c, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
f, err := os.CreateTemp(dataDir, ".dr-node-")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer os.Remove(f.Name())
|
|
if _, err = f.Write(append(b, '\n')); err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
if err = f.Sync(); err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
if err = f.Close(); err != nil {
|
|
return err
|
|
}
|
|
if err = os.Rename(f.Name(), filepath.Join(dataDir, ConfigFile)); err != nil {
|
|
return err
|
|
}
|
|
return syncDir(dataDir)
|
|
}
|
|
|
|
// Lock is shared for a running server/CLI, exclusive for offline maintenance.
|
|
// Advisory locking cannot fence older binaries or an independent SQLite client.
|
|
func Lock(dataDir string, exclusive bool) (*os.File, error) {
|
|
// Shared locks also work with a read-only standby mount.
|
|
path := filepath.Join(dataDir, LockFile)
|
|
f, err := os.Open(path)
|
|
if exclusive || errors.Is(err, os.ErrNotExist) {
|
|
if f != nil {
|
|
f.Close()
|
|
}
|
|
f, err = os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600)
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
mode := syscall.LOCK_SH
|
|
if exclusive {
|
|
mode = syscall.LOCK_EX
|
|
}
|
|
if err = syscall.Flock(int(f.Fd()), mode|syscall.LOCK_NB); err != nil {
|
|
f.Close()
|
|
return nil, fmt.Errorf("DR data directory is in use; stop all writers/readers before maintenance: %w", err)
|
|
}
|
|
return f, nil
|
|
}
|
|
|
|
func Init(dataDir string) (*Config, error) {
|
|
l, err := Lock(dataDir, true)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer l.Close()
|
|
if c, err := Load(dataDir); err != nil || c != nil {
|
|
return nil, errors.New("DR configuration exists or cannot be read; refusing initialization")
|
|
}
|
|
id := uuid.NewString()
|
|
c := Config{Version: 1, ClusterID: uuid.NewString(), NodeID: id, PrimaryNodeID: id, Role: "primary", Epoch: 1, MaxReadAgeSeconds: 3600, WorkersEnabled: true}
|
|
return &c, Save(dataDir, c)
|
|
}
|
|
|
|
// Promote records an operator's fencing attestation. It does not contact,
|
|
// power off, or independently verify the former primary. No automatic promotion.
|
|
func Promote(dataDir, fencedNode, reference string, acceptLoss bool) (*Config, error) {
|
|
l, err := Lock(dataDir, true)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer l.Close()
|
|
c, err := Load(dataDir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if c == nil || c.Role != "standby" {
|
|
return nil, errors.New("only an offline standby can be promoted")
|
|
}
|
|
if err = checkAuthKey(dataDir); err != nil {
|
|
return nil, err
|
|
}
|
|
if fencedNode != c.PrimaryNodeID || !acceptLoss || len(strings.TrimSpace(reference)) < 4 || len(reference) > 1024 {
|
|
return nil, errors.New("promotion requires exact former primary ID, fencing reference and explicit acceptance of snapshot data loss")
|
|
}
|
|
if c.Epoch == ^uint64(0) {
|
|
return nil, errors.New("DR epoch exhausted")
|
|
}
|
|
c.Epoch++
|
|
c.History = append(c.History, Promotion{At: time.Now().UTC(), PreviousNode: fencedNode, FenceReference: strings.TrimSpace(reference), SnapshotID: c.SnapshotID, Epoch: c.Epoch})
|
|
c.Role, c.PrimaryNodeID = "primary", c.NodeID
|
|
// Replaying pre-snapshot jobs can duplicate notifications or external writes.
|
|
// Resume them only after an operator reviews the restored queue.
|
|
c.WorkersEnabled = false
|
|
return c, Save(dataDir, *c)
|
|
}
|
|
|
|
func checkAuthKey(dataDir string) error {
|
|
if strings.TrimSpace(os.Getenv("PAGESWIKI_AUTH_KEY")) != "" {
|
|
return nil
|
|
}
|
|
key, err := os.ReadFile(filepath.Join(dataDir, "auth.key"))
|
|
if err != nil || len(key) != 32 {
|
|
return errors.New("original auth.key or PAGESWIKI_AUTH_KEY is required; refusing implicit key regeneration")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ResumeWorkers(dataDir string, reviewed bool) (*Config, error) {
|
|
if !reviewed {
|
|
return nil, errors.New("review restored jobs and federation state before resuming workers")
|
|
}
|
|
l, err := Lock(dataDir, true)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer l.Close()
|
|
c, err := Load(dataDir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if c == nil || c.Role != "primary" {
|
|
return nil, errors.New("only an offline primary can resume workers")
|
|
}
|
|
c.WorkersEnabled = true
|
|
c.WorkersResumedAt = time.Now().UTC()
|
|
return c, Save(dataDir, *c)
|
|
}
|
|
|
|
func syncDir(path string) error {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
return f.Sync()
|
|
}
|