// Package dr implements offline, operator-fenced recovery, not leader election. package dr import ( "encoding/json" "errors" "fmt" "io" "os" "path/filepath" "strings" "syscall" "time" "github.com/google/uuid" ) const ConfigFile = "dr-node.json" const LockFile = ".pageswiki-instance.lock" // RequireRole is an optional deployment guard against mounting an empty/wrong // data volume on a DR host and accidentally starting a new writable instance. func RequireRole(c *Config, expected string) error { if expected == "" { return nil } if expected != "primary" && expected != "standby" { return errors.New("PAGESWIKI_DR_EXPECT_ROLE must be primary or standby") } if c == nil || c.Role != expected { return errors.New("DR role does not match PAGESWIKI_DR_EXPECT_ROLE; refusing startup") } return nil } type Promotion struct { At time.Time `json:"at"` PreviousNode string `json:"previous_node"` FenceReference string `json:"fence_reference"` SnapshotID string `json:"snapshot_id"` Epoch uint64 `json:"epoch"` } // Config is node-local and MUST NOT be overwritten by a primary's snapshot. // Epoch is an audit generation only; it is NOT a distributed fencing token. type Config struct { Version int `json:"version"` ClusterID string `json:"cluster_id"` NodeID string `json:"node_id"` PrimaryNodeID string `json:"primary_node_id"` Role string `json:"role"` Epoch uint64 `json:"epoch"` SnapshotID string `json:"snapshot_id,omitempty"` CapturedAt time.Time `json:"captured_at,omitempty"` MaxReadAgeSeconds int64 `json:"max_read_age_seconds"` WorkersEnabled bool `json:"workers_enabled"` WorkersResumedAt time.Time `json:"workers_resumed_at,omitempty"` History []Promotion `json:"promotion_history,omitempty"` } func (c Config) Validate() error { for _, id := range []string{c.ClusterID, c.NodeID, c.PrimaryNodeID} { u, err := uuid.Parse(id) if err != nil || u == uuid.Nil || u.String() != id { return errors.New("DR identity must be a canonical nonzero UUID") } } if c.Version != 1 || c.Epoch == 0 || c.MaxReadAgeSeconds < 1 || c.MaxReadAgeSeconds > 86400 { return errors.New("invalid DR version, epoch or max_read_age_seconds (1..86400)") } if c.Role != "primary" && c.Role != "standby" { return errors.New("DR role must be primary or standby") } if (c.Role == "primary") != (c.PrimaryNodeID == c.NodeID) { return errors.New("DR role and primary identity disagree") } if c.Role == "standby" && (c.SnapshotID == "" || c.CapturedAt.IsZero()) { return errors.New("standby requires a verified snapshot") } if c.Role == "standby" && c.WorkersEnabled { return errors.New("standby cannot enable background workers") } if c.SnapshotID != "" { u, err := uuid.Parse(c.SnapshotID) if err != nil || u == uuid.Nil || u.String() != c.SnapshotID { return errors.New("invalid snapshot identity") } } return nil } func (c Config) Readable(now time.Time) bool { return c.Role == "standby" && !c.CapturedAt.After(now.Add(time.Minute)) && now.Sub(c.CapturedAt) <= time.Duration(c.MaxReadAgeSeconds)*time.Second } func (c Config) ReadUntil() time.Time { if c.CapturedAt.After(time.Now().Add(time.Minute)) { return time.Time{} } return c.CapturedAt.Add(time.Duration(c.MaxReadAgeSeconds) * time.Second) } func Load(dataDir string) (*Config, error) { f, err := os.Open(filepath.Join(dataDir, ConfigFile)) if errors.Is(err, os.ErrNotExist) { return nil, nil } if err != nil { return nil, err } defer f.Close() info, err := f.Stat() if err != nil { return nil, err } if !info.Mode().IsRegular() || info.Size() > 1<<20 { return nil, errors.New("invalid or oversized DR configuration") } var c Config d := json.NewDecoder(io.LimitReader(f, 1<<20)) d.DisallowUnknownFields() if err = d.Decode(&c); err != nil { return nil, err } if err = d.Decode(new(any)); err != io.EOF { return nil, errors.New("trailing DR configuration data") } return &c, c.Validate() } func Save(dataDir string, c Config) error { if err := c.Validate(); err != nil { return err } b, err := json.MarshalIndent(c, "", " ") if err != nil { return err } f, err := os.CreateTemp(dataDir, ".dr-node-") if err != nil { return err } defer os.Remove(f.Name()) if _, err = f.Write(append(b, '\n')); err != nil { f.Close() return err } if err = f.Sync(); err != nil { f.Close() return err } if err = f.Close(); err != nil { return err } if err = os.Rename(f.Name(), filepath.Join(dataDir, ConfigFile)); err != nil { return err } return syncDir(dataDir) } // Lock is shared for a running server/CLI, exclusive for offline maintenance. // Advisory locking cannot fence older binaries or an independent SQLite client. func Lock(dataDir string, exclusive bool) (*os.File, error) { // Shared locks also work with a read-only standby mount. path := filepath.Join(dataDir, LockFile) f, err := os.Open(path) if exclusive || errors.Is(err, os.ErrNotExist) { if f != nil { f.Close() } f, err = os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600) } if err != nil { return nil, err } mode := syscall.LOCK_SH if exclusive { mode = syscall.LOCK_EX } if err = syscall.Flock(int(f.Fd()), mode|syscall.LOCK_NB); err != nil { f.Close() return nil, fmt.Errorf("DR data directory is in use; stop all writers/readers before maintenance: %w", err) } return f, nil } func Init(dataDir string) (*Config, error) { l, err := Lock(dataDir, true) if err != nil { return nil, err } defer l.Close() if c, err := Load(dataDir); err != nil || c != nil { return nil, errors.New("DR configuration exists or cannot be read; refusing initialization") } id := uuid.NewString() c := Config{Version: 1, ClusterID: uuid.NewString(), NodeID: id, PrimaryNodeID: id, Role: "primary", Epoch: 1, MaxReadAgeSeconds: 3600, WorkersEnabled: true} return &c, Save(dataDir, c) } // Promote records an operator's fencing attestation. It does not contact, // power off, or independently verify the former primary. No automatic promotion. func Promote(dataDir, fencedNode, reference string, acceptLoss bool) (*Config, error) { l, err := Lock(dataDir, true) if err != nil { return nil, err } defer l.Close() c, err := Load(dataDir) if err != nil { return nil, err } if c == nil || c.Role != "standby" { return nil, errors.New("only an offline standby can be promoted") } if err = checkAuthKey(dataDir); err != nil { return nil, err } if fencedNode != c.PrimaryNodeID || !acceptLoss || len(strings.TrimSpace(reference)) < 4 || len(reference) > 1024 { return nil, errors.New("promotion requires exact former primary ID, fencing reference and explicit acceptance of snapshot data loss") } if c.Epoch == ^uint64(0) { return nil, errors.New("DR epoch exhausted") } c.Epoch++ c.History = append(c.History, Promotion{At: time.Now().UTC(), PreviousNode: fencedNode, FenceReference: strings.TrimSpace(reference), SnapshotID: c.SnapshotID, Epoch: c.Epoch}) c.Role, c.PrimaryNodeID = "primary", c.NodeID // Replaying pre-snapshot jobs can duplicate notifications or external writes. // Resume them only after an operator reviews the restored queue. c.WorkersEnabled = false return c, Save(dataDir, *c) } func checkAuthKey(dataDir string) error { if strings.TrimSpace(os.Getenv("PAGESWIKI_AUTH_KEY")) != "" { return nil } key, err := os.ReadFile(filepath.Join(dataDir, "auth.key")) if err != nil || len(key) != 32 { return errors.New("original auth.key or PAGESWIKI_AUTH_KEY is required; refusing implicit key regeneration") } return nil } func ResumeWorkers(dataDir string, reviewed bool) (*Config, error) { if !reviewed { return nil, errors.New("review restored jobs and federation state before resuming workers") } l, err := Lock(dataDir, true) if err != nil { return nil, err } defer l.Close() c, err := Load(dataDir) if err != nil { return nil, err } if c == nil || c.Role != "primary" { return nil, errors.New("only an offline primary can resume workers") } c.WorkersEnabled = true c.WorkersResumedAt = time.Now().UTC() return c, Save(dataDir, *c) } func syncDir(path string) error { f, err := os.Open(path) if err != nil { return err } defer f.Close() return f.Sync() }