Refactor architecture into eight components and consolidate reliability work
Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts. Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation. Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
This commit is contained in:
1 parent
9dd425f765
commit
d5d21b7841
302 files changed
+20037
-3492
No files matched your search
@@ -0,0 +1,273 @@
|
||||
// Package dr implements offline, operator-fenced recovery, not leader election.
|
||||
package dr
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const ConfigFile = "dr-node.json"
|
||||
const LockFile = ".pageswiki-instance.lock"
|
||||
|
||||
// RequireRole is an optional deployment guard against mounting an empty/wrong
|
||||
// data volume on a DR host and accidentally starting a new writable instance.
|
||||
func RequireRole(c *Config, expected string) error {
|
||||
if expected == "" {
|
||||
return nil
|
||||
}
|
||||
if expected != "primary" && expected != "standby" {
|
||||
return errors.New("PAGESWIKI_DR_EXPECT_ROLE must be primary or standby")
|
||||
}
|
||||
if c == nil || c.Role != expected {
|
||||
return errors.New("DR role does not match PAGESWIKI_DR_EXPECT_ROLE; refusing startup")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Promotion struct {
|
||||
At time.Time `json:"at"`
|
||||
PreviousNode string `json:"previous_node"`
|
||||
FenceReference string `json:"fence_reference"`
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
}
|
||||
|
||||
// Config is node-local and MUST NOT be overwritten by a primary's snapshot.
|
||||
// Epoch is an audit generation only; it is NOT a distributed fencing token.
|
||||
type Config struct {
|
||||
Version int `json:"version"`
|
||||
ClusterID string `json:"cluster_id"`
|
||||
NodeID string `json:"node_id"`
|
||||
PrimaryNodeID string `json:"primary_node_id"`
|
||||
Role string `json:"role"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
SnapshotID string `json:"snapshot_id,omitempty"`
|
||||
CapturedAt time.Time `json:"captured_at,omitempty"`
|
||||
MaxReadAgeSeconds int64 `json:"max_read_age_seconds"`
|
||||
WorkersEnabled bool `json:"workers_enabled"`
|
||||
WorkersResumedAt time.Time `json:"workers_resumed_at,omitempty"`
|
||||
History []Promotion `json:"promotion_history,omitempty"`
|
||||
}
|
||||
|
||||
func (c Config) Validate() error {
|
||||
for _, id := range []string{c.ClusterID, c.NodeID, c.PrimaryNodeID} {
|
||||
u, err := uuid.Parse(id)
|
||||
if err != nil || u == uuid.Nil || u.String() != id {
|
||||
return errors.New("DR identity must be a canonical nonzero UUID")
|
||||
}
|
||||
}
|
||||
if c.Version != 1 || c.Epoch == 0 || c.MaxReadAgeSeconds < 1 || c.MaxReadAgeSeconds > 86400 {
|
||||
return errors.New("invalid DR version, epoch or max_read_age_seconds (1..86400)")
|
||||
}
|
||||
if c.Role != "primary" && c.Role != "standby" {
|
||||
return errors.New("DR role must be primary or standby")
|
||||
}
|
||||
if (c.Role == "primary") != (c.PrimaryNodeID == c.NodeID) {
|
||||
return errors.New("DR role and primary identity disagree")
|
||||
}
|
||||
if c.Role == "standby" && (c.SnapshotID == "" || c.CapturedAt.IsZero()) {
|
||||
return errors.New("standby requires a verified snapshot")
|
||||
}
|
||||
if c.Role == "standby" && c.WorkersEnabled {
|
||||
return errors.New("standby cannot enable background workers")
|
||||
}
|
||||
if c.SnapshotID != "" {
|
||||
u, err := uuid.Parse(c.SnapshotID)
|
||||
if err != nil || u == uuid.Nil || u.String() != c.SnapshotID {
|
||||
return errors.New("invalid snapshot identity")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c Config) Readable(now time.Time) bool {
|
||||
return c.Role == "standby" && !c.CapturedAt.After(now.Add(time.Minute)) && now.Sub(c.CapturedAt) <= time.Duration(c.MaxReadAgeSeconds)*time.Second
|
||||
}
|
||||
|
||||
func (c Config) ReadUntil() time.Time {
|
||||
if c.CapturedAt.After(time.Now().Add(time.Minute)) {
|
||||
return time.Time{}
|
||||
}
|
||||
return c.CapturedAt.Add(time.Duration(c.MaxReadAgeSeconds) * time.Second)
|
||||
}
|
||||
|
||||
func Load(dataDir string) (*Config, error) {
|
||||
f, err := os.Open(filepath.Join(dataDir, ConfigFile))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Size() > 1<<20 {
|
||||
return nil, errors.New("invalid or oversized DR configuration")
|
||||
}
|
||||
var c Config
|
||||
d := json.NewDecoder(io.LimitReader(f, 1<<20))
|
||||
d.DisallowUnknownFields()
|
||||
if err = d.Decode(&c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = d.Decode(new(any)); err != io.EOF {
|
||||
return nil, errors.New("trailing DR configuration data")
|
||||
}
|
||||
return &c, c.Validate()
|
||||
}
|
||||
|
||||
func Save(dataDir string, c Config) error {
|
||||
if err := c.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.MarshalIndent(c, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.CreateTemp(dataDir, ".dr-node-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
if _, err = f.Write(append(b, '\n')); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err = f.Sync(); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err = f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = os.Rename(f.Name(), filepath.Join(dataDir, ConfigFile)); err != nil {
|
||||
return err
|
||||
}
|
||||
return syncDir(dataDir)
|
||||
}
|
||||
|
||||
// Lock is shared for a running server/CLI, exclusive for offline maintenance.
|
||||
// Advisory locking cannot fence older binaries or an independent SQLite client.
|
||||
func Lock(dataDir string, exclusive bool) (*os.File, error) {
|
||||
// Shared locks also work with a read-only standby mount.
|
||||
path := filepath.Join(dataDir, LockFile)
|
||||
f, err := os.Open(path)
|
||||
if exclusive || errors.Is(err, os.ErrNotExist) {
|
||||
if f != nil {
|
||||
f.Close()
|
||||
}
|
||||
f, err = os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mode := syscall.LOCK_SH
|
||||
if exclusive {
|
||||
mode = syscall.LOCK_EX
|
||||
}
|
||||
if err = syscall.Flock(int(f.Fd()), mode|syscall.LOCK_NB); err != nil {
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("DR data directory is in use; stop all writers/readers before maintenance: %w", err)
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func Init(dataDir string) (*Config, error) {
|
||||
l, err := Lock(dataDir, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer l.Close()
|
||||
if c, err := Load(dataDir); err != nil || c != nil {
|
||||
return nil, errors.New("DR configuration exists or cannot be read; refusing initialization")
|
||||
}
|
||||
id := uuid.NewString()
|
||||
c := Config{Version: 1, ClusterID: uuid.NewString(), NodeID: id, PrimaryNodeID: id, Role: "primary", Epoch: 1, MaxReadAgeSeconds: 3600, WorkersEnabled: true}
|
||||
return &c, Save(dataDir, c)
|
||||
}
|
||||
|
||||
// Promote records an operator's fencing attestation. It does not contact,
|
||||
// power off, or independently verify the former primary. No automatic promotion.
|
||||
func Promote(dataDir, fencedNode, reference string, acceptLoss bool) (*Config, error) {
|
||||
l, err := Lock(dataDir, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer l.Close()
|
||||
c, err := Load(dataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c == nil || c.Role != "standby" {
|
||||
return nil, errors.New("only an offline standby can be promoted")
|
||||
}
|
||||
if err = checkAuthKey(dataDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fencedNode != c.PrimaryNodeID || !acceptLoss || len(strings.TrimSpace(reference)) < 4 || len(reference) > 1024 {
|
||||
return nil, errors.New("promotion requires exact former primary ID, fencing reference and explicit acceptance of snapshot data loss")
|
||||
}
|
||||
if c.Epoch == ^uint64(0) {
|
||||
return nil, errors.New("DR epoch exhausted")
|
||||
}
|
||||
c.Epoch++
|
||||
c.History = append(c.History, Promotion{At: time.Now().UTC(), PreviousNode: fencedNode, FenceReference: strings.TrimSpace(reference), SnapshotID: c.SnapshotID, Epoch: c.Epoch})
|
||||
c.Role, c.PrimaryNodeID = "primary", c.NodeID
|
||||
// Replaying pre-snapshot jobs can duplicate notifications or external writes.
|
||||
// Resume them only after an operator reviews the restored queue.
|
||||
c.WorkersEnabled = false
|
||||
return c, Save(dataDir, *c)
|
||||
}
|
||||
|
||||
func checkAuthKey(dataDir string) error {
|
||||
if strings.TrimSpace(os.Getenv("PAGESWIKI_AUTH_KEY")) != "" {
|
||||
return nil
|
||||
}
|
||||
key, err := os.ReadFile(filepath.Join(dataDir, "auth.key"))
|
||||
if err != nil || len(key) != 32 {
|
||||
return errors.New("original auth.key or PAGESWIKI_AUTH_KEY is required; refusing implicit key regeneration")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ResumeWorkers(dataDir string, reviewed bool) (*Config, error) {
|
||||
if !reviewed {
|
||||
return nil, errors.New("review restored jobs and federation state before resuming workers")
|
||||
}
|
||||
l, err := Lock(dataDir, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer l.Close()
|
||||
c, err := Load(dataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c == nil || c.Role != "primary" {
|
||||
return nil, errors.New("only an offline primary can resume workers")
|
||||
}
|
||||
c.WorkersEnabled = true
|
||||
c.WorkersResumedAt = time.Now().UTC()
|
||||
return c, Save(dataDir, *c)
|
||||
}
|
||||
|
||||
func syncDir(path string) error {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
return f.Sync()
|
||||
}
|
||||
Reference in new issue
Block a user