Refactor architecture into eight components and consolidate reliability work
Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts. Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation. Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
This commit is contained in:
1 parent
9dd425f765
commit
d5d21b7841
302 files changed
+20037
-3492
No files matched your search
@@ -0,0 +1,228 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
var documentBackfillMigrations = []db.SQLMigration{{Version: 1, Name: "document_backfill_progress", Statements: []string{
|
||||
`CREATE TABLE document_backfills (
|
||||
store_id TEXT PRIMARY KEY, last_uuid TEXT NOT NULL DEFAULT '',
|
||||
processed INTEGER NOT NULL DEFAULT 0, completed INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at_ms INTEGER NOT NULL)`,
|
||||
}}, {Version: 2, Name: "verify_backfill_contract", Assertions: []string{
|
||||
schemaColumns("document_backfills", "store_id,last_uuid,processed,completed,updated_at_ms"),
|
||||
schemaUnique("document_backfills", "store_id"),
|
||||
}}}
|
||||
|
||||
type DocumentBackfillProgress struct {
|
||||
StoreID string
|
||||
LastUUID string
|
||||
Processed int64
|
||||
Completed bool
|
||||
}
|
||||
|
||||
// BackfillDocuments performs one restartable, bounded batch. It is a maintenance
|
||||
// operation: legacy DB-only writers must be stopped. Normal application saves
|
||||
// persist canonical bytes before inserting revisions and need no backfill.
|
||||
// The UUID cursor is stable across SQLite VACUUM (rowid is not).
|
||||
func (s *Store) BackfillDocuments(ctx context.Context, batchSize int) (DocumentBackfillProgress, error) {
|
||||
var progress DocumentBackfillProgress
|
||||
if s.ReadOnly {
|
||||
return progress, ErrReadOnly
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
if s.DocumentRoot == "" {
|
||||
progress.Completed = true
|
||||
return progress, nil
|
||||
}
|
||||
if batchSize <= 0 || batchSize > 128 {
|
||||
batchSize = 128
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(s.DocumentRoot, "sha256"), 0755); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
root, err := os.OpenRoot(s.DocumentRoot)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
defer root.Close()
|
||||
unlock, err := lockDocumentBackfill(ctx, root)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
defer unlock()
|
||||
progress.StoreID, err = documentStoreIdentity(root)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
if err := db.ApplySQLiteMigrations(ctx, s.DB, "document-store", documentBackfillMigrations); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
if _, err := s.DB.ExecContext(ctx, `INSERT INTO document_backfills(store_id,updated_at_ms) VALUES(?,?) ON CONFLICT(store_id) DO NOTHING`, progress.StoreID, time.Now().UnixMilli()); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
if err := s.DB.QueryRowContext(ctx, `SELECT last_uuid,processed,completed FROM document_backfills WHERE store_id=?`, progress.StoreID).Scan(&progress.LastUUID, &progress.Processed, &progress.Completed); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
if progress.Completed {
|
||||
return progress, nil
|
||||
}
|
||||
rows, err := s.DB.QueryContext(ctx, `SELECT uuid,document_hash FROM revisions WHERE uuid>? AND document_hash IS NOT NULL AND document_hash!='' ORDER BY uuid LIMIT ?`, progress.LastUUID, batchSize)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
type ref struct{ uuid, hash string }
|
||||
var refs []ref
|
||||
for rows.Next() {
|
||||
var r ref
|
||||
if err := rows.Scan(&r.uuid, &r.hash); err != nil {
|
||||
rows.Close()
|
||||
return progress, err
|
||||
}
|
||||
refs = append(refs, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return progress, err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
backend, err := s.documentStorage()
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
for _, ref := range refs {
|
||||
// Existing canonical bytes outrank a stale/missing compatibility body.
|
||||
// Never overwrite a corrupt canonical file with a cache implicitly.
|
||||
err := backend.Verify(ctx, ref.hash)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
var body string
|
||||
if err = s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE uuid=? AND document_hash=?`, ref.uuid, ref.hash).Scan(&body); err == nil {
|
||||
err = backend.PutIfAbsent(ctx, ref.hash, int64(len(body)), strings.NewReader(body))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return progress, fmt.Errorf("Document backfill revision=%s: %w", ref.uuid, err)
|
||||
}
|
||||
}
|
||||
next := progress
|
||||
if len(refs) != 0 {
|
||||
next.LastUUID = refs[len(refs)-1].uuid
|
||||
}
|
||||
next.Processed += int64(len(refs))
|
||||
next.Completed = len(refs) < batchSize
|
||||
if _, err := s.DB.ExecContext(ctx, `UPDATE document_backfills SET last_uuid=?,processed=?,completed=?,updated_at_ms=? WHERE store_id=?`, next.LastUUID, next.Processed, next.Completed, time.Now().UnixMilli(), next.StoreID); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func lockDocumentBackfill(ctx context.Context, root *os.Root) (func(), error) {
|
||||
file, err := root.OpenFile(".document-backfill.lock", os.O_CREATE|os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
file.Close()
|
||||
return nil, err
|
||||
}
|
||||
err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
if err == nil {
|
||||
return func() { _ = syscall.Flock(int(file.Fd()), syscall.LOCK_UN); file.Close() }, nil
|
||||
}
|
||||
if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) {
|
||||
file.Close()
|
||||
return nil, err
|
||||
}
|
||||
timer := time.NewTimer(20 * time.Millisecond)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
file.Close()
|
||||
return nil, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Identity follows the physical store, not its pathname. A replaced/empty root
|
||||
// gets a new identity and cannot inherit the DB's previous completion marker.
|
||||
func documentStoreIdentity(root *os.Root) (string, error) {
|
||||
file, err := root.Open(".document-store-id")
|
||||
if err == nil {
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Size() > 64 {
|
||||
return "", errors.New("invalid Document store identity file")
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(file, 128))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
id := strings.TrimSpace(string(body))
|
||||
if decoded, err := hex.DecodeString(id); err != nil || len(decoded) != 16 {
|
||||
return "", errors.New("invalid Document store identity")
|
||||
}
|
||||
// Also sync on resume: the previous process may have published the
|
||||
// identity then crashed before syncing its directory.
|
||||
return id, syncDocumentRoot(root)
|
||||
}
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return "", err
|
||||
}
|
||||
var nonce [16]byte
|
||||
if _, err := rand.Read(nonce[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
id := hex.EncodeToString(nonce[:])
|
||||
tmp := ".document-store-id-" + id
|
||||
file, err = root.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer root.Remove(tmp)
|
||||
_, err = file.WriteString(id + "\n")
|
||||
if err == nil {
|
||||
err = file.Sync()
|
||||
}
|
||||
closeErr := file.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := root.Link(tmp, ".document-store-id"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return id, syncDocumentRoot(root)
|
||||
}
|
||||
|
||||
func syncDocumentRoot(root *os.Root) error {
|
||||
dir, err := root.Open(".")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = dir.Sync()
|
||||
dir.Close()
|
||||
return err
|
||||
}
|
||||
Reference in new issue
Block a user