Files
pages/internal/store/document_backfill.go
T
Anya Tokugawa d5d21b7841
Build and publish container images / images (push) Successful in 9m34s
Release container images / release (push) Successful in 36s
Refactor architecture into eight components and consolidate reliability work
Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts.

Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation.

Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
2026-11-01 15:23:31 +09:00

229 lines
6.7 KiB
Go

package store
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"syscall"
"time"
"pageswiki/internal/db"
)
var documentBackfillMigrations = []db.SQLMigration{{Version: 1, Name: "document_backfill_progress", Statements: []string{
`CREATE TABLE document_backfills (
store_id TEXT PRIMARY KEY, last_uuid TEXT NOT NULL DEFAULT '',
processed INTEGER NOT NULL DEFAULT 0, completed INTEGER NOT NULL DEFAULT 0,
updated_at_ms INTEGER NOT NULL)`,
}}, {Version: 2, Name: "verify_backfill_contract", Assertions: []string{
schemaColumns("document_backfills", "store_id,last_uuid,processed,completed,updated_at_ms"),
schemaUnique("document_backfills", "store_id"),
}}}
type DocumentBackfillProgress struct {
StoreID string
LastUUID string
Processed int64
Completed bool
}
// BackfillDocuments performs one restartable, bounded batch. It is a maintenance
// operation: legacy DB-only writers must be stopped. Normal application saves
// persist canonical bytes before inserting revisions and need no backfill.
// The UUID cursor is stable across SQLite VACUUM (rowid is not).
func (s *Store) BackfillDocuments(ctx context.Context, batchSize int) (DocumentBackfillProgress, error) {
var progress DocumentBackfillProgress
if s.ReadOnly {
return progress, ErrReadOnly
}
if err := ctx.Err(); err != nil {
return progress, err
}
if s.DocumentRoot == "" {
progress.Completed = true
return progress, nil
}
if batchSize <= 0 || batchSize > 128 {
batchSize = 128
}
if err := os.MkdirAll(filepath.Join(s.DocumentRoot, "sha256"), 0755); err != nil {
return progress, err
}
root, err := os.OpenRoot(s.DocumentRoot)
if err != nil {
return progress, err
}
defer root.Close()
unlock, err := lockDocumentBackfill(ctx, root)
if err != nil {
return progress, err
}
defer unlock()
progress.StoreID, err = documentStoreIdentity(root)
if err != nil {
return progress, err
}
if err := db.ApplySQLiteMigrations(ctx, s.DB, "document-store", documentBackfillMigrations); err != nil {
return progress, err
}
if _, err := s.DB.ExecContext(ctx, `INSERT INTO document_backfills(store_id,updated_at_ms) VALUES(?,?) ON CONFLICT(store_id) DO NOTHING`, progress.StoreID, time.Now().UnixMilli()); err != nil {
return progress, err
}
if err := s.DB.QueryRowContext(ctx, `SELECT last_uuid,processed,completed FROM document_backfills WHERE store_id=?`, progress.StoreID).Scan(&progress.LastUUID, &progress.Processed, &progress.Completed); err != nil {
return progress, err
}
if progress.Completed {
return progress, nil
}
rows, err := s.DB.QueryContext(ctx, `SELECT uuid,document_hash FROM revisions WHERE uuid>? AND document_hash IS NOT NULL AND document_hash!='' ORDER BY uuid LIMIT ?`, progress.LastUUID, batchSize)
if err != nil {
return progress, err
}
type ref struct{ uuid, hash string }
var refs []ref
for rows.Next() {
var r ref
if err := rows.Scan(&r.uuid, &r.hash); err != nil {
rows.Close()
return progress, err
}
refs = append(refs, r)
}
if err := rows.Err(); err != nil {
rows.Close()
return progress, err
}
if err := rows.Close(); err != nil {
return progress, err
}
backend, err := s.documentStorage()
if err != nil {
return progress, err
}
for _, ref := range refs {
// Existing canonical bytes outrank a stale/missing compatibility body.
// Never overwrite a corrupt canonical file with a cache implicitly.
err := backend.Verify(ctx, ref.hash)
if errors.Is(err, os.ErrNotExist) {
var body string
if err = s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE uuid=? AND document_hash=?`, ref.uuid, ref.hash).Scan(&body); err == nil {
err = backend.PutIfAbsent(ctx, ref.hash, int64(len(body)), strings.NewReader(body))
}
}
if err != nil {
return progress, fmt.Errorf("Document backfill revision=%s: %w", ref.uuid, err)
}
}
next := progress
if len(refs) != 0 {
next.LastUUID = refs[len(refs)-1].uuid
}
next.Processed += int64(len(refs))
next.Completed = len(refs) < batchSize
if _, err := s.DB.ExecContext(ctx, `UPDATE document_backfills SET last_uuid=?,processed=?,completed=?,updated_at_ms=? WHERE store_id=?`, next.LastUUID, next.Processed, next.Completed, time.Now().UnixMilli(), next.StoreID); err != nil {
return progress, err
}
return next, nil
}
func lockDocumentBackfill(ctx context.Context, root *os.Root) (func(), error) {
file, err := root.OpenFile(".document-backfill.lock", os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return nil, err
}
for {
if err := ctx.Err(); err != nil {
file.Close()
return nil, err
}
err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
if err == nil {
return func() { _ = syscall.Flock(int(file.Fd()), syscall.LOCK_UN); file.Close() }, nil
}
if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) {
file.Close()
return nil, err
}
timer := time.NewTimer(20 * time.Millisecond)
select {
case <-ctx.Done():
timer.Stop()
file.Close()
return nil, ctx.Err()
case <-timer.C:
}
}
}
// Identity follows the physical store, not its pathname. A replaced/empty root
// gets a new identity and cannot inherit the DB's previous completion marker.
func documentStoreIdentity(root *os.Root) (string, error) {
file, err := root.Open(".document-store-id")
if err == nil {
defer file.Close()
info, err := file.Stat()
if err != nil {
return "", err
}
if !info.Mode().IsRegular() || info.Size() > 64 {
return "", errors.New("invalid Document store identity file")
}
body, err := io.ReadAll(io.LimitReader(file, 128))
if err != nil {
return "", err
}
id := strings.TrimSpace(string(body))
if decoded, err := hex.DecodeString(id); err != nil || len(decoded) != 16 {
return "", errors.New("invalid Document store identity")
}
// Also sync on resume: the previous process may have published the
// identity then crashed before syncing its directory.
return id, syncDocumentRoot(root)
}
if !errors.Is(err, os.ErrNotExist) {
return "", err
}
var nonce [16]byte
if _, err := rand.Read(nonce[:]); err != nil {
return "", err
}
id := hex.EncodeToString(nonce[:])
tmp := ".document-store-id-" + id
file, err = root.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return "", err
}
defer root.Remove(tmp)
_, err = file.WriteString(id + "\n")
if err == nil {
err = file.Sync()
}
closeErr := file.Close()
if err == nil {
err = closeErr
}
if err != nil {
return "", err
}
if err := root.Link(tmp, ".document-store-id"); err != nil {
return "", err
}
return id, syncDocumentRoot(root)
}
func syncDocumentRoot(root *os.Root) error {
dir, err := root.Open(".")
if err != nil {
return err
}
err = dir.Sync()
dir.Close()
return err
}