Files
pages/internal/store/document_backfill.go
T
2026-11-15 05:47:14 +09:00

232 lines
6.8 KiB
Go

package store
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"syscall"
"time"
"pageswiki/internal/db"
)
var documentBackfillMigrations = []db.SQLMigration{{Version: 1, Name: "document_backfill_progress", Statements: []string{
`CREATE TABLE document_backfills (
store_id TEXT PRIMARY KEY, last_uuid TEXT NOT NULL DEFAULT '',
processed INTEGER NOT NULL DEFAULT 0, completed INTEGER NOT NULL DEFAULT 0,
updated_at_ms INTEGER NOT NULL)`,
}}, {Version: 2, Name: "verify_backfill_contract", Assertions: []string{
schemaColumns("document_backfills", "store_id,last_uuid,processed,completed,updated_at_ms"),
schemaUnique("document_backfills", "store_id"),
}}}
type DocumentBackfillProgress struct {
StoreID string
LastUUID string
Processed int64
Completed bool
}
// BackfillDocuments performs one restartable, bounded batch. It is a maintenance
// operation: legacy DB-only writers must be stopped. Normal application saves
// persist canonical bytes before inserting revisions and need no backfill.
// The UUID cursor is stable across SQLite VACUUM (rowid is not).
func (s *Store) BackfillDocuments(ctx context.Context, batchSize int) (DocumentBackfillProgress, error) {
var progress DocumentBackfillProgress
if s.sqlBackend == storePostgreSQL {
return progress, ErrBackendFeatureUnavailable
}
if s.ReadOnly {
return progress, ErrReadOnly
}
if err := ctx.Err(); err != nil {
return progress, err
}
if s.DocumentRoot == "" {
progress.Completed = true
return progress, nil
}
if batchSize <= 0 || batchSize > 128 {
batchSize = 128
}
if err := os.MkdirAll(filepath.Join(s.DocumentRoot, "sha256"), 0755); err != nil {
return progress, err
}
root, err := os.OpenRoot(s.DocumentRoot)
if err != nil {
return progress, err
}
defer root.Close()
unlock, err := lockDocumentBackfill(ctx, root)
if err != nil {
return progress, err
}
defer unlock()
progress.StoreID, err = documentStoreIdentity(root)
if err != nil {
return progress, err
}
if err := db.ApplySQLiteMigrations(ctx, s.DB, "document-store", documentBackfillMigrations); err != nil {
return progress, err
}
if _, err := s.DB.ExecContext(ctx, `INSERT INTO document_backfills(store_id,updated_at_ms) VALUES(?,?) ON CONFLICT(store_id) DO NOTHING`, progress.StoreID, time.Now().UnixMilli()); err != nil {
return progress, err
}
if err := s.DB.QueryRowContext(ctx, `SELECT last_uuid,processed,completed FROM document_backfills WHERE store_id=?`, progress.StoreID).Scan(&progress.LastUUID, &progress.Processed, &progress.Completed); err != nil {
return progress, err
}
if progress.Completed {
return progress, nil
}
rows, err := s.DB.QueryContext(ctx, `SELECT uuid,document_hash FROM revisions WHERE uuid>? AND document_hash IS NOT NULL AND document_hash!='' ORDER BY uuid LIMIT ?`, progress.LastUUID, batchSize)
if err != nil {
return progress, err
}
type ref struct{ uuid, hash string }
var refs []ref
for rows.Next() {
var r ref
if err := rows.Scan(&r.uuid, &r.hash); err != nil {
rows.Close()
return progress, err
}
refs = append(refs, r)
}
if err := rows.Err(); err != nil {
rows.Close()
return progress, err
}
if err := rows.Close(); err != nil {
return progress, err
}
backend, err := s.documentStorage()
if err != nil {
return progress, err
}
for _, ref := range refs {
// Existing canonical bytes outrank a stale/missing compatibility body.
// Never overwrite a corrupt canonical file with a cache implicitly.
err := backend.Verify(ctx, ref.hash)
if errors.Is(err, os.ErrNotExist) {
var body string
if err = s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE uuid=? AND document_hash=?`, ref.uuid, ref.hash).Scan(&body); err == nil {
err = backend.PutIfAbsent(ctx, ref.hash, int64(len(body)), strings.NewReader(body))
}
}
if err != nil {
return progress, fmt.Errorf("Document backfill revision=%s: %w", ref.uuid, err)
}
}
next := progress
if len(refs) != 0 {
next.LastUUID = refs[len(refs)-1].uuid
}
next.Processed += int64(len(refs))
next.Completed = len(refs) < batchSize
if _, err := s.DB.ExecContext(ctx, `UPDATE document_backfills SET last_uuid=?,processed=?,completed=?,updated_at_ms=? WHERE store_id=?`, next.LastUUID, next.Processed, next.Completed, time.Now().UnixMilli(), next.StoreID); err != nil {
return progress, err
}
return next, nil
}
func lockDocumentBackfill(ctx context.Context, root *os.Root) (func(), error) {
file, err := root.OpenFile(".document-backfill.lock", os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return nil, err
}
for {
if err := ctx.Err(); err != nil {
file.Close()
return nil, err
}
err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
if err == nil {
return func() { _ = syscall.Flock(int(file.Fd()), syscall.LOCK_UN); file.Close() }, nil
}
if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) {
file.Close()
return nil, err
}
timer := time.NewTimer(20 * time.Millisecond)
select {
case <-ctx.Done():
timer.Stop()
file.Close()
return nil, ctx.Err()
case <-timer.C:
}
}
}
// Identity follows the physical store, not its pathname. A replaced/empty root
// gets a new identity and cannot inherit the DB's previous completion marker.
func documentStoreIdentity(root *os.Root) (string, error) {
file, err := root.Open(".document-store-id")
if err == nil {
defer file.Close()
info, err := file.Stat()
if err != nil {
return "", err
}
if !info.Mode().IsRegular() || info.Size() > 64 {
return "", errors.New("invalid Document store identity file")
}
body, err := io.ReadAll(io.LimitReader(file, 128))
if err != nil {
return "", err
}
id := strings.TrimSpace(string(body))
if decoded, err := hex.DecodeString(id); err != nil || len(decoded) != 16 {
return "", errors.New("invalid Document store identity")
}
// Also sync on resume: the previous process may have published the
// identity then crashed before syncing its directory.
return id, syncDocumentRoot(root)
}
if !errors.Is(err, os.ErrNotExist) {
return "", err
}
var nonce [16]byte
if _, err := rand.Read(nonce[:]); err != nil {
return "", err
}
id := hex.EncodeToString(nonce[:])
tmp := ".document-store-id-" + id
file, err = root.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return "", err
}
defer root.Remove(tmp)
_, err = file.WriteString(id + "\n")
if err == nil {
err = file.Sync()
}
closeErr := file.Close()
if err == nil {
err = closeErr
}
if err != nil {
return "", err
}
if err := root.Link(tmp, ".document-store-id"); err != nil {
return "", err
}
return id, syncDocumentRoot(root)
}
func syncDocumentRoot(root *os.Root) error {
dir, err := root.Open(".")
if err != nil {
return err
}
err = dir.Sync()
dir.Close()
return err
}