232 lines
6.8 KiB
Go
232 lines
6.8 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"pageswiki/internal/db"
|
|
)
|
|
|
|
var documentBackfillMigrations = []db.SQLMigration{{Version: 1, Name: "document_backfill_progress", Statements: []string{
|
|
`CREATE TABLE document_backfills (
|
|
store_id TEXT PRIMARY KEY, last_uuid TEXT NOT NULL DEFAULT '',
|
|
processed INTEGER NOT NULL DEFAULT 0, completed INTEGER NOT NULL DEFAULT 0,
|
|
updated_at_ms INTEGER NOT NULL)`,
|
|
}}, {Version: 2, Name: "verify_backfill_contract", Assertions: []string{
|
|
schemaColumns("document_backfills", "store_id,last_uuid,processed,completed,updated_at_ms"),
|
|
schemaUnique("document_backfills", "store_id"),
|
|
}}}
|
|
|
|
type DocumentBackfillProgress struct {
|
|
StoreID string
|
|
LastUUID string
|
|
Processed int64
|
|
Completed bool
|
|
}
|
|
|
|
// BackfillDocuments performs one restartable, bounded batch. It is a maintenance
|
|
// operation: legacy DB-only writers must be stopped. Normal application saves
|
|
// persist canonical bytes before inserting revisions and need no backfill.
|
|
// The UUID cursor is stable across SQLite VACUUM (rowid is not).
|
|
func (s *Store) BackfillDocuments(ctx context.Context, batchSize int) (DocumentBackfillProgress, error) {
|
|
var progress DocumentBackfillProgress
|
|
if s.sqlBackend == storePostgreSQL {
|
|
return progress, ErrBackendFeatureUnavailable
|
|
}
|
|
if s.ReadOnly {
|
|
return progress, ErrReadOnly
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return progress, err
|
|
}
|
|
if s.DocumentRoot == "" {
|
|
progress.Completed = true
|
|
return progress, nil
|
|
}
|
|
if batchSize <= 0 || batchSize > 128 {
|
|
batchSize = 128
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(s.DocumentRoot, "sha256"), 0755); err != nil {
|
|
return progress, err
|
|
}
|
|
root, err := os.OpenRoot(s.DocumentRoot)
|
|
if err != nil {
|
|
return progress, err
|
|
}
|
|
defer root.Close()
|
|
unlock, err := lockDocumentBackfill(ctx, root)
|
|
if err != nil {
|
|
return progress, err
|
|
}
|
|
defer unlock()
|
|
progress.StoreID, err = documentStoreIdentity(root)
|
|
if err != nil {
|
|
return progress, err
|
|
}
|
|
if err := db.ApplySQLiteMigrations(ctx, s.DB, "document-store", documentBackfillMigrations); err != nil {
|
|
return progress, err
|
|
}
|
|
if _, err := s.DB.ExecContext(ctx, `INSERT INTO document_backfills(store_id,updated_at_ms) VALUES(?,?) ON CONFLICT(store_id) DO NOTHING`, progress.StoreID, time.Now().UnixMilli()); err != nil {
|
|
return progress, err
|
|
}
|
|
if err := s.DB.QueryRowContext(ctx, `SELECT last_uuid,processed,completed FROM document_backfills WHERE store_id=?`, progress.StoreID).Scan(&progress.LastUUID, &progress.Processed, &progress.Completed); err != nil {
|
|
return progress, err
|
|
}
|
|
if progress.Completed {
|
|
return progress, nil
|
|
}
|
|
rows, err := s.DB.QueryContext(ctx, `SELECT uuid,document_hash FROM revisions WHERE uuid>? AND document_hash IS NOT NULL AND document_hash!='' ORDER BY uuid LIMIT ?`, progress.LastUUID, batchSize)
|
|
if err != nil {
|
|
return progress, err
|
|
}
|
|
type ref struct{ uuid, hash string }
|
|
var refs []ref
|
|
for rows.Next() {
|
|
var r ref
|
|
if err := rows.Scan(&r.uuid, &r.hash); err != nil {
|
|
rows.Close()
|
|
return progress, err
|
|
}
|
|
refs = append(refs, r)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
rows.Close()
|
|
return progress, err
|
|
}
|
|
if err := rows.Close(); err != nil {
|
|
return progress, err
|
|
}
|
|
backend, err := s.documentStorage()
|
|
if err != nil {
|
|
return progress, err
|
|
}
|
|
for _, ref := range refs {
|
|
// Existing canonical bytes outrank a stale/missing compatibility body.
|
|
// Never overwrite a corrupt canonical file with a cache implicitly.
|
|
err := backend.Verify(ctx, ref.hash)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
var body string
|
|
if err = s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE uuid=? AND document_hash=?`, ref.uuid, ref.hash).Scan(&body); err == nil {
|
|
err = backend.PutIfAbsent(ctx, ref.hash, int64(len(body)), strings.NewReader(body))
|
|
}
|
|
}
|
|
if err != nil {
|
|
return progress, fmt.Errorf("Document backfill revision=%s: %w", ref.uuid, err)
|
|
}
|
|
}
|
|
next := progress
|
|
if len(refs) != 0 {
|
|
next.LastUUID = refs[len(refs)-1].uuid
|
|
}
|
|
next.Processed += int64(len(refs))
|
|
next.Completed = len(refs) < batchSize
|
|
if _, err := s.DB.ExecContext(ctx, `UPDATE document_backfills SET last_uuid=?,processed=?,completed=?,updated_at_ms=? WHERE store_id=?`, next.LastUUID, next.Processed, next.Completed, time.Now().UnixMilli(), next.StoreID); err != nil {
|
|
return progress, err
|
|
}
|
|
return next, nil
|
|
}
|
|
|
|
func lockDocumentBackfill(ctx context.Context, root *os.Root) (func(), error) {
|
|
file, err := root.OpenFile(".document-backfill.lock", os.O_CREATE|os.O_RDWR, 0600)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for {
|
|
if err := ctx.Err(); err != nil {
|
|
file.Close()
|
|
return nil, err
|
|
}
|
|
err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
|
if err == nil {
|
|
return func() { _ = syscall.Flock(int(file.Fd()), syscall.LOCK_UN); file.Close() }, nil
|
|
}
|
|
if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) {
|
|
file.Close()
|
|
return nil, err
|
|
}
|
|
timer := time.NewTimer(20 * time.Millisecond)
|
|
select {
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
file.Close()
|
|
return nil, ctx.Err()
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
// Identity follows the physical store, not its pathname. A replaced/empty root
|
|
// gets a new identity and cannot inherit the DB's previous completion marker.
|
|
func documentStoreIdentity(root *os.Root) (string, error) {
|
|
file, err := root.Open(".document-store-id")
|
|
if err == nil {
|
|
defer file.Close()
|
|
info, err := file.Stat()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if !info.Mode().IsRegular() || info.Size() > 64 {
|
|
return "", errors.New("invalid Document store identity file")
|
|
}
|
|
body, err := io.ReadAll(io.LimitReader(file, 128))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
id := strings.TrimSpace(string(body))
|
|
if decoded, err := hex.DecodeString(id); err != nil || len(decoded) != 16 {
|
|
return "", errors.New("invalid Document store identity")
|
|
}
|
|
// Also sync on resume: the previous process may have published the
|
|
// identity then crashed before syncing its directory.
|
|
return id, syncDocumentRoot(root)
|
|
}
|
|
if !errors.Is(err, os.ErrNotExist) {
|
|
return "", err
|
|
}
|
|
var nonce [16]byte
|
|
if _, err := rand.Read(nonce[:]); err != nil {
|
|
return "", err
|
|
}
|
|
id := hex.EncodeToString(nonce[:])
|
|
tmp := ".document-store-id-" + id
|
|
file, err = root.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer root.Remove(tmp)
|
|
_, err = file.WriteString(id + "\n")
|
|
if err == nil {
|
|
err = file.Sync()
|
|
}
|
|
closeErr := file.Close()
|
|
if err == nil {
|
|
err = closeErr
|
|
}
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if err := root.Link(tmp, ".document-store-id"); err != nil {
|
|
return "", err
|
|
}
|
|
return id, syncDocumentRoot(root)
|
|
}
|
|
|
|
func syncDocumentRoot(root *os.Root) error {
|
|
dir, err := root.Open(".")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = dir.Sync()
|
|
dir.Close()
|
|
return err
|
|
}
|