package store import ( "context" "crypto/rand" "encoding/hex" "errors" "fmt" "io" "os" "path/filepath" "strings" "syscall" "time" "pageswiki/internal/db" ) var documentBackfillMigrations = []db.SQLMigration{{Version: 1, Name: "document_backfill_progress", Statements: []string{ `CREATE TABLE document_backfills ( store_id TEXT PRIMARY KEY, last_uuid TEXT NOT NULL DEFAULT '', processed INTEGER NOT NULL DEFAULT 0, completed INTEGER NOT NULL DEFAULT 0, updated_at_ms INTEGER NOT NULL)`, }}, {Version: 2, Name: "verify_backfill_contract", Assertions: []string{ schemaColumns("document_backfills", "store_id,last_uuid,processed,completed,updated_at_ms"), schemaUnique("document_backfills", "store_id"), }}} type DocumentBackfillProgress struct { StoreID string LastUUID string Processed int64 Completed bool } // BackfillDocuments performs one restartable, bounded batch. It is a maintenance // operation: legacy DB-only writers must be stopped. Normal application saves // persist canonical bytes before inserting revisions and need no backfill. // The UUID cursor is stable across SQLite VACUUM (rowid is not). func (s *Store) BackfillDocuments(ctx context.Context, batchSize int) (DocumentBackfillProgress, error) { var progress DocumentBackfillProgress if s.ReadOnly { return progress, ErrReadOnly } if err := ctx.Err(); err != nil { return progress, err } if s.DocumentRoot == "" { progress.Completed = true return progress, nil } if batchSize <= 0 || batchSize > 128 { batchSize = 128 } if err := os.MkdirAll(filepath.Join(s.DocumentRoot, "sha256"), 0755); err != nil { return progress, err } root, err := os.OpenRoot(s.DocumentRoot) if err != nil { return progress, err } defer root.Close() unlock, err := lockDocumentBackfill(ctx, root) if err != nil { return progress, err } defer unlock() progress.StoreID, err = documentStoreIdentity(root) if err != nil { return progress, err } if err := db.ApplySQLiteMigrations(ctx, s.DB, "document-store", documentBackfillMigrations); err != nil { return progress, err } if _, err := s.DB.ExecContext(ctx, `INSERT INTO document_backfills(store_id,updated_at_ms) VALUES(?,?) ON CONFLICT(store_id) DO NOTHING`, progress.StoreID, time.Now().UnixMilli()); err != nil { return progress, err } if err := s.DB.QueryRowContext(ctx, `SELECT last_uuid,processed,completed FROM document_backfills WHERE store_id=?`, progress.StoreID).Scan(&progress.LastUUID, &progress.Processed, &progress.Completed); err != nil { return progress, err } if progress.Completed { return progress, nil } rows, err := s.DB.QueryContext(ctx, `SELECT uuid,document_hash FROM revisions WHERE uuid>? AND document_hash IS NOT NULL AND document_hash!='' ORDER BY uuid LIMIT ?`, progress.LastUUID, batchSize) if err != nil { return progress, err } type ref struct{ uuid, hash string } var refs []ref for rows.Next() { var r ref if err := rows.Scan(&r.uuid, &r.hash); err != nil { rows.Close() return progress, err } refs = append(refs, r) } if err := rows.Err(); err != nil { rows.Close() return progress, err } if err := rows.Close(); err != nil { return progress, err } backend, err := s.documentStorage() if err != nil { return progress, err } for _, ref := range refs { // Existing canonical bytes outrank a stale/missing compatibility body. // Never overwrite a corrupt canonical file with a cache implicitly. err := backend.Verify(ctx, ref.hash) if errors.Is(err, os.ErrNotExist) { var body string if err = s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE uuid=? AND document_hash=?`, ref.uuid, ref.hash).Scan(&body); err == nil { err = backend.PutIfAbsent(ctx, ref.hash, int64(len(body)), strings.NewReader(body)) } } if err != nil { return progress, fmt.Errorf("Document backfill revision=%s: %w", ref.uuid, err) } } next := progress if len(refs) != 0 { next.LastUUID = refs[len(refs)-1].uuid } next.Processed += int64(len(refs)) next.Completed = len(refs) < batchSize if _, err := s.DB.ExecContext(ctx, `UPDATE document_backfills SET last_uuid=?,processed=?,completed=?,updated_at_ms=? WHERE store_id=?`, next.LastUUID, next.Processed, next.Completed, time.Now().UnixMilli(), next.StoreID); err != nil { return progress, err } return next, nil } func lockDocumentBackfill(ctx context.Context, root *os.Root) (func(), error) { file, err := root.OpenFile(".document-backfill.lock", os.O_CREATE|os.O_RDWR, 0600) if err != nil { return nil, err } for { if err := ctx.Err(); err != nil { file.Close() return nil, err } err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) if err == nil { return func() { _ = syscall.Flock(int(file.Fd()), syscall.LOCK_UN); file.Close() }, nil } if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) { file.Close() return nil, err } timer := time.NewTimer(20 * time.Millisecond) select { case <-ctx.Done(): timer.Stop() file.Close() return nil, ctx.Err() case <-timer.C: } } } // Identity follows the physical store, not its pathname. A replaced/empty root // gets a new identity and cannot inherit the DB's previous completion marker. func documentStoreIdentity(root *os.Root) (string, error) { file, err := root.Open(".document-store-id") if err == nil { defer file.Close() info, err := file.Stat() if err != nil { return "", err } if !info.Mode().IsRegular() || info.Size() > 64 { return "", errors.New("invalid Document store identity file") } body, err := io.ReadAll(io.LimitReader(file, 128)) if err != nil { return "", err } id := strings.TrimSpace(string(body)) if decoded, err := hex.DecodeString(id); err != nil || len(decoded) != 16 { return "", errors.New("invalid Document store identity") } // Also sync on resume: the previous process may have published the // identity then crashed before syncing its directory. return id, syncDocumentRoot(root) } if !errors.Is(err, os.ErrNotExist) { return "", err } var nonce [16]byte if _, err := rand.Read(nonce[:]); err != nil { return "", err } id := hex.EncodeToString(nonce[:]) tmp := ".document-store-id-" + id file, err = root.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) if err != nil { return "", err } defer root.Remove(tmp) _, err = file.WriteString(id + "\n") if err == nil { err = file.Sync() } closeErr := file.Close() if err == nil { err = closeErr } if err != nil { return "", err } if err := root.Link(tmp, ".document-store-id"); err != nil { return "", err } return id, syncDocumentRoot(root) } func syncDocumentRoot(root *os.Root) error { dir, err := root.Open(".") if err != nil { return err } err = dir.Sync() dir.Close() return err }