package store_test import ( "context" "crypto/sha256" "encoding/hex" "errors" "os" "path/filepath" "sync" "syscall" "testing" "time" "pageswiki/internal/store" ) func backfillFixture(t *testing.T, st *store.Store, id, body string) string { t.Helper() sum := sha256.Sum256([]byte(body)) hash := hex.EncodeToString(sum[:]) _, err := st.DB.Exec(`INSERT INTO revisions(uuid,object_uuid,body,document_hash,lang,content_type,filename,message,created_at_ms) VALUES(?,'backfill-object',?,?,'und','text/plain','','',0)`, id, body, hash) if err != nil { t.Fatal(err) } return hash } func TestDocumentBackfillResumesAndDoesNotRescanCompletedStore(t *testing.T) { st := newTopologyStore(t) hashA := backfillFixture(t, st, "a", "first") hashB := backfillFixture(t, st, "b", "second") st.DocumentRoot = t.TempDir() first, err := st.BackfillDocuments(context.Background(), 1) if err != nil || first.LastUUID != "a" || first.Processed != 1 || first.Completed { t.Fatalf("first=%+v err=%v", first, err) } if _, err := st.DB.Exec(`UPDATE revisions SET body='broken cache' WHERE uuid='a'`); err != nil { t.Fatal(err) } if _, err := st.DB.Exec(`VACUUM`); err != nil { t.Fatal(err) } // A new Store instance resumes the durable UUID cursor, not in-memory state. restarted := &store.Store{DB: st.DB, DocumentRoot: st.DocumentRoot} second, err := restarted.BackfillDocuments(context.Background(), 1) if err != nil || second.LastUUID != "b" || second.Processed != 2 { t.Fatalf("second=%+v err=%v", second, err) } final, err := restarted.BackfillDocuments(context.Background(), 1) if err != nil || !final.Completed { t.Fatalf("final=%+v err=%v", final, err) } for _, hash := range []string{hashA, hashB} { if _, err := os.Stat(filepath.Join(st.DocumentRoot, "sha256", hash[:2], hash)); err != nil { t.Fatal(err) } } // Renaming the revisions table proves the completed path does not even // query it. This is synthetic data in a temporary DB, never production. if _, err := st.DB.Exec(`ALTER TABLE revisions RENAME TO hidden_revisions`); err != nil { t.Fatal(err) } again, err := restarted.BackfillDocuments(context.Background(), 128) if err != nil || again != final { t.Fatalf("completed progress=%+v err=%v", again, err) } } func TestDocumentBackfillRootReplacementAndFailedBatch(t *testing.T) { st := newTopologyStore(t) hash := backfillFixture(t, st, "a", "original") st.DocumentRoot = filepath.Join(t.TempDir(), "documents") first, err := st.BackfillDocuments(context.Background(), 128) if err != nil || !first.Completed { t.Fatal(err) } // Keep exactly the same pathname, but replace its storage. Path-based // completion markers would incorrectly skip this empty replacement. if err := os.Rename(st.DocumentRoot, st.DocumentRoot+"-old"); err != nil { t.Fatal(err) } if _, err := st.DB.Exec(`UPDATE revisions SET body='wrong' WHERE uuid='a'`); err != nil { t.Fatal(err) } failed, err := st.BackfillDocuments(context.Background(), 128) if !errors.Is(err, store.ErrDocumentHashMismatch) || failed.Processed != 0 || failed.Completed || failed.StoreID == first.StoreID { t.Fatalf("failed=%+v err=%v", failed, err) } if _, err := st.DB.Exec(`UPDATE revisions SET body='original' WHERE uuid='a'`); err != nil { t.Fatal(err) } retried, err := st.BackfillDocuments(context.Background(), 128) if err != nil || !retried.Completed || retried.Processed != 1 { t.Fatalf("retry=%+v err=%v", retried, err) } if _, err := os.Stat(filepath.Join(st.DocumentRoot, "sha256", hash[:2], hash)); err != nil { t.Fatal(err) } } func TestDocumentBackfillPrefersCanonicalAndNeverRepairsCorruptionImplicitly(t *testing.T) { st := newTopologyStore(t) hash := backfillFixture(t, st, "a", "canonical") st.DocumentRoot = t.TempDir() dir := filepath.Join(st.DocumentRoot, "sha256", hash[:2]) if err := os.MkdirAll(dir, 0755); err != nil { t.Fatal(err) } filename := filepath.Join(dir, hash) if err := os.WriteFile(filename, []byte("canonical"), 0644); err != nil { t.Fatal(err) } if _, err := st.DB.Exec(`UPDATE revisions SET body='' WHERE uuid='a'`); err != nil { t.Fatal(err) } progress, err := st.BackfillDocuments(context.Background(), 128) if err != nil || !progress.Completed { t.Fatalf("canonical rejected: %v", err) } if _, err := st.DB.Exec(`DELETE FROM document_backfills`); err != nil { t.Fatal(err) } if err := os.WriteFile(filename, []byte("broken"), 0644); err != nil { t.Fatal(err) } if _, err := st.BackfillDocuments(context.Background(), 128); !errors.Is(err, store.ErrDocumentHashMismatch) { t.Fatalf("corrupt canonical: %v", err) } data, _ := os.ReadFile(filename) if string(data) != "broken" { t.Fatal("corruption overwritten") } } func TestDocumentBackfillCancellationAndReadOnlyDoNotCreateFiles(t *testing.T) { st := newTopologyStore(t) st.DocumentRoot = filepath.Join(t.TempDir(), "new-root") ctx, cancel := context.WithCancel(context.Background()) cancel() if err := st.EnsureDocumentStoreContext(ctx); !errors.Is(err, context.Canceled) { t.Fatal(err) } st.ReadOnly = true if err := st.EnsureDocumentStore(); !errors.Is(err, store.ErrReadOnly) { t.Fatal(err) } if _, err := os.Stat(st.DocumentRoot); !os.IsNotExist(err) { t.Fatal("cancelled/standby operation created root") } } func TestDocumentBackfillSerializesAndLockWaitHonorsCancellation(t *testing.T) { st := newTopologyStore(t) backfillFixture(t, st, "a", "first") backfillFixture(t, st, "b", "second") st.DocumentRoot = t.TempDir() lock, err := os.OpenFile(filepath.Join(st.DocumentRoot, ".document-backfill.lock"), os.O_CREATE|os.O_RDWR, 0600) if err != nil { t.Fatal(err) } defer lock.Close() if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() if _, err := st.BackfillDocuments(ctx, 1); !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("lock cancellation: %v", err) } if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_UN); err != nil { t.Fatal(err) } var wg sync.WaitGroup errs := make(chan error, 2) for i := 0; i < 2; i++ { wg.Add(1) go func() { defer wg.Done() for { progress, err := st.BackfillDocuments(context.Background(), 1) if err != nil || progress.Completed { errs <- err return } } }() } wg.Wait() close(errs) for err := range errs { if err != nil { t.Fatal(err) } } progress, err := st.BackfillDocuments(context.Background(), 1) if err != nil || !progress.Completed || progress.Processed != 2 { t.Fatalf("double-counted migration: %+v %v", progress, err) } }