package geduldh import ( "bytes" "context" "errors" "os" "path/filepath" "sync/atomic" "testing" "time" ) type materializeSource struct { body []byte size int64 gate chan struct{} started chan struct{} opens atomic.Int64 } func (s *materializeSource) Stat(ctx context.Context, hash string) (Info, error) { return Info{Hash: hash, Size: s.size}, ctx.Err() } func (s *materializeSource) Open(ctx context.Context, hash string) (Reader, error) { s.opens.Add(1) if s.started != nil { s.started <- struct{}{} } if s.gate != nil { select { case <-s.gate: case <-ctx.Done(): return nil, ctx.Err() } } return materializeReader{bytes.NewReader(s.body)}, ctx.Err() } type materializeReader struct{ *bytes.Reader } func (materializeReader) Close() error { return nil } func newTestMaterializer(t *testing.T, source Source, quota int64) (*Materializer, string) { t.Helper() dir := t.TempDir() m, err := NewMaterializer(source, MaterializeOptions{TempDir: dir, MaxBytes: quota, Concurrent: 1, Timeout: 5 * time.Second}) if err != nil { t.Fatal(err) } return m, dir } func waitMaterializer(t *testing.T, m *Materializer, predicate func() bool) { t.Helper() deadline := time.Now().Add(3 * time.Second) for time.Now().Before(deadline) { m.mu.Lock() ok := predicate() m.mu.Unlock() if ok { return } time.Sleep(time.Millisecond) } t.Fatal("materializer condition did not become true") } func TestMaterializerSharesPinnedCopiesAndCleansLastLease(t *testing.T) { body := []byte("materialized 日本語") s := &materializeSource{body: body, size: int64(len(body))} m, root := newTestMaterializer(t, s, 1024) a, err := m.Acquire(context.Background(), digest(body), 1024) if err != nil { t.Fatal(err) } b, err := m.Acquire(context.Background(), digest(body), 1024) if err != nil { t.Fatal(err) } if a.Path != b.Path || s.opens.Load() != 1 { t.Fatal("copy was not shared") } data, err := os.ReadFile(a.Path) if err != nil || !bytes.Equal(data, body) { t.Fatalf("body=%q err=%v", data, err) } info, _ := os.Stat(a.Path) if info.Mode().Perm() != 0400 { t.Fatalf("permissions=%v", info.Mode()) } dirInfo, _ := os.Stat(filepath.Dir(a.Path)) if dirInfo.Mode().Perm() != 0700 { t.Fatalf("directory permissions=%v", dirInfo.Mode()) } a.Close() a.Close() if _, err := os.Stat(b.Path); err != nil { t.Fatal("active lease lost its file", err) } b.Close() items, _ := os.ReadDir(root) if len(items) != 0 || m.reserved != 0 || m.active != 0 { t.Fatal("scratch was not reclaimed") } } func TestMaterializerCoalescesDownloadsAndCancelsOnlyLastWaiter(t *testing.T) { body := []byte("shared") s := &materializeSource{body: body, size: int64(len(body)), gate: make(chan struct{}), started: make(chan struct{}, 2)} m, _ := newTestMaterializer(t, s, 1024) ctx, cancel := context.WithCancel(context.Background()) defer cancel() errA := make(chan error, 1) go func() { lease, err := m.Acquire(ctx, digest(body), 1024) if lease != nil { lease.Close() } errA <- err }() <-s.started leaseB := make(chan *Lease, 1) errB := make(chan error, 1) go func() { lease, err := m.Acquire(context.Background(), digest(body), 1024) leaseB <- lease errB <- err }() waitMaterializer(t, m, func() bool { return m.entries[digest(body)].refs == 2 }) cancel() if err := <-errA; !errors.Is(err, context.Canceled) { t.Fatalf("first waiter: %v", err) } close(s.gate) b := <-leaseB if err := <-errB; err != nil { t.Fatal(err) } if s.opens.Load() != 1 { t.Fatal("concurrent transfer not coalesced") } b.Close() s.gate = make(chan struct{}) ctx, cancel = context.WithCancel(context.Background()) go func() { lease, err := m.Acquire(ctx, digest(body), 1024) if lease != nil { lease.Close() } errA <- err }() <-s.started cancel() if err := <-errA; !errors.Is(err, context.Canceled) { t.Fatalf("last waiter: %v", err) } waitMaterializer(t, m, func() bool { return m.reserved == 0 && m.active == 0 }) } func TestMaterializerRejectsCorruptionSizeAndCapacity(t *testing.T) { for _, tc := range []struct { name string body []byte size int64 hash string want error }{ {"hash", []byte("bad"), 3, digest([]byte("yes")), ErrHashMismatch}, {"short", []byte("x"), 2, digest([]byte("x")), ErrSizeMismatch}, {"long", []byte("xyz"), 2, digest([]byte("xyz")), ErrSizeMismatch}, {"oversize", []byte("xyz"), 3, digest([]byte("xyz")), ErrMaterializeLimit}, } { t.Run(tc.name, func(t *testing.T) { s := &materializeSource{body: tc.body, size: tc.size} m, root := newTestMaterializer(t, s, 10) limit := int64(10) if tc.name == "oversize" { limit = 2 } lease, err := m.Acquire(context.Background(), tc.hash, limit) if lease != nil || !errors.Is(err, tc.want) { t.Fatalf("lease=%v err=%v", lease, err) } items, _ := os.ReadDir(root) if len(items) != 0 || m.reserved != 0 { t.Fatal("failed input leaked scratch") } }) } s := &materializeSource{body: []byte("x"), size: 1} m, _ := newTestMaterializer(t, s, 1) lease, err := m.Acquire(context.Background(), digest(s.body), 1) if err != nil { t.Fatal(err) } defer lease.Close() if _, err := m.Acquire(context.Background(), digest([]byte("y")), 1); !errors.Is(err, ErrMaterializeLimit) { t.Fatalf("quota: %v", err) } if s.opens.Load() != 1 { t.Fatal("over-quota transfer started") } } func TestMaterializerTimeoutAndEntryBound(t *testing.T) { s := &materializeSource{body: []byte("x"), size: 1, gate: make(chan struct{}), started: make(chan struct{}, 3)} m, err := NewMaterializer(s, MaterializeOptions{TempDir: t.TempDir(), MaxBytes: 100, Concurrent: 1, MaxEntries: 1, Timeout: 50 * time.Millisecond}) if err != nil { t.Fatal(err) } done := make(chan error, 1) go func() { _, err := m.Acquire(context.Background(), digest(s.body), 10); done <- err }() <-s.started if _, err := m.Acquire(context.Background(), digest([]byte("y")), 10); !errors.Is(err, ErrMaterializeLimit) { t.Fatalf("entry bound: %v", err) } if err := <-done; !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("timeout: %v", err) } waitMaterializer(t, m, func() bool { return m.active == 0 }) ctx, cancel := context.WithCancel(context.Background()) cancel() if _, err := m.Acquire(ctx, digest(s.body), 10); !errors.Is(err, context.Canceled) { t.Fatal(err) } } func TestMaterializerConcurrencyBoundAndCanonicalIsolation(t *testing.T) { s := &materializeSource{body: []byte("x"), size: 1, gate: make(chan struct{}), started: make(chan struct{}, 3)} m, _ := newTestMaterializer(t, s, 100) done := make(chan error, 2) for _, data := range []string{"x", "y"} { go func(data string) { lease, err := m.Acquire(context.Background(), digest([]byte(data)), 100) if lease != nil { lease.Close() } done <- err }(data) } <-s.started waitMaterializer(t, m, func() bool { return m.active == 2 }) if s.opens.Load() != 1 { t.Fatal("concurrency limit exceeded") } close(s.gate) for i := 0; i < 2; i++ { if err := <-done; err != nil && !errors.Is(err, ErrHashMismatch) { t.Fatal(err) } } waitMaterializer(t, m, func() bool { return m.active == 0 }) canonical, root := local(t) body := []byte("canonical") hash := digest(body) if err := canonical.PutIfAbsent(context.Background(), hash, int64(len(body)), bytes.NewReader(body)); err != nil { t.Fatal(err) } m, _ = newTestMaterializer(t, canonical, 100) lease, err := m.Acquire(context.Background(), hash, 100) if err != nil { t.Fatal(err) } defer lease.Close() key, _ := Key(hash) sourceInfo, _ := os.Stat(filepath.Join(root, filepath.FromSlash(key))) tempInfo, _ := os.Stat(lease.Path) if os.SameFile(sourceInfo, tempInfo) { t.Fatal("canonical inode exposed to external tools") } if err := os.Chmod(lease.Path, 0600); err != nil { t.Fatal(err) } if err := os.WriteFile(lease.Path, []byte("modified!"), 0600); err != nil { t.Fatal(err) } if err := canonical.Verify(context.Background(), hash); err != nil { t.Fatal("scratch mutation affected canonical data", err) } }