Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts. Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation. Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
284 lines
7.9 KiB
Go
284 lines
7.9 KiB
Go
package geduldh
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type materializeSource struct {
|
|
body []byte
|
|
size int64
|
|
gate chan struct{}
|
|
started chan struct{}
|
|
opens atomic.Int64
|
|
}
|
|
|
|
func (s *materializeSource) Stat(ctx context.Context, hash string) (Info, error) {
|
|
return Info{Hash: hash, Size: s.size}, ctx.Err()
|
|
}
|
|
func (s *materializeSource) Open(ctx context.Context, hash string) (Reader, error) {
|
|
s.opens.Add(1)
|
|
if s.started != nil {
|
|
s.started <- struct{}{}
|
|
}
|
|
if s.gate != nil {
|
|
select {
|
|
case <-s.gate:
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
return materializeReader{bytes.NewReader(s.body)}, ctx.Err()
|
|
}
|
|
|
|
type materializeReader struct{ *bytes.Reader }
|
|
|
|
func (materializeReader) Close() error { return nil }
|
|
|
|
func newTestMaterializer(t *testing.T, source Source, quota int64) (*Materializer, string) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
m, err := NewMaterializer(source, MaterializeOptions{TempDir: dir, MaxBytes: quota, Concurrent: 1, Timeout: 5 * time.Second})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return m, dir
|
|
}
|
|
|
|
func waitMaterializer(t *testing.T, m *Materializer, predicate func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
m.mu.Lock()
|
|
ok := predicate()
|
|
m.mu.Unlock()
|
|
if ok {
|
|
return
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
t.Fatal("materializer condition did not become true")
|
|
}
|
|
|
|
func TestMaterializerSharesPinnedCopiesAndCleansLastLease(t *testing.T) {
|
|
body := []byte("materialized 日本語")
|
|
s := &materializeSource{body: body, size: int64(len(body))}
|
|
m, root := newTestMaterializer(t, s, 1024)
|
|
a, err := m.Acquire(context.Background(), digest(body), 1024)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b, err := m.Acquire(context.Background(), digest(body), 1024)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if a.Path != b.Path || s.opens.Load() != 1 {
|
|
t.Fatal("copy was not shared")
|
|
}
|
|
data, err := os.ReadFile(a.Path)
|
|
if err != nil || !bytes.Equal(data, body) {
|
|
t.Fatalf("body=%q err=%v", data, err)
|
|
}
|
|
info, _ := os.Stat(a.Path)
|
|
if info.Mode().Perm() != 0400 {
|
|
t.Fatalf("permissions=%v", info.Mode())
|
|
}
|
|
dirInfo, _ := os.Stat(filepath.Dir(a.Path))
|
|
if dirInfo.Mode().Perm() != 0700 {
|
|
t.Fatalf("directory permissions=%v", dirInfo.Mode())
|
|
}
|
|
a.Close()
|
|
a.Close()
|
|
if _, err := os.Stat(b.Path); err != nil {
|
|
t.Fatal("active lease lost its file", err)
|
|
}
|
|
b.Close()
|
|
items, _ := os.ReadDir(root)
|
|
if len(items) != 0 || m.reserved != 0 || m.active != 0 {
|
|
t.Fatal("scratch was not reclaimed")
|
|
}
|
|
}
|
|
|
|
func TestMaterializerCoalescesDownloadsAndCancelsOnlyLastWaiter(t *testing.T) {
|
|
body := []byte("shared")
|
|
s := &materializeSource{body: body, size: int64(len(body)), gate: make(chan struct{}), started: make(chan struct{}, 2)}
|
|
m, _ := newTestMaterializer(t, s, 1024)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
errA := make(chan error, 1)
|
|
go func() {
|
|
lease, err := m.Acquire(ctx, digest(body), 1024)
|
|
if lease != nil {
|
|
lease.Close()
|
|
}
|
|
errA <- err
|
|
}()
|
|
<-s.started
|
|
leaseB := make(chan *Lease, 1)
|
|
errB := make(chan error, 1)
|
|
go func() {
|
|
lease, err := m.Acquire(context.Background(), digest(body), 1024)
|
|
leaseB <- lease
|
|
errB <- err
|
|
}()
|
|
waitMaterializer(t, m, func() bool { return m.entries[digest(body)].refs == 2 })
|
|
cancel()
|
|
if err := <-errA; !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("first waiter: %v", err)
|
|
}
|
|
close(s.gate)
|
|
b := <-leaseB
|
|
if err := <-errB; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if s.opens.Load() != 1 {
|
|
t.Fatal("concurrent transfer not coalesced")
|
|
}
|
|
b.Close()
|
|
|
|
s.gate = make(chan struct{})
|
|
ctx, cancel = context.WithCancel(context.Background())
|
|
go func() {
|
|
lease, err := m.Acquire(ctx, digest(body), 1024)
|
|
if lease != nil {
|
|
lease.Close()
|
|
}
|
|
errA <- err
|
|
}()
|
|
<-s.started
|
|
cancel()
|
|
if err := <-errA; !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("last waiter: %v", err)
|
|
}
|
|
waitMaterializer(t, m, func() bool { return m.reserved == 0 && m.active == 0 })
|
|
}
|
|
|
|
func TestMaterializerRejectsCorruptionSizeAndCapacity(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
body []byte
|
|
size int64
|
|
hash string
|
|
want error
|
|
}{
|
|
{"hash", []byte("bad"), 3, digest([]byte("yes")), ErrHashMismatch},
|
|
{"short", []byte("x"), 2, digest([]byte("x")), ErrSizeMismatch},
|
|
{"long", []byte("xyz"), 2, digest([]byte("xyz")), ErrSizeMismatch},
|
|
{"oversize", []byte("xyz"), 3, digest([]byte("xyz")), ErrMaterializeLimit},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
s := &materializeSource{body: tc.body, size: tc.size}
|
|
m, root := newTestMaterializer(t, s, 10)
|
|
limit := int64(10)
|
|
if tc.name == "oversize" {
|
|
limit = 2
|
|
}
|
|
lease, err := m.Acquire(context.Background(), tc.hash, limit)
|
|
if lease != nil || !errors.Is(err, tc.want) {
|
|
t.Fatalf("lease=%v err=%v", lease, err)
|
|
}
|
|
items, _ := os.ReadDir(root)
|
|
if len(items) != 0 || m.reserved != 0 {
|
|
t.Fatal("failed input leaked scratch")
|
|
}
|
|
})
|
|
}
|
|
s := &materializeSource{body: []byte("x"), size: 1}
|
|
m, _ := newTestMaterializer(t, s, 1)
|
|
lease, err := m.Acquire(context.Background(), digest(s.body), 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer lease.Close()
|
|
if _, err := m.Acquire(context.Background(), digest([]byte("y")), 1); !errors.Is(err, ErrMaterializeLimit) {
|
|
t.Fatalf("quota: %v", err)
|
|
}
|
|
if s.opens.Load() != 1 {
|
|
t.Fatal("over-quota transfer started")
|
|
}
|
|
}
|
|
|
|
func TestMaterializerTimeoutAndEntryBound(t *testing.T) {
|
|
s := &materializeSource{body: []byte("x"), size: 1, gate: make(chan struct{}), started: make(chan struct{}, 3)}
|
|
m, err := NewMaterializer(s, MaterializeOptions{TempDir: t.TempDir(), MaxBytes: 100, Concurrent: 1, MaxEntries: 1, Timeout: 50 * time.Millisecond})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
done := make(chan error, 1)
|
|
go func() { _, err := m.Acquire(context.Background(), digest(s.body), 10); done <- err }()
|
|
<-s.started
|
|
if _, err := m.Acquire(context.Background(), digest([]byte("y")), 10); !errors.Is(err, ErrMaterializeLimit) {
|
|
t.Fatalf("entry bound: %v", err)
|
|
}
|
|
if err := <-done; !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Fatalf("timeout: %v", err)
|
|
}
|
|
waitMaterializer(t, m, func() bool { return m.active == 0 })
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if _, err := m.Acquire(ctx, digest(s.body), 10); !errors.Is(err, context.Canceled) {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestMaterializerConcurrencyBoundAndCanonicalIsolation(t *testing.T) {
|
|
s := &materializeSource{body: []byte("x"), size: 1, gate: make(chan struct{}), started: make(chan struct{}, 3)}
|
|
m, _ := newTestMaterializer(t, s, 100)
|
|
done := make(chan error, 2)
|
|
for _, data := range []string{"x", "y"} {
|
|
go func(data string) {
|
|
lease, err := m.Acquire(context.Background(), digest([]byte(data)), 100)
|
|
if lease != nil {
|
|
lease.Close()
|
|
}
|
|
done <- err
|
|
}(data)
|
|
}
|
|
<-s.started
|
|
waitMaterializer(t, m, func() bool { return m.active == 2 })
|
|
if s.opens.Load() != 1 {
|
|
t.Fatal("concurrency limit exceeded")
|
|
}
|
|
close(s.gate)
|
|
for i := 0; i < 2; i++ {
|
|
if err := <-done; err != nil && !errors.Is(err, ErrHashMismatch) {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
waitMaterializer(t, m, func() bool { return m.active == 0 })
|
|
|
|
canonical, root := local(t)
|
|
body := []byte("canonical")
|
|
hash := digest(body)
|
|
if err := canonical.PutIfAbsent(context.Background(), hash, int64(len(body)), bytes.NewReader(body)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m, _ = newTestMaterializer(t, canonical, 100)
|
|
lease, err := m.Acquire(context.Background(), hash, 100)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer lease.Close()
|
|
key, _ := Key(hash)
|
|
sourceInfo, _ := os.Stat(filepath.Join(root, filepath.FromSlash(key)))
|
|
tempInfo, _ := os.Stat(lease.Path)
|
|
if os.SameFile(sourceInfo, tempInfo) {
|
|
t.Fatal("canonical inode exposed to external tools")
|
|
}
|
|
if err := os.Chmod(lease.Path, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(lease.Path, []byte("modified!"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := canonical.Verify(context.Background(), hash); err != nil {
|
|
t.Fatal("scratch mutation affected canonical data", err)
|
|
}
|
|
}
|