Refactor architecture into eight components and consolidate reliability work
Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts. Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation. Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
This commit is contained in:
1 parent
9dd425f765
commit
d5d21b7841
302 files changed
+20037
-3492
No files matched your search
@@ -0,0 +1,283 @@
|
||||
package geduldh
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type materializeSource struct {
|
||||
body []byte
|
||||
size int64
|
||||
gate chan struct{}
|
||||
started chan struct{}
|
||||
opens atomic.Int64
|
||||
}
|
||||
|
||||
func (s *materializeSource) Stat(ctx context.Context, hash string) (Info, error) {
|
||||
return Info{Hash: hash, Size: s.size}, ctx.Err()
|
||||
}
|
||||
func (s *materializeSource) Open(ctx context.Context, hash string) (Reader, error) {
|
||||
s.opens.Add(1)
|
||||
if s.started != nil {
|
||||
s.started <- struct{}{}
|
||||
}
|
||||
if s.gate != nil {
|
||||
select {
|
||||
case <-s.gate:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
return materializeReader{bytes.NewReader(s.body)}, ctx.Err()
|
||||
}
|
||||
|
||||
type materializeReader struct{ *bytes.Reader }
|
||||
|
||||
func (materializeReader) Close() error { return nil }
|
||||
|
||||
func newTestMaterializer(t *testing.T, source Source, quota int64) (*Materializer, string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
m, err := NewMaterializer(source, MaterializeOptions{TempDir: dir, MaxBytes: quota, Concurrent: 1, Timeout: 5 * time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, dir
|
||||
}
|
||||
|
||||
func waitMaterializer(t *testing.T, m *Materializer, predicate func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
m.mu.Lock()
|
||||
ok := predicate()
|
||||
m.mu.Unlock()
|
||||
if ok {
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
t.Fatal("materializer condition did not become true")
|
||||
}
|
||||
|
||||
func TestMaterializerSharesPinnedCopiesAndCleansLastLease(t *testing.T) {
|
||||
body := []byte("materialized 日本語")
|
||||
s := &materializeSource{body: body, size: int64(len(body))}
|
||||
m, root := newTestMaterializer(t, s, 1024)
|
||||
a, err := m.Acquire(context.Background(), digest(body), 1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := m.Acquire(context.Background(), digest(body), 1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.Path != b.Path || s.opens.Load() != 1 {
|
||||
t.Fatal("copy was not shared")
|
||||
}
|
||||
data, err := os.ReadFile(a.Path)
|
||||
if err != nil || !bytes.Equal(data, body) {
|
||||
t.Fatalf("body=%q err=%v", data, err)
|
||||
}
|
||||
info, _ := os.Stat(a.Path)
|
||||
if info.Mode().Perm() != 0400 {
|
||||
t.Fatalf("permissions=%v", info.Mode())
|
||||
}
|
||||
dirInfo, _ := os.Stat(filepath.Dir(a.Path))
|
||||
if dirInfo.Mode().Perm() != 0700 {
|
||||
t.Fatalf("directory permissions=%v", dirInfo.Mode())
|
||||
}
|
||||
a.Close()
|
||||
a.Close()
|
||||
if _, err := os.Stat(b.Path); err != nil {
|
||||
t.Fatal("active lease lost its file", err)
|
||||
}
|
||||
b.Close()
|
||||
items, _ := os.ReadDir(root)
|
||||
if len(items) != 0 || m.reserved != 0 || m.active != 0 {
|
||||
t.Fatal("scratch was not reclaimed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializerCoalescesDownloadsAndCancelsOnlyLastWaiter(t *testing.T) {
|
||||
body := []byte("shared")
|
||||
s := &materializeSource{body: body, size: int64(len(body)), gate: make(chan struct{}), started: make(chan struct{}, 2)}
|
||||
m, _ := newTestMaterializer(t, s, 1024)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
errA := make(chan error, 1)
|
||||
go func() {
|
||||
lease, err := m.Acquire(ctx, digest(body), 1024)
|
||||
if lease != nil {
|
||||
lease.Close()
|
||||
}
|
||||
errA <- err
|
||||
}()
|
||||
<-s.started
|
||||
leaseB := make(chan *Lease, 1)
|
||||
errB := make(chan error, 1)
|
||||
go func() {
|
||||
lease, err := m.Acquire(context.Background(), digest(body), 1024)
|
||||
leaseB <- lease
|
||||
errB <- err
|
||||
}()
|
||||
waitMaterializer(t, m, func() bool { return m.entries[digest(body)].refs == 2 })
|
||||
cancel()
|
||||
if err := <-errA; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("first waiter: %v", err)
|
||||
}
|
||||
close(s.gate)
|
||||
b := <-leaseB
|
||||
if err := <-errB; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.opens.Load() != 1 {
|
||||
t.Fatal("concurrent transfer not coalesced")
|
||||
}
|
||||
b.Close()
|
||||
|
||||
s.gate = make(chan struct{})
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
go func() {
|
||||
lease, err := m.Acquire(ctx, digest(body), 1024)
|
||||
if lease != nil {
|
||||
lease.Close()
|
||||
}
|
||||
errA <- err
|
||||
}()
|
||||
<-s.started
|
||||
cancel()
|
||||
if err := <-errA; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("last waiter: %v", err)
|
||||
}
|
||||
waitMaterializer(t, m, func() bool { return m.reserved == 0 && m.active == 0 })
|
||||
}
|
||||
|
||||
func TestMaterializerRejectsCorruptionSizeAndCapacity(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body []byte
|
||||
size int64
|
||||
hash string
|
||||
want error
|
||||
}{
|
||||
{"hash", []byte("bad"), 3, digest([]byte("yes")), ErrHashMismatch},
|
||||
{"short", []byte("x"), 2, digest([]byte("x")), ErrSizeMismatch},
|
||||
{"long", []byte("xyz"), 2, digest([]byte("xyz")), ErrSizeMismatch},
|
||||
{"oversize", []byte("xyz"), 3, digest([]byte("xyz")), ErrMaterializeLimit},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := &materializeSource{body: tc.body, size: tc.size}
|
||||
m, root := newTestMaterializer(t, s, 10)
|
||||
limit := int64(10)
|
||||
if tc.name == "oversize" {
|
||||
limit = 2
|
||||
}
|
||||
lease, err := m.Acquire(context.Background(), tc.hash, limit)
|
||||
if lease != nil || !errors.Is(err, tc.want) {
|
||||
t.Fatalf("lease=%v err=%v", lease, err)
|
||||
}
|
||||
items, _ := os.ReadDir(root)
|
||||
if len(items) != 0 || m.reserved != 0 {
|
||||
t.Fatal("failed input leaked scratch")
|
||||
}
|
||||
})
|
||||
}
|
||||
s := &materializeSource{body: []byte("x"), size: 1}
|
||||
m, _ := newTestMaterializer(t, s, 1)
|
||||
lease, err := m.Acquire(context.Background(), digest(s.body), 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer lease.Close()
|
||||
if _, err := m.Acquire(context.Background(), digest([]byte("y")), 1); !errors.Is(err, ErrMaterializeLimit) {
|
||||
t.Fatalf("quota: %v", err)
|
||||
}
|
||||
if s.opens.Load() != 1 {
|
||||
t.Fatal("over-quota transfer started")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializerTimeoutAndEntryBound(t *testing.T) {
|
||||
s := &materializeSource{body: []byte("x"), size: 1, gate: make(chan struct{}), started: make(chan struct{}, 3)}
|
||||
m, err := NewMaterializer(s, MaterializeOptions{TempDir: t.TempDir(), MaxBytes: 100, Concurrent: 1, MaxEntries: 1, Timeout: 50 * time.Millisecond})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := m.Acquire(context.Background(), digest(s.body), 10); done <- err }()
|
||||
<-s.started
|
||||
if _, err := m.Acquire(context.Background(), digest([]byte("y")), 10); !errors.Is(err, ErrMaterializeLimit) {
|
||||
t.Fatalf("entry bound: %v", err)
|
||||
}
|
||||
if err := <-done; !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("timeout: %v", err)
|
||||
}
|
||||
waitMaterializer(t, m, func() bool { return m.active == 0 })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := m.Acquire(ctx, digest(s.body), 10); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializerConcurrencyBoundAndCanonicalIsolation(t *testing.T) {
|
||||
s := &materializeSource{body: []byte("x"), size: 1, gate: make(chan struct{}), started: make(chan struct{}, 3)}
|
||||
m, _ := newTestMaterializer(t, s, 100)
|
||||
done := make(chan error, 2)
|
||||
for _, data := range []string{"x", "y"} {
|
||||
go func(data string) {
|
||||
lease, err := m.Acquire(context.Background(), digest([]byte(data)), 100)
|
||||
if lease != nil {
|
||||
lease.Close()
|
||||
}
|
||||
done <- err
|
||||
}(data)
|
||||
}
|
||||
<-s.started
|
||||
waitMaterializer(t, m, func() bool { return m.active == 2 })
|
||||
if s.opens.Load() != 1 {
|
||||
t.Fatal("concurrency limit exceeded")
|
||||
}
|
||||
close(s.gate)
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := <-done; err != nil && !errors.Is(err, ErrHashMismatch) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
waitMaterializer(t, m, func() bool { return m.active == 0 })
|
||||
|
||||
canonical, root := local(t)
|
||||
body := []byte("canonical")
|
||||
hash := digest(body)
|
||||
if err := canonical.PutIfAbsent(context.Background(), hash, int64(len(body)), bytes.NewReader(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, _ = newTestMaterializer(t, canonical, 100)
|
||||
lease, err := m.Acquire(context.Background(), hash, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer lease.Close()
|
||||
key, _ := Key(hash)
|
||||
sourceInfo, _ := os.Stat(filepath.Join(root, filepath.FromSlash(key)))
|
||||
tempInfo, _ := os.Stat(lease.Path)
|
||||
if os.SameFile(sourceInfo, tempInfo) {
|
||||
t.Fatal("canonical inode exposed to external tools")
|
||||
}
|
||||
if err := os.Chmod(lease.Path, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(lease.Path, []byte("modified!"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := canonical.Verify(context.Background(), hash); err != nil {
|
||||
t.Fatal("scratch mutation affected canonical data", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user