Refactor architecture into eight components and consolidate reliability work
Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts. Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation. Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
This commit is contained in:
1 parent
9dd425f765
commit
d5d21b7841
302 files changed
+20037
-3492
No files matched your search
@@ -0,0 +1,219 @@
|
||||
package geduldh
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrMaterializeLimit = errors.New("Document materialization capacity exceeded")
|
||||
|
||||
// Source deliberately excludes mutation. Authorization precedes Acquire.
|
||||
type Source interface {
|
||||
Stat(context.Context, string) (Info, error)
|
||||
Open(context.Context, string) (Reader, error)
|
||||
}
|
||||
|
||||
type MaterializeOptions struct {
|
||||
// TempDir must be node-local scratch, never canonical storage. Empty uses TMPDIR.
|
||||
TempDir string
|
||||
MaxBytes int64
|
||||
Concurrent int
|
||||
// MaxEntries bounds queued/leased objects too, including empty payloads.
|
||||
// Zero selects 64.
|
||||
MaxEntries int
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Materializer shares in-flight/leased copies only. The last lease removes the
|
||||
// copy; it is not a persistent cache and needs no idle cache eviction goroutine.
|
||||
// Limits include both copies being downloaded and copies in use by tools.
|
||||
type Materializer struct {
|
||||
source Source
|
||||
options MaterializeOptions
|
||||
slots chan struct{}
|
||||
mu sync.Mutex
|
||||
entries map[string]*materializedEntry
|
||||
reserved int64
|
||||
active int
|
||||
}
|
||||
|
||||
type materializedEntry struct {
|
||||
hash string
|
||||
size int64
|
||||
refs int
|
||||
ready chan struct{}
|
||||
cancel context.CancelFunc
|
||||
finished bool
|
||||
dir, path string
|
||||
err error
|
||||
}
|
||||
|
||||
// Lease pins an immutable scratch copy until Close. Tools must finish before
|
||||
// Close; never give its path to a detached process. Close is idempotent.
|
||||
type Lease struct {
|
||||
Path string
|
||||
Size int64
|
||||
once sync.Once
|
||||
release func()
|
||||
}
|
||||
|
||||
func (l *Lease) Close() { l.once.Do(l.release) }
|
||||
|
||||
func NewMaterializer(source Source, options MaterializeOptions) (*Materializer, error) {
|
||||
if source == nil || options.MaxBytes <= 0 || options.Concurrent <= 0 || options.Timeout <= 0 {
|
||||
return nil, errors.New("invalid Document materializer configuration")
|
||||
}
|
||||
if options.MaxEntries == 0 {
|
||||
options.MaxEntries = 64
|
||||
}
|
||||
if options.MaxEntries < 0 {
|
||||
return nil, errors.New("invalid materializer entry limit")
|
||||
}
|
||||
return &Materializer{source: source, options: options, slots: make(chan struct{}, options.Concurrent), entries: make(map[string]*materializedEntry)}, nil
|
||||
}
|
||||
|
||||
func (m *Materializer) Acquire(ctx context.Context, hash string, maxBytes int64) (*Lease, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := Key(hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hash = strings.ToLower(strings.TrimSpace(hash))
|
||||
info, err := m.source.Stat(ctx, hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.Size < 0 || maxBytes < 0 || info.Size > maxBytes || info.Size > m.options.MaxBytes {
|
||||
return nil, ErrMaterializeLimit
|
||||
}
|
||||
m.mu.Lock()
|
||||
e := m.entries[hash]
|
||||
if e != nil && e.size != info.Size {
|
||||
m.mu.Unlock()
|
||||
return nil, ErrSizeMismatch
|
||||
}
|
||||
if e == nil {
|
||||
if info.Size > m.options.MaxBytes-m.reserved || m.active >= m.options.MaxEntries {
|
||||
m.mu.Unlock()
|
||||
return nil, ErrMaterializeLimit
|
||||
}
|
||||
// One caller leaving must not cancel another caller's transfer. The
|
||||
// shared deadline is propagated to backend operations. Backends must
|
||||
// honor cancellation; it cannot interrupt an uninterruptible OS syscall.
|
||||
loadCtx, cancel := context.WithTimeout(context.Background(), m.options.Timeout)
|
||||
e = &materializedEntry{hash: hash, size: info.Size, ready: make(chan struct{}), cancel: cancel}
|
||||
m.entries[hash] = e
|
||||
m.reserved += info.Size
|
||||
m.active++
|
||||
go m.load(loadCtx, e)
|
||||
}
|
||||
e.refs++
|
||||
m.mu.Unlock()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
m.release(e)
|
||||
return nil, ctx.Err()
|
||||
case <-e.ready:
|
||||
if err := ctx.Err(); err != nil {
|
||||
m.release(e)
|
||||
return nil, err
|
||||
}
|
||||
if e.err != nil {
|
||||
m.release(e)
|
||||
return nil, e.err
|
||||
}
|
||||
return &Lease{Path: e.path, Size: e.size, release: func() { m.release(e) }}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Materializer) load(ctx context.Context, e *materializedEntry) {
|
||||
defer e.cancel()
|
||||
var dir, filename string
|
||||
var err error
|
||||
select {
|
||||
case m.slots <- struct{}{}:
|
||||
dir, filename, err = m.copy(ctx, e.hash, e.size)
|
||||
<-m.slots
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
}
|
||||
m.mu.Lock()
|
||||
e.dir, e.path, e.err, e.finished = dir, filename, err, true
|
||||
close(e.ready)
|
||||
if e.refs == 0 {
|
||||
m.remove(e)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *Materializer) release(e *materializedEntry) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
e.refs--
|
||||
if e.refs == 0 {
|
||||
e.cancel()
|
||||
// A later caller must not join an already-cancelled download.
|
||||
if m.entries[e.hash] == e {
|
||||
delete(m.entries, e.hash)
|
||||
}
|
||||
if e.finished {
|
||||
m.remove(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Materializer) remove(e *materializedEntry) {
|
||||
if m.entries[e.hash] == e {
|
||||
delete(m.entries, e.hash)
|
||||
}
|
||||
// Failed removal keeps the byte reservation: do not pretend disk space
|
||||
// was reclaimed. Recovery of crash leftovers is an operator/scratch task.
|
||||
if e.dir != "" {
|
||||
if err := os.RemoveAll(e.dir); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
m.reserved -= e.size
|
||||
m.active--
|
||||
}
|
||||
|
||||
func (m *Materializer) copy(ctx context.Context, hash string, size int64) (dir, filename string, err error) {
|
||||
reader, err := m.source.Open(ctx, hash)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer reader.Close()
|
||||
dir, err = os.MkdirTemp(m.options.TempDir, "pageswiki-document-*")
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
// Failed partial copies are retained in dir until load performs cleanup.
|
||||
filename = filepath.Join(dir, hash)
|
||||
file, err := os.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return dir, "", err
|
||||
}
|
||||
err = copyValidated(ctx, hash, size, reader, file)
|
||||
if err == nil {
|
||||
err = file.Chmod(0400)
|
||||
}
|
||||
closeErr := file.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
return dir, "", fmt.Errorf("materialize Document: %w", err)
|
||||
}
|
||||
// Scratch is reproducible, not durable canonical storage: no fsync here.
|
||||
return dir, filename, ctx.Err()
|
||||
}
|
||||
Reference in new issue
Block a user