Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts. Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation. Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
220 lines
5.7 KiB
Go
220 lines
5.7 KiB
Go
package geduldh
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
var ErrMaterializeLimit = errors.New("Document materialization capacity exceeded")
|
|
|
|
// Source deliberately excludes mutation. Authorization precedes Acquire.
|
|
type Source interface {
|
|
Stat(context.Context, string) (Info, error)
|
|
Open(context.Context, string) (Reader, error)
|
|
}
|
|
|
|
type MaterializeOptions struct {
|
|
// TempDir must be node-local scratch, never canonical storage. Empty uses TMPDIR.
|
|
TempDir string
|
|
MaxBytes int64
|
|
Concurrent int
|
|
// MaxEntries bounds queued/leased objects too, including empty payloads.
|
|
// Zero selects 64.
|
|
MaxEntries int
|
|
Timeout time.Duration
|
|
}
|
|
|
|
// Materializer shares in-flight/leased copies only. The last lease removes the
|
|
// copy; it is not a persistent cache and needs no idle cache eviction goroutine.
|
|
// Limits include both copies being downloaded and copies in use by tools.
|
|
type Materializer struct {
|
|
source Source
|
|
options MaterializeOptions
|
|
slots chan struct{}
|
|
mu sync.Mutex
|
|
entries map[string]*materializedEntry
|
|
reserved int64
|
|
active int
|
|
}
|
|
|
|
type materializedEntry struct {
|
|
hash string
|
|
size int64
|
|
refs int
|
|
ready chan struct{}
|
|
cancel context.CancelFunc
|
|
finished bool
|
|
dir, path string
|
|
err error
|
|
}
|
|
|
|
// Lease pins an immutable scratch copy until Close. Tools must finish before
|
|
// Close; never give its path to a detached process. Close is idempotent.
|
|
type Lease struct {
|
|
Path string
|
|
Size int64
|
|
once sync.Once
|
|
release func()
|
|
}
|
|
|
|
func (l *Lease) Close() { l.once.Do(l.release) }
|
|
|
|
func NewMaterializer(source Source, options MaterializeOptions) (*Materializer, error) {
|
|
if source == nil || options.MaxBytes <= 0 || options.Concurrent <= 0 || options.Timeout <= 0 {
|
|
return nil, errors.New("invalid Document materializer configuration")
|
|
}
|
|
if options.MaxEntries == 0 {
|
|
options.MaxEntries = 64
|
|
}
|
|
if options.MaxEntries < 0 {
|
|
return nil, errors.New("invalid materializer entry limit")
|
|
}
|
|
return &Materializer{source: source, options: options, slots: make(chan struct{}, options.Concurrent), entries: make(map[string]*materializedEntry)}, nil
|
|
}
|
|
|
|
func (m *Materializer) Acquire(ctx context.Context, hash string, maxBytes int64) (*Lease, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := Key(hash); err != nil {
|
|
return nil, err
|
|
}
|
|
hash = strings.ToLower(strings.TrimSpace(hash))
|
|
info, err := m.source.Stat(ctx, hash)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if info.Size < 0 || maxBytes < 0 || info.Size > maxBytes || info.Size > m.options.MaxBytes {
|
|
return nil, ErrMaterializeLimit
|
|
}
|
|
m.mu.Lock()
|
|
e := m.entries[hash]
|
|
if e != nil && e.size != info.Size {
|
|
m.mu.Unlock()
|
|
return nil, ErrSizeMismatch
|
|
}
|
|
if e == nil {
|
|
if info.Size > m.options.MaxBytes-m.reserved || m.active >= m.options.MaxEntries {
|
|
m.mu.Unlock()
|
|
return nil, ErrMaterializeLimit
|
|
}
|
|
// One caller leaving must not cancel another caller's transfer. The
|
|
// shared deadline is propagated to backend operations. Backends must
|
|
// honor cancellation; it cannot interrupt an uninterruptible OS syscall.
|
|
loadCtx, cancel := context.WithTimeout(context.Background(), m.options.Timeout)
|
|
e = &materializedEntry{hash: hash, size: info.Size, ready: make(chan struct{}), cancel: cancel}
|
|
m.entries[hash] = e
|
|
m.reserved += info.Size
|
|
m.active++
|
|
go m.load(loadCtx, e)
|
|
}
|
|
e.refs++
|
|
m.mu.Unlock()
|
|
select {
|
|
case <-ctx.Done():
|
|
m.release(e)
|
|
return nil, ctx.Err()
|
|
case <-e.ready:
|
|
if err := ctx.Err(); err != nil {
|
|
m.release(e)
|
|
return nil, err
|
|
}
|
|
if e.err != nil {
|
|
m.release(e)
|
|
return nil, e.err
|
|
}
|
|
return &Lease{Path: e.path, Size: e.size, release: func() { m.release(e) }}, nil
|
|
}
|
|
}
|
|
|
|
func (m *Materializer) load(ctx context.Context, e *materializedEntry) {
|
|
defer e.cancel()
|
|
var dir, filename string
|
|
var err error
|
|
select {
|
|
case m.slots <- struct{}{}:
|
|
dir, filename, err = m.copy(ctx, e.hash, e.size)
|
|
<-m.slots
|
|
case <-ctx.Done():
|
|
err = ctx.Err()
|
|
}
|
|
m.mu.Lock()
|
|
e.dir, e.path, e.err, e.finished = dir, filename, err, true
|
|
close(e.ready)
|
|
if e.refs == 0 {
|
|
m.remove(e)
|
|
}
|
|
m.mu.Unlock()
|
|
}
|
|
|
|
func (m *Materializer) release(e *materializedEntry) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
e.refs--
|
|
if e.refs == 0 {
|
|
e.cancel()
|
|
// A later caller must not join an already-cancelled download.
|
|
if m.entries[e.hash] == e {
|
|
delete(m.entries, e.hash)
|
|
}
|
|
if e.finished {
|
|
m.remove(e)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (m *Materializer) remove(e *materializedEntry) {
|
|
if m.entries[e.hash] == e {
|
|
delete(m.entries, e.hash)
|
|
}
|
|
// Failed removal keeps the byte reservation: do not pretend disk space
|
|
// was reclaimed. Recovery of crash leftovers is an operator/scratch task.
|
|
if e.dir != "" {
|
|
if err := os.RemoveAll(e.dir); err != nil {
|
|
return
|
|
}
|
|
}
|
|
m.reserved -= e.size
|
|
m.active--
|
|
}
|
|
|
|
func (m *Materializer) copy(ctx context.Context, hash string, size int64) (dir, filename string, err error) {
|
|
reader, err := m.source.Open(ctx, hash)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
defer reader.Close()
|
|
dir, err = os.MkdirTemp(m.options.TempDir, "pageswiki-document-*")
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
// Failed partial copies are retained in dir until load performs cleanup.
|
|
filename = filepath.Join(dir, hash)
|
|
file, err := os.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
|
if err != nil {
|
|
return dir, "", err
|
|
}
|
|
err = copyValidated(ctx, hash, size, reader, file)
|
|
if err == nil {
|
|
err = file.Chmod(0400)
|
|
}
|
|
closeErr := file.Close()
|
|
if err == nil {
|
|
err = closeErr
|
|
}
|
|
if err != nil {
|
|
return dir, "", fmt.Errorf("materialize Document: %w", err)
|
|
}
|
|
// Scratch is reproducible, not durable canonical storage: no fsync here.
|
|
return dir, filename, ctx.Err()
|
|
}
|