Files
pages/internal/geduldh/materialize.go
T
Anya Tokugawa d5d21b7841
Build and publish container images / images (push) Successful in 9m34s
Release container images / release (push) Successful in 36s
Refactor architecture into eight components and consolidate reliability work
Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts.

Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation.

Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
2026-11-01 15:23:31 +09:00

220 lines
5.7 KiB
Go

package geduldh
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
)
var ErrMaterializeLimit = errors.New("Document materialization capacity exceeded")
// Source deliberately excludes mutation. Authorization precedes Acquire.
type Source interface {
Stat(context.Context, string) (Info, error)
Open(context.Context, string) (Reader, error)
}
type MaterializeOptions struct {
// TempDir must be node-local scratch, never canonical storage. Empty uses TMPDIR.
TempDir string
MaxBytes int64
Concurrent int
// MaxEntries bounds queued/leased objects too, including empty payloads.
// Zero selects 64.
MaxEntries int
Timeout time.Duration
}
// Materializer shares in-flight/leased copies only. The last lease removes the
// copy; it is not a persistent cache and needs no idle cache eviction goroutine.
// Limits include both copies being downloaded and copies in use by tools.
type Materializer struct {
source Source
options MaterializeOptions
slots chan struct{}
mu sync.Mutex
entries map[string]*materializedEntry
reserved int64
active int
}
type materializedEntry struct {
hash string
size int64
refs int
ready chan struct{}
cancel context.CancelFunc
finished bool
dir, path string
err error
}
// Lease pins an immutable scratch copy until Close. Tools must finish before
// Close; never give its path to a detached process. Close is idempotent.
type Lease struct {
Path string
Size int64
once sync.Once
release func()
}
func (l *Lease) Close() { l.once.Do(l.release) }
func NewMaterializer(source Source, options MaterializeOptions) (*Materializer, error) {
if source == nil || options.MaxBytes <= 0 || options.Concurrent <= 0 || options.Timeout <= 0 {
return nil, errors.New("invalid Document materializer configuration")
}
if options.MaxEntries == 0 {
options.MaxEntries = 64
}
if options.MaxEntries < 0 {
return nil, errors.New("invalid materializer entry limit")
}
return &Materializer{source: source, options: options, slots: make(chan struct{}, options.Concurrent), entries: make(map[string]*materializedEntry)}, nil
}
func (m *Materializer) Acquire(ctx context.Context, hash string, maxBytes int64) (*Lease, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if _, err := Key(hash); err != nil {
return nil, err
}
hash = strings.ToLower(strings.TrimSpace(hash))
info, err := m.source.Stat(ctx, hash)
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
if info.Size < 0 || maxBytes < 0 || info.Size > maxBytes || info.Size > m.options.MaxBytes {
return nil, ErrMaterializeLimit
}
m.mu.Lock()
e := m.entries[hash]
if e != nil && e.size != info.Size {
m.mu.Unlock()
return nil, ErrSizeMismatch
}
if e == nil {
if info.Size > m.options.MaxBytes-m.reserved || m.active >= m.options.MaxEntries {
m.mu.Unlock()
return nil, ErrMaterializeLimit
}
// One caller leaving must not cancel another caller's transfer. The
// shared deadline is propagated to backend operations. Backends must
// honor cancellation; it cannot interrupt an uninterruptible OS syscall.
loadCtx, cancel := context.WithTimeout(context.Background(), m.options.Timeout)
e = &materializedEntry{hash: hash, size: info.Size, ready: make(chan struct{}), cancel: cancel}
m.entries[hash] = e
m.reserved += info.Size
m.active++
go m.load(loadCtx, e)
}
e.refs++
m.mu.Unlock()
select {
case <-ctx.Done():
m.release(e)
return nil, ctx.Err()
case <-e.ready:
if err := ctx.Err(); err != nil {
m.release(e)
return nil, err
}
if e.err != nil {
m.release(e)
return nil, e.err
}
return &Lease{Path: e.path, Size: e.size, release: func() { m.release(e) }}, nil
}
}
func (m *Materializer) load(ctx context.Context, e *materializedEntry) {
defer e.cancel()
var dir, filename string
var err error
select {
case m.slots <- struct{}{}:
dir, filename, err = m.copy(ctx, e.hash, e.size)
<-m.slots
case <-ctx.Done():
err = ctx.Err()
}
m.mu.Lock()
e.dir, e.path, e.err, e.finished = dir, filename, err, true
close(e.ready)
if e.refs == 0 {
m.remove(e)
}
m.mu.Unlock()
}
func (m *Materializer) release(e *materializedEntry) {
m.mu.Lock()
defer m.mu.Unlock()
e.refs--
if e.refs == 0 {
e.cancel()
// A later caller must not join an already-cancelled download.
if m.entries[e.hash] == e {
delete(m.entries, e.hash)
}
if e.finished {
m.remove(e)
}
}
}
func (m *Materializer) remove(e *materializedEntry) {
if m.entries[e.hash] == e {
delete(m.entries, e.hash)
}
// Failed removal keeps the byte reservation: do not pretend disk space
// was reclaimed. Recovery of crash leftovers is an operator/scratch task.
if e.dir != "" {
if err := os.RemoveAll(e.dir); err != nil {
return
}
}
m.reserved -= e.size
m.active--
}
func (m *Materializer) copy(ctx context.Context, hash string, size int64) (dir, filename string, err error) {
reader, err := m.source.Open(ctx, hash)
if err != nil {
return "", "", err
}
defer reader.Close()
dir, err = os.MkdirTemp(m.options.TempDir, "pageswiki-document-*")
if err != nil {
return "", "", err
}
// Failed partial copies are retained in dir until load performs cleanup.
filename = filepath.Join(dir, hash)
file, err := os.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return dir, "", err
}
err = copyValidated(ctx, hash, size, reader, file)
if err == nil {
err = file.Chmod(0400)
}
closeErr := file.Close()
if err == nil {
err = closeErr
}
if err != nil {
return dir, "", fmt.Errorf("materialize Document: %w", err)
}
// Scratch is reproducible, not durable canonical storage: no fsync here.
return dir, filename, ctx.Err()
}