package geduldh import ( "context" "errors" "fmt" "os" "path/filepath" "strings" "sync" "time" ) var ErrMaterializeLimit = errors.New("Document materialization capacity exceeded") // Source deliberately excludes mutation. Authorization precedes Acquire. type Source interface { Stat(context.Context, string) (Info, error) Open(context.Context, string) (Reader, error) } type MaterializeOptions struct { // TempDir must be node-local scratch, never canonical storage. Empty uses TMPDIR. TempDir string MaxBytes int64 Concurrent int // MaxEntries bounds queued/leased objects too, including empty payloads. // Zero selects 64. MaxEntries int Timeout time.Duration } // Materializer shares in-flight/leased copies only. The last lease removes the // copy; it is not a persistent cache and needs no idle cache eviction goroutine. // Limits include both copies being downloaded and copies in use by tools. type Materializer struct { source Source options MaterializeOptions slots chan struct{} mu sync.Mutex entries map[string]*materializedEntry reserved int64 active int } type materializedEntry struct { hash string size int64 refs int ready chan struct{} cancel context.CancelFunc finished bool dir, path string err error } // Lease pins an immutable scratch copy until Close. Tools must finish before // Close; never give its path to a detached process. Close is idempotent. type Lease struct { Path string Size int64 once sync.Once release func() } func (l *Lease) Close() { l.once.Do(l.release) } func NewMaterializer(source Source, options MaterializeOptions) (*Materializer, error) { if source == nil || options.MaxBytes <= 0 || options.Concurrent <= 0 || options.Timeout <= 0 { return nil, errors.New("invalid Document materializer configuration") } if options.MaxEntries == 0 { options.MaxEntries = 64 } if options.MaxEntries < 0 { return nil, errors.New("invalid materializer entry limit") } return &Materializer{source: source, options: options, slots: make(chan struct{}, options.Concurrent), entries: make(map[string]*materializedEntry)}, nil } func (m *Materializer) Acquire(ctx context.Context, hash string, maxBytes int64) (*Lease, error) { if err := ctx.Err(); err != nil { return nil, err } if _, err := Key(hash); err != nil { return nil, err } hash = strings.ToLower(strings.TrimSpace(hash)) info, err := m.source.Stat(ctx, hash) if err != nil { return nil, err } if err := ctx.Err(); err != nil { return nil, err } if info.Size < 0 || maxBytes < 0 || info.Size > maxBytes || info.Size > m.options.MaxBytes { return nil, ErrMaterializeLimit } m.mu.Lock() e := m.entries[hash] if e != nil && e.size != info.Size { m.mu.Unlock() return nil, ErrSizeMismatch } if e == nil { if info.Size > m.options.MaxBytes-m.reserved || m.active >= m.options.MaxEntries { m.mu.Unlock() return nil, ErrMaterializeLimit } // One caller leaving must not cancel another caller's transfer. The // shared deadline is propagated to backend operations. Backends must // honor cancellation; it cannot interrupt an uninterruptible OS syscall. loadCtx, cancel := context.WithTimeout(context.Background(), m.options.Timeout) e = &materializedEntry{hash: hash, size: info.Size, ready: make(chan struct{}), cancel: cancel} m.entries[hash] = e m.reserved += info.Size m.active++ go m.load(loadCtx, e) } e.refs++ m.mu.Unlock() select { case <-ctx.Done(): m.release(e) return nil, ctx.Err() case <-e.ready: if err := ctx.Err(); err != nil { m.release(e) return nil, err } if e.err != nil { m.release(e) return nil, e.err } return &Lease{Path: e.path, Size: e.size, release: func() { m.release(e) }}, nil } } func (m *Materializer) load(ctx context.Context, e *materializedEntry) { defer e.cancel() var dir, filename string var err error select { case m.slots <- struct{}{}: dir, filename, err = m.copy(ctx, e.hash, e.size) <-m.slots case <-ctx.Done(): err = ctx.Err() } m.mu.Lock() e.dir, e.path, e.err, e.finished = dir, filename, err, true close(e.ready) if e.refs == 0 { m.remove(e) } m.mu.Unlock() } func (m *Materializer) release(e *materializedEntry) { m.mu.Lock() defer m.mu.Unlock() e.refs-- if e.refs == 0 { e.cancel() // A later caller must not join an already-cancelled download. if m.entries[e.hash] == e { delete(m.entries, e.hash) } if e.finished { m.remove(e) } } } func (m *Materializer) remove(e *materializedEntry) { if m.entries[e.hash] == e { delete(m.entries, e.hash) } // Failed removal keeps the byte reservation: do not pretend disk space // was reclaimed. Recovery of crash leftovers is an operator/scratch task. if e.dir != "" { if err := os.RemoveAll(e.dir); err != nil { return } } m.reserved -= e.size m.active-- } func (m *Materializer) copy(ctx context.Context, hash string, size int64) (dir, filename string, err error) { reader, err := m.source.Open(ctx, hash) if err != nil { return "", "", err } defer reader.Close() dir, err = os.MkdirTemp(m.options.TempDir, "pageswiki-document-*") if err != nil { return "", "", err } // Failed partial copies are retained in dir until load performs cleanup. filename = filepath.Join(dir, hash) file, err := os.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) if err != nil { return dir, "", err } err = copyValidated(ctx, hash, size, reader, file) if err == nil { err = file.Chmod(0400) } closeErr := file.Close() if err == nil { err = closeErr } if err != nil { return dir, "", fmt.Errorf("materialize Document: %w", err) } // Scratch is reproducible, not durable canonical storage: no fsync here. return dir, filename, ctx.Err() }