Files
pages/internal/server/projection_metadata.go
Anya Tokugawa d5d21b7841
Build and publish container images / images (push) Successful in 9m34s
Release container images / release (push) Successful in 36s
Refactor architecture into eight components and consolidate reliability work
Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts.

Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation.

Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
2026-11-01 15:23:31 +09:00

273 lines
9.6 KiB
Go

package server
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"pageswiki/internal/store"
)
const (
metadataProjectionEventType = "projection.metadata.generate"
metadataProjectionVersion = 1
maxArchiveMetadataEntries = 2000
maxArchiveMetadataNameBytes = 256 << 10
maxMetadataInputBytes = 512 << 20
metadataFailureBackoff = 5 * time.Minute
)
type metadataProjectionPayload struct {
DocumentHash string `json:"document_hash"`
Kind string `json:"kind"`
Version int `json:"version"`
}
type metadataProjectionResult struct {
Status string `json:"status"`
PDFPages int `json:"pdf_pages,omitempty"`
Entries []archiveEntryView `json:"entries,omitempty"`
Truncated bool `json:"truncated,omitempty"`
Error string `json:"error,omitempty"`
RetryAtMS int64 `json:"retry_at_ms,omitempty"`
}
func (s *Server) requestMetadataProjection(hash, kind string) (metadataProjectionResult, error) {
result := metadataProjectionResult{Status: "pending"}
entry, err := s.store.GetProjectionMetadata(hash, kind, metadataProjectionVersion)
if err == nil {
if entry.Error != "" && entry.RetryAtMS > time.Now().UnixMilli() {
return metadataProjectionResult{Status: "failed", Error: entry.Error, RetryAtMS: entry.RetryAtMS}, nil
}
if entry.Error == "" {
if err := json.Unmarshal(entry.Payload, &result); err != nil {
return result, err
}
return result, nil
}
} else if !errors.Is(err, sql.ErrNoRows) {
return result, err
}
payload, err := json.Marshal(metadataProjectionPayload{hash, kind, metadataProjectionVersion})
if err != nil {
return result, err
}
key := fmt.Sprintf("%d:%s:%s", metadataProjectionVersion, kind, hash)
queued, err := s.store.EnqueueEventWithOptions("", metadataProjectionEventType, key, string(payload), time.Now(), -5, 1)
if queued {
s.wakeEventScheduler()
}
return result, err
}
func metadataProjectionURL(kind, revisionUUID, semanticPath string) string {
query := url.Values{"rev": {revisionUUID}, "metadata": {"1"}}
if kind == "pdf" {
return "/v/pdf-page?" + query.Encode()
}
query.Set("path", semanticPath)
return "/v/archive?" + query.Encode()
}
func (s *Server) applyMetadataProjection(view *contentView, semanticPath string) {
view.MetadataURL = metadataProjectionURL(view.Kind, view.RevisionUUID, semanticPath)
result, err := s.requestMetadataProjection(view.DocumentHash, view.Kind)
if err != nil {
view.MetadataStatus = "unavailable"
view.ParseError = "Metadata queue is unavailable. Download remains available."
return
}
view.MetadataStatus, view.ArchiveTruncated = result.Status, result.Truncated
view.PDFPages, view.ParseError = result.PDFPages, result.Error
view.ArchiveEntries = archiveMetadataLinks(result.Entries, semanticPath, view.RevisionUUID)
}
func archiveMetadataLinks(entries []archiveEntryView, semanticPath, revisionUUID string) []archiveEntryView {
result := make([]archiveEntryView, len(entries))
copy(result, entries)
for index := range result {
query := url.Values{"path": {semanticPath}, "rev": {revisionUUID}, "entry": {result[index].Name}}
result[index].URL = "/v/archive?" + query.Encode()
}
return result
}
func (s *Server) serveMetadataProjection(w http.ResponseWriter, r *http.Request, hash, kind, semanticPath, revisionUUID string) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Cache-Control", "private, no-store")
w.Header().Set("Vary", "Authorization, Cookie")
w.Header().Set("X-Content-Type-Options", "nosniff")
result, err := s.requestMetadataProjection(hash, kind)
if err != nil {
w.Header().Set("Retry-After", "5")
w.WriteHeader(http.StatusServiceUnavailable)
_ = json.NewEncoder(w).Encode(metadataProjectionResult{Status: "unavailable", Error: "Metadata queue is unavailable"})
return
}
result.Entries = archiveMetadataLinks(result.Entries, semanticPath, revisionUUID)
if result.Status == "pending" {
w.Header().Set("Retry-After", "2")
w.WriteHeader(http.StatusAccepted)
} else if result.Status == "failed" {
w.Header().Set("Retry-After", strconv.FormatInt(max(1, (result.RetryAtMS-time.Now().UnixMilli())/1000), 10))
w.WriteHeader(http.StatusUnprocessableEntity)
}
_ = json.NewEncoder(w).Encode(result)
}
func (s *Server) generateMetadataProjection(payload metadataProjectionPayload) error {
if payload.Version != metadataProjectionVersion || (payload.Kind != "pdf" && payload.Kind != "archive") {
return errors.New("unsupported metadata projection")
}
if cached, err := s.store.GetProjectionMetadata(payload.DocumentHash, payload.Kind, payload.Version); err == nil && cached.Error == "" {
return nil
}
result, runErr := s.inspectProjectionMetadata(payload)
entry := store.ProjectionMetadata{DocumentHash: payload.DocumentHash, Kind: payload.Kind, Version: payload.Version}
if runErr != nil {
entry.Error = payload.Kind + " metadata inspection failed; retry is available after the cooldown"
entry.RetryAtMS = time.Now().Add(metadataFailureBackoff).UnixMilli()
entry.Payload = json.RawMessage(`{}`)
} else {
var err error
entry.Payload, err = json.Marshal(result)
if err != nil {
return err
}
}
if err := s.store.SaveProjectionMetadata(entry); err != nil {
return err
}
return runErr
}
func (s *Server) inspectProjectionMetadata(payload metadataProjectionPayload) (metadataProjectionResult, error) {
result := metadataProjectionResult{Status: "ready"}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
lease, err := s.store.MaterializeDocument(ctx, payload.DocumentHash, maxMetadataInputBytes)
if err != nil {
return result, err
}
defer lease.Close()
source := lease.Path
if payload.Kind == "pdf" {
output, truncated, err := runMetadataCommand(ctx, 64<<10, "pdfinfo", "-enc", "UTF-8", source)
if err != nil || truncated {
return result, fmt.Errorf("bounded pdfinfo failed: %w", err)
}
for _, line := range strings.Split(string(output), "\n") {
if strings.HasPrefix(line, "Pages:") {
result.PDFPages, err = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "Pages:")))
if err == nil && result.PDFPages > 0 && result.PDFPages <= 10000 {
return result, nil
}
}
}
return result, errors.New("PDF page count is missing or exceeds 10000 pages")
}
output, truncated, err := runMetadataCommand(ctx, 4<<20, "7z", "l", "-slt", "-ba", "-sccUTF-8", "-mmt1", "-p-", "--", source)
if err != nil && !truncated {
return result, fmt.Errorf("bounded archive listing failed: %w", err)
}
return parseArchiveMetadata(output, truncated)
}
type boundedMetadataOutput struct {
buffer bytes.Buffer
limit int
truncated bool
cancel context.CancelFunc
}
func (output *boundedMetadataOutput) Write(data []byte) (int, error) {
if len(data) <= output.limit-output.buffer.Len() {
return output.buffer.Write(data)
}
remaining := max(0, output.limit-output.buffer.Len())
_, _ = output.buffer.Write(data[:remaining])
output.truncated = true
output.cancel()
return remaining, errors.New("metadata output limit exceeded")
}
func runMetadataCommand(ctx context.Context, limit int, name string, args ...string) ([]byte, bool, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
output := &boundedMetadataOutput{limit: limit, cancel: cancel}
command := exec.CommandContext(ctx, "nice", append([]string{"-n", "15", name}, args...)...)
command.Stdout, command.Stderr = output, io.Discard
command.Stdin = nil
command.WaitDelay = time.Second
err := command.Run()
return output.buffer.Bytes(), output.truncated, err
}
func parseArchiveMetadata(output []byte, truncated bool) (metadataProjectionResult, error) {
result := metadataProjectionResult{Status: "ready", Truncated: truncated}
if truncated {
// Never expose a final incomplete filename or numeric field.
if end := bytes.LastIndex(output, []byte("\n\n")); end >= 0 {
output = output[:end]
} else {
return result, errors.New("archive metadata record exceeds output limit")
}
}
entry := archiveEntryView{}
nameBytes := 0
flush := func() bool {
if entry.Name == "" {
return true
}
if len(result.Entries) >= maxArchiveMetadataEntries || nameBytes+len(entry.Name) > maxArchiveMetadataNameBytes {
result.Truncated = true
return false
}
ext := strings.ToLower(filepath.Ext(entry.Name))
entry.IsImage = !entry.IsDir && (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif" || ext == ".webp" || ext == ".avif")
result.Entries = append(result.Entries, entry)
nameBytes += len(entry.Name)
entry = archiveEntryView{}
return true
}
for _, line := range strings.Split(string(output), "\n") {
line = strings.TrimSuffix(line, "\r")
switch {
case strings.TrimSpace(line) == "":
if !flush() {
return result, nil
}
case strings.HasPrefix(line, "Path = "):
entry.Name = strings.TrimPrefix(line, "Path = ")
if len(entry.Name) > 4096 || !utf8.ValidString(entry.Name) || strings.IndexFunc(entry.Name, unicode.IsControl) >= 0 {
return result, errors.New("invalid archive entry name")
}
case strings.HasPrefix(line, "Size = "):
var err error
entry.Size, err = strconv.ParseUint(strings.TrimSpace(strings.TrimPrefix(line, "Size = ")), 10, 64)
if err != nil {
return result, errors.New("invalid archive entry size")
}
case strings.HasPrefix(line, "Folder = "):
entry.IsDir = strings.TrimSpace(strings.TrimPrefix(line, "Folder = ")) == "+"
case strings.HasPrefix(line, "Attributes = "):
entry.IsDir = entry.IsDir || strings.Contains(strings.TrimPrefix(line, "Attributes = "), "D")
}
}
flush()
return result, nil
}