Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts. Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation. Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
273 lines
9.6 KiB
Go
273 lines
9.6 KiB
Go
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
"pageswiki/internal/store"
|
|
)
|
|
|
|
const (
|
|
metadataProjectionEventType = "projection.metadata.generate"
|
|
metadataProjectionVersion = 1
|
|
maxArchiveMetadataEntries = 2000
|
|
maxArchiveMetadataNameBytes = 256 << 10
|
|
maxMetadataInputBytes = 512 << 20
|
|
metadataFailureBackoff = 5 * time.Minute
|
|
)
|
|
|
|
type metadataProjectionPayload struct {
|
|
DocumentHash string `json:"document_hash"`
|
|
Kind string `json:"kind"`
|
|
Version int `json:"version"`
|
|
}
|
|
|
|
type metadataProjectionResult struct {
|
|
Status string `json:"status"`
|
|
PDFPages int `json:"pdf_pages,omitempty"`
|
|
Entries []archiveEntryView `json:"entries,omitempty"`
|
|
Truncated bool `json:"truncated,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
RetryAtMS int64 `json:"retry_at_ms,omitempty"`
|
|
}
|
|
|
|
func (s *Server) requestMetadataProjection(hash, kind string) (metadataProjectionResult, error) {
|
|
result := metadataProjectionResult{Status: "pending"}
|
|
entry, err := s.store.GetProjectionMetadata(hash, kind, metadataProjectionVersion)
|
|
if err == nil {
|
|
if entry.Error != "" && entry.RetryAtMS > time.Now().UnixMilli() {
|
|
return metadataProjectionResult{Status: "failed", Error: entry.Error, RetryAtMS: entry.RetryAtMS}, nil
|
|
}
|
|
if entry.Error == "" {
|
|
if err := json.Unmarshal(entry.Payload, &result); err != nil {
|
|
return result, err
|
|
}
|
|
return result, nil
|
|
}
|
|
} else if !errors.Is(err, sql.ErrNoRows) {
|
|
return result, err
|
|
}
|
|
payload, err := json.Marshal(metadataProjectionPayload{hash, kind, metadataProjectionVersion})
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
key := fmt.Sprintf("%d:%s:%s", metadataProjectionVersion, kind, hash)
|
|
queued, err := s.store.EnqueueEventWithOptions("", metadataProjectionEventType, key, string(payload), time.Now(), -5, 1)
|
|
if queued {
|
|
s.wakeEventScheduler()
|
|
}
|
|
return result, err
|
|
}
|
|
|
|
func metadataProjectionURL(kind, revisionUUID, semanticPath string) string {
|
|
query := url.Values{"rev": {revisionUUID}, "metadata": {"1"}}
|
|
if kind == "pdf" {
|
|
return "/v/pdf-page?" + query.Encode()
|
|
}
|
|
query.Set("path", semanticPath)
|
|
return "/v/archive?" + query.Encode()
|
|
}
|
|
|
|
func (s *Server) applyMetadataProjection(view *contentView, semanticPath string) {
|
|
view.MetadataURL = metadataProjectionURL(view.Kind, view.RevisionUUID, semanticPath)
|
|
result, err := s.requestMetadataProjection(view.DocumentHash, view.Kind)
|
|
if err != nil {
|
|
view.MetadataStatus = "unavailable"
|
|
view.ParseError = "Metadata queue is unavailable. Download remains available."
|
|
return
|
|
}
|
|
view.MetadataStatus, view.ArchiveTruncated = result.Status, result.Truncated
|
|
view.PDFPages, view.ParseError = result.PDFPages, result.Error
|
|
view.ArchiveEntries = archiveMetadataLinks(result.Entries, semanticPath, view.RevisionUUID)
|
|
}
|
|
|
|
func archiveMetadataLinks(entries []archiveEntryView, semanticPath, revisionUUID string) []archiveEntryView {
|
|
result := make([]archiveEntryView, len(entries))
|
|
copy(result, entries)
|
|
for index := range result {
|
|
query := url.Values{"path": {semanticPath}, "rev": {revisionUUID}, "entry": {result[index].Name}}
|
|
result[index].URL = "/v/archive?" + query.Encode()
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (s *Server) serveMetadataProjection(w http.ResponseWriter, r *http.Request, hash, kind, semanticPath, revisionUUID string) {
|
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
|
w.Header().Set("Cache-Control", "private, no-store")
|
|
w.Header().Set("Vary", "Authorization, Cookie")
|
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
result, err := s.requestMetadataProjection(hash, kind)
|
|
if err != nil {
|
|
w.Header().Set("Retry-After", "5")
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
_ = json.NewEncoder(w).Encode(metadataProjectionResult{Status: "unavailable", Error: "Metadata queue is unavailable"})
|
|
return
|
|
}
|
|
result.Entries = archiveMetadataLinks(result.Entries, semanticPath, revisionUUID)
|
|
if result.Status == "pending" {
|
|
w.Header().Set("Retry-After", "2")
|
|
w.WriteHeader(http.StatusAccepted)
|
|
} else if result.Status == "failed" {
|
|
w.Header().Set("Retry-After", strconv.FormatInt(max(1, (result.RetryAtMS-time.Now().UnixMilli())/1000), 10))
|
|
w.WriteHeader(http.StatusUnprocessableEntity)
|
|
}
|
|
_ = json.NewEncoder(w).Encode(result)
|
|
}
|
|
|
|
func (s *Server) generateMetadataProjection(payload metadataProjectionPayload) error {
|
|
if payload.Version != metadataProjectionVersion || (payload.Kind != "pdf" && payload.Kind != "archive") {
|
|
return errors.New("unsupported metadata projection")
|
|
}
|
|
if cached, err := s.store.GetProjectionMetadata(payload.DocumentHash, payload.Kind, payload.Version); err == nil && cached.Error == "" {
|
|
return nil
|
|
}
|
|
result, runErr := s.inspectProjectionMetadata(payload)
|
|
entry := store.ProjectionMetadata{DocumentHash: payload.DocumentHash, Kind: payload.Kind, Version: payload.Version}
|
|
if runErr != nil {
|
|
entry.Error = payload.Kind + " metadata inspection failed; retry is available after the cooldown"
|
|
entry.RetryAtMS = time.Now().Add(metadataFailureBackoff).UnixMilli()
|
|
entry.Payload = json.RawMessage(`{}`)
|
|
} else {
|
|
var err error
|
|
entry.Payload, err = json.Marshal(result)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := s.store.SaveProjectionMetadata(entry); err != nil {
|
|
return err
|
|
}
|
|
return runErr
|
|
}
|
|
|
|
func (s *Server) inspectProjectionMetadata(payload metadataProjectionPayload) (metadataProjectionResult, error) {
|
|
result := metadataProjectionResult{Status: "ready"}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
lease, err := s.store.MaterializeDocument(ctx, payload.DocumentHash, maxMetadataInputBytes)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
defer lease.Close()
|
|
source := lease.Path
|
|
if payload.Kind == "pdf" {
|
|
output, truncated, err := runMetadataCommand(ctx, 64<<10, "pdfinfo", "-enc", "UTF-8", source)
|
|
if err != nil || truncated {
|
|
return result, fmt.Errorf("bounded pdfinfo failed: %w", err)
|
|
}
|
|
for _, line := range strings.Split(string(output), "\n") {
|
|
if strings.HasPrefix(line, "Pages:") {
|
|
result.PDFPages, err = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "Pages:")))
|
|
if err == nil && result.PDFPages > 0 && result.PDFPages <= 10000 {
|
|
return result, nil
|
|
}
|
|
}
|
|
}
|
|
return result, errors.New("PDF page count is missing or exceeds 10000 pages")
|
|
}
|
|
output, truncated, err := runMetadataCommand(ctx, 4<<20, "7z", "l", "-slt", "-ba", "-sccUTF-8", "-mmt1", "-p-", "--", source)
|
|
if err != nil && !truncated {
|
|
return result, fmt.Errorf("bounded archive listing failed: %w", err)
|
|
}
|
|
return parseArchiveMetadata(output, truncated)
|
|
}
|
|
|
|
type boundedMetadataOutput struct {
|
|
buffer bytes.Buffer
|
|
limit int
|
|
truncated bool
|
|
cancel context.CancelFunc
|
|
}
|
|
|
|
func (output *boundedMetadataOutput) Write(data []byte) (int, error) {
|
|
if len(data) <= output.limit-output.buffer.Len() {
|
|
return output.buffer.Write(data)
|
|
}
|
|
remaining := max(0, output.limit-output.buffer.Len())
|
|
_, _ = output.buffer.Write(data[:remaining])
|
|
output.truncated = true
|
|
output.cancel()
|
|
return remaining, errors.New("metadata output limit exceeded")
|
|
}
|
|
|
|
func runMetadataCommand(ctx context.Context, limit int, name string, args ...string) ([]byte, bool, error) {
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
output := &boundedMetadataOutput{limit: limit, cancel: cancel}
|
|
command := exec.CommandContext(ctx, "nice", append([]string{"-n", "15", name}, args...)...)
|
|
command.Stdout, command.Stderr = output, io.Discard
|
|
command.Stdin = nil
|
|
command.WaitDelay = time.Second
|
|
err := command.Run()
|
|
return output.buffer.Bytes(), output.truncated, err
|
|
}
|
|
|
|
func parseArchiveMetadata(output []byte, truncated bool) (metadataProjectionResult, error) {
|
|
result := metadataProjectionResult{Status: "ready", Truncated: truncated}
|
|
if truncated {
|
|
// Never expose a final incomplete filename or numeric field.
|
|
if end := bytes.LastIndex(output, []byte("\n\n")); end >= 0 {
|
|
output = output[:end]
|
|
} else {
|
|
return result, errors.New("archive metadata record exceeds output limit")
|
|
}
|
|
}
|
|
entry := archiveEntryView{}
|
|
nameBytes := 0
|
|
flush := func() bool {
|
|
if entry.Name == "" {
|
|
return true
|
|
}
|
|
if len(result.Entries) >= maxArchiveMetadataEntries || nameBytes+len(entry.Name) > maxArchiveMetadataNameBytes {
|
|
result.Truncated = true
|
|
return false
|
|
}
|
|
ext := strings.ToLower(filepath.Ext(entry.Name))
|
|
entry.IsImage = !entry.IsDir && (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif" || ext == ".webp" || ext == ".avif")
|
|
result.Entries = append(result.Entries, entry)
|
|
nameBytes += len(entry.Name)
|
|
entry = archiveEntryView{}
|
|
return true
|
|
}
|
|
for _, line := range strings.Split(string(output), "\n") {
|
|
line = strings.TrimSuffix(line, "\r")
|
|
switch {
|
|
case strings.TrimSpace(line) == "":
|
|
if !flush() {
|
|
return result, nil
|
|
}
|
|
case strings.HasPrefix(line, "Path = "):
|
|
entry.Name = strings.TrimPrefix(line, "Path = ")
|
|
if len(entry.Name) > 4096 || !utf8.ValidString(entry.Name) || strings.IndexFunc(entry.Name, unicode.IsControl) >= 0 {
|
|
return result, errors.New("invalid archive entry name")
|
|
}
|
|
case strings.HasPrefix(line, "Size = "):
|
|
var err error
|
|
entry.Size, err = strconv.ParseUint(strings.TrimSpace(strings.TrimPrefix(line, "Size = ")), 10, 64)
|
|
if err != nil {
|
|
return result, errors.New("invalid archive entry size")
|
|
}
|
|
case strings.HasPrefix(line, "Folder = "):
|
|
entry.IsDir = strings.TrimSpace(strings.TrimPrefix(line, "Folder = ")) == "+"
|
|
case strings.HasPrefix(line, "Attributes = "):
|
|
entry.IsDir = entry.IsDir || strings.Contains(strings.TrimPrefix(line, "Attributes = "), "D")
|
|
}
|
|
}
|
|
flush()
|
|
return result, nil
|
|
}
|