Files
pages/internal/server/projection_metadata.go
T
Anya Tokugawa c1175d389f Implement Knowledge City platform, organizer and DAV integration
Capture accumulated application, UI, CLI, regression tests and design documentation changes, including task project mapping and default CalDAV list naming.

Keep deployment data, backups, local credentials and build artifacts out of Git. Move the Compose enrollment token to an ignored local environment file.
2026-09-27 14:44:43 +09:00

279 lines
9.9 KiB
Go

package server
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"pageswiki/internal/store"
)
const (
metadataProjectionEventType = "projection.metadata.generate"
metadataProjectionVersion = 1
maxArchiveMetadataEntries = 2000
maxArchiveMetadataNameBytes = 256 << 10
maxMetadataInputBytes = 512 << 20
metadataFailureBackoff = 5 * time.Minute
)
type metadataProjectionPayload struct {
DocumentHash string `json:"document_hash"`
Kind string `json:"kind"`
Version int `json:"version"`
}
type metadataProjectionResult struct {
Status string `json:"status"`
PDFPages int `json:"pdf_pages,omitempty"`
Entries []archiveEntryView `json:"entries,omitempty"`
Truncated bool `json:"truncated,omitempty"`
Error string `json:"error,omitempty"`
RetryAtMS int64 `json:"retry_at_ms,omitempty"`
}
func (s *Server) requestMetadataProjection(hash, kind string) (metadataProjectionResult, error) {
result := metadataProjectionResult{Status: "pending"}
entry, err := s.store.GetProjectionMetadata(hash, kind, metadataProjectionVersion)
if err == nil {
if entry.Error != "" && entry.RetryAtMS > time.Now().UnixMilli() {
return metadataProjectionResult{Status: "failed", Error: entry.Error, RetryAtMS: entry.RetryAtMS}, nil
}
if entry.Error == "" {
if err := json.Unmarshal(entry.Payload, &result); err != nil {
return result, err
}
return result, nil
}
} else if !errors.Is(err, sql.ErrNoRows) {
return result, err
}
payload, err := json.Marshal(metadataProjectionPayload{hash, kind, metadataProjectionVersion})
if err != nil {
return result, err
}
key := fmt.Sprintf("%d:%s:%s", metadataProjectionVersion, kind, hash)
queued, err := s.store.EnqueueEventWithOptions("", metadataProjectionEventType, key, string(payload), time.Now(), -5, 1)
if queued {
s.wakeEventScheduler()
}
return result, err
}
func metadataProjectionURL(kind, revisionUUID, semanticPath string) string {
query := url.Values{"rev": {revisionUUID}, "metadata": {"1"}}
if kind == "pdf" {
return "/v/pdf-page?" + query.Encode()
}
query.Set("path", semanticPath)
return "/v/archive?" + query.Encode()
}
func (s *Server) applyMetadataProjection(view *contentView, semanticPath string) {
view.MetadataURL = metadataProjectionURL(view.Kind, view.RevisionUUID, semanticPath)
result, err := s.requestMetadataProjection(view.DocumentHash, view.Kind)
if err != nil {
view.MetadataStatus = "unavailable"
view.ParseError = "Metadata queue is unavailable. Download remains available."
return
}
view.MetadataStatus, view.ArchiveTruncated = result.Status, result.Truncated
view.PDFPages, view.ParseError = result.PDFPages, result.Error
view.ArchiveEntries = archiveMetadataLinks(result.Entries, semanticPath, view.RevisionUUID)
}
func archiveMetadataLinks(entries []archiveEntryView, semanticPath, revisionUUID string) []archiveEntryView {
result := make([]archiveEntryView, len(entries))
copy(result, entries)
for index := range result {
query := url.Values{"path": {semanticPath}, "rev": {revisionUUID}, "entry": {result[index].Name}}
result[index].URL = "/v/archive?" + query.Encode()
}
return result
}
func (s *Server) serveMetadataProjection(w http.ResponseWriter, r *http.Request, hash, kind, semanticPath, revisionUUID string) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Cache-Control", "private, no-store")
w.Header().Set("Vary", "Authorization, Cookie")
w.Header().Set("X-Content-Type-Options", "nosniff")
result, err := s.requestMetadataProjection(hash, kind)
if err != nil {
w.Header().Set("Retry-After", "5")
w.WriteHeader(http.StatusServiceUnavailable)
_ = json.NewEncoder(w).Encode(metadataProjectionResult{Status: "unavailable", Error: "Metadata queue is unavailable"})
return
}
result.Entries = archiveMetadataLinks(result.Entries, semanticPath, revisionUUID)
if result.Status == "pending" {
w.Header().Set("Retry-After", "2")
w.WriteHeader(http.StatusAccepted)
} else if result.Status == "failed" {
w.Header().Set("Retry-After", strconv.FormatInt(max(1, (result.RetryAtMS-time.Now().UnixMilli())/1000), 10))
w.WriteHeader(http.StatusUnprocessableEntity)
}
_ = json.NewEncoder(w).Encode(result)
}
func (s *Server) generateMetadataProjection(payload metadataProjectionPayload) error {
if payload.Version != metadataProjectionVersion || (payload.Kind != "pdf" && payload.Kind != "archive") {
return errors.New("unsupported metadata projection")
}
if cached, err := s.store.GetProjectionMetadata(payload.DocumentHash, payload.Kind, payload.Version); err == nil && cached.Error == "" {
return nil
}
result, runErr := s.inspectProjectionMetadata(payload)
entry := store.ProjectionMetadata{DocumentHash: payload.DocumentHash, Kind: payload.Kind, Version: payload.Version}
if runErr != nil {
entry.Error = payload.Kind + " metadata inspection failed; retry is available after the cooldown"
entry.RetryAtMS = time.Now().Add(metadataFailureBackoff).UnixMilli()
entry.Payload = json.RawMessage(`{}`)
} else {
var err error
entry.Payload, err = json.Marshal(result)
if err != nil {
return err
}
}
if err := s.store.SaveProjectionMetadata(entry); err != nil {
return err
}
return runErr
}
func (s *Server) inspectProjectionMetadata(payload metadataProjectionPayload) (metadataProjectionResult, error) {
result := metadataProjectionResult{Status: "ready"}
source, err := s.store.DocumentPath(payload.DocumentHash)
if err != nil || source == "" {
return result, errors.New("metadata projection requires immutable Document storage")
}
info, err := os.Stat(source)
if err != nil {
return result, err
}
if !info.Mode().IsRegular() || info.Size() > maxMetadataInputBytes {
return result, errors.New("metadata source exceeds the 512 MiB limit")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if payload.Kind == "pdf" {
output, truncated, err := runMetadataCommand(ctx, 64<<10, "pdfinfo", "-enc", "UTF-8", source)
if err != nil || truncated {
return result, fmt.Errorf("bounded pdfinfo failed: %w", err)
}
for _, line := range strings.Split(string(output), "\n") {
if strings.HasPrefix(line, "Pages:") {
result.PDFPages, err = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "Pages:")))
if err == nil && result.PDFPages > 0 && result.PDFPages <= 10000 {
return result, nil
}
}
}
return result, errors.New("PDF page count is missing or exceeds 10000 pages")
}
output, truncated, err := runMetadataCommand(ctx, 4<<20, "7z", "l", "-slt", "-ba", "-sccUTF-8", "-mmt1", "-p-", "--", source)
if err != nil && !truncated {
return result, fmt.Errorf("bounded archive listing failed: %w", err)
}
return parseArchiveMetadata(output, truncated)
}
type boundedMetadataOutput struct {
buffer bytes.Buffer
limit int
truncated bool
cancel context.CancelFunc
}
func (output *boundedMetadataOutput) Write(data []byte) (int, error) {
if len(data) <= output.limit-output.buffer.Len() {
return output.buffer.Write(data)
}
remaining := max(0, output.limit-output.buffer.Len())
_, _ = output.buffer.Write(data[:remaining])
output.truncated = true
output.cancel()
return remaining, errors.New("metadata output limit exceeded")
}
func runMetadataCommand(ctx context.Context, limit int, name string, args ...string) ([]byte, bool, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
output := &boundedMetadataOutput{limit: limit, cancel: cancel}
command := exec.CommandContext(ctx, "nice", append([]string{"-n", "15", name}, args...)...)
command.Stdout, command.Stderr = output, io.Discard
command.Stdin = nil
command.WaitDelay = time.Second
err := command.Run()
return output.buffer.Bytes(), output.truncated, err
}
func parseArchiveMetadata(output []byte, truncated bool) (metadataProjectionResult, error) {
result := metadataProjectionResult{Status: "ready", Truncated: truncated}
if truncated {
// Never expose a final incomplete filename or numeric field.
if end := bytes.LastIndex(output, []byte("\n\n")); end >= 0 {
output = output[:end]
} else {
return result, errors.New("archive metadata record exceeds output limit")
}
}
entry := archiveEntryView{}
nameBytes := 0
flush := func() bool {
if entry.Name == "" {
return true
}
if len(result.Entries) >= maxArchiveMetadataEntries || nameBytes+len(entry.Name) > maxArchiveMetadataNameBytes {
result.Truncated = true
return false
}
ext := strings.ToLower(filepath.Ext(entry.Name))
entry.IsImage = !entry.IsDir && (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif" || ext == ".webp" || ext == ".avif")
result.Entries = append(result.Entries, entry)
nameBytes += len(entry.Name)
entry = archiveEntryView{}
return true
}
for _, line := range strings.Split(string(output), "\n") {
line = strings.TrimSuffix(line, "\r")
switch {
case strings.TrimSpace(line) == "":
if !flush() {
return result, nil
}
case strings.HasPrefix(line, "Path = "):
entry.Name = strings.TrimPrefix(line, "Path = ")
if len(entry.Name) > 4096 || !utf8.ValidString(entry.Name) || strings.IndexFunc(entry.Name, unicode.IsControl) >= 0 {
return result, errors.New("invalid archive entry name")
}
case strings.HasPrefix(line, "Size = "):
var err error
entry.Size, err = strconv.ParseUint(strings.TrimSpace(strings.TrimPrefix(line, "Size = ")), 10, 64)
if err != nil {
return result, errors.New("invalid archive entry size")
}
case strings.HasPrefix(line, "Folder = "):
entry.IsDir = strings.TrimSpace(strings.TrimPrefix(line, "Folder = ")) == "+"
case strings.HasPrefix(line, "Attributes = "):
entry.IsDir = entry.IsDir || strings.Contains(strings.TrimPrefix(line, "Attributes = "), "D")
}
}
flush()
return result, nil
}