Capture accumulated application, UI, CLI, regression tests and design documentation changes, including task project mapping and default CalDAV list naming. Keep deployment data, backups, local credentials and build artifacts out of Git. Move the Compose enrollment token to an ignored local environment file.
279 lines
9.9 KiB
Go
279 lines
9.9 KiB
Go
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
"pageswiki/internal/store"
|
|
)
|
|
|
|
const (
|
|
metadataProjectionEventType = "projection.metadata.generate"
|
|
metadataProjectionVersion = 1
|
|
maxArchiveMetadataEntries = 2000
|
|
maxArchiveMetadataNameBytes = 256 << 10
|
|
maxMetadataInputBytes = 512 << 20
|
|
metadataFailureBackoff = 5 * time.Minute
|
|
)
|
|
|
|
type metadataProjectionPayload struct {
|
|
DocumentHash string `json:"document_hash"`
|
|
Kind string `json:"kind"`
|
|
Version int `json:"version"`
|
|
}
|
|
|
|
type metadataProjectionResult struct {
|
|
Status string `json:"status"`
|
|
PDFPages int `json:"pdf_pages,omitempty"`
|
|
Entries []archiveEntryView `json:"entries,omitempty"`
|
|
Truncated bool `json:"truncated,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
RetryAtMS int64 `json:"retry_at_ms,omitempty"`
|
|
}
|
|
|
|
func (s *Server) requestMetadataProjection(hash, kind string) (metadataProjectionResult, error) {
|
|
result := metadataProjectionResult{Status: "pending"}
|
|
entry, err := s.store.GetProjectionMetadata(hash, kind, metadataProjectionVersion)
|
|
if err == nil {
|
|
if entry.Error != "" && entry.RetryAtMS > time.Now().UnixMilli() {
|
|
return metadataProjectionResult{Status: "failed", Error: entry.Error, RetryAtMS: entry.RetryAtMS}, nil
|
|
}
|
|
if entry.Error == "" {
|
|
if err := json.Unmarshal(entry.Payload, &result); err != nil {
|
|
return result, err
|
|
}
|
|
return result, nil
|
|
}
|
|
} else if !errors.Is(err, sql.ErrNoRows) {
|
|
return result, err
|
|
}
|
|
payload, err := json.Marshal(metadataProjectionPayload{hash, kind, metadataProjectionVersion})
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
key := fmt.Sprintf("%d:%s:%s", metadataProjectionVersion, kind, hash)
|
|
queued, err := s.store.EnqueueEventWithOptions("", metadataProjectionEventType, key, string(payload), time.Now(), -5, 1)
|
|
if queued {
|
|
s.wakeEventScheduler()
|
|
}
|
|
return result, err
|
|
}
|
|
|
|
func metadataProjectionURL(kind, revisionUUID, semanticPath string) string {
|
|
query := url.Values{"rev": {revisionUUID}, "metadata": {"1"}}
|
|
if kind == "pdf" {
|
|
return "/v/pdf-page?" + query.Encode()
|
|
}
|
|
query.Set("path", semanticPath)
|
|
return "/v/archive?" + query.Encode()
|
|
}
|
|
|
|
func (s *Server) applyMetadataProjection(view *contentView, semanticPath string) {
|
|
view.MetadataURL = metadataProjectionURL(view.Kind, view.RevisionUUID, semanticPath)
|
|
result, err := s.requestMetadataProjection(view.DocumentHash, view.Kind)
|
|
if err != nil {
|
|
view.MetadataStatus = "unavailable"
|
|
view.ParseError = "Metadata queue is unavailable. Download remains available."
|
|
return
|
|
}
|
|
view.MetadataStatus, view.ArchiveTruncated = result.Status, result.Truncated
|
|
view.PDFPages, view.ParseError = result.PDFPages, result.Error
|
|
view.ArchiveEntries = archiveMetadataLinks(result.Entries, semanticPath, view.RevisionUUID)
|
|
}
|
|
|
|
func archiveMetadataLinks(entries []archiveEntryView, semanticPath, revisionUUID string) []archiveEntryView {
|
|
result := make([]archiveEntryView, len(entries))
|
|
copy(result, entries)
|
|
for index := range result {
|
|
query := url.Values{"path": {semanticPath}, "rev": {revisionUUID}, "entry": {result[index].Name}}
|
|
result[index].URL = "/v/archive?" + query.Encode()
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (s *Server) serveMetadataProjection(w http.ResponseWriter, r *http.Request, hash, kind, semanticPath, revisionUUID string) {
|
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
|
w.Header().Set("Cache-Control", "private, no-store")
|
|
w.Header().Set("Vary", "Authorization, Cookie")
|
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
result, err := s.requestMetadataProjection(hash, kind)
|
|
if err != nil {
|
|
w.Header().Set("Retry-After", "5")
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
_ = json.NewEncoder(w).Encode(metadataProjectionResult{Status: "unavailable", Error: "Metadata queue is unavailable"})
|
|
return
|
|
}
|
|
result.Entries = archiveMetadataLinks(result.Entries, semanticPath, revisionUUID)
|
|
if result.Status == "pending" {
|
|
w.Header().Set("Retry-After", "2")
|
|
w.WriteHeader(http.StatusAccepted)
|
|
} else if result.Status == "failed" {
|
|
w.Header().Set("Retry-After", strconv.FormatInt(max(1, (result.RetryAtMS-time.Now().UnixMilli())/1000), 10))
|
|
w.WriteHeader(http.StatusUnprocessableEntity)
|
|
}
|
|
_ = json.NewEncoder(w).Encode(result)
|
|
}
|
|
|
|
func (s *Server) generateMetadataProjection(payload metadataProjectionPayload) error {
|
|
if payload.Version != metadataProjectionVersion || (payload.Kind != "pdf" && payload.Kind != "archive") {
|
|
return errors.New("unsupported metadata projection")
|
|
}
|
|
if cached, err := s.store.GetProjectionMetadata(payload.DocumentHash, payload.Kind, payload.Version); err == nil && cached.Error == "" {
|
|
return nil
|
|
}
|
|
result, runErr := s.inspectProjectionMetadata(payload)
|
|
entry := store.ProjectionMetadata{DocumentHash: payload.DocumentHash, Kind: payload.Kind, Version: payload.Version}
|
|
if runErr != nil {
|
|
entry.Error = payload.Kind + " metadata inspection failed; retry is available after the cooldown"
|
|
entry.RetryAtMS = time.Now().Add(metadataFailureBackoff).UnixMilli()
|
|
entry.Payload = json.RawMessage(`{}`)
|
|
} else {
|
|
var err error
|
|
entry.Payload, err = json.Marshal(result)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := s.store.SaveProjectionMetadata(entry); err != nil {
|
|
return err
|
|
}
|
|
return runErr
|
|
}
|
|
|
|
func (s *Server) inspectProjectionMetadata(payload metadataProjectionPayload) (metadataProjectionResult, error) {
|
|
result := metadataProjectionResult{Status: "ready"}
|
|
source, err := s.store.DocumentPath(payload.DocumentHash)
|
|
if err != nil || source == "" {
|
|
return result, errors.New("metadata projection requires immutable Document storage")
|
|
}
|
|
info, err := os.Stat(source)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
if !info.Mode().IsRegular() || info.Size() > maxMetadataInputBytes {
|
|
return result, errors.New("metadata source exceeds the 512 MiB limit")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
if payload.Kind == "pdf" {
|
|
output, truncated, err := runMetadataCommand(ctx, 64<<10, "pdfinfo", "-enc", "UTF-8", source)
|
|
if err != nil || truncated {
|
|
return result, fmt.Errorf("bounded pdfinfo failed: %w", err)
|
|
}
|
|
for _, line := range strings.Split(string(output), "\n") {
|
|
if strings.HasPrefix(line, "Pages:") {
|
|
result.PDFPages, err = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "Pages:")))
|
|
if err == nil && result.PDFPages > 0 && result.PDFPages <= 10000 {
|
|
return result, nil
|
|
}
|
|
}
|
|
}
|
|
return result, errors.New("PDF page count is missing or exceeds 10000 pages")
|
|
}
|
|
output, truncated, err := runMetadataCommand(ctx, 4<<20, "7z", "l", "-slt", "-ba", "-sccUTF-8", "-mmt1", "-p-", "--", source)
|
|
if err != nil && !truncated {
|
|
return result, fmt.Errorf("bounded archive listing failed: %w", err)
|
|
}
|
|
return parseArchiveMetadata(output, truncated)
|
|
}
|
|
|
|
type boundedMetadataOutput struct {
|
|
buffer bytes.Buffer
|
|
limit int
|
|
truncated bool
|
|
cancel context.CancelFunc
|
|
}
|
|
|
|
func (output *boundedMetadataOutput) Write(data []byte) (int, error) {
|
|
if len(data) <= output.limit-output.buffer.Len() {
|
|
return output.buffer.Write(data)
|
|
}
|
|
remaining := max(0, output.limit-output.buffer.Len())
|
|
_, _ = output.buffer.Write(data[:remaining])
|
|
output.truncated = true
|
|
output.cancel()
|
|
return remaining, errors.New("metadata output limit exceeded")
|
|
}
|
|
|
|
func runMetadataCommand(ctx context.Context, limit int, name string, args ...string) ([]byte, bool, error) {
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
output := &boundedMetadataOutput{limit: limit, cancel: cancel}
|
|
command := exec.CommandContext(ctx, "nice", append([]string{"-n", "15", name}, args...)...)
|
|
command.Stdout, command.Stderr = output, io.Discard
|
|
command.Stdin = nil
|
|
command.WaitDelay = time.Second
|
|
err := command.Run()
|
|
return output.buffer.Bytes(), output.truncated, err
|
|
}
|
|
|
|
func parseArchiveMetadata(output []byte, truncated bool) (metadataProjectionResult, error) {
|
|
result := metadataProjectionResult{Status: "ready", Truncated: truncated}
|
|
if truncated {
|
|
// Never expose a final incomplete filename or numeric field.
|
|
if end := bytes.LastIndex(output, []byte("\n\n")); end >= 0 {
|
|
output = output[:end]
|
|
} else {
|
|
return result, errors.New("archive metadata record exceeds output limit")
|
|
}
|
|
}
|
|
entry := archiveEntryView{}
|
|
nameBytes := 0
|
|
flush := func() bool {
|
|
if entry.Name == "" {
|
|
return true
|
|
}
|
|
if len(result.Entries) >= maxArchiveMetadataEntries || nameBytes+len(entry.Name) > maxArchiveMetadataNameBytes {
|
|
result.Truncated = true
|
|
return false
|
|
}
|
|
ext := strings.ToLower(filepath.Ext(entry.Name))
|
|
entry.IsImage = !entry.IsDir && (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif" || ext == ".webp" || ext == ".avif")
|
|
result.Entries = append(result.Entries, entry)
|
|
nameBytes += len(entry.Name)
|
|
entry = archiveEntryView{}
|
|
return true
|
|
}
|
|
for _, line := range strings.Split(string(output), "\n") {
|
|
line = strings.TrimSuffix(line, "\r")
|
|
switch {
|
|
case strings.TrimSpace(line) == "":
|
|
if !flush() {
|
|
return result, nil
|
|
}
|
|
case strings.HasPrefix(line, "Path = "):
|
|
entry.Name = strings.TrimPrefix(line, "Path = ")
|
|
if len(entry.Name) > 4096 || !utf8.ValidString(entry.Name) || strings.IndexFunc(entry.Name, unicode.IsControl) >= 0 {
|
|
return result, errors.New("invalid archive entry name")
|
|
}
|
|
case strings.HasPrefix(line, "Size = "):
|
|
var err error
|
|
entry.Size, err = strconv.ParseUint(strings.TrimSpace(strings.TrimPrefix(line, "Size = ")), 10, 64)
|
|
if err != nil {
|
|
return result, errors.New("invalid archive entry size")
|
|
}
|
|
case strings.HasPrefix(line, "Folder = "):
|
|
entry.IsDir = strings.TrimSpace(strings.TrimPrefix(line, "Folder = ")) == "+"
|
|
case strings.HasPrefix(line, "Attributes = "):
|
|
entry.IsDir = entry.IsDir || strings.Contains(strings.TrimPrefix(line, "Attributes = "), "D")
|
|
}
|
|
}
|
|
flush()
|
|
return result, nil
|
|
}
|