package server import ( "bytes" "context" "database/sql" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "os/exec" "path/filepath" "strconv" "strings" "time" "unicode" "unicode/utf8" "pageswiki/internal/store" ) const ( metadataProjectionEventType = "projection.metadata.generate" metadataProjectionVersion = 1 maxArchiveMetadataEntries = 2000 maxArchiveMetadataNameBytes = 256 << 10 maxMetadataInputBytes = 512 << 20 metadataFailureBackoff = 5 * time.Minute ) type metadataProjectionPayload struct { DocumentHash string `json:"document_hash"` Kind string `json:"kind"` Version int `json:"version"` } type metadataProjectionResult struct { Status string `json:"status"` PDFPages int `json:"pdf_pages,omitempty"` Entries []archiveEntryView `json:"entries,omitempty"` Truncated bool `json:"truncated,omitempty"` Error string `json:"error,omitempty"` RetryAtMS int64 `json:"retry_at_ms,omitempty"` } func (s *Server) requestMetadataProjection(hash, kind string) (metadataProjectionResult, error) { result := metadataProjectionResult{Status: "pending"} entry, err := s.store.GetProjectionMetadata(hash, kind, metadataProjectionVersion) if err == nil { if entry.Error != "" && entry.RetryAtMS > time.Now().UnixMilli() { return metadataProjectionResult{Status: "failed", Error: entry.Error, RetryAtMS: entry.RetryAtMS}, nil } if entry.Error == "" { if err := json.Unmarshal(entry.Payload, &result); err != nil { return result, err } return result, nil } } else if !errors.Is(err, sql.ErrNoRows) { return result, err } payload, err := json.Marshal(metadataProjectionPayload{hash, kind, metadataProjectionVersion}) if err != nil { return result, err } key := fmt.Sprintf("%d:%s:%s", metadataProjectionVersion, kind, hash) queued, err := s.store.EnqueueEventWithOptions("", metadataProjectionEventType, key, string(payload), time.Now(), -5, 1) if queued { s.wakeEventScheduler() } return result, err } func metadataProjectionURL(kind, revisionUUID, semanticPath string) string { query := url.Values{"rev": {revisionUUID}, "metadata": {"1"}} if kind == "pdf" { return "/v/pdf-page?" + query.Encode() } query.Set("path", semanticPath) return "/v/archive?" + query.Encode() } func (s *Server) applyMetadataProjection(view *contentView, semanticPath string) { view.MetadataURL = metadataProjectionURL(view.Kind, view.RevisionUUID, semanticPath) result, err := s.requestMetadataProjection(view.DocumentHash, view.Kind) if err != nil { view.MetadataStatus = "unavailable" view.ParseError = "Metadata queue is unavailable. Download remains available." return } view.MetadataStatus, view.ArchiveTruncated = result.Status, result.Truncated view.PDFPages, view.ParseError = result.PDFPages, result.Error view.ArchiveEntries = archiveMetadataLinks(result.Entries, semanticPath, view.RevisionUUID) } func archiveMetadataLinks(entries []archiveEntryView, semanticPath, revisionUUID string) []archiveEntryView { result := make([]archiveEntryView, len(entries)) copy(result, entries) for index := range result { query := url.Values{"path": {semanticPath}, "rev": {revisionUUID}, "entry": {result[index].Name}} result[index].URL = "/v/archive?" + query.Encode() } return result } func (s *Server) serveMetadataProjection(w http.ResponseWriter, r *http.Request, hash, kind, semanticPath, revisionUUID string) { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("Cache-Control", "private, no-store") w.Header().Set("Vary", "Authorization, Cookie") w.Header().Set("X-Content-Type-Options", "nosniff") result, err := s.requestMetadataProjection(hash, kind) if err != nil { w.Header().Set("Retry-After", "5") w.WriteHeader(http.StatusServiceUnavailable) _ = json.NewEncoder(w).Encode(metadataProjectionResult{Status: "unavailable", Error: "Metadata queue is unavailable"}) return } result.Entries = archiveMetadataLinks(result.Entries, semanticPath, revisionUUID) if result.Status == "pending" { w.Header().Set("Retry-After", "2") w.WriteHeader(http.StatusAccepted) } else if result.Status == "failed" { w.Header().Set("Retry-After", strconv.FormatInt(max(1, (result.RetryAtMS-time.Now().UnixMilli())/1000), 10)) w.WriteHeader(http.StatusUnprocessableEntity) } _ = json.NewEncoder(w).Encode(result) } func (s *Server) generateMetadataProjection(payload metadataProjectionPayload) error { if payload.Version != metadataProjectionVersion || (payload.Kind != "pdf" && payload.Kind != "archive") { return errors.New("unsupported metadata projection") } if cached, err := s.store.GetProjectionMetadata(payload.DocumentHash, payload.Kind, payload.Version); err == nil && cached.Error == "" { return nil } result, runErr := s.inspectProjectionMetadata(payload) entry := store.ProjectionMetadata{DocumentHash: payload.DocumentHash, Kind: payload.Kind, Version: payload.Version} if runErr != nil { entry.Error = payload.Kind + " metadata inspection failed; retry is available after the cooldown" entry.RetryAtMS = time.Now().Add(metadataFailureBackoff).UnixMilli() entry.Payload = json.RawMessage(`{}`) } else { var err error entry.Payload, err = json.Marshal(result) if err != nil { return err } } if err := s.store.SaveProjectionMetadata(entry); err != nil { return err } return runErr } func (s *Server) inspectProjectionMetadata(payload metadataProjectionPayload) (metadataProjectionResult, error) { result := metadataProjectionResult{Status: "ready"} ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() lease, err := s.store.MaterializeDocument(ctx, payload.DocumentHash, maxMetadataInputBytes) if err != nil { return result, err } defer lease.Close() source := lease.Path if payload.Kind == "pdf" { output, truncated, err := runMetadataCommand(ctx, 64<<10, "pdfinfo", "-enc", "UTF-8", source) if err != nil || truncated { return result, fmt.Errorf("bounded pdfinfo failed: %w", err) } for _, line := range strings.Split(string(output), "\n") { if strings.HasPrefix(line, "Pages:") { result.PDFPages, err = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "Pages:"))) if err == nil && result.PDFPages > 0 && result.PDFPages <= 10000 { return result, nil } } } return result, errors.New("PDF page count is missing or exceeds 10000 pages") } output, truncated, err := runMetadataCommand(ctx, 4<<20, "7z", "l", "-slt", "-ba", "-sccUTF-8", "-mmt1", "-p-", "--", source) if err != nil && !truncated { return result, fmt.Errorf("bounded archive listing failed: %w", err) } return parseArchiveMetadata(output, truncated) } type boundedMetadataOutput struct { buffer bytes.Buffer limit int truncated bool cancel context.CancelFunc } func (output *boundedMetadataOutput) Write(data []byte) (int, error) { if len(data) <= output.limit-output.buffer.Len() { return output.buffer.Write(data) } remaining := max(0, output.limit-output.buffer.Len()) _, _ = output.buffer.Write(data[:remaining]) output.truncated = true output.cancel() return remaining, errors.New("metadata output limit exceeded") } func runMetadataCommand(ctx context.Context, limit int, name string, args ...string) ([]byte, bool, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() output := &boundedMetadataOutput{limit: limit, cancel: cancel} command := exec.CommandContext(ctx, "nice", append([]string{"-n", "15", name}, args...)...) command.Stdout, command.Stderr = output, io.Discard command.Stdin = nil command.WaitDelay = time.Second err := command.Run() return output.buffer.Bytes(), output.truncated, err } func parseArchiveMetadata(output []byte, truncated bool) (metadataProjectionResult, error) { result := metadataProjectionResult{Status: "ready", Truncated: truncated} if truncated { // Never expose a final incomplete filename or numeric field. if end := bytes.LastIndex(output, []byte("\n\n")); end >= 0 { output = output[:end] } else { return result, errors.New("archive metadata record exceeds output limit") } } entry := archiveEntryView{} nameBytes := 0 flush := func() bool { if entry.Name == "" { return true } if len(result.Entries) >= maxArchiveMetadataEntries || nameBytes+len(entry.Name) > maxArchiveMetadataNameBytes { result.Truncated = true return false } ext := strings.ToLower(filepath.Ext(entry.Name)) entry.IsImage = !entry.IsDir && (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif" || ext == ".webp" || ext == ".avif") result.Entries = append(result.Entries, entry) nameBytes += len(entry.Name) entry = archiveEntryView{} return true } for _, line := range strings.Split(string(output), "\n") { line = strings.TrimSuffix(line, "\r") switch { case strings.TrimSpace(line) == "": if !flush() { return result, nil } case strings.HasPrefix(line, "Path = "): entry.Name = strings.TrimPrefix(line, "Path = ") if len(entry.Name) > 4096 || !utf8.ValidString(entry.Name) || strings.IndexFunc(entry.Name, unicode.IsControl) >= 0 { return result, errors.New("invalid archive entry name") } case strings.HasPrefix(line, "Size = "): var err error entry.Size, err = strconv.ParseUint(strings.TrimSpace(strings.TrimPrefix(line, "Size = ")), 10, 64) if err != nil { return result, errors.New("invalid archive entry size") } case strings.HasPrefix(line, "Folder = "): entry.IsDir = strings.TrimSpace(strings.TrimPrefix(line, "Folder = ")) == "+" case strings.HasPrefix(line, "Attributes = "): entry.IsDir = entry.IsDir || strings.Contains(strings.TrimPrefix(line, "Attributes = "), "D") } } flush() return result, nil }