package server import ( "archive/zip" "context" "database/sql" "encoding/json" "errors" "fmt" "html" "image" _ "image/gif" "image/jpeg" _ "image/png" "io" "log" "net/http" "os" "os/exec" "path/filepath" "strconv" "strings" "time" "pageswiki/internal/store" ) const previewEventType = "preview.generate" const previewSweepEventType = "preview.sweep" const cityReconcileEventType = "city.reconcile" var cityActivitySubscription = store.ActivitySubscription{ConsumerID: "city", EventTypes: []string{"page.created", "revision.created", "path.renamed", "path.unlinked", "object.deleted"}} type previewEventPayload struct { RevisionUUID string `json:"revision_uuid"` DocumentHash string `json:"document_hash"` ContentType string `json:"content_type"` Filename string `json:"filename"` Kind string `json:"kind"` Page int `json:"page,omitempty"` } type cityReconcilePayload struct { DomainUUID string `json:"domain_uuid"` } func (s *Server) initializePreviewScheduler() { if s.store == nil || s.store.DocumentRoot == "" { return } s.previewRoot = filepath.Join(filepath.Dir(s.store.DocumentRoot), "previews") if err := os.MkdirAll(s.previewRoot, 0755); err != nil { log.Printf("preview cache initialization failed: %v", err) s.previewRoot = "" } } func (s *Server) startEventScheduler() { if s.store == nil { return } if err := s.store.EnsureBuiltinResourceEvent(); err != nil { log.Printf("resource monitor initialization failed: %v", err) } if err := s.ensureCalendarSweep(); err != nil { log.Printf("calendar scheduler initialization failed: %v", err) } if err := s.store.EnsureTaskActivityBridgeEvent(); err != nil { log.Printf("task journal bridge initialization failed: %v", err) } if err := s.store.EnsureTaskReminderEvent(); err != nil { log.Printf("task reminder initialization failed: %v", err) } if err := s.store.EnsureTaskFocusEvent(); err != nil { log.Printf("focus timer initialization failed: %v", err) } if err := s.store.EnsureNotificationPushEvent(); err != nil { log.Printf("notification push initialization failed: %v", err) } go func() { initialPreviewSweep := time.NewTimer(5 * time.Second) defer initialPreviewSweep.Stop() previewSweep := time.NewTicker(15 * time.Minute) defer previewSweep.Stop() dispatch := time.NewTicker(time.Second) defer dispatch.Stop() cleanup := time.NewTicker(time.Hour) defer cleanup.Stop() activitySweep := time.NewTicker(10 * time.Second) defer activitySweep.Stop() for { select { case <-initialPreviewSweep.C: s.enqueueAllPreviewEvents() case <-previewSweep.C: s.enqueueAllPreviewEvents() case <-dispatch.C: case <-cleanup.C: if settings, err := s.store.GetQueueSettings(); err == nil { if err := s.store.PurgeEventHistory(settings.RetentionHours); err != nil { log.Printf("event history cleanup failed: %v", err) } } case <-activitySweep.C: if _, err := s.store.ExpireApprovalRequests(context.Background(), time.Now().UnixMilli(), 100); err != nil { log.Printf("approval request expiry failed: %v", err) } s.enqueueCityActivityEvents() case <-s.schedulerWake: if _, err := s.store.ExpireApprovalRequests(context.Background(), time.Now().UnixMilli(), 100); err != nil { log.Printf("approval request expiry failed: %v", err) } s.enqueueCityActivityEvents() } if _, err := s.store.EnqueueDueDefinitions(time.Now()); err != nil { log.Printf("event schedule evaluation failed: %v", err) } s.runDueEvents() } }() } func (s *Server) wakeEventScheduler() { if s.schedulerWake == nil { return } select { case s.schedulerWake <- struct{}{}: default: } } func (s *Server) runDueEvents() { for { settings, err := s.store.GetQueueSettings() if err != nil { log.Printf("event scheduler settings failed: %v", err) return } if settings.Paused || s.schedulerRunning.Load() >= int64(settings.Concurrency) { return } // Keep one reserved background slot while foreground work is active. // A global stop lets long-lived editor/WebSocket requests starve jobs. if s.foregroundRequests.Load() > 0 && s.schedulerRunning.Load() > 0 { return } job, err := s.store.ClaimDueEvent() if errors.Is(err, sql.ErrNoRows) || errors.Is(err, context.Canceled) { return } if err != nil { log.Printf("event scheduler claim failed: %v", err) return } s.schedulerRunning.Add(1) _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", "execution started") go func(job *store.EventJob) { defer func() { s.schedulerRunning.Add(-1) s.wakeEventScheduler() }() var runErr error if s.store.EventCancellationRequested(job.UUID) { runErr = errors.New("cancelled before execution") } else { runErr = s.runEvent(job) if runErr == nil && s.store.EventCancellationRequested(job.UUID) { runErr = errors.New("cancelled during execution") } } if finishErr := s.store.FinishEvent(job.UUID, runErr); finishErr != nil { log.Printf("event scheduler finish failed job=%s: %v", job.UUID, finishErr) } if runErr != nil { log.Printf("event job failed type=%s key=%s: %v", job.Type, job.Key, runErr) } }(job) } } func (s *Server) runEvent(job *store.EventJob) error { switch job.Type { case store.NotificationPushEventType: return s.deliverNotificationPush(job) case store.TaskFocusEventType: processed, err := s.store.CompleteTaskFocus(context.Background(), time.Now(), 100) if err == nil && processed > 0 { _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("focus timers completed=%d", processed)) } return err case store.TaskReminderEventType: checked, sent, err := s.store.DeliverTaskReminders(context.Background(), time.Now(), 100) if err == nil && checked > 0 { _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("task reminders checked=%d delivered=%d", checked, sent)) } return err case store.TaskActivityBridgeEventType: processed, published, err := s.store.PublishTaskActivity(context.Background(), 100) if err == nil && processed > 0 { _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("journal entries checked=%d published=%d", processed, published)) } return err case calendarSweepType: return s.sweepCalendarSources() case calendarSyncType: return s.runCalendarSource(job) case metadataProjectionEventType: var payload metadataProjectionPayload if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { return err } return s.generateMetadataProjection(payload) case store.ResourceEventType: snapshot := s.resourceStatus() payload, err := json.Marshal(snapshot) if err != nil { return err } return s.store.SaveResourceSample(job.UUID, snapshot["timestamp_ms"].(int64), payload) case previewSweepEventType: s.enqueueAllPreviewEvents() return nil case previewEventType: var payload previewEventPayload if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { return err } return s.generatePreview(payload) case cityReconcileEventType: var payload cityReconcilePayload if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { return err } return s.reconcileCityActivity(payload.DomainUUID) case "pdf.page.generate": var payload previewEventPayload if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { return err } return s.generatePDFPage(payload) case importAnalyzeEventType: var payload importAnalyzePayload if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { return err } return s.analyzeImport(payload) case importCommitEventType: var payload importCommitPayload if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { return err } return s.commitImport(payload) default: return fmt.Errorf("unknown event type %q", job.Type) } } func (s *Server) enqueueCityActivityEvents() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() domains, err := s.store.PendingActivityConsumerDomains(ctx, cityActivitySubscription, 100) if err != nil { log.Printf("activity outbox domain scan failed: %v", err) return } for _, domain := range domains { payload, err := json.Marshal(cityReconcilePayload{DomainUUID: domain}) if err != nil { continue } _, err = s.store.EnqueueEventWithOptions("", cityReconcileEventType, "city:"+domain, string(payload), time.Now(), -2, 3) if err != nil { log.Printf("city activity enqueue failed domain=%s: %v", domain, err) } } } func (s *Server) reconcileCityActivity(domainUUID string) error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() events, err := s.store.ActivityOutboxForConsumer(ctx, cityActivitySubscription, domainUUID, 500) if err != nil { return err } if len(events) == 0 { return nil } if _, err := s.applyCityAutoPlan(domainUUID, "system:city"); err != nil { return err } return s.store.CompleteActivityOutboxThrough("city", domainUUID, events[len(events)-1].Sequence) } func (s *Server) generatePDFPage(payload previewEventPayload) error { if payload.Page < 1 || payload.DocumentHash == "" { return fmt.Errorf("invalid PDF page job") } target := filepath.Join(s.previewRoot, fmt.Sprintf("%s-p%d.jpg", payload.DocumentHash, payload.Page)) if info, err := os.Stat(target); err == nil && info.Size() > 0 { return nil } source, err := s.store.DocumentPath(payload.DocumentHash) if err != nil { return err } workDir, err := os.MkdirTemp(s.previewRoot, ".pdf-page-*") if err != nil { return err } defer os.RemoveAll(workDir) base := filepath.Join(workDir, "page") if err := runPreviewCommand("pdftoppm", "-f", strconv.Itoa(payload.Page), "-l", strconv.Itoa(payload.Page), "-singlefile", "-jpeg", "-scale-to", "1600", source, base); err != nil { return err } return os.Rename(base+".jpg", target) } func (s *Server) previewCachePath(hash, kind string) string { extension := ".jpg" if kind == "excalidraw" { extension = ".svg" } return filepath.Join(s.previewRoot, hash+extension) } func (s *Server) previewCacheURL(hash, kind string) string { extension := ".jpg" if kind == "excalidraw" { extension = ".svg" } return "/v/preview-cache/" + hash + extension } func (s *Server) handlePreviewCache(w http.ResponseWriter, r *http.Request) { if s.previewRoot == "" { http.NotFound(w, r) return } name := filepath.Base(strings.TrimPrefix(r.URL.Path, "/v/preview-cache/")) if name == "." || name == "" || (!strings.HasSuffix(name, ".jpg") && !strings.HasSuffix(name, ".svg")) { http.NotFound(w, r) return } path := filepath.Join(s.previewRoot, name) if _, err := os.Stat(path); err != nil { http.NotFound(w, r) return } w.Header().Set("Cache-Control", "private, max-age=31536000, immutable") w.Header().Set("Vary", "Authorization, Cookie") w.Header().Set("X-Content-Type-Options", "nosniff") if strings.HasSuffix(name, ".svg") { w.Header().Set("Content-Security-Policy", "sandbox") } http.ServeFile(w, r, path) } func (s *Server) enqueuePreview(entry store.BrowserEntry) (string, bool) { if s.previewRoot == "" || entry.DocumentHash == "" { return "", false } filename := entry.Filename if filename == "" { filename = filepath.Base(entry.Path) } adapter := selectProjectionAdapter(contentDescriptor{ MediaType: strings.ToLower(strings.Split(entry.ContentType, ";")[0]), Filename: filename, }) kind := adapter.ID if kind != "image" && kind != "pdf" && kind != "video" && kind != "excalidraw" && kind != "archive" { return "", false } cachePath := s.previewCachePath(entry.DocumentHash, kind) cacheURL := s.previewCacheURL(entry.DocumentHash, kind) if info, err := os.Stat(cachePath); err == nil && info.Size() > 0 { return cacheURL, true } payload, err := json.Marshal(previewEventPayload{ RevisionUUID: entry.RevisionUUID, DocumentHash: entry.DocumentHash, ContentType: entry.ContentType, Filename: filename, Kind: kind, }) if err == nil { if queued, queueErr := s.store.EnqueueEvent( previewEventType, entry.DocumentHash, string(payload), time.Now(), ); queueErr == nil && queued { s.wakeEventScheduler() } } return cacheURL, false } func (s *Server) enqueueAllPreviewEvents() int { domains, err := s.store.ListDomains() if err != nil { return 0 } count := 0 for _, domain := range domains { if !domain.Enabled || domain.Placement != "local" { continue } entries, err := s.store.ListPreviewCandidatesInDomain(domain.UUID) if err != nil { continue } for _, entry := range entries { if _, cached := s.enqueuePreview(entry); !cached { count++ } } } s.wakeEventScheduler() return count } func (s *Server) generatePreview(payload previewEventPayload) error { target := s.previewCachePath(payload.DocumentHash, payload.Kind) if info, err := os.Stat(target); err == nil && info.Size() > 0 { return nil } source, err := s.store.DocumentPath(payload.DocumentHash) if err != nil { return err } workDir, err := os.MkdirTemp(s.previewRoot, ".job-*") if err != nil { return err } defer os.RemoveAll(workDir) var generated string switch payload.Kind { case "image": generated = filepath.Join(workDir, "preview.jpg") err = generateImageThumbnail(source, generated) case "pdf": generated = filepath.Join(workDir, "preview.jpg") err = runPreviewCommand( "pdftoppm", "-f", "1", "-singlefile", "-jpeg", "-scale-to", "480", source, strings.TrimSuffix(generated, ".jpg"), ) case "video": generated = filepath.Join(workDir, "preview.jpg") err = runPreviewCommand( "ffmpeg", "-hide_banner", "-loglevel", "error", "-ss", "1", "-i", source, "-frames:v", "1", "-vf", "scale=480:-2", "-y", generated, ) case "excalidraw": generated = filepath.Join(workDir, "preview.svg") err = generateExcalidrawSVG(source, generated) case "archive": generated = filepath.Join(workDir, "preview.jpg") err = generateArchiveThumbnail(source, generated, payload.Filename) default: return fmt.Errorf("unsupported preview kind %q", payload.Kind) } if err != nil { return err } if err := os.Chmod(generated, 0644); err != nil { return err } return os.Rename(generated, target) } func runPreviewCommand(name string, args ...string) error { ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() command := exec.CommandContext(ctx, "nice", append([]string{"-n", "15", name}, args...)...) output, err := command.CombinedOutput() if ctx.Err() != nil { return ctx.Err() } if err != nil { return fmt.Errorf("%s failed: %w: %s", name, err, strings.TrimSpace(string(output))) } return nil } func generateImageThumbnail(source, target string) error { file, err := os.Open(source) if err != nil { return err } defer file.Close() return generateImageThumbnailFromReader(file, target) } func generateArchiveThumbnail(source, target, filename string) error { if strings.EqualFold(filepath.Ext(filename), ".zip") { return generateZIPThumbnail(source, target) } return store.ErrEventSkipped } func generateZIPThumbnail(source, target string) error { reader, err := zip.OpenReader(source) if err != nil { return err } defer reader.Close() imageCount, fileCount := 0, 0 for _, file := range reader.File { if file.FileInfo().IsDir() { continue } fileCount++ if archiveImageName(file.Name) { imageCount++ } } if imageCount < 3 || imageCount*2 < fileCount { return fmt.Errorf("%w: archive is not image-dominant", store.ErrEventSkipped) } for _, file := range reader.File { if file.FileInfo().IsDir() || !archiveImageName(file.Name) { continue } input, err := file.Open() if err != nil { return err } err = generateImageThumbnailFromReader(input, target) input.Close() if err != nil { return err } return nil } return fmt.Errorf("%w: archive has no supported comic images", store.ErrEventSkipped) } func archiveImageName(name string) bool { switch strings.ToLower(filepath.Ext(name)) { case ".jpg", ".jpeg", ".png", ".gif": return true default: return false } } func generateImageThumbnailFromReader(input io.Reader, target string) error { decoded, _, err := image.Decode(input) if err != nil { return err } bounds := decoded.Bounds() width, height := bounds.Dx(), bounds.Dy() if width <= 0 || height <= 0 { return errors.New("invalid image dimensions") } scale := min(480.0/float64(width), 300.0/float64(height), 1) outWidth := max(1, int(float64(width)*scale)) outHeight := max(1, int(float64(height)*scale)) thumbnail := image.NewRGBA(image.Rect(0, 0, outWidth, outHeight)) for y := 0; y < outHeight; y++ { for x := 0; x < outWidth; x++ { sourceX := bounds.Min.X + x*width/outWidth sourceY := bounds.Min.Y + y*height/outHeight thumbnail.Set(x, y, decoded.At(sourceX, sourceY)) } } output, err := os.Create(target) if err != nil { return err } defer output.Close() return jpeg.Encode(output, thumbnail, &jpeg.Options{Quality: 78}) } func generateExcalidrawSVG(source, target string) error { data, err := os.ReadFile(source) if err != nil { return err } var scene struct { Elements []struct { Type string `json:"type"` X float64 `json:"x"` Y float64 `json:"y"` Width float64 `json:"width"` Height float64 `json:"height"` Text string `json:"text"` StrokeColor string `json:"strokeColor"` BackgroundColor string `json:"backgroundColor"` IsDeleted bool `json:"isDeleted"` } `json:"elements"` } if err := json.Unmarshal(data, &scene); err != nil { return err } var body strings.Builder body.WriteString(``) body.WriteString(``) for _, element := range scene.Elements { if element.IsDeleted { continue } stroke := element.StrokeColor if stroke == "" { stroke = "#1b1b1f" } fill := element.BackgroundColor if fill == "" || fill == "transparent" { fill = "none" } switch element.Type { case "ellipse": fmt.Fprintf(&body, ``, element.X+element.Width/2, element.Y+element.Height/2, element.Width/2, element.Height/2, html.EscapeString(fill), html.EscapeString(stroke)) case "text": fmt.Fprintf(&body, `%s`, element.X, element.Y+20, html.EscapeString(stroke), html.EscapeString(element.Text)) default: fmt.Fprintf(&body, ``, element.X, element.Y, element.Width, element.Height, html.EscapeString(fill), html.EscapeString(stroke)) } } body.WriteString("") return os.WriteFile(target, []byte(body.String()), 0644) }