Files
pages/internal/server/event_scheduler.go
T
2026-11-15 05:47:14 +09:00

555 lines
16 KiB
Go

package server
import (
"archive/zip"
"context"
"encoding/json"
"errors"
"fmt"
"html"
"image"
_ "image/gif"
"image/jpeg"
_ "image/png"
"io"
"log"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
"pageswiki/internal/store"
)
const previewEventType = "preview.generate"
const previewSweepEventType = "preview.sweep"
const cityReconcileEventType = "city.reconcile"
var cityActivitySubscription = store.ActivitySubscription{ConsumerID: "city", EventTypes: []string{"page.created", "revision.created", "path.renamed", "path.unlinked", "object.deleted"}}
type previewEventPayload struct {
RevisionUUID string `json:"revision_uuid"`
DocumentHash string `json:"document_hash"`
ContentType string `json:"content_type"`
Filename string `json:"filename"`
Kind string `json:"kind"`
Page int `json:"page,omitempty"`
}
type cityReconcilePayload struct {
DomainUUID string `json:"domain_uuid"`
}
func (s *Server) initializePreviewScheduler() {
if s.store == nil || s.store.DocumentRoot == "" {
return
}
s.previewRoot = filepath.Join(filepath.Dir(s.store.DocumentRoot), "previews")
if err := os.MkdirAll(s.previewRoot, 0755); err != nil {
log.Printf("preview cache initialization failed: %v", err)
s.previewRoot = ""
}
}
func (s *Server) startEventScheduler() error {
if s.store == nil {
return nil
}
release, err := s.store.AcquireEventWorker(s.backgroundCtx)
if err != nil {
return err
}
ready := false
defer func() {
if !ready {
_ = release()
}
}()
if err := s.store.EnsureFeedEvent(); err != nil {
return fmt.Errorf("feed scheduler initialization: %w", err)
}
if err := s.store.EnsureBuiltinResourceEvent(); err != nil {
return fmt.Errorf("resource monitor initialization: %w", err)
}
if err := s.ensureCalendarSweep(); err != nil {
return fmt.Errorf("calendar scheduler initialization: %w", err)
}
if err := s.store.EnsureTaskActivityBridgeEvent(); err != nil {
return fmt.Errorf("task journal bridge initialization: %w", err)
}
if err := s.store.EnsureTaskReminderEvent(); err != nil {
return fmt.Errorf("task reminder initialization: %w", err)
}
if err := s.store.EnsureTaskFocusEvent(); err != nil {
return fmt.Errorf("focus timer initialization: %w", err)
}
if err := s.store.EnsureTaskDigestEvent(); err != nil {
return fmt.Errorf("task digest initialization: %w", err)
}
if s.store.SupportsNotificationPush() {
if err := s.store.EnsureNotificationPushEvent(); err != nil {
return fmt.Errorf("notification push initialization: %w", err)
}
}
s.workerRelease = release
ready = true
s.backgroundWG.Add(1)
go func() {
defer s.backgroundWG.Done()
if err := s.schedulerLoop().Run(s.backgroundCtx); err != nil && !errors.Is(err, context.Canceled) {
log.Printf("event scheduler loop stopped: %v", err)
}
}()
return nil
}
func (s *Server) wakeEventScheduler() {
if s.schedulerWake == nil {
return
}
select {
case s.schedulerWake <- struct{}{}:
default:
}
}
func (s *Server) enqueueCityActivityEvents() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
domains, err := s.store.PendingActivityConsumerDomains(ctx, cityActivitySubscription, 100)
if err != nil {
log.Printf("activity outbox domain scan failed: %v", err)
return
}
for _, domain := range domains {
payload, err := json.Marshal(cityReconcilePayload{DomainUUID: domain})
if err != nil {
continue
}
_, err = s.store.EnqueueEventWithOptions("", cityReconcileEventType, "city:"+domain, string(payload), time.Now(), -2, 3)
if err != nil {
log.Printf("city activity enqueue failed domain=%s: %v", domain, err)
}
}
}
func (s *Server) reconcileCityActivity(domainUUID string) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
events, err := s.store.ActivityOutboxForConsumer(ctx, cityActivitySubscription, domainUUID, 500)
if err != nil {
return err
}
if len(events) == 0 {
return nil
}
if _, err := s.applyCityAutoPlan(domainUUID, "system:city"); err != nil {
return err
}
return s.store.CompleteActivityOutboxThrough("city", domainUUID, events[len(events)-1].Sequence)
}
func (s *Server) generatePDFPage(payload previewEventPayload) error {
if payload.Page < 1 || payload.DocumentHash == "" {
return fmt.Errorf("invalid PDF page job")
}
target := filepath.Join(s.previewRoot, fmt.Sprintf("%s-p%d.jpg", payload.DocumentHash, payload.Page))
if info, err := os.Stat(target); err == nil && info.Size() > 0 {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
lease, err := s.store.MaterializeDocument(ctx, payload.DocumentHash, maxMetadataInputBytes)
if err != nil {
return err
}
defer lease.Close()
workDir, err := os.MkdirTemp(s.previewRoot, ".pdf-page-*")
if err != nil {
return err
}
defer os.RemoveAll(workDir)
base := filepath.Join(workDir, "page")
if err := runPreviewCommandContext(ctx, "pdftoppm", "-f", strconv.Itoa(payload.Page), "-l", strconv.Itoa(payload.Page), "-singlefile", "-jpeg", "-scale-to", "1600", lease.Path, base); err != nil {
return err
}
return os.Rename(base+".jpg", target)
}
func (s *Server) previewCachePath(hash, kind string) string {
extension := ".jpg"
if kind == "excalidraw" {
extension = ".svg"
}
return filepath.Join(s.previewRoot, hash+extension)
}
func (s *Server) previewCacheURL(hash, kind string) string {
extension := ".jpg"
if kind == "excalidraw" {
extension = ".svg"
}
return "/v/preview-cache/" + hash + extension
}
func (s *Server) handlePreviewCache(w http.ResponseWriter, r *http.Request) {
if s.previewRoot == "" {
http.NotFound(w, r)
return
}
name := filepath.Base(strings.TrimPrefix(r.URL.Path, "/v/preview-cache/"))
if name == "." || name == "" ||
(!strings.HasSuffix(name, ".jpg") && !strings.HasSuffix(name, ".svg")) {
http.NotFound(w, r)
return
}
path := filepath.Join(s.previewRoot, name)
if _, err := os.Stat(path); err != nil {
http.NotFound(w, r)
return
}
w.Header().Set("Cache-Control", "private, max-age=31536000, immutable")
w.Header().Set("Vary", "Authorization, Cookie")
w.Header().Set("X-Content-Type-Options", "nosniff")
if strings.HasSuffix(name, ".svg") {
w.Header().Set("Content-Security-Policy", "sandbox")
}
http.ServeFile(w, r, path)
}
func (s *Server) enqueuePreview(entry store.BrowserEntry) (string, bool) {
if s.previewRoot == "" || entry.DocumentHash == "" {
return "", false
}
filename := entry.Filename
if filename == "" {
filename = filepath.Base(entry.Path)
}
adapter := selectProjectionAdapter(contentDescriptor{
MediaType: strings.ToLower(strings.Split(entry.ContentType, ";")[0]),
Filename: filename,
})
kind := adapter.ID
if kind != "image" && kind != "pdf" &&
kind != "video" && kind != "excalidraw" && kind != "archive" {
return "", false
}
cachePath := s.previewCachePath(entry.DocumentHash, kind)
cacheURL := s.previewCacheURL(entry.DocumentHash, kind)
if info, err := os.Stat(cachePath); err == nil && info.Size() > 0 {
return cacheURL, true
}
payload, err := json.Marshal(previewEventPayload{
RevisionUUID: entry.RevisionUUID,
DocumentHash: entry.DocumentHash,
ContentType: entry.ContentType,
Filename: filename,
Kind: kind,
})
if err == nil {
if queued, queueErr := s.store.EnqueueEvent(
previewEventType,
entry.DocumentHash,
string(payload),
time.Now(),
); queueErr == nil && queued {
s.wakeEventScheduler()
}
}
return cacheURL, false
}
func (s *Server) enqueueAllPreviewEvents() int {
domains, err := s.store.ListDomains()
if err != nil {
return 0
}
count := 0
for _, domain := range domains {
if !domain.Enabled || domain.Placement != "local" {
continue
}
entries, err := s.store.ListPreviewCandidatesInDomain(domain.UUID)
if err != nil {
continue
}
for _, entry := range entries {
if _, cached := s.enqueuePreview(entry); !cached {
count++
}
}
}
s.wakeEventScheduler()
return count
}
func (s *Server) generatePreview(payload previewEventPayload) error {
target := s.previewCachePath(payload.DocumentHash, payload.Kind)
if info, err := os.Stat(target); err == nil && info.Size() > 0 {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
// Image and ZIP decoders consume readers directly. Only subprocesses and
// the legacy Excalidraw renderer need a verified local scratch pathname.
var source string
if payload.Kind == "pdf" || payload.Kind == "video" || payload.Kind == "excalidraw" {
limit := int64(maxMetadataInputBytes)
if payload.Kind == "video" {
limit = 1 << 30
}
if payload.Kind == "excalidraw" {
limit = 16 << 20
}
lease, err := s.store.MaterializeDocument(ctx, payload.DocumentHash, limit)
if err != nil {
return err
}
defer lease.Close()
source = lease.Path
}
workDir, err := os.MkdirTemp(s.previewRoot, ".job-*")
if err != nil {
return err
}
defer os.RemoveAll(workDir)
var generated string
switch payload.Kind {
case "image":
generated = filepath.Join(workDir, "preview.jpg")
input, openErr := s.store.OpenDocumentContext(ctx, payload.DocumentHash)
if openErr != nil {
return openErr
}
defer input.Close()
err = generateImageThumbnailFromReader(input, generated)
case "pdf":
generated = filepath.Join(workDir, "preview.jpg")
err = runPreviewCommandContext(ctx,
"pdftoppm",
"-f", "1", "-singlefile", "-jpeg",
"-scale-to", "480",
source,
strings.TrimSuffix(generated, ".jpg"),
)
case "video":
generated = filepath.Join(workDir, "preview.jpg")
err = runPreviewCommandContext(ctx,
"ffmpeg", "-hide_banner", "-loglevel", "error",
"-ss", "1", "-i", source,
"-frames:v", "1", "-vf", "scale=480:-2",
"-y", generated,
)
case "excalidraw":
generated = filepath.Join(workDir, "preview.svg")
err = generateExcalidrawSVG(source, generated)
case "archive":
generated = filepath.Join(workDir, "preview.jpg")
if !strings.EqualFold(filepath.Ext(payload.Filename), ".zip") {
return store.ErrEventSkipped
}
input, openErr := s.store.OpenDocumentContext(ctx, payload.DocumentHash)
if openErr != nil {
return openErr
}
defer input.Close()
size, seekErr := input.Seek(0, io.SeekEnd)
if seekErr != nil {
return seekErr
}
reader, zipErr := zip.NewReader(input, size)
if zipErr != nil {
return zipErr
}
err = generateZIPThumbnailFromReader(reader, generated)
default:
return fmt.Errorf("unsupported preview kind %q", payload.Kind)
}
if err != nil {
return err
}
if err := os.Chmod(generated, 0644); err != nil {
return err
}
return os.Rename(generated, target)
}
func runPreviewCommand(name string, args ...string) error {
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
return runPreviewCommandContext(ctx, name, args...)
}
func runPreviewCommandContext(ctx context.Context, name string, args ...string) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
command := exec.CommandContext(ctx, "nice", append([]string{"-n", "15", name}, args...)...)
output := &boundedMetadataOutput{limit: 2 << 20, cancel: cancel}
command.Stdout, command.Stderr = output, output
command.WaitDelay = time.Second
err := command.Run()
if ctx.Err() != nil {
return ctx.Err()
}
if err != nil {
return fmt.Errorf("%s failed: %w: %s", name, err, strings.TrimSpace(output.buffer.String()))
}
return nil
}
func generateImageThumbnail(source, target string) error {
file, err := os.Open(source)
if err != nil {
return err
}
defer file.Close()
return generateImageThumbnailFromReader(file, target)
}
func generateArchiveThumbnail(source, target, filename string) error {
if strings.EqualFold(filepath.Ext(filename), ".zip") {
return generateZIPThumbnail(source, target)
}
return store.ErrEventSkipped
}
func generateZIPThumbnail(source, target string) error {
reader, err := zip.OpenReader(source)
if err != nil {
return err
}
defer reader.Close()
return generateZIPThumbnailFromReader(&reader.Reader, target)
}
func generateZIPThumbnailFromReader(reader *zip.Reader, target string) error {
imageCount, fileCount := 0, 0
for _, file := range reader.File {
if file.FileInfo().IsDir() {
continue
}
fileCount++
if archiveImageName(file.Name) {
imageCount++
}
}
if imageCount < 3 || imageCount*2 < fileCount {
return fmt.Errorf("%w: archive is not image-dominant", store.ErrEventSkipped)
}
for _, file := range reader.File {
if file.FileInfo().IsDir() || !archiveImageName(file.Name) {
continue
}
input, err := file.Open()
if err != nil {
return err
}
err = generateImageThumbnailFromReader(input, target)
input.Close()
if err != nil {
return err
}
return nil
}
return fmt.Errorf("%w: archive has no supported comic images", store.ErrEventSkipped)
}
func archiveImageName(name string) bool {
switch strings.ToLower(filepath.Ext(name)) {
case ".jpg", ".jpeg", ".png", ".gif":
return true
default:
return false
}
}
func generateImageThumbnailFromReader(input io.Reader, target string) error {
decoded, _, err := image.Decode(input)
if err != nil {
return err
}
bounds := decoded.Bounds()
width, height := bounds.Dx(), bounds.Dy()
if width <= 0 || height <= 0 {
return errors.New("invalid image dimensions")
}
scale := min(480.0/float64(width), 300.0/float64(height), 1)
outWidth := max(1, int(float64(width)*scale))
outHeight := max(1, int(float64(height)*scale))
thumbnail := image.NewRGBA(image.Rect(0, 0, outWidth, outHeight))
for y := 0; y < outHeight; y++ {
for x := 0; x < outWidth; x++ {
sourceX := bounds.Min.X + x*width/outWidth
sourceY := bounds.Min.Y + y*height/outHeight
thumbnail.Set(x, y, decoded.At(sourceX, sourceY))
}
}
output, err := os.Create(target)
if err != nil {
return err
}
defer output.Close()
return jpeg.Encode(output, thumbnail, &jpeg.Options{Quality: 78})
}
func generateExcalidrawSVG(source, target string) error {
data, err := os.ReadFile(source)
if err != nil {
return err
}
var scene struct {
Elements []struct {
Type string `json:"type"`
X float64 `json:"x"`
Y float64 `json:"y"`
Width float64 `json:"width"`
Height float64 `json:"height"`
Text string `json:"text"`
StrokeColor string `json:"strokeColor"`
BackgroundColor string `json:"backgroundColor"`
IsDeleted bool `json:"isDeleted"`
} `json:"elements"`
}
if err := json.Unmarshal(data, &scene); err != nil {
return err
}
var body strings.Builder
body.WriteString(`<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 500">`)
body.WriteString(`<rect width="100%" height="100%" fill="#fff"/>`)
for _, element := range scene.Elements {
if element.IsDeleted {
continue
}
stroke := element.StrokeColor
if stroke == "" {
stroke = "#1b1b1f"
}
fill := element.BackgroundColor
if fill == "" || fill == "transparent" {
fill = "none"
}
switch element.Type {
case "ellipse":
fmt.Fprintf(&body,
`<ellipse cx="%g" cy="%g" rx="%g" ry="%g" fill="%s" stroke="%s"/>`,
element.X+element.Width/2, element.Y+element.Height/2,
element.Width/2, element.Height/2, html.EscapeString(fill), html.EscapeString(stroke))
case "text":
fmt.Fprintf(&body,
`<text x="%g" y="%g" fill="%s" font-family="sans-serif" font-size="20">%s</text>`,
element.X, element.Y+20, html.EscapeString(stroke), html.EscapeString(element.Text))
default:
fmt.Fprintf(&body,
`<rect x="%g" y="%g" width="%g" height="%g" fill="%s" stroke="%s"/>`,
element.X, element.Y, element.Width, element.Height,
html.EscapeString(fill), html.EscapeString(stroke))
}
}
body.WriteString("</svg>")
return os.WriteFile(target, []byte(body.String()), 0644)
}