"+block.token+"
", block.html, 1) + } + return out +} + +func htmlStructure(source string) string { + n, err := html.Parse(strings.NewReader(source)) + if err != nil { + return source + } + var out bytes.Buffer + var visit func(*html.Node, bool) + visit = func(n *html.Node, literal bool) { + literal = literal || n.Data == "pre" || n.Data == "code" || n.Data == "script" || n.Data == "style" + data := n.Data + if n.Type == html.TextNode && !literal { + if strings.TrimSpace(data) == "" && (n.Parent == nil || n.Parent.Data == "body" || n.Parent.Data == "html") { + return + } + // Inline whitespace can separate words (including around an inline + // element). Never normalize it away when deciding conversion safety. + } + attrs := append([]html.Attribute(nil), n.Attr...) + sort.Slice(attrs, func(i, j int) bool { return attrs[i].Namespace+":"+attrs[i].Key < attrs[j].Namespace+":"+attrs[j].Key }) + b, _ := json.Marshal([]any{n.Type, n.Namespace, data, attrs}) + out.Write(b) + for c := n.FirstChild; c != nil; c = c.NextSibling { + visit(c, literal) + } + out.WriteByte(')') + } + visit(n, false) + return out.String() +} diff --git a/internal/grammaratur/authoring_test.go b/internal/grammaratur/authoring_test.go new file mode 100644 index 0000000..2e0efba --- /dev/null +++ b/internal/grammaratur/authoring_test.go @@ -0,0 +1,67 @@ +package grammaratur + +import ( + "strings" + "testing" +) + +func TestAuthoringRoundTrip(t *testing.T) { + for _, source := range []string{ + "Hello world.
", + "{{children:/guide}}
", + "text
", + "| a | b |
| c |
a b c
", + " print('x')\n",
+ "one
\n\ntwo
\nHello
`, `a b c
`} { + md, err := HTMLToMarkdownAuthoring(source) + if err != nil { + t.Fatal(err) + } + if htmlStructure(MarkdownAuthoringHTML(md)) != htmlStructure(source) { + t.Fatal("whitespace/attributes lost") + } + } +} + +func TestAuthoringProtectedHTML(t *testing.T) { + for _, raw := range []string{"```\n{=html}\n````\n", "", "line\r\n
"} { + md := rawHTMLBlock(raw) + if got := MarkdownAuthoringHTML(md); got != raw { + t.Fatalf("raw bytes changed: %q want %q", got, raw) + } + mixed := MarkdownAuthoringHTML("# Added\n\n" + md + "\n\nTail") + if !strings.Contains(mixed, raw) || !strings.Contains(mixed, "Tail
") { + t.Fatalf("mixed blocks: %q", mixed) + } + } + code := "````text\n```{=html}\n\n```\n````" + if strings.Contains(MarkdownAuthoringHTML(code), ""} { + want, err := htmltomarkdown.ConvertString(source) + if err != nil { + t.Fatal(err) + } + for _, convert := range []func(string) (string, error){grammaratur.HTMLToMarkdown, compiler.HTMLToMarkdown} { + got, err := convert(source) + if err != nil || got != want { + t.Fatalf("ordinary conversion changed: %q %v", got, err) + } + } + protected, err := grammaratur.HTMLToMarkdownAuthoring(source) + if err != nil { + t.Fatal(err) + } + compat, err := compiler.HTMLToMarkdownAuthoring(source) + if err != nil || protected != compat { + t.Fatal("authoring facade changed", err) + } + if compiler.MarkdownAuthoringHTML(compat) != grammaratur.MarkdownAuthoringHTML(protected) { + t.Fatal("protected render facade changed") + } + } +} + +func TestAuthoringIsExplicitAndNotASanitizer(t *testing.T) { + const source = "```{=html}\n\n```" + if strings.Contains(grammaratur.MarkdownToHTML(source), "" { + t.Fatalf("authoring removed source: %q", got) + } + // Authorization, macro expansion and sanitization stay in the projection + // layer. Grammaratur returns source, not HTML trusted for a browser sink. + if got := grammaratur.MarkdownToHTML("{{children}}"); !strings.Contains(got, "{{children}}") { + t.Fatal("macro unexpectedly expanded") + } +} diff --git a/internal/grammaratur/doc.go b/internal/grammaratur/doc.go new file mode 100644 index 0000000..93e68fd --- /dev/null +++ b/internal/grammaratur/doc.go @@ -0,0 +1,16 @@ +// Package grammaratur owns document-format conversion and authoring fidelity. +// +// Ordinary conversion and protected authoring conversion are distinct contracts: +// callers must opt into {=html} fences. Neither path authorizes content, expands +// macros, sanitizes a display projection, persists revisions, or fetches URLs. +// Those responsibilities remain with the caller. In particular, preserving +// source HTML never grants permission to execute its scripts or event handlers. +// +// Grammaratur is an in-process component, not a service or a new public API. +// The name denotes the words/format-conversion responsibility in PagesWiki's +// architecture; see docs/design/ja/component-names.md. +package grammaratur + +// Component is the stable component label for diagnostics and instrumentation. +// It is not a replacement for API routes, provider IDs or persisted settings. +const Component = "grammaratur" diff --git a/internal/grammaratur/markdown.go b/internal/grammaratur/markdown.go new file mode 100644 index 0000000..d434343 --- /dev/null +++ b/internal/grammaratur/markdown.go @@ -0,0 +1,18 @@ +package grammaratur + +import ( + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2" + "github.com/gomarkdown/markdown" +) + +// HTMLToMarkdown is the existing ordinary conversion, used by integrations +// which do not understand protected authoring blocks. It can be lossy. +func HTMLToMarkdown(source string) (string, error) { + return htmltomarkdown.ConvertString(source) +} + +// MarkdownToHTML preserves the existing Markdown dialect and renderer options. +// It neither interprets authoring fences nor sanitizes the returned HTML. +func MarkdownToHTML(source string) string { + return string(markdown.ToHTML([]byte(source), nil, nil)) +} diff --git a/internal/kunstzeal/adapters.go b/internal/kunstzeal/adapters.go new file mode 100644 index 0000000..3d560f4 --- /dev/null +++ b/internal/kunstzeal/adapters.go @@ -0,0 +1,230 @@ +// Package kunstzeal selects projections and executes bounded rendering providers. +// Authentication, source provenance and canonical storage remain caller-owned. +package kunstzeal + +import ( + "path/filepath" + "strings" +) + +type Delivery string + +const ( + DeliveryHTML Delivery = "html" + DeliveryText Delivery = "text" + DeliveryRawMedia Delivery = "raw-media" + DeliveryRawFrame Delivery = "raw-frame" + DeliveryDownload Delivery = "download" +) + +type Descriptor struct { + MediaType string + Filename string + SizeBytes int64 +} + +type Adapter struct { + ID string + Kind string + Delivery Delivery + NeedsText bool + WindowableText bool + Match func(Descriptor) bool +} + +const InlineTextWindowThreshold = int64(2 << 20) + +var projectionAdapters = []Adapter{ + { + ID: "html", + Kind: "html", + Delivery: DeliveryHTML, + Match: mediaTypes("text/html"), + }, + { + ID: "excalidraw", + Kind: "excalidraw", + Delivery: DeliveryText, + NeedsText: true, + WindowableText: true, + Match: extensions(".excalidraw"), + }, + { + ID: "csv", + Kind: "csv", + Delivery: DeliveryText, + NeedsText: true, + WindowableText: true, + Match: matchAny( + mediaTypes("text/csv", "application/csv"), + extensions(".csv"), + ), + }, + { + ID: "archive", + Kind: "archive", + Delivery: DeliveryDownload, + Match: matchAny( + mediaTypes( + "application/zip", "application/x-zip-compressed", + "application/x-rar-compressed", "application/vnd.rar", + "application/x-7z-compressed", "application/x-tar", + "application/gzip", "application/x-xz", + ), + extensions(".zip", ".rar", ".7z", ".tar", ".gz", ".tgz", ".xz", ".txz"), + ), + }, + { + ID: "debian-package", + Kind: "metadata", + Delivery: DeliveryDownload, + Match: matchAny( + mediaTypes( + "application/vnd.debian.binary-package", + "application/x-debian-package", + ), + extensions(".deb"), + ), + }, + { + ID: "eml", + Kind: "email", + Delivery: DeliveryText, + Match: matchAny( + mediaTypes("message/rfc822"), + extensions(".eml"), + ), + }, + { + ID: "pe", + Kind: "metadata", + Delivery: DeliveryDownload, + Match: extensions(".exe", ".dll", ".sys"), + }, + { + ID: "json", + Kind: "json", + Delivery: DeliveryText, + WindowableText: true, + Match: matchAny(mediaTypes("application/json", "application/ld+json"), extensions(".json", ".jsonl")), + }, + { + ID: "text", + Kind: "code", + Delivery: DeliveryText, + NeedsText: true, + WindowableText: true, + Match: textMediaType, + }, + { + ID: "image", + Kind: "image", + Delivery: DeliveryRawMedia, + Match: matchAny( + mediaTypePrefix("image/"), + extensions(".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif", ".svg"), + ), + }, + { + ID: "audio", + Kind: "audio", + Delivery: DeliveryRawMedia, + Match: matchAny( + mediaTypePrefix("audio/"), + extensions(".mp3", ".flac", ".wav", ".ogg", ".m4a", ".aac", ".opus"), + ), + }, + { + ID: "video", + Kind: "video", + Delivery: DeliveryRawMedia, + Match: matchAny( + mediaTypePrefix("video/"), + extensions(".mp4", ".webm", ".mkv", ".mov", ".m4v", ".ogv"), + ), + }, + { + ID: "pdf", + Kind: "pdf", + Delivery: DeliveryRawFrame, + Match: matchAny( + mediaTypes("application/pdf"), + extensions(".pdf"), + ), + }, + { + ID: "download", + Kind: "download", + Delivery: DeliveryDownload, + Match: func(Descriptor) bool { return true }, + }, +} + +func SelectAdapter(descriptor Descriptor) Adapter { + for _, adapter := range projectionAdapters { + if adapter.Match(descriptor) { + return adapter + } + } + return projectionAdapters[len(projectionAdapters)-1] +} + +func mediaTypes(types ...string) func(Descriptor) bool { + return func(descriptor Descriptor) bool { + for _, candidate := range types { + if descriptor.MediaType == candidate { + return true + } + } + return false + } +} + +func mediaTypePrefix(prefix string) func(Descriptor) bool { + return func(descriptor Descriptor) bool { + return strings.HasPrefix(descriptor.MediaType, prefix) + } +} + +func extensions(values ...string) func(Descriptor) bool { + return func(descriptor Descriptor) bool { + extension := strings.ToLower(filepath.Ext(descriptor.Filename)) + for _, candidate := range values { + if extension == candidate { + return true + } + } + return false + } +} + +func matchAny(matchers ...func(Descriptor) bool) func(Descriptor) bool { + return func(descriptor Descriptor) bool { + for _, matcher := range matchers { + if matcher(descriptor) { + return true + } + } + return false + } +} + +func textMediaType(descriptor Descriptor) bool { + switch descriptor.MediaType { + case "application/json", + "application/xml", + "application/javascript", + "application/x-javascript", + "application/x-sh", + "application/x-shellscript", + "application/x-yaml", + "application/yaml", + "application/toml": + return true + default: + return strings.HasPrefix(descriptor.MediaType, "text/") + } +} + +// Catalog returns a copy; callers cannot reorder or disable the shared registry. +func Catalog() []Adapter { return append([]Adapter(nil), projectionAdapters...) } diff --git a/internal/kunstzeal/adapters_test.go b/internal/kunstzeal/adapters_test.go new file mode 100644 index 0000000..d912bbf --- /dev/null +++ b/internal/kunstzeal/adapters_test.go @@ -0,0 +1,105 @@ +package kunstzeal + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" +) + +func TestAdapterSelectionAndOrdering(t *testing.T) { + for _, test := range []struct { + mime, filename, id, kind string + delivery Delivery + needsText, window bool + }{ + {"text/html", "scene.excalidraw", "html", "html", DeliveryHTML, false, false}, + {"application/json", "scene.EXCALIDRAW", "excalidraw", "excalidraw", DeliveryText, true, true}, + {"text/plain", "data.CSV", "csv", "csv", DeliveryText, true, true}, + {"application/zip", "archive", "archive", "archive", DeliveryDownload, false, false}, + {"application/octet-stream", "package.deb", "debian-package", "metadata", DeliveryDownload, false, false}, + {"text/plain", "mail.eml", "eml", "email", DeliveryText, false, false}, + {"application/octet-stream", "program.exe", "pe", "metadata", DeliveryDownload, false, false}, + {"application/json", "data.json", "json", "json", DeliveryText, false, true}, + {"text/x-python", "code.py", "text", "code", DeliveryText, true, true}, + {"image/png", "image", "image", "image", DeliveryRawMedia, false, false}, + {"audio/flac", "music", "audio", "audio", DeliveryRawMedia, false, false}, + {"video/mp4", "movie", "video", "video", DeliveryRawMedia, false, false}, + {"application/pdf", "manual", "pdf", "pdf", DeliveryRawFrame, false, false}, + {"application/octet-stream", "unknown", "download", "download", DeliveryDownload, false, false}, + } { + got := SelectAdapter(Descriptor{MediaType: test.mime, Filename: test.filename, SizeBytes: InlineTextWindowThreshold + 1}) + if got.ID != test.id || got.Kind != test.kind || got.Delivery != test.delivery || got.NeedsText != test.needsText || got.WindowableText != test.window { + t.Fatalf("%s/%s: %+v", test.mime, test.filename, got) + } + } + if got := SelectAdapter(Descriptor{}); got.ID != "download" { + t.Fatal(got) + } +} + +func TestAdapterCatalogIsCopyAndConcurrent(t *testing.T) { + catalog := Catalog() + ids := []string{} + for _, a := range catalog { + ids = append(ids, a.ID) + } + if strings.Join(ids, ",") != "html,excalidraw,csv,archive,debian-package,eml,pe,json,text,image,audio,video,pdf,download" { + t.Fatal("catalog order changed", ids) + } + catalog[0].ID = "corrupt" + catalog[0].Match = func(Descriptor) bool { return false } + if Catalog()[0].ID != "html" || SelectAdapter(Descriptor{MediaType: "text/html"}).ID != "html" { + t.Fatal("catalog mutation changed shared selection") + } + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + Catalog()[0].ID = "copy" + if SelectAdapter(Descriptor{MediaType: "text/html"}).ID != "html" { + t.Error("registry changed") + } + } + }() + } + wg.Wait() +} + +func TestComponentBoundary(t *testing.T) { + if Component != "kunstzeal" { + t.Fatal("component identity changed") + } + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, spec := range ast.Imports { + path, err := strconv.Unquote(spec.Path.Value) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"pageswiki/internal", "database", "os/exec"} { + if path == forbidden || strings.HasPrefix(path, forbidden+"/") { + t.Errorf("%s imports %s: providers must not own application state or arbitrary process execution", file, path) + } + } + if file == "adapters.go" && (strings.HasPrefix(path, "net") || path == "os") { + t.Error("adapter selection acquired I/O") + } + } + } +} diff --git a/internal/kunstzeal/cache.go b/internal/kunstzeal/cache.go new file mode 100644 index 0000000..b71de64 --- /dev/null +++ b/internal/kunstzeal/cache.go @@ -0,0 +1,55 @@ +package kunstzeal + +import ( + "errors" + "io" + "os" + "path/filepath" +) + +var ErrDiagramCacheLimit = errors.New("diagram cache exceeds limit") + +// ReadDiagramCache reads only a bounded derived artifact. filename must be +// constructed by the application after validating kind/hash and authorizing +// source provenance. A cache hit is not an access grant or a sanitized SVG. +func ReadDiagramCache(filename string) ([]byte, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + b, err := io.ReadAll(io.LimitReader(f, DiagramOutputLimit+1)) + if len(b) > DiagramOutputLimit { + return nil, ErrDiagramCacheLimit + } + return b, err +} + +// WriteDiagramCache publishes a complete artifact with same-directory rename. +// Concurrent readers observe the old or new file, never a truncated write. +// This is disposable cache storage, not a durable canonical Document commit. +func WriteDiagramCache(filename string, data []byte) error { + if len(data) > DiagramOutputLimit { + return ErrDiagramCacheLimit + } + dir := filepath.Dir(filename) + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + f, err := os.CreateTemp(dir, ".diagram-*.tmp") + if err != nil { + return err + } + defer os.Remove(f.Name()) + defer f.Close() + if _, err = f.Write(data); err != nil { + return err + } + if err = f.Chmod(0644); err != nil { + return err + } + if err = f.Close(); err != nil { + return err + } + return os.Rename(f.Name(), filename) +} diff --git a/internal/kunstzeal/cache_test.go b/internal/kunstzeal/cache_test.go new file mode 100644 index 0000000..8b2a2f2 --- /dev/null +++ b/internal/kunstzeal/cache_test.go @@ -0,0 +1,84 @@ +package kunstzeal + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "sync" + "testing" +) + +func TestDiagramCacheAtomicReplacement(t *testing.T) { + filename := filepath.Join(t.TempDir(), "mermaid", "fixture.svg") + a, b := bytes.Repeat([]byte("A"), 64<<10), bytes.Repeat([]byte("B"), 64<<10) + if err := WriteDiagramCache(filename, a); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + for n := 0; n < 20; n++ { + if i%2 == 0 { + data := a + if n%2 == 0 { + data = b + } + if err := WriteDiagramCache(filename, data); err != nil { + t.Error(err) + } + } else { + data, err := ReadDiagramCache(filename) + if err != nil || (!bytes.Equal(data, a) && !bytes.Equal(data, b)) { + t.Errorf("partial cache: length=%d err=%v", len(data), err) + return + } + } + } + }(i) + } + wg.Wait() + files, err := filepath.Glob(filepath.Join(filepath.Dir(filename), ".diagram-*.tmp")) + if err != nil || len(files) != 0 { + t.Fatal("temporary artifacts leaked", files, err) + } + info, err := os.Stat(filename) + if err != nil || info.Mode().Perm() != 0644 { + t.Fatal(info, err) + } +} + +func TestDiagramCacheBudgetsAndFailures(t *testing.T) { + root := t.TempDir() + filename := filepath.Join(root, "fixture.svg") + if err := WriteDiagramCache(filename, []byte("complete")); err != nil { + t.Fatal(err) + } + large := bytes.Repeat([]byte("x"), DiagramOutputLimit+1) + if err := WriteDiagramCache(filename, large); !errors.Is(err, ErrDiagramCacheLimit) { + t.Fatal(err) + } + b, err := ReadDiagramCache(filename) + if err != nil || string(b) != "complete" { + t.Fatal("failed write destroyed cache", err) + } + if err = os.WriteFile(filename, large, 0644); err != nil { + t.Fatal(err) + } + if b, err = ReadDiagramCache(filename); !errors.Is(err, ErrDiagramCacheLimit) || b != nil { + t.Fatal("oversized cache returned", err) + } + if _, err = ReadDiagramCache(filepath.Join(root, "missing")); !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + if err = WriteDiagramCache(root, []byte("cannot replace a directory")); err == nil { + t.Fatal("rename failure hidden") + } + // Failed rename removes its temporary artifact too. + files, err := filepath.Glob(filepath.Join(filepath.Dir(root), ".diagram-*.tmp")) + if err != nil || len(files) != 0 { + t.Fatal(files, err) + } +} diff --git a/internal/kunstzeal/diagram.go b/internal/kunstzeal/diagram.go new file mode 100644 index 0000000..7898e84 --- /dev/null +++ b/internal/kunstzeal/diagram.go @@ -0,0 +1,116 @@ +package kunstzeal + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +const Component = "kunstzeal" +const DiagramSourceLimit = 1 << 20 +const DiagramOutputLimit = 4 << 20 +const DiagramInlineLimit = 32 << 10 +const DiagramTimeout = 15 * time.Second + +var ( + ErrDiagramKind = errors.New("unsupported diagram type") + ErrDiagramSource = errors.New("diagram source exceeds limit") + ErrDiagramOutput = errors.New("diagram output unavailable or exceeds limit") + ErrProviderConfig = errors.New("diagram renderer configuration invalid") + ErrProviderUnavailable = errors.New("diagram renderer unavailable") + ErrProviderFailed = errors.New("diagram renderer failed") +) + +// DiagramProvider does not authorize a request or trust its SVG output. +// Callers must retain source access checks and the sandbox response policy. +type DiagramProvider interface { + Render(context.Context, string, string) ([]byte, error) +} + +func DiagramHash(kind, source, format string) string { + sum := sha256.Sum256([]byte(kind + "\x00" + format + "\x00" + source)) + return hex.EncodeToString(sum[:]) +} + +func AllowedDiagramType(kind string) bool { + switch kind { + case "mermaid", "plantuml", "graphviz", "dot", "d2": + return true + } + return false +} + +func NormalizeDiagramType(kind string) string { + if kind == "dot" { + return "graphviz" + } + return kind +} + +type HTTPClient interface { + Do(*http.Request) (*http.Response, error) +} + +// Kroki receives an administrator-configured service address, never a URL from +// document content. Private service-network endpoints are intentional here. +// Client must honor context cancellation and reject redirects. No default +// client is used, and provider failures never expose its URL or response body. +type Kroki struct { + Endpoint string + Client HTTPClient +} + +func (k Kroki) Render(parent context.Context, kind, source string) ([]byte, error) { + if err := parent.Err(); err != nil { + return nil, err + } + kind = NormalizeDiagramType(kind) + if !AllowedDiagramType(kind) { + return nil, ErrDiagramKind + } + if len(source) > DiagramSourceLimit { + return nil, ErrDiagramSource + } + u, err := url.Parse(k.Endpoint) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || k.Client == nil { + return nil, ErrProviderConfig + } + ctx, cancel := context.WithTimeout(parent, DiagramTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(k.Endpoint, "/")+"/"+kind+"/svg", strings.NewReader(source)) + if err != nil { + return nil, ErrProviderConfig + } + req.Header.Set("Content-Type", "text/plain; charset=utf-8") + resp, err := k.Client.Do(req) + if err != nil { + if resp != nil && resp.Body != nil { + resp.Body.Close() + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, ErrProviderUnavailable + } + if resp == nil || resp.Body == nil { + return nil, ErrProviderUnavailable + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, ErrProviderFailed + } + b, err := io.ReadAll(io.LimitReader(resp.Body, DiagramOutputLimit+1)) + if ctx.Err() != nil { + return nil, ctx.Err() + } + if err != nil || len(b) > DiagramOutputLimit { + return nil, ErrDiagramOutput + } + return b, nil +} diff --git a/internal/kunstzeal/diagram_test.go b/internal/kunstzeal/diagram_test.go new file mode 100644 index 0000000..9f18d3e --- /dev/null +++ b/internal/kunstzeal/diagram_test.go @@ -0,0 +1,167 @@ +package kunstzeal + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "strings" + "testing" + "time" +) + +var _ DiagramProvider = Kroki{} + +type clientFixture func(*http.Request) (*http.Response, error) + +func (f clientFixture) Do(r *http.Request) (*http.Response, error) { return f(r) } + +type trackedReader struct { + io.Reader + closed bool +} + +func (r *trackedReader) Close() error { r.closed = true; return nil } + +func TestDiagramIdentityAndKinds(t *testing.T) { + for _, kind := range []string{"mermaid", "plantuml", "graphviz", "dot", "d2"} { + if !AllowedDiagramType(kind) { + t.Fatal(kind) + } + for _, source := range []string{"", "graph LR; A-->B;", "日本語\x00source"} { + sum := sha256.Sum256([]byte(kind + "\x00svg\x00" + source)) + if DiagramHash(kind, source, "svg") != hex.EncodeToString(sum[:]) { + t.Fatal("cache identity changed") + } + } + } + for _, kind := range []string{"", "nwdiag", "../mermaid", "Mermaid", "svg", "graphviz/svg"} { + if AllowedDiagramType(kind) { + t.Fatal("unknown kind", kind) + } + } + if NormalizeDiagramType("dot") != "graphviz" || NormalizeDiagramType("mermaid") != "mermaid" { + t.Fatal("normalization changed") + } +} + +func TestKrokiBoundedRequestAndOutput(t *testing.T) { + for _, kind := range []string{"mermaid", "dot", "d2"} { + body := &trackedReader{Reader: strings.NewReader("")} + k := Kroki{Endpoint: "http://renderer.example/prefix/", Client: clientFixture(func(r *http.Request) (*http.Response, error) { + b, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + if r.Method != "POST" || r.URL.String() != "http://renderer.example/prefix/"+NormalizeDiagramType(kind)+"/svg" || string(b) != "source" || r.Header.Get("Content-Type") != "text/plain; charset=utf-8" { + t.Fatal("request contract changed") + } + if r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" { + t.Fatal("credentials forwarded") + } + deadline, ok := r.Context().Deadline() + if !ok || time.Until(deadline) > DiagramTimeout { + t.Fatal("unbounded provider") + } + return &http.Response{StatusCode: 200, Body: body}, nil + })} + b, err := k.Render(context.Background(), kind, "source") + if err != nil || string(b) != "" || !body.closed { + t.Fatal(string(b), err, body.closed) + } + } + for _, size := range []int{DiagramOutputLimit, DiagramOutputLimit + 1} { + body := &trackedReader{Reader: strings.NewReader(strings.Repeat("x", size))} + k := Kroki{Endpoint: "https://renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) { return &http.Response{StatusCode: 200, Body: body}, nil })} + b, err := k.Render(context.Background(), "mermaid", strings.Repeat("x", DiagramSourceLimit)) + if size == DiagramOutputLimit { + if err != nil || len(b) != size { + t.Fatal(len(b), err) + } + } else if !errors.Is(err, ErrDiagramOutput) || b != nil { + t.Fatal("oversized output escaped", err) + } + if !body.closed { + t.Fatal("response leaked") + } + } +} + +func TestKrokiFailsClosedBeforeIO(t *testing.T) { + client := clientFixture(func(*http.Request) (*http.Response, error) { t.Fatal("invalid request performed I/O"); return nil, nil }) + for _, endpoint := range []string{"", "file:///etc/passwd", "https://user:secret@renderer.example", "https://renderer.example?token=secret", "https://renderer.example#fragment", "https://renderer.example?", "https:///path", "http://%zz"} { + b, err := (Kroki{Endpoint: endpoint, Client: client}).Render(context.Background(), "mermaid", "source") + if !errors.Is(err, ErrProviderConfig) || b != nil { + t.Fatal("bad provider accepted", err) + } + } + k := Kroki{Endpoint: "http://renderer.example", Client: client} + if _, err := k.Render(context.Background(), "../../secret", "source"); !errors.Is(err, ErrDiagramKind) { + t.Fatal(err) + } + if _, err := k.Render(context.Background(), "mermaid", strings.Repeat("x", DiagramSourceLimit+1)); !errors.Is(err, ErrDiagramSource) { + t.Fatal(err) + } + if _, err := (Kroki{Endpoint: k.Endpoint}).Render(context.Background(), "mermaid", "source"); !errors.Is(err, ErrProviderConfig) { + t.Fatal("default HTTP client used", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := k.Render(ctx, "mermaid", "source"); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} + +func TestKrokiFailuresAreOpaqueAndCloseBodies(t *testing.T) { + for _, code := range []int{301, 302, 307, 308, 400, 401, 403, 404, 429, 500} { + body := &trackedReader{Reader: strings.NewReader("PRIVATE SOURCE OR INTERNAL DIAGNOSTICS")} + k := Kroki{Endpoint: "https://private-renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) { return &http.Response{StatusCode: code, Body: body}, nil })} + b, err := k.Render(context.Background(), "mermaid", "PRIVATE SOURCE") + if !errors.Is(err, ErrProviderFailed) || b != nil || !body.closed { + t.Fatal(code, err, body.closed) + } + } + for _, mode := range []string{"network", "nil-response", "nil-body", "read-error"} { + k := Kroki{Endpoint: "https://private-renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) { + switch mode { + case "network": + return nil, errors.New("https://private-renderer.example: secret") + case "nil-response": + return nil, nil + case "nil-body": + return &http.Response{StatusCode: 200}, nil + default: + return &http.Response{StatusCode: 200, Body: io.NopCloser(errorReader{})}, nil + } + })} + b, err := k.Render(context.Background(), "mermaid", "PRIVATE SOURCE") + if b != nil || err == nil || strings.Contains(err.Error(), "secret") || strings.Contains(err.Error(), "private-renderer") { + t.Fatal(mode, err) + } + } +} + +type errorReader struct{} + +func (errorReader) Read([]byte) (int, error) { return 0, errors.New("secret body read failure") } + +func TestKrokiCancellationDuringSendAndRead(t *testing.T) { + for _, duringRead := range []bool{false, true} { + ctx, cancel := context.WithCancel(context.Background()) + body := &trackedReader{Reader: strings.NewReader("")} + k := Kroki{Endpoint: "https://renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) { + cancel() + if duringRead { + return &http.Response{StatusCode: 200, Body: body}, nil + } + return &http.Response{StatusCode: 200, Body: body}, errors.New("cancelled network") + })} + b, err := k.Render(ctx, "mermaid", "source") + cancel() + if !errors.Is(err, context.Canceled) || b != nil || !body.closed { + t.Fatal(duringRead, err, body.closed) + } + } +} diff --git a/internal/mestionora/boundary_test.go b/internal/mestionora/boundary_test.go new file mode 100644 index 0000000..3024c05 --- /dev/null +++ b/internal/mestionora/boundary_test.go @@ -0,0 +1,44 @@ +package mestionora + +import ( + "go/parser" + "go/token" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" +) + +func TestComponentBoundary(t *testing.T) { + if Component != "mestionora" { + t.Fatal("component identity changed") + } + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, spec := range ast.Imports { + path, err := strconv.Unquote(spec.Path.Value) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"pageswiki/internal", "net", "os"} { + if path == forbidden || strings.HasPrefix(path, forbidden+"/") { + t.Errorf("%s imports %s: search must not own HTTP, filesystem payloads or application policy", file, path) + } + } + } + } + if _, exists := reflect.TypeFor[*SQLite]().MethodByName("SearchPathsInDomain"); exists { + t.Fatal("implicit allow-all belongs only in the legacy trusted-caller facade") + } +} diff --git a/internal/mestionora/city_display.go b/internal/mestionora/city_display.go new file mode 100644 index 0000000..8ecc8b4 --- /dev/null +++ b/internal/mestionora/city_display.go @@ -0,0 +1,50 @@ +package mestionora + +import ( + "path" + "strings" +) + +// A display projection only: names, boundaries and manual titles remain stored +// unchanged. Legacy generated Spots have no nested Spot semantics. +func IsUnclassifiedCitySpot(name, boundary string) bool { + return strings.HasSuffix(name, "-undefined") || strings.HasSuffix(boundary, "/_undefined") +} + +func CitySpotDisplayTitle(name, title, boundary, parentTitle string) string { + if !IsUnclassifiedCitySpot(name, boundary) { + if title != "" { + return title + } + return name + } + switch strings.ToLower(strings.TrimSpace(title)) { + case "", "undefined", "未定義", "未分類のpage", "unclassified pages", strings.ToLower(name): + if parentTitle != "" { + return parentTitle + } + if parent := path.Base(path.Dir(boundary)); parent != "/" && parent != "." { + return parent + } + return "Spot" + default: + return title // A manual display title takes precedence. + } +} + +func (r CitySearchResult) DisplayTitle() string { + if r.Kind == "spot" { + return CitySpotDisplayTitle(r.Name, r.Title, r.PathPrefix, r.StreetTitle) + } + if r.Title != "" { + return r.Title + } + return r.Name +} + +func (r CitySearchResult) ContentPath() string { + if r.NavigationPath != "" { + return r.NavigationPath + } + return r.PathPrefix +} diff --git a/internal/mestionora/city_query_plan_test.go b/internal/mestionora/city_query_plan_test.go new file mode 100644 index 0000000..942ba5a --- /dev/null +++ b/internal/mestionora/city_query_plan_test.go @@ -0,0 +1,38 @@ +package mestionora + +import ( + "database/sql" + "strings" + "testing" +) + +func TestCityContentExistenceUsesPathIndex(t *testing.T) { + database := newSearchTestDB(t) + for _, kind := range []string{"quartier", "street", "spot"} { + rows, err := database.Query("EXPLAIN QUERY PLAN "+citySearchSQL(kind, ""), sql.Named("domain", "d"), sql.Named("scope", "/"), sql.Named("prefix", "/"), sql.Named("upper", "0"), sql.Named("query", "needle"), sql.Named("limit", 1000)) + if err != nil { + t.Fatal(err) + } + indexed := false + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + if strings.Contains(detail, "SEARCH cp USING") { + indexed = true + } + if strings.Contains(detail, "SCAN cp") { + t.Fatalf("unbounded Path lookup: %s", detail) + } + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + rows.Close() + if !indexed { + t.Fatalf("no indexed portal lookup for %s", kind) + } + } +} diff --git a/internal/mestionora/city_search.go b/internal/mestionora/city_search.go new file mode 100644 index 0000000..f036f6e --- /dev/null +++ b/internal/mestionora/city_search.go @@ -0,0 +1,212 @@ +package mestionora + +import ( + "context" + "database/sql" + "errors" + "sort" + "strings" + "time" +) + +const ( + CitySearchCandidateLimit = 1000 // Per locality kind, before authorization. + CitySearchResultLimit = 30 + citySearchTimeout = 2 * time.Second +) + +// CitySearchResult describes a locality, not its contained Pages. Description +// is a bounded plain-text excerpt around the match and is never trusted HTML. +type CitySearchResult struct { + UUID, Kind, Name, Title, Description, PathPrefix string + QuartierTitle, StreetTitle, QuartierName string + CreatedAtMS, UpdatedAtMS int64 + NameMatch, PathMatch, DescriptionMatch bool + NavigationPath string + HasPage bool +} + +type citySearchCandidate struct { + CitySearchResult + quartierPath, streetPath string +} + +// SearchCityInDomain searches locality metadata only. It does not construct a +// CityMap, expand descendants, read Page bodies, or count hidden localities. +// Scope selects a locality's own boundary, not an ancestor of the scope. Every +// boundary in its parent chain must be readable before the result is exposed. +func (s *SQLite) SearchCityInDomain(parent context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []CitySearchResult, searchErr error) { + if err := options.Validate(); err != nil { + return nil, err + } + query = strings.TrimSpace(query) + if len(query) > 512 || len(strings.Fields(query)) > 16 { + return nil, errors.New("search query is too long") + } + // File filters are intentionally not applied to locality metadata: a Spot + // containing a PDF is not itself a PDF and must not claim to match one. + if allowed == nil || options.FileType != "" || options.Extension != "" || (query == "" && !options.HasFilters()) { + return nil, nil + } + if limit <= 0 || limit > CitySearchResultLimit { + limit = CitySearchResultLimit + } + ctx, cancel := context.WithTimeout(parent, citySearchTimeout) + defer cancel() + defer func() { + if searchErr != nil && ctx.Err() != nil { + searchErr = ctx.Err() + } + }() + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return nil, err + } + defer tx.Rollback() + scope = normalizePath(scope) + prefix, upper := strings.TrimRight(scope, "/")+"/", strings.TrimRight(scope, "/")+"0" + args := []any{ + sql.Named("domain", domainUUID), sql.Named("scope", scope), + sql.Named("prefix", prefix), sql.Named("upper", upper), + sql.Named("query", query), sql.Named("limit", CitySearchCandidateLimit), + } + dateColumn := "n.updated_at_ms" + if options.DateField == "created" { + dateColumn = "n.created_at_ms" + } + dateWhere := "" + if options.FromMS != nil { + dateWhere += " AND " + dateColumn + ">=:from_ms" + args = append(args, sql.Named("from_ms", *options.FromMS)) + } + if options.BeforeMS != nil { + dateWhere += " AND " + dateColumn + "<:before_ms" + args = append(args, sql.Named("before_ms", *options.BeforeMS)) + } + candidates := make([]citySearchCandidate, 0) + for _, kind := range []string{"quartier", "street", "spot"} { + rows, err := tx.QueryContext(ctx, citySearchSQL(kind, dateWhere), args...) + if err != nil { + return nil, err + } + for rows.Next() { + var candidate citySearchCandidate + candidate.Kind = kind + if err := rows.Scan(&candidate.UUID, &candidate.Name, &candidate.Title, &candidate.Description, + &candidate.PathPrefix, &candidate.CreatedAtMS, &candidate.UpdatedAtMS, + &candidate.NameMatch, &candidate.PathMatch, &candidate.DescriptionMatch, + &candidate.QuartierTitle, &candidate.StreetTitle, &candidate.quartierPath, &candidate.streetPath, &candidate.QuartierName, + &candidate.NavigationPath, &candidate.HasPage); err != nil { + rows.Close() + return nil, err + } + candidates = append(candidates, candidate) + } + err = rows.Err() + closeErr := rows.Close() + if err != nil { + return nil, err + } + if closeErr != nil { + return nil, closeErr + } + } + // Release the sole read connection before calling authorization, which may + // itself need a database connection. HTTP normally passes its cached policy. + if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { + return nil, err + } + sort.SliceStable(candidates, func(i, j int) bool { + a, b := candidates[i], candidates[j] + if a.Kind != b.Kind { + return citySearchKindOrder(a.Kind) < citySearchKindOrder(b.Kind) + } + if a.NameMatch != b.NameMatch { + return a.NameMatch + } + if a.PathMatch != b.PathMatch { + return a.PathMatch + } + if a.PathPrefix != b.PathPrefix { + return a.PathPrefix < b.PathPrefix + } + return a.UUID < b.UUID + }) + readable := make(map[string]bool) + canRead := func(path string) bool { + if value, exists := readable[path]; exists { + return value + } + value := allowed(path) + readable[path] = value + return value + } + for _, candidate := range candidates { + if err := ctx.Err(); err != nil { + return nil, err + } + if !canRead(candidate.PathPrefix) || (candidate.quartierPath != "" && !canRead(candidate.quartierPath)) || (candidate.streetPath != "" && !canRead(candidate.streetPath)) { + continue + } + found = append(found, candidate.CitySearchResult) + if len(found) == limit { + break + } + } + if err := ctx.Err(); err != nil { + return nil, err + } + return found, nil +} + +func citySearchKindOrder(kind string) int { + switch kind { + case "quartier": + return 0 + case "street": + return 1 + default: + return 2 + } +} + +// All interpolated SQL fragments are package-owned constants. User input is +// bound, so %, _, quotes and backslashes are literal substring characters. +func citySearchSQL(kind, dateWhere string) string { + from, description := "zones n", "n.description" + navigationPath := "n.path_prefix" + parentColumns := "'','','','',''" + where := "n.kind='quartier'" + switch kind { + case "street": + from += ` JOIN zones q ON q.uuid=n.parent_uuid AND q.domain_uuid=n.domain_uuid AND q.kind='quartier' AND q.state!='archived'` + parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),'',q.path_prefix,'',q.name" + where = "n.kind='street' AND " + citySearchAbsolutePathSQL("q.path_prefix") + case "spot": + navigationPath = `CASE WHEN substr(n.path_prefix,-11)='/_undefined' THEN st.path_prefix ELSE n.path_prefix END` + from = `spots n JOIN street_spots ss ON ss.spot_uuid=n.uuid + JOIN zones st ON st.uuid=ss.street_uuid AND st.domain_uuid=n.domain_uuid AND st.kind='street' AND st.state!='archived' + JOIN zones q ON q.uuid=st.parent_uuid AND q.domain_uuid=st.domain_uuid AND q.kind='quartier' AND q.state!='archived'` + description = "n.summary" + parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),substr(COALESCE(NULLIF(st.title,''),st.name),1,256),q.path_prefix,st.path_prefix,q.name" + where = citySearchAbsolutePathSQL("q.path_prefix") + " AND " + citySearchAbsolutePathSQL("st.path_prefix") + } + nameMatch := `(:query!='' AND (instr(lower(n.name),lower(:query))>0 OR instr(lower(n.title),lower(:query))>0))` + pathMatch := `(:query!='' AND instr(lower(n.path_prefix),lower(:query))>0)` + descriptionMatch := `(:query!='' AND instr(lower(` + description + `),lower(:query))>0)` + return `SELECT n.uuid,substr(n.name,1,256),substr(n.title,1,256), + substr(` + description + `,max(1,instr(lower(` + description + `),lower(:query))-160),512), + n.path_prefix,n.created_at_ms,n.updated_at_ms,` + nameMatch + `,` + pathMatch + `,` + descriptionMatch + `,` + parentColumns + `,` + navigationPath + `, + EXISTS(SELECT 1 FROM paths cp WHERE cp.domain_uuid=n.domain_uuid AND cp.path=(` + navigationPath + `)) + FROM ` + from + ` WHERE n.domain_uuid=:domain AND n.state!='archived' AND ` + where + ` + AND ` + citySearchAbsolutePathSQL("n.path_prefix") + ` + AND (n.path_prefix COLLATE BINARY=:scope OR (n.path_prefix COLLATE BINARY>=:prefix AND n.path_prefix COLLATE BINARY<:upper)) + AND (:query='' OR ` + nameMatch + ` OR ` + pathMatch + ` OR ` + descriptionMatch + `)` + dateWhere + ` + ORDER BY ` + nameMatch + ` DESC,` + pathMatch + ` DESC,n.path_prefix COLLATE BINARY,n.uuid LIMIT :limit` +} + +func citySearchAbsolutePathSQL(column string) string { + // Keep authorization paths intact rather than truncating them. Malformed + // legacy boundaries and impractically large metadata are not navigable hits. + return `(substr(` + column + `,1,1)='/' AND length(` + column + `)<=4096 AND instr(` + column + `,char(0))=0)` +} diff --git a/internal/mestionora/fixture_test.go b/internal/mestionora/fixture_test.go new file mode 100644 index 0000000..777a109 --- /dev/null +++ b/internal/mestionora/fixture_test.go @@ -0,0 +1,40 @@ +package mestionora + +import ( + "database/sql" + "testing" + + "pageswiki/internal/db" +) + +func newSearchTestDB(t *testing.T) *sql.DB { + t.Helper() + database, err := db.Open(t.TempDir() + "/search.db") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { database.Close() }) + initSearchFixture(t, database) + return database +} + +// Only the application columns/indexes consumed by search, not application +// bootstrap, auth keys, job recovery or external calendar initialization. +func initSearchFixture(t *testing.T, database *sql.DB) { + t.Helper() + _, err := database.Exec(` + CREATE TABLE objects(uuid TEXT PRIMARY KEY, created_at INTEGER); + CREATE TABLE revisions(uuid TEXT PRIMARY KEY,object_uuid TEXT,filename TEXT,body TEXT,content_type TEXT,document_hash TEXT,created_at_ms INTEGER,created_at INTEGER); + CREATE TABLE paths(domain_uuid TEXT NOT NULL,path TEXT NOT NULL,object_uuid TEXT,PRIMARY KEY(domain_uuid,path)); + CREATE INDEX idx_paths_object_domain_page ON paths(object_uuid,domain_uuid,path); + CREATE TABLE zones(uuid TEXT PRIMARY KEY,domain_uuid TEXT,kind TEXT,parent_uuid TEXT,name TEXT,title TEXT,description TEXT,path_prefix TEXT,state TEXT,created_at_ms INTEGER,updated_at_ms INTEGER); + CREATE TABLE spots(uuid TEXT PRIMARY KEY,domain_uuid TEXT,name TEXT,title TEXT,summary TEXT,path_prefix TEXT,state TEXT,created_at_ms INTEGER,updated_at_ms INTEGER); + CREATE TABLE street_spots(street_uuid TEXT,spot_uuid TEXT); + `) + if err != nil { + t.Fatal(err) + } + if err := NewSQLite(database).EnsureSearchIndexSchema(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/mestionora/heading.go b/internal/mestionora/heading.go new file mode 100644 index 0000000..b3356a8 --- /dev/null +++ b/internal/mestionora/heading.go @@ -0,0 +1,36 @@ +package mestionora + +import ( + "golang.org/x/net/html" + "io" + "strings" +) + +// FirstHTMLHeading extracts untrusted title text. Callers bound input before +// extraction and escape output at rendering time; this is not a sanitizer. +func FirstHTMLHeading(source string) string { + t := html.NewTokenizer(strings.NewReader(source)) + in := false + var out strings.Builder + for { + kind := t.Next() + if kind == html.ErrorToken { + if t.Err() == io.EOF { + break + } + break + } + tok := t.Token() + if kind == html.StartTagToken && strings.EqualFold(tok.Data, "h1") { + in = true + continue + } + if kind == html.EndTagToken && strings.EqualFold(tok.Data, "h1") { + break + } + if in && kind == html.TextToken { + out.WriteString(tok.Data) + } + } + return strings.TrimSpace(out.String()) +} diff --git a/internal/mestionora/search.go b/internal/mestionora/search.go new file mode 100644 index 0000000..d9f31a7 --- /dev/null +++ b/internal/mestionora/search.go @@ -0,0 +1,377 @@ +package mestionora + +import ( + "context" + "database/sql" + "errors" + "sort" + "strings" + "time" +) + +const SearchCandidateLimit = 1000 +const SearchExcerptDocumentBytes = 64 << 10 +const SearchExcerptTotalBytes = 1 << 20 +const searchExcerptTimeout = 400 * time.Millisecond + +const searchResultColumns = `p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.revision_uuid,''),COALESCE(r.created_at_ms/1000,0)` +const searchPathBoundary = `(p.path=? OR (p.path>=? AND p.path))` +const searchPathCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,0 + FROM paths p LEFT JOIN latest_revision_index r ON r.object_uuid=p.object_uuid + WHERE p.domain_uuid=? AND ` + searchPathBoundary +const searchPathQueryPredicate = ` AND (instr(lower(p.path),lower(?))>0 OR instr(lower(COALESCE(r.filename,'')),lower(?))>0)` +const searchPathCandidatesSQL = searchPathCandidatesBaseSQL + searchPathQueryPredicate + ` ORDER BY p.path LIMIT ?` + +// CROSS JOIN is intentional: revision_uuid is an UNINDEXED FTS payload column. +// A Path-first plan re-runs the complete MATCH scan for every Path. Start with +// MATCH exactly once, then seek current Revision metadata and Domain bindings +// through their B-tree indexes. Keep scope filtering before the candidate cap. +const searchFTSCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,revision_search.rowid + FROM revision_search + CROSS JOIN latest_revision_index r ON r.revision_uuid=revision_search.revision_uuid + CROSS JOIN paths p ON p.object_uuid=r.object_uuid + WHERE revision_search MATCH ? AND p.domain_uuid=? AND ` + searchPathBoundary +const searchFTSCandidatesSQL = searchFTSCandidatesBaseSQL + ` LIMIT ?` + +func searchBodyExcerptsSQL(count int) string { + // UUID is not indexed by FTS5. Restrict the MATCH/snippet work using the + // integer rowids collected from the same read snapshot, not payload UUIDs. + return `SELECT revision_uuid,substr(snippet(revision_search,3,'','',' … ',36),1,4096) + FROM revision_search WHERE revision_search MATCH ? AND rowid IN (` + strings.TrimRight(strings.Repeat("?,", count), ",") + `)` +} + +// Excerpts are optional enrichment. Even small documents can have expensive +// prefix hit-position scans, so their time budget is separate from finding +// results. A spent excerpt budget omits snippets, not already-authorized hits. +func loadSearchBodyExcerpts(parent context.Context, tx *sql.Tx, expression string, ids []any, budget time.Duration) (excerpts map[string]string, err error) { + ctx, cancel := context.WithTimeout(parent, budget) + defer cancel() + defer func() { + if err != nil && ctx.Err() != nil { + if parent.Err() != nil { + err = parent.Err() + } else { + excerpts, err = nil, nil + } + } + }() + args := append([]any{"body : (" + expression + ")"}, ids...) + rows, err := tx.QueryContext(ctx, searchBodyExcerptsSQL(len(ids)), args...) + if err != nil { + return nil, err + } + defer rows.Close() + excerpts = make(map[string]string) + for rows.Next() { + var id, excerpt string + if err := rows.Scan(&id, &excerpt); err != nil { + return nil, err + } + excerpts[id] = excerpt + } + return excerpts, rows.Err() +} + +type SearchResult struct { + Path string + ContentType string + Filename string + DocumentHash string + RevisionUUID string + Size int64 // Kept for API compatibility; search does not stat Document files. + UpdatedAt int64 + PathMatch bool + FilenameMatch bool + BodyMatch bool + Excerpt string // Untrusted source text, never HTML-safe markup. + ExcerptOmitted bool // A body match remains visible when snippet work exceeds its budget. +} + +type PathSuggestion struct { + SearchResult + Title string +} + +// SearchPathsInDomainFiltered bounds SQL candidates before materialization and +// authorizes each Path before loading an excerpt. It never computes a global hit +// count or stats Document files. A narrow scope improves both recall and cost. +// allowed runs inside a read snapshot. Prefer a preloaded request-local policy; +// a callback doing DB reads requires spare pool capacity (not a one-connection +// pool). Keeping policy I/O outside this snapshot is separate follow-up work. +func (s *SQLite) SearchPathsInDomainFiltered(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]SearchResult, error) { + return s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}) +} + +// SearchPathsInDomainWithOptions also permits filter-only searches. File/date +// predicates apply before SQL candidate caps, while ACL checks stay ahead of all +// snippet enrichment. Zero options preserve the original indexed query plans. +func (s *SQLite) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, searchErr error) { + return s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options, true) +} + +// SearchPathSuggestionsInDomain shares the current-Revision FTS and ACL path +// with full search, but deliberately skips snippet generation. Typeahead calls +// are frequent and only need a bounded display label, never a body excerpt. +func (s *SQLite) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) { + results, err := s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}, false) + if err != nil || len(results) == 0 { + return nil, err + } + ids := make([]any, 0, len(results)) + seen := make(map[string]bool, len(results)) + for _, result := range results { + if result.RevisionUUID != "" && strings.HasPrefix(strings.ToLower(result.ContentType), "text/html") && !seen[result.RevisionUUID] { + ids = append(ids, result.RevisionUUID) + seen[result.RevisionUUID] = true + } + } + headings := make(map[string]string, len(ids)) + if len(ids) > 0 { + rows, queryErr := s.db.QueryContext(ctx, `SELECT uuid,substr(CAST(body AS TEXT),1,65536) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...) + if queryErr != nil { + return nil, queryErr + } + for rows.Next() { + var revision, source string + if queryErr = rows.Scan(&revision, &source); queryErr != nil { + rows.Close() + return nil, queryErr + } + headings[revision] = FirstHTMLHeading(source) + } + queryErr = rows.Err() + rows.Close() + if queryErr != nil { + return nil, queryErr + } + } + out := make([]PathSuggestion, 0, len(results)) + for _, result := range results { + out = append(out, PathSuggestion{SearchResult: result, Title: headings[result.RevisionUUID]}) + } + return out, nil +} + +func (s *SQLite) searchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions, loadExcerpts bool) (found []SearchResult, searchErr error) { + if err := options.Validate(); err != nil { + return nil, err + } + scope, query = normalizePath(scope), strings.TrimSpace(query) + if query == "" && !options.HasFilters() || allowed == nil { + return nil, nil + } + if len(query) > 512 || len(strings.Fields(query)) > 16 { + return nil, errors.New("search query is too long") + } + if limit <= 0 || limit > 200 { + limit = 100 + } + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + defer func() { + // SQLite can report an interrupted statement instead of wrapping the + // context error. Preserve cancellation/deadline semantics for HTTP. + if ctx.Err() != nil { + found = nil + searchErr = ctx.Err() + } + }() + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return nil, err + } + defer tx.Rollback() + prefix := strings.TrimRight(scope, "/") + "/" + upper := strings.TrimRight(scope, "/") + "0" + results := make(map[string]SearchResult) + ftsRows := make(map[string]int64) + collect := func(statement string, body, filename bool, args ...any) error { + rows, err := tx.QueryContext(ctx, statement, args...) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var result SearchResult + var rowID int64 + if err := rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt, &rowID); err != nil { + return err + } + if !allowed(result.Path) { + continue + } + if rowID != 0 { + ftsRows[result.RevisionUUID] = rowID + } + result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query)) + result.FilenameMatch = filename || results[result.Path].FilenameMatch || query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query)) + result.BodyMatch = body || results[result.Path].BodyMatch + results[result.Path] = result + } + return rows.Err() + } + filterSQL, filterArgs := searchOptionsPredicate(options) + pathSQL := searchPathCandidatesBaseSQL + pathArgs := []any{domainUUID, scope, prefix, upper} + if query != "" { + pathSQL += searchPathQueryPredicate + pathArgs = append(pathArgs, query, query) + } + pathSQL += filterSQL + ` ORDER BY p.path LIMIT ?` + pathArgs = append(pathArgs, filterArgs...) + pathArgs = append(pathArgs, SearchCandidateLimit) + if err := collect(pathSQL, false, false, pathArgs...); err != nil { + return nil, err + } + expression := ftsPrefixExpression(query) + if expression != "" { + // No global ORDER BY or unbounded IN materialization: stop the FTS scan + // after at most this many current, in-Domain, in-scope Path candidates. + for _, field := range []string{"body", "filename"} { + args := []any{field + " : (" + expression + ")", domainUUID, scope, prefix, upper} + args = append(args, filterArgs...) + args = append(args, SearchCandidateLimit) + if err := collect(searchFTSCandidatesBaseSQL+filterSQL+` LIMIT ?`, field == "body", field == "filename", args...); err != nil { + return nil, err + } + } + } + out := make([]SearchResult, 0, len(results)) + for _, result := range results { + out = append(out, result) + } + sort.Slice(out, func(i, j int) bool { + // Direct name/Path matches precede body-only hits; all ties are stable. + iName, jName := out[i].PathMatch || out[i].FilenameMatch, out[j].PathMatch || out[j].FilenameMatch + if iName != jName { + return iName + } + if len(out[i].Path) != len(out[j].Path) { + return len(out[i].Path) < len(out[j].Path) + } + return out[i].Path < out[j].Path + }) + if len(out) > limit { + out = out[:limit] + } + if !loadExcerpts { + return out, nil + } + // Only authorized, displayed results receive text excerpts. Batch the FTS + // lookup so a broad query is not re-scanned once for every visible result. + ids := make([]any, 0, len(out)) + seen := make(map[string]bool) + for _, result := range out { + if result.BodyMatch && !seen[result.RevisionUUID] { + ids = append(ids, ftsRows[result.RevisionUUID]) + seen[result.RevisionUUID] = true + } + } + if len(ids) != 0 { + // snippet's output token/character limit does not bound its input work: + // FTS5 may tokenize the entire matched document. Read byte lengths only + // for authorized displayed rows before spending the snippet budget. + rows, err := tx.QueryContext(ctx, `SELECT rowid,octet_length(body) + FROM revision_search WHERE rowid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...) + if err != nil { + return nil, err + } + sizes := make(map[int64]int64) + for rows.Next() { + var rowID, size int64 + if err := rows.Scan(&rowID, &size); err != nil { + rows.Close() + return nil, err + } + sizes[rowID] = size + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + ids = nil + selected := make(map[string]bool) + remaining := int64(SearchExcerptTotalBytes) + for i := range out { + if !out[i].BodyMatch { + continue + } + revision := out[i].RevisionUUID + include, decided := selected[revision] + if !decided { + rowID := ftsRows[revision] + size, exists := sizes[rowID] + include = exists && size >= 0 && size <= SearchExcerptDocumentBytes && size <= remaining + if include { + ids = append(ids, rowID) + remaining -= size + } + selected[revision] = include + } + out[i].ExcerptOmitted = !include + } + } + if len(ids) != 0 { + excerpts, err := loadSearchBodyExcerpts(ctx, tx, expression, ids, searchExcerptTimeout) + if err != nil { + return nil, err + } + for i := range out { + if out[i].BodyMatch { + var present bool + out[i].Excerpt, present = excerpts[out[i].RevisionUUID] + out[i].ExcerptOmitted = !present + } + } + } + // A filename/Path-only hit still benefits from a short Page summary. Read + // only a bounded prefix from displayed textual revisions, in one batch. + ids, seen = nil, make(map[string]bool) + for _, result := range out { + media := strings.ToLower(result.ContentType) + textual := strings.HasPrefix(media, "text/") || strings.HasPrefix(media, "application/json") || strings.HasPrefix(media, "application/xml") || strings.HasPrefix(media, "application/javascript") || strings.HasPrefix(media, "application/yaml") || strings.HasPrefix(media, "application/x-yaml") + if !result.BodyMatch && textual && result.RevisionUUID != "" && !seen[result.RevisionUUID] { + ids = append(ids, result.RevisionUUID) + seen[result.RevisionUUID] = true + } + } + if len(ids) != 0 { + rows, err := tx.QueryContext(ctx, `SELECT uuid,substr(body,1,1024) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...) + if err != nil { + return nil, err + } + excerpts := make(map[string]string) + for rows.Next() { + var id, excerpt string + if err := rows.Scan(&id, &excerpt); err != nil { + rows.Close() + return nil, err + } + excerpts[id] = excerpt + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + for i := range out { + if !out[i].BodyMatch { + out[i].Excerpt = excerpts[out[i].RevisionUUID] + } + } + } + return out, nil +} + +func ftsPrefixExpression(query string) string { + words := strings.Fields(query) + terms := make([]string, 0, len(words)) + for _, word := range words { + word = strings.ReplaceAll(word, `"`, `""`) + if word != "" { + terms = append(terms, `"`+word+`"*`) + } + } + return strings.Join(terms, " AND ") +} diff --git a/internal/mestionora/search_filters.go b/internal/mestionora/search_filters.go new file mode 100644 index 0000000..4bd9379 --- /dev/null +++ b/internal/mestionora/search_filters.go @@ -0,0 +1,135 @@ +package mestionora + +import ( + "errors" + "fmt" + "strings" +) + +var ErrInvalidSearchOptions = errors.New("invalid search options") + +// SearchOptions applies to current Page metadata, not historical Revisions. +// Dates are Unix milliseconds: FromMS is inclusive and BeforeMS is exclusive. +// The created field refers to Object creation, not creation of a Path alias. +// FileType and Extension are normalized lower-case values; Extension has no dot +// prefix and may contain a compound suffix such as tar.gz. +type SearchOptions struct { + FileType string + Extension string + DateField string + FromMS *int64 + BeforeMS *int64 +} + +func (o SearchOptions) HasFilters() bool { + return o.FileType != "" || o.Extension != "" || o.FromMS != nil || o.BeforeMS != nil +} + +func (o SearchOptions) Validate() error { + switch o.FileType { + case "", "html", "text", "image", "audio", "video", "pdf", "archive", "csv", "json", "code", "office", "other": + default: + return fmt.Errorf("%w: unknown file type", ErrInvalidSearchOptions) + } + switch o.DateField { + case "", "updated", "created": + default: + return fmt.Errorf("%w: unknown date field", ErrInvalidSearchOptions) + } + if len(o.Extension) > 32 { + return fmt.Errorf("%w: extension is too long", ErrInvalidSearchOptions) + } + for i, c := range o.Extension { + alnum := c >= 'a' && c <= 'z' || c >= '0' && c <= '9' + if !alnum && (i == 0 || !strings.ContainsRune("._+-", c)) { + return fmt.Errorf("%w: invalid extension", ErrInvalidSearchOptions) + } + } + if o.FromMS != nil && o.BeforeMS != nil && *o.FromMS >= *o.BeforeMS { + return fmt.Errorf("%w: empty or reversed date range", ErrInvalidSearchOptions) + } + return nil +} + +const searchMIMESQL = `lower(trim(substr(COALESCE(r.content_type,''),1,instr(COALESCE(r.content_type,'')||';',';')-1)))` + +// The original filename wins over the Path suffix when present. Names remain +// data: even extension patterns are bound arguments and do not become SQL. +const searchFilenameSQL = `lower(CASE WHEN COALESCE(r.filename,'')<>'' THEN r.filename ELSE p.path END)` + +func searchSuffixesSQL(suffixes ...string) string { + clauses := make([]string, 0, len(suffixes)) + for _, suffix := range suffixes { + // Only package-owned ASCII constants reach this helper. GLOB has no + // escaping ambiguity for underscores, unlike LIKE. + clauses = append(clauses, searchFilenameSQL+` GLOB '*.`+suffix+`'`) + } + return "(" + strings.Join(clauses, " OR ") + ")" +} + +// Specific MIME types are authoritative. Generic text/binary types may use a +// known suffix (e.g. a Python source uploaded as text/plain); unrecognized, +// specific MIME types stay Other instead of pretending to be a different type. +// This uses metadata only and never hydrates a Document to sniff its bytes. +var searchFileTypeSQL = `CASE + WHEN ` + searchMIMESQL + ` IN ('text/html','application/xhtml+xml') THEN 'html' + WHEN ` + searchMIMESQL + `='application/pdf' THEN 'pdf' + WHEN ` + searchMIMESQL + ` LIKE 'image/%' THEN 'image' + WHEN ` + searchMIMESQL + ` LIKE 'audio/%' THEN 'audio' + WHEN ` + searchMIMESQL + ` LIKE 'video/%' THEN 'video' + WHEN ` + searchMIMESQL + ` IN ('text/csv','application/csv','text/tab-separated-values') THEN 'csv' + WHEN ` + searchMIMESQL + ` IN ('application/json','text/json') OR ` + searchMIMESQL + ` GLOB '*+json' THEN 'json' + WHEN ` + searchMIMESQL + ` IN ('application/zip','application/x-zip-compressed','application/x-7z-compressed','application/vnd.rar','application/x-rar','application/x-rar-compressed','application/x-tar','application/gzip','application/x-gzip','application/x-xz','application/x-bzip2','application/zstd','application/x-zstd','application/x-compress','application/x-cpio') THEN 'archive' + WHEN ` + searchMIMESQL + ` IN ('application/msword','application/vnd.ms-excel','application/vnd.ms-powerpoint','application/rtf','text/rtf') OR ` + searchMIMESQL + ` GLOB 'application/vnd.openxmlformats-officedocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.oasis.opendocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.ms-*.macroenabled.*' THEN 'office' + WHEN ` + searchMIMESQL + ` IN ('text/markdown','text/x-markdown') THEN 'text' + WHEN ` + searchMIMESQL + ` IN ('application/javascript','application/x-javascript','text/javascript','application/typescript','text/typescript','application/xml','text/xml','application/yaml','application/x-yaml','text/yaml','application/toml','application/x-sh','application/x-shellscript','application/x-python','application/x-python-code','application/x-httpd-php','application/sql','text/css') OR ` + searchMIMESQL + ` GLOB 'text/x-*' OR ` + searchMIMESQL + ` GLOB '*+xml' THEN 'code' + WHEN ` + searchMIMESQL + ` NOT IN ('','text/plain','application/octet-stream','binary/octet-stream','application/x-empty') THEN CASE WHEN ` + searchMIMESQL + ` LIKE 'text/%' THEN 'text' ELSE 'other' END + WHEN ` + searchSuffixesSQL("html", "htm", "xhtml") + ` THEN 'html' + WHEN ` + searchSuffixesSQL("pdf") + ` THEN 'pdf' + WHEN ` + searchSuffixesSQL("png", "jpg", "jpeg", "gif", "webp", "avif", "bmp", "svg", "tif", "tiff", "ico", "heic", "heif") + ` THEN 'image' + WHEN ` + searchSuffixesSQL("mp3", "flac", "wav", "ogg", "oga", "opus", "m4a", "aac", "aiff", "alac", "mid", "midi") + ` THEN 'audio' + WHEN ` + searchSuffixesSQL("mp4", "webm", "mkv", "mov", "avi", "m4v", "ogv", "mpeg", "mpg") + ` THEN 'video' + WHEN ` + searchSuffixesSQL("zip", "7z", "rar", "tar", "gz", "tgz", "xz", "txz", "bz2", "tbz", "tbz2", "zst", "tzst", "cbz", "cbr", "cpio") + ` THEN 'archive' + WHEN ` + searchSuffixesSQL("csv", "tsv") + ` THEN 'csv' + WHEN ` + searchSuffixesSQL("json", "jsonl", "ndjson", "excalidraw", "ipynb") + ` THEN 'json' + WHEN ` + searchSuffixesSQL("doc", "docx", "docm", "xls", "xlsx", "xlsm", "ppt", "pptx", "pptm", "odt", "ods", "odp", "rtf") + ` THEN 'office' + WHEN ` + searchSuffixesSQL("py", "pyw", "sh", "bash", "zsh", "fish", "ps1", "bat", "cmd", "js", "mjs", "cjs", "jsx", "ts", "tsx", "go", "rs", "c", "h", "cpp", "hpp", "cc", "cs", "java", "kt", "kts", "swift", "rb", "php", "pl", "pm", "lua", "r", "sql", "css", "scss", "sass", "less", "xml", "xsl", "yaml", "yml", "toml", "ini", "conf", "vue", "svelte", "dart", "ex", "exs", "erl", "hs", "scala", "clj", "lisp", "el", "vim", "wasm") + ` THEN 'code' + WHEN ` + searchMIMESQL + `='text/plain' OR ` + searchSuffixesSQL("txt", "text", "log", "md", "markdown", "rst", "adoc", "org") + ` THEN 'text' + ELSE 'other' END` + +// The returned predicate is inserted before each candidate stream's LIMIT, +// including FTS-first streams. Filtering after their cap silently loses hits. +func searchOptionsPredicate(options SearchOptions) (string, []any) { + var predicate strings.Builder + var args []any + if options.FromMS != nil || options.BeforeMS != nil { + dateSQL := "r.created_at_ms" + if options.DateField == "created" { + // Objects.uuid is a primary-key lookup. Avoid joining Objects in the + // normal search plan when Object creation is not being filtered. + predicate.WriteString(" AND EXISTS (SELECT 1 FROM objects o WHERE o.uuid=p.object_uuid") + dateSQL = "o.created_at*1000" + } + if options.FromMS != nil { + predicate.WriteString(" AND " + dateSQL + ">=?") + args = append(args, *options.FromMS) + } + if options.BeforeMS != nil { + predicate.WriteString(" AND " + dateSQL + "") + args = append(args, *options.BeforeMS) + } + if options.DateField == "created" { + predicate.WriteString(")") + } + } + // Evaluate inexpensive date/suffix exclusions before the metadata category. + if options.Extension != "" { + predicate.WriteString(" AND " + searchFilenameSQL + " GLOB ?") + args = append(args, "*."+options.Extension) + } + if options.FileType != "" { + predicate.WriteString(" AND (" + searchFileTypeSQL + ")=?") + args = append(args, options.FileType) + } + return predicate.String(), args +} diff --git a/internal/mestionora/search_index.go b/internal/mestionora/search_index.go new file mode 100644 index 0000000..50aaf81 --- /dev/null +++ b/internal/mestionora/search_index.go @@ -0,0 +1,211 @@ +package mestionora + +import "database/sql" + +// EnsureSearchIndexSchema installs an FTS5 index over immutable Revisions. +// Paths stay outside the index because links can be renamed independently; the +// search query joins matching latest Revisions to the current domain Paths. +func (s *SQLite) EnsureSearchIndexSchema() error { + var revisionsTable string + if err := s.db.QueryRow(` + SELECT name FROM sqlite_master + WHERE type = 'table' AND name = 'revisions' + `).Scan(&revisionsTable); err != nil { + if err == sql.ErrNoRows { + return nil + } + return err + } + if _, err := s.db.Exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS revision_search USING fts5( + revision_uuid UNINDEXED, + object_uuid UNINDEXED, + filename, + body, + tokenize = 'unicode61' + ) + `); err != nil { + return err + } + if _, err := s.db.Exec(` + CREATE TABLE IF NOT EXISTS revision_search_state ( + revision_uuid TEXT PRIMARY KEY + ) + `); err != nil { + return err + } + if _, err := s.db.Exec(` + CREATE TABLE IF NOT EXISTS latest_revision_index ( + object_uuid TEXT PRIMARY KEY, + revision_uuid TEXT NOT NULL, + content_type TEXT NOT NULL, + filename TEXT NOT NULL, + document_hash TEXT NOT NULL, + created_at_ms INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS search_schema_state ( + name TEXT PRIMARY KEY + ); + CREATE INDEX IF NOT EXISTS idx_latest_revision_search_uuid ON latest_revision_index(revision_uuid) + `); err != nil { + return err + } + var latestBackfilled int + if err := s.db.QueryRow(` + SELECT EXISTS( + SELECT 1 FROM search_schema_state + WHERE name = 'latest_revision_index_v1' + ) + `).Scan(&latestBackfilled); err != nil { + return err + } + if latestBackfilled == 0 { + if _, err := s.db.Exec(` + INSERT OR REPLACE INTO latest_revision_index ( + object_uuid, revision_uuid, content_type, filename, + document_hash, created_at_ms + ) + SELECT + object_uuid, uuid, COALESCE(content_type, ''), + COALESCE(filename, ''), COALESCE(document_hash, ''), + COALESCE(created_at_ms, created_at * 1000, 0) + FROM ( + SELECT + uuid, object_uuid, content_type, filename, + document_hash, created_at_ms, created_at, + ROW_NUMBER() OVER ( + PARTITION BY object_uuid + ORDER BY COALESCE(created_at_ms, created_at * 1000) DESC, + rowid DESC + ) AS rn + FROM revisions + ) + WHERE rn = 1; + INSERT OR IGNORE INTO search_schema_state (name) + VALUES ('latest_revision_index_v1'); + `); err != nil { + return err + } + } + // Populate the cheap membership table once for installations that already + // created revision_search before revision_search_state was introduced. + var stateExists int + if err := s.db.QueryRow(` + SELECT EXISTS(SELECT 1 FROM revision_search_state LIMIT 1) + `).Scan(&stateExists); err != nil { + return err + } + if stateExists == 0 { + if _, err := s.db.Exec(` + INSERT OR IGNORE INTO revision_search_state (revision_uuid) + SELECT revision_uuid FROM revision_search + `); err != nil { + return err + } + } + // Trigger definitions evolve with the projection schema; recreate them so + // existing installations receive the current maintenance logic. + if _, err := s.db.Exec(` + DROP TRIGGER IF EXISTS revision_search_insert; + DROP TRIGGER IF EXISTS revision_search_delete; + `); err != nil { + return err + } + if _, err := s.db.Exec(` + CREATE TRIGGER IF NOT EXISTS revision_search_insert + AFTER INSERT ON revisions BEGIN + INSERT INTO revision_search ( + revision_uuid, object_uuid, filename, body + ) VALUES ( + new.uuid, + new.object_uuid, + COALESCE(new.filename, ''), + CASE + WHEN lower(COALESCE(new.content_type, '')) LIKE 'text/%' + OR lower(COALESCE(new.content_type, '')) LIKE 'application/json%' + OR lower(COALESCE(new.content_type, '')) LIKE 'application/xml%' + OR lower(COALESCE(new.content_type, '')) LIKE 'application/javascript%' + OR lower(COALESCE(new.content_type, '')) LIKE 'application/x-yaml%' + OR lower(COALESCE(new.content_type, '')) LIKE 'application/yaml%' + THEN COALESCE(new.body, '') + ELSE '' + END + ); + INSERT OR IGNORE INTO revision_search_state (revision_uuid) + VALUES (new.uuid); + INSERT INTO latest_revision_index ( + object_uuid, revision_uuid, content_type, filename, + document_hash, created_at_ms + ) VALUES ( + new.object_uuid, new.uuid, COALESCE(new.content_type, ''), + COALESCE(new.filename, ''), COALESCE(new.document_hash, ''), + COALESCE(new.created_at_ms, new.created_at * 1000, 0) + ) + ON CONFLICT(object_uuid) DO UPDATE SET + revision_uuid = excluded.revision_uuid, + content_type = excluded.content_type, + filename = excluded.filename, + document_hash = excluded.document_hash, + created_at_ms = excluded.created_at_ms + WHERE excluded.created_at_ms >= latest_revision_index.created_at_ms; + END + `); err != nil { + return err + } + if _, err := s.db.Exec(` + CREATE TRIGGER IF NOT EXISTS revision_search_delete + AFTER DELETE ON revisions BEGIN + DELETE FROM revision_search WHERE revision_uuid = old.uuid; + DELETE FROM revision_search_state WHERE revision_uuid = old.uuid; + DELETE FROM latest_revision_index + WHERE object_uuid = old.object_uuid + AND revision_uuid = old.uuid; + INSERT OR REPLACE INTO latest_revision_index ( + object_uuid, revision_uuid, content_type, filename, + document_hash, created_at_ms + ) + SELECT + object_uuid, uuid, COALESCE(content_type, ''), + COALESCE(filename, ''), COALESCE(document_hash, ''), + COALESCE(created_at_ms, created_at * 1000, 0) + FROM revisions + WHERE object_uuid = old.object_uuid + ORDER BY COALESCE(created_at_ms, created_at * 1000) DESC, rowid DESC + LIMIT 1; + END + `); err != nil { + return err + } + _, err := s.db.Exec(` + INSERT INTO revision_search ( + revision_uuid, object_uuid, filename, body + ) + SELECT + r.uuid, + r.object_uuid, + COALESCE(r.filename, ''), + CASE + WHEN lower(COALESCE(r.content_type, '')) LIKE 'text/%' + OR lower(COALESCE(r.content_type, '')) LIKE 'application/json%' + OR lower(COALESCE(r.content_type, '')) LIKE 'application/xml%' + OR lower(COALESCE(r.content_type, '')) LIKE 'application/javascript%' + OR lower(COALESCE(r.content_type, '')) LIKE 'application/x-yaml%' + OR lower(COALESCE(r.content_type, '')) LIKE 'application/yaml%' + THEN COALESCE(r.body, '') + ELSE '' + END + FROM revisions r + WHERE NOT EXISTS ( + SELECT 1 FROM revision_search_state state + WHERE state.revision_uuid = r.uuid + ) + `) + if err != nil { + return err + } + _, err = s.db.Exec(` + INSERT OR IGNORE INTO revision_search_state (revision_uuid) + SELECT uuid FROM revisions + `) + return err +} diff --git a/internal/store/search_query_plan_test.go b/internal/mestionora/search_query_plan_test.go similarity index 95% rename from internal/store/search_query_plan_test.go rename to internal/mestionora/search_query_plan_test.go index 1f00a69..bc41838 100644 --- a/internal/store/search_query_plan_test.go +++ b/internal/mestionora/search_query_plan_test.go @@ -1,4 +1,4 @@ -package store +package mestionora import ( "context" @@ -39,10 +39,7 @@ func TestSearchExcerptDeadlineIsOptionalButParentCancellationIsNot(t *testing.T) t.Fatal(err) } defer database.Close() - if err := db.Migrate(database); err != nil { - t.Fatal(err) - } - New(database) + initSearchFixture(t, database) tx, err := database.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true}) if err != nil { t.Fatal(err) @@ -72,10 +69,7 @@ func TestSearchFTSPlanScansMatchesOnce(t *testing.T) { t.Fatal(err) } defer database.Close() - if err := db.Migrate(database); err != nil { - t.Fatal(err) - } - New(database) + initSearchFixture(t, database) for _, field := range []string{"body", "filename"} { plan := searchQueryPlan(t, database, searchFTSCandidatesSQL, field+` : ("needle"*)`, "domain", "/", "/", "0", SearchCandidateLimit) @@ -96,10 +90,7 @@ func TestSearchFilteredFTSPlanKeepsMatchFirstAndIndexedSeeks(t *testing.T) { t.Fatal(err) } defer database.Close() - if err := db.Migrate(database); err != nil { - t.Fatal(err) - } - New(database) + initSearchFixture(t, database) from, before := int64(1000), int64(2000) for _, options := range []SearchOptions{ {FileType: "code"}, {Extension: "py"}, {FromMS: &from, BeforeMS: &before}, diff --git a/internal/mestionora/security_test.go b/internal/mestionora/security_test.go new file mode 100644 index 0000000..fc4eba1 --- /dev/null +++ b/internal/mestionora/security_test.go @@ -0,0 +1,102 @@ +package mestionora + +import ( + "context" + "errors" + "testing" +) + +func TestMissingPolicyDoesNotReadOrInitialize(t *testing.T) { + // A nil pool makes accidental construction, search or maintenance I/O fail. + engine := NewSQLite(nil) + ctx := context.Background() + if hits, err := engine.SearchPathsInDomainFiltered(ctx, "d", "/", "needle", 20, nil); err != nil || len(hits) != 0 { + t.Fatalf("missing page policy: %v %v", hits, err) + } + if hits, err := engine.SearchPathSuggestionsInDomain(ctx, "d", "/", "needle", 8, nil); err != nil || len(hits) != 0 { + t.Fatalf("missing suggestion policy: %v %v", hits, err) + } + if hits, err := engine.SearchCityInDomain(ctx, "d", "/", "needle", 30, nil, SearchOptions{}); err != nil || len(hits) != 0 { + t.Fatalf("missing city policy: %v %v", hits, err) + } +} + +func TestDeniedPagesDoNotLoadSummaryOrHeading(t *testing.T) { + database := newSearchTestDB(t) + _, err := database.Exec(` + INSERT INTO objects(uuid,created_at) VALUES('o',1); + INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('d','/needle','o'); + INSERT INTO revisions(uuid,object_uuid,filename,body,content_type,document_hash,created_at_ms,created_at) + VALUES('r','o','needle.html','PRIVATE_SUMMARY
','text/html','hash',1000,1); + DROP TABLE revisions; + `) + if err != nil { + t.Fatal(err) + } + engine := NewSQLite(database) + deny := func(string) bool { return false } + ctx := context.Background() + // Candidates still exist in the independent index. Any attempt to hydrate + // denied summaries/headings now fails because their source table is absent. + if hits, err := engine.SearchPathsInDomainFiltered(ctx, "d", "/", "needle", 20, deny); err != nil || len(hits) != 0 { + t.Fatalf("denied summary loaded: %v %v", hits, err) + } + if hits, err := engine.SearchPathSuggestionsInDomain(ctx, "d", "/", "needle", 8, deny); err != nil || len(hits) != 0 { + t.Fatalf("denied heading loaded: %v %v", hits, err) + } + allow := func(string) bool { return true } + if _, err := engine.SearchPathsInDomainFiltered(ctx, "d", "/", "needle", 20, allow); err == nil { + t.Fatal("fixture did not exercise summary loading") + } + if _, err := engine.SearchPathSuggestionsInDomain(ctx, "d", "/", "needle", 8, allow); err == nil { + t.Fatal("fixture did not exercise heading loading") + } +} + +func TestCancellationDuringPolicyNeverReturnsPartialSuccess(t *testing.T) { + database := newSearchTestDB(t) + if _, err := database.Exec(`INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('d','/needle','o')`); err != nil { + t.Fatal(err) + } + for _, excerpts := range []bool{false, true} { + ctx, cancel := context.WithCancel(context.Background()) + allowed := func(string) bool { cancel(); return true } + // Filter-only search has no later FTS statement to notice cancellation. + hits, err := NewSQLite(database).searchPathsInDomainWithOptions(ctx, "d", "/", "", 20, allowed, SearchOptions{FileType: "other"}, excerpts) + cancel() + if !errors.Is(err, context.Canceled) || len(hits) != 0 { + t.Fatalf("partial success after cancellation: %v %v", hits, err) + } + } +} + +func TestFTSExpressionTreatsOperatorsAsData(t *testing.T) { + for input, want := range map[string]string{ + "": "", "alpha beta": `"alpha"* AND "beta"*`, + `a"b OR`: `"a""b"* AND "OR"*`, + `body:NEAR(x) -secret`: `"body:NEAR(x)"* AND "-secret"*`, + "日本語": `"日本語"*`, + } { + if got := ftsPrefixExpression(input); got != want { + t.Fatalf("%q: got %q want %q", input, got, want) + } + } +} + +func TestSearchDoesNotMaintainIndexOnReadOnlyConnection(t *testing.T) { + database := newSearchTestDB(t) + database.SetMaxOpenConns(1) + if _, err := database.Exec(`INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('d','/needle','o'); PRAGMA query_only=ON`); err != nil { + t.Fatal(err) + } + engine := NewSQLite(database) + results, err := engine.SearchPathsInDomainFiltered(context.Background(), "d", "/", "needle", 20, func(string) bool { return true }) + if err != nil || len(results) != 1 { + t.Fatalf("read-only search: %v %v", results, err) + } + // Schema maintenance is deliberately explicit, not performed by NewSQLite + // or a query. An accidental write in those paths would fail the test above. + if err := engine.EnsureSearchIndexSchema(); err == nil { + t.Fatal("read-only fixture unexpectedly accepted maintenance") + } +} diff --git a/internal/mestionora/sqlite.go b/internal/mestionora/sqlite.go new file mode 100644 index 0000000..a2ddc38 --- /dev/null +++ b/internal/mestionora/sqlite.go @@ -0,0 +1,34 @@ +// Package mestionora owns bounded search, search projections and the SQLite +// search index. It consumes read-policy decisions; it never grants permissions. +// Search results and excerpts are untrusted text, not HTML-safe output. +package mestionora + +import ( + "database/sql" + "strings" +) + +const Component = "mestionora" + +// SQLite borrows a metadata connection pool. Construction performs no I/O or +// schema initialization. EnsureSearchIndexSchema is a separate startup step. +// The caller owns the pool lifetime and provisions the application schema. +type SQLite struct{ db *sql.DB } + +func NewSQLite(db *sql.DB) *SQLite { return &SQLite{db: db} } + +// Match the established Path scope rules, without filesystem path cleaning or +// URL decoding. Percent/underscore characters remain literal search data. +func normalizePath(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "/" + } + if !strings.HasPrefix(value, "/") { + value = "/" + value + } + if value != "/" { + value = strings.TrimRight(value, "/") + } + return value +} diff --git a/internal/ordoschnellic/boundary_test.go b/internal/ordoschnellic/boundary_test.go new file mode 100644 index 0000000..5c7f185 --- /dev/null +++ b/internal/ordoschnellic/boundary_test.go @@ -0,0 +1,40 @@ +package ordoschnellic + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestComponentBoundary(t *testing.T) { + if Component != "ordoschnellic" || PushEventType != "notification.push.deliver" { + t.Fatal("component/event identity changed") + } + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, spec := range ast.Imports { + path, err := strconv.Unquote(spec.Path.Value) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"pageswiki/internal", "database", "os"} { + if path == forbidden || strings.HasPrefix(path, forbidden+"/") { + t.Errorf("%s imports %s: delivery must not own application policy, DB transactions or files", file, path) + } + } + } + } +} diff --git a/internal/ordoschnellic/dispatcher.go b/internal/ordoschnellic/dispatcher.go new file mode 100644 index 0000000..2712f25 --- /dev/null +++ b/internal/ordoschnellic/dispatcher.go @@ -0,0 +1,82 @@ +package ordoschnellic + +import ( + "context" + "errors" + "time" +) + +// Queue rechecks session/recipient eligibility in Claim. Empty Subscription +// means the claimed item is no longer deliverable and should be finalized. +// No outbound network operation runs inside a Queue transaction. +type Queue interface { + Cleanup(context.Context, time.Time) error + Claim(context.Context, time.Time) (*PushDelivery, error) + DeferFocused(context.Context, *PushDelivery, time.Time) error + Finish(context.Context, *PushDelivery, Outcome, time.Time) error +} + +type Sender interface { + Send(context.Context, *PushDelivery) Outcome +} + +type Stats struct{ Sent, Failed int } + +type Dispatcher struct { + Queue Queue + Sender Sender + Now func() time.Time +} + +func (d Dispatcher) Run(parent context.Context) (Stats, error) { + var stats Stats + if d.Queue == nil || d.Sender == nil { + return stats, errors.New("notification delivery dependencies are required") + } + if err := parent.Err(); err != nil { + return stats, err + } + now := d.Now + if now == nil { + now = time.Now + } + ctx, cancel := context.WithTimeout(parent, PushBatchTimeout) + defer cancel() + if err := d.Queue.Cleanup(ctx, now()); err != nil { + return stats, err + } + for i := 0; i < PushBatchLimit && ctx.Err() == nil; i++ { + item, err := d.Queue.Claim(ctx, now()) + if err != nil { + return stats, err + } + if item == nil { + break + } + if item.Focused { + if err := d.Queue.DeferFocused(ctx, item, now()); err != nil { + return stats, err + } + continue + } + var outcome Outcome + if item.Subscription != "" { + outcome = d.Sender.Send(ctx, item) + if outcome.Attempted { + if outcome.Success { + stats.Sent++ + } else { + stats.Failed++ + } + } + } + // Preserve completion when the caller or outbound request is cancelled. + finishCtx, stop := context.WithTimeout(context.Background(), PushFinishTimeout) + err = d.Queue.Finish(finishCtx, item, outcome, now()) + stop() + if err != nil { + return stats, err + } + } + return stats, parent.Err() +} diff --git a/internal/ordoschnellic/dispatcher_test.go b/internal/ordoschnellic/dispatcher_test.go new file mode 100644 index 0000000..0aa79b4 --- /dev/null +++ b/internal/ordoschnellic/dispatcher_test.go @@ -0,0 +1,150 @@ +package ordoschnellic + +import ( + "context" + "errors" + "fmt" + "testing" + "time" +) + +type queueFixture struct { + items []*PushDelivery + calls []string + errAt string + err error + finished []Outcome + finishContext func(context.Context) + at time.Time + t *testing.T +} + +func (q *queueFixture) call(op string, at time.Time) error { + q.t.Helper() + if !at.Equal(q.at) { + q.t.Fatalf("clock for %s: %v", op, at) + } + q.calls = append(q.calls, op) + if op == q.errAt { + return q.err + } + return nil +} +func (q *queueFixture) Cleanup(_ context.Context, at time.Time) error { return q.call("cleanup", at) } +func (q *queueFixture) Claim(_ context.Context, at time.Time) (*PushDelivery, error) { + if err := q.call("claim", at); err != nil { + return nil, err + } + if len(q.items) == 0 { + return nil, nil + } + d := q.items[0] + q.items = q.items[1:] + return d, nil +} +func (q *queueFixture) DeferFocused(_ context.Context, _ *PushDelivery, at time.Time) error { + return q.call("defer", at) +} +func (q *queueFixture) Finish(ctx context.Context, _ *PushDelivery, o Outcome, at time.Time) error { + if q.finishContext != nil { + q.finishContext(ctx) + } + q.finished = append(q.finished, o) + return q.call("finish", at) +} + +type senderFixture func(context.Context, *PushDelivery) Outcome + +func (f senderFixture) Send(ctx context.Context, d *PushDelivery) Outcome { return f(ctx, d) } + +func newQueueFixture(t *testing.T) *queueFixture { + return &queueFixture{t: t, at: time.Unix(1800000000, 0), err: errors.New("queue failure")} +} + +func TestDispatcherBoundedBatchAndCounters(t *testing.T) { + q := newQueueFixture(t) + q.items = append(q.items, &PushDelivery{Focused: true}, &PushDelivery{}) + for i := 0; i < 25; i++ { + q.items = append(q.items, &PushDelivery{Subscription: "valid"}) + } + sends := 0 + sender := senderFixture(func(context.Context, *PushDelivery) Outcome { + sends++ + return Outcome{Attempted: true, Success: sends%2 == 0, Retry: sends%2 != 0} + }) + stats, err := (Dispatcher{Queue: q, Sender: sender, Now: func() time.Time { return q.at }}).Run(context.Background()) + if err != nil || stats != (Stats{Sent: 9, Failed: 9}) || sends != 18 || len(q.items) != 7 || len(q.finished) != 19 { + t.Fatalf("stats=%+v sends=%d pending=%d finished=%d err=%v", stats, sends, len(q.items), len(q.finished), err) + } + if q.finished[0] != (Outcome{}) { + t.Fatal("ineligible delivery was sent/retried") + } + if fmt.Sprint(q.calls[:5]) != "[cleanup claim defer claim finish]" { + t.Fatal(q.calls) + } +} + +func TestDispatcherStopsOnQueueErrors(t *testing.T) { + for _, op := range []string{"cleanup", "claim", "defer", "finish"} { + t.Run(op, func(t *testing.T) { + q := newQueueFixture(t) + q.errAt = op + q.items = []*PushDelivery{{Subscription: "valid", Focused: op == "defer"}, {Subscription: "valid"}} + sends := 0 + _, err := (Dispatcher{Queue: q, Sender: senderFixture(func(context.Context, *PushDelivery) Outcome { sends++; return Outcome{Attempted: true, Success: true} }), Now: func() time.Time { return q.at }}).Run(context.Background()) + if !errors.Is(err, q.err) || q.calls[len(q.calls)-1] != op || sends > 1 { + t.Fatalf("calls=%v sends=%d err=%v", q.calls, sends, err) + } + }) + } +} + +func TestDispatcherCancellationFinalizesClaim(t *testing.T) { + q := newQueueFixture(t) + q.items = []*PushDelivery{{Subscription: "valid"}, {Subscription: "valid"}} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + q.finishContext = func(ctx context.Context) { + if ctx.Err() != nil { + t.Fatal("completion inherited cancellation") + } + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) > PushFinishTimeout { + t.Fatal("completion is not bounded") + } + } + stats, err := (Dispatcher{Queue: q, Now: func() time.Time { return q.at }, Sender: senderFixture(func(ctx context.Context, _ *PushDelivery) Outcome { + if _, ok := ctx.Deadline(); !ok { + t.Fatal("send has no deadline") + } + cancel() + return Outcome{Attempted: true, Retry: true} + })}).Run(ctx) + if !errors.Is(err, context.Canceled) || stats.Failed != 1 || len(q.finished) != 1 || len(q.items) != 1 || !q.finished[0].Retry { + t.Fatalf("%+v %v %+v", stats, err, q.finished) + } + q.calls = nil + _, err = (Dispatcher{Queue: q, Sender: senderFixture(func(context.Context, *PushDelivery) Outcome { t.Fatal("cancelled batch sent"); return Outcome{} })}).Run(ctx) + if !errors.Is(err, context.Canceled) || len(q.calls) != 0 { + t.Fatal("cancelled batch accessed queue", err, q.calls) + } +} + +func TestDeliveryPolicy(t *testing.T) { + at := time.Unix(1800000000, 0) + for _, attempt := range []int{-1, 0, 1, 2, 3, 4, 5, 1000000} { + next, ok := RetryAt(attempt, at) + if ok != (attempt >= 1 && attempt <= 4) { + t.Fatalf("attempt %d", attempt) + } + if ok && next.Sub(at) != time.Duration(1<# comment
+def hello():
+ return "<script>literal</script>"`)
+ createUserSpacePage(t, s, domain, "/notes/alpha", "Canonical body.
", "test", "und") + if err != nil { + t.Fatal(err) + } + if err = database.Close(); err != nil { + t.Fatal(err) + } + if _, err = dr.Init(root); err != nil { + t.Fatal(err) + } + snap := filepath.Join(t.TempDir(), "snapshot") + _, digest, err := dr.Snapshot(context.Background(), root, snap) + if err != nil { + t.Fatal(err) + } + standby := filepath.Join(t.TempDir(), "standby") + node, err := dr.Restore(context.Background(), snap, digest, standby) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(standby, "pages.db") + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + ro, err := db.OpenReadOnly(path) + if err != nil { + t.Fatal(err) + } + defer ro.Close() + readonly, err := store.NewReadOnlyWithDocumentRoot(ro, filepath.Join(standby, "documents")) + if err != nil { + t.Fatal(err) + } + readonly.StandbyReadUntil = node.ReadUntil() + s := New(readonly) + if s.previewRoot != "" || s.importRoot != "" || s.schedulerRunning.Load() != 0 { + t.Fatal("started standby worker/cache writers") + } + request := func(method, path, auth string) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, "http://localhost"+path, nil) + if auth != "" { + r.Header.Set("Authorization", "Bearer "+auth) + } + w := httptest.NewRecorder() + s.handler().ServeHTTP(w, r) + return w + } + for _, path := range []string{"/v/raw/rev:" + rev.UUID, "/dr-fixture"} { + w := request("GET", path, token) + if w.Code != 200 || !strings.Contains(w.Body.String(), "Canonical body.") { + t.Fatalf("read %s: %d %s", path, w.Code, w.Body.String()) + } + if w.Header().Get("Cache-Control") != "private, no-store" { + t.Fatalf("stale authorization cache permitted: %s", w.Header().Get("Cache-Control")) + } + } + if w := request("GET", "/v/raw/rev:"+rev.UUID, ""); w.Code == 200 { + t.Fatal("read-only bypassed authorization") + } + if w := request("PUT", "/_/api/v1/objects/"+object, token); w.Code != 503 { + t.Fatal("standby write not rejected") + } + if _, err = readonly.SaveRevisionForLanguage(object, "must never persist", "test", "und"); err == nil { + t.Fatal("store allowed write") + } + if _, err = store.NewReadOnlyWithDocumentRoot(ro, filepath.Join(t.TempDir(), "documents")); err == nil { + t.Fatal("regenerated missing auth key") + } + readonly.StandbyReadUntil = time.Now().Add(-time.Second) + if w := request("GET", "/dr-fixture", token); w.Code != 503 { + t.Fatal("stale snapshot served") + } + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if sha256.Sum256(before) != sha256.Sum256(after) { + t.Fatal("standby changed database bytes") + } + for _, name := range []string{"previews", "imports", "pages.db-wal", "pages.db-shm"} { + if _, err = os.Stat(filepath.Join(standby, name)); !os.IsNotExist(err) { + t.Fatalf("created standby artifact %s", name) + } + } +} diff --git a/internal/server/dregarnuhr_queue.go b/internal/server/dregarnuhr_queue.go new file mode 100644 index 0000000..0faf3b3 --- /dev/null +++ b/internal/server/dregarnuhr_queue.go @@ -0,0 +1,99 @@ +package server + +import ( + "context" + "database/sql" + "errors" + "log" + "time" + + "pageswiki/internal/dregarnuhr" + "pageswiki/internal/store" +) + +// This adapter preserves SQL/storage ownership and all existing queue IDs. +type schedulerQueue struct{ store *store.Store } + +func (q schedulerQueue) Settings(ctx context.Context) (dregarnuhr.Settings, error) { + if err := ctx.Err(); err != nil { + return dregarnuhr.Settings{}, err + } + if q.store.ReadOnly || q.store.BackgroundJobsDisabled { + return dregarnuhr.Settings{Paused: true}, nil + } + return q.store.GetQueueSettings() +} +func (q schedulerQueue) Claim(ctx context.Context, now time.Time) (*dregarnuhr.Job, error) { + job, err := q.store.ClaimDueEventAt(ctx, now) + if errors.Is(err, sql.ErrNoRows) { + return nil, dregarnuhr.ErrNoJob + } + return job, err +} +func (q schedulerQueue) Cancellation(ctx context.Context, id string) (bool, error) { + return q.store.EventCancellationState(ctx, id) +} +func (q schedulerQueue) Started(ctx context.Context, job *dregarnuhr.Job) error { + return q.store.AddEventLog(job.UUID, job.EventUUID, "info", "execution started") +} +func (q schedulerQueue) Finish(ctx context.Context, job *dregarnuhr.Job, err error, now time.Time) error { + return q.store.FinishEventAt(ctx, job.UUID, err, now) +} + +func (s *Server) eventRunner() *dregarnuhr.Runner { + s.schedulerOnce.Do(func() { + s.schedulerRunner = dregarnuhr.NewRunner(schedulerQueue{s.store}, s.registeredEventHandlers(), dregarnuhr.RunnerOptions{ + Active: &s.schedulerRunning, + Foreground: func() bool { return s.foregroundRequests.Load() > 0 }, + Wake: s.wakeEventScheduler, + Report: func(stage string, job *dregarnuhr.Job, err error) { + if job != nil { + log.Printf("component=%s event scheduler %s failed job=%s type=%s: %v", dregarnuhr.Component, stage, job.UUID, job.Type, err) + } else { + log.Printf("component=%s event scheduler %s failed: %v", dregarnuhr.Component, stage, err) + } + }, + }) + }) + return s.schedulerRunner +} + +func (s *Server) runDueEvents() { + if s.store == nil { + return + } + s.eventRunner().Dispatch(context.Background()) +} + +func (s *Server) schedulerActivity(ctx context.Context, now time.Time) error { + _, err := s.store.ExpireApprovalRequests(ctx, now.UnixMilli(), 100) + // Preserve independent City processing even if approval expiry fails. + s.enqueueCityActivityEvents() + return err +} + +func (s *Server) schedulerLoop() dregarnuhr.Loop { + return dregarnuhr.Loop{ + Every: time.Second, Wake: s.schedulerWake, OnWake: s.schedulerActivity, + Triggers: []dregarnuhr.Trigger{ + {Name: "initial-preview-sweep", After: 5 * time.Second, Run: func(context.Context, time.Time) error { s.enqueueAllPreviewEvents(); return nil }}, + {Name: "preview-sweep", Every: 15 * time.Minute, Run: func(context.Context, time.Time) error { s.enqueueAllPreviewEvents(); return nil }}, + {Name: "history-cleanup", Every: time.Hour, Run: func(context.Context, time.Time) error { + settings, err := s.store.GetQueueSettings() + if err != nil { + return err + } + return s.store.PurgeEventHistory(settings.RetentionHours) + }}, + {Name: "activity-sweep", Every: 10 * time.Second, Run: s.schedulerActivity}, + }, + Step: func(ctx context.Context, now time.Time) error { + _, err := s.store.EnqueueDueDefinitions(now) + s.eventRunner().Dispatch(ctx) + return err + }, + Report: func(stage string, err error) { + log.Printf("component=%s event scheduler %s failed: %v", dregarnuhr.Component, stage, err) + }, + } +} diff --git a/internal/server/dregarnuhr_queue_test.go b/internal/server/dregarnuhr_queue_test.go new file mode 100644 index 0000000..9826417 --- /dev/null +++ b/internal/server/dregarnuhr_queue_test.go @@ -0,0 +1,135 @@ +package server + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "pageswiki/internal/db" + "pageswiki/internal/dregarnuhr" + "pageswiki/internal/store" +) + +func TestDregarnuhrBuiltinsRemainRegistered(t *testing.T) { + s := &Server{} + registry := s.registeredEventHandlers() + // Duplicate registration proves presence without executing a business handler + // or reading any external calendar/source content. + for _, kind := range []string{ + store.NotificationPushEventType, store.TaskFocusEventType, store.TaskReminderEventType, + store.TaskActivityBridgeEventType, calendarSweepType, calendarSyncType, + metadataProjectionEventType, store.ResourceEventType, previewSweepEventType, + previewEventType, cityReconcileEventType, "pdf.page.generate", importAnalyzeEventType, importCommitEventType, + } { + if err := registry.Register(kind, func(context.Context, *dregarnuhr.Job) error { return nil }); err == nil { + t.Errorf("missing builtin: %s", kind) + } + } + if s.registeredEventHandlers() != registry { + t.Fatal("registry recreated") + } + if err := s.runEvent(&store.EventJob{Type: "test.unknown"}); err == nil { + t.Fatal("unknown event accepted") + } + loop := s.schedulerLoop() + if loop.Every != time.Second || loop.OnWake == nil { + t.Fatal("dispatch cadence or activity hook changed") + } + want := map[string][2]time.Duration{ + "initial-preview-sweep": {5 * time.Second, 0}, "preview-sweep": {0, 15 * time.Minute}, + "history-cleanup": {0, time.Hour}, "activity-sweep": {0, 10 * time.Second}, + } + if len(loop.Triggers) != len(want) { + t.Fatal("maintenance hooks changed") + } + for _, trigger := range loop.Triggers { + v, ok := want[trigger.Name] + if !ok || v != [2]time.Duration{trigger.After, trigger.Every} || trigger.Run == nil { + t.Fatalf("unexpected trigger: %+v", trigger) + } + } +} + +func TestDregarnuhrServerQueueIntegration(t *testing.T) { + database, err := db.Open(t.TempDir() + "/queue.db") + if err != nil { + t.Fatal(err) + } + defer database.Close() + st := &store.Store{DB: database} + if err := st.EnsureEventJobsSchema(); err != nil { + t.Fatal(err) + } + settings, err := st.GetQueueSettings() + if err != nil { + t.Fatal(err) + } + settings.Concurrency = 2 + if err := st.SetQueueSettings(settings); err != nil { + t.Fatal(err) + } + s := &Server{store: st, schedulerWake: make(chan struct{}, 8)} + started := make(chan struct{}, 8) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := s.registeredEventHandlers().Register("test.synthetic", func(ctx context.Context, job *dregarnuhr.Job) error { + started <- struct{}{} + <-ctx.Done() + return nil + }); err != nil { + t.Fatal(err) + } + for n := 0; n < 4; n++ { + if _, err := st.EnqueueEvent("test.synthetic", fmt.Sprint(n), "{}", time.Now().Add(-time.Minute)); err != nil { + t.Fatal(err) + } + } + var wg sync.WaitGroup + for range 16 { + wg.Add(1) + go func() { defer wg.Done(); s.eventRunner().Dispatch(ctx) }() + } + wg.Wait() + // Ensure workers have reached the synthetic handler before cancellation. + for range 2 { + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("handler not started") + } + } + if got := s.schedulerRunning.Load(); got != 2 { + t.Errorf("monitor running=%d, want=2", got) + } + cancel() + for range 2 { + select { + case <-s.schedulerWake: + case <-time.After(5 * time.Second): + t.Fatal("completion not persisted") + } + } + if got := s.schedulerRunning.Load(); got != 0 { + t.Fatalf("monitor leaked running=%d", got) + } + var running, attempted int + if err := database.QueryRow(`SELECT count(*) FROM event_jobs WHERE status='running'`).Scan(&running); err != nil { + t.Fatal(err) + } + if err := database.QueryRow(`SELECT count(*) FROM event_jobs WHERE attempts=1`).Scan(&attempted); err != nil { + t.Fatal(err) + } + if running != 0 || attempted != 2 { + t.Fatalf("persisted running=%d attempts=%d", running, attempted) + } + // Both deployment guards must stop claim before any registered handler runs. + for _, mode := range []string{"readonly", "disabled"} { + st.ReadOnly, st.BackgroundJobsDisabled = mode == "readonly", mode == "disabled" + s.runDueEvents() + if got := s.schedulerRunning.Load(); got != 0 { + t.Fatalf("%s started worker", mode) + } + } +} diff --git a/internal/server/editor_i18n.go b/internal/server/editor_i18n.go new file mode 100644 index 0000000..76d8602 --- /dev/null +++ b/internal/server/editor_i18n.go @@ -0,0 +1,12 @@ +package server + +func init() { + for lang, entries := range map[string]map[string]string{ + "en": {"editor.tools": "Insert component / drafts", "editor.language": "Edit / fork language", "editor.restore": "Restore a revision into the editor"}, + "ja": {"editor.tools": "コンポーネント挿入・下書き", "editor.language": "言語の編集・分岐", "editor.restore": "過去のリビジョンをエディタに復元"}, + } { + for key, value := range entries { + translations[lang][key] = value + } + } +} diff --git a/internal/server/editor_plain_test.go b/internal/server/editor_plain_test.go new file mode 100644 index 0000000..7786f3b --- /dev/null +++ b/internal/server/editor_plain_test.go @@ -0,0 +1,86 @@ +package server + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestPlainTextEditorCanonicalAndExistingHTML(t *testing.T) { + s, token, domain, htmlObject := editorFixture(t) + page := configRequest(s, token, "GET", "/e/-/notes.txt?edittype=plain", nil) + if page.Code != 200 || !strings.Contains(page.Body.String(), `value="plain"`) || !strings.Contains(page.Body.String(), `value="text/plain; charset=utf-8"`) { + t.Fatalf("plain editor: %d %s", page.Code, page.Body.String()) + } + object, err := s.store.ResolvePathInDomain(domain, "/notes.txt") + if err != nil { + t.Fatal(err) + } + body := "\n# Not Markdown\n{{children}}\n日本語 & <>\n" + payload, _ := json.Marshal(map[string]any{"object_uuid": object, "body": body, "lang": "und", "content_type": "text/plain; charset=utf-8", "filename": "notes.txt", "base_revision_uuid": ""}) + r := httptest.NewRequest("POST", "http://localhost/_/api/v1/revisions/save", bytes.NewReader(payload)) + r.Header.Set("Authorization", "Bearer "+token) + r.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + s.withAuthorization(s.mux).ServeHTTP(w, r) + if w.Code != 200 { + t.Fatalf("plain save: %d %s", w.Code, w.Body.String()) + } + rev, err := s.store.GetRevisionForExactLanguage(object, "und") + if err != nil { + t.Fatal(err) + } + if rev.Body != body || rev.ContentType != "text/plain; charset=utf-8" { + t.Fatalf("plain content mutated: %#v", rev) + } + if view := revisionContentView(rev, "/notes.txt"); view.Kind != "code" { + t.Fatalf("plain rendered as %s", view.Kind) + } + page = configRequest(s, token, "GET", "/e/-/notes.txt?edittype=markdown", nil) + if page.Code != 200 || !strings.Contains(page.Body.String(), `value="plain"`) || strings.Contains(page.Body.String(), `data-editor-action="markdown"`) { + t.Fatal("reopen switched plain file to HTML") + } + before, _ := s.store.GetRevisionForExactLanguage(htmlObject, "und") + page = configRequest(s, token, "GET", "/e/-/editor-fixture?edittype=plain", nil) + if page.Code != http.StatusConflict { + t.Fatalf("existing HTML converted: %d", page.Code) + } + after, _ := s.store.GetRevisionForExactLanguage(htmlObject, "und") + if before.UUID != after.UUID { + t.Fatal("existing HTML changed") + } + page = configRequest(s, token, "GET", "/e/-/invalid-format?edittype=not-a-format", nil) + if page.Code != http.StatusBadRequest { + t.Fatal("invalid editor format accepted") + } + if _, err = s.store.ResolvePathInDomain(domain, "/invalid-format"); err == nil { + t.Fatal("invalid format created a Page") + } +} + +func TestBrowserEditorCreationRequiresPermission(t *testing.T) { + s, _, domain, _ := editorFixture(t) + user, err := s.store.CreatePasswordAccount(domain, "reader", "a long test password", "Reader", "") + if err != nil { + t.Fatal(err) + } + if err := s.store.SetPrincipalState(domain, user.UUID, "readonly"); err != nil { + t.Fatal(err) + } + token, err := s.store.CreateAuthSession(domain, user.UUID) + if err != nil { + t.Fatal(err) + } + for _, format := range []string{"html", "plain"} { + response := configRequest(s, token, "GET", "/e/-/denied?edittype="+format, nil) + if response.Code != http.StatusForbidden { + t.Fatalf("read-only create: %d", response.Code) + } + if _, err := s.store.ResolvePathInDomain(domain, "/denied"); err == nil { + t.Fatal("denied editor created a Page") + } + } +} diff --git a/internal/server/editor_resilience_test.go b/internal/server/editor_resilience_test.go new file mode 100644 index 0000000..860453a --- /dev/null +++ b/internal/server/editor_resilience_test.go @@ -0,0 +1,147 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +func editorFixture(t *testing.T) (*Server, string, string, string) { + s, token, domain := configFixture(t) + s.mux.HandleFunc("/e/", s.handleEditorRoute) + s.mux.HandleFunc("/_/api/v1/editor/state", s.handleEditorState) + s.mux.HandleFunc("/_/api/v1/editor/convert", s.handleEditorConvert) + s.mux.HandleFunc("/_/api/v1/revisions/save", s.handleSaveRevisionAPI) + s.mux.HandleFunc("/_/api/v1/revisions/list", s.handleRevisionListAPI) + s.mux.HandleFunc("/_/api/v1/render/markdown", s.handleMarkdownRenderAPI) + object, err := s.store.CreatePageAtPathInDomain(domain, "/editor-fixture") + if err != nil { + t.Fatal(err) + } + if _, err = s.store.SaveRevisionContent(object, []byte("one two three
"), "base", "und", "text/html", ""); err != nil { + t.Fatal(err) + } + return s, token, domain, object +} + +func TestEditorResilienceAPI(t *testing.T) { + s, token, domain, object := editorFixture(t) + principal, err := s.store.PrincipalForSession(domain, token) + if err != nil { + t.Fatal(err) + } + base, err := s.store.GetRevisionForExactLanguage(object, "und") + if err != nil { + t.Fatal(err) + } + payload := map[string]any{"object_uuid": object, "body": "