Files
Anya Tokugawa d5d21b7841
Build and publish container images / images (push) Successful in 9m34s
Release container images / release (push) Successful in 36s
Refactor architecture into eight components and consolidate reliability work
Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts.

Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation.

Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
2026-11-01 15:23:31 +09:00

378 lines
14 KiB
Go

package mestionora
import (
"context"
"database/sql"
"errors"
"sort"
"strings"
"time"
)
const SearchCandidateLimit = 1000
const SearchExcerptDocumentBytes = 64 << 10
const SearchExcerptTotalBytes = 1 << 20
const searchExcerptTimeout = 400 * time.Millisecond
const searchResultColumns = `p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.revision_uuid,''),COALESCE(r.created_at_ms/1000,0)`
const searchPathBoundary = `(p.path=? OR (p.path>=? AND p.path<?))`
const searchPathCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,0
FROM paths p LEFT JOIN latest_revision_index r ON r.object_uuid=p.object_uuid
WHERE p.domain_uuid=? AND ` + searchPathBoundary
const searchPathQueryPredicate = ` AND (instr(lower(p.path),lower(?))>0 OR instr(lower(COALESCE(r.filename,'')),lower(?))>0)`
const searchPathCandidatesSQL = searchPathCandidatesBaseSQL + searchPathQueryPredicate + ` ORDER BY p.path LIMIT ?`
// CROSS JOIN is intentional: revision_uuid is an UNINDEXED FTS payload column.
// A Path-first plan re-runs the complete MATCH scan for every Path. Start with
// MATCH exactly once, then seek current Revision metadata and Domain bindings
// through their B-tree indexes. Keep scope filtering before the candidate cap.
const searchFTSCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,revision_search.rowid
FROM revision_search
CROSS JOIN latest_revision_index r ON r.revision_uuid=revision_search.revision_uuid
CROSS JOIN paths p ON p.object_uuid=r.object_uuid
WHERE revision_search MATCH ? AND p.domain_uuid=? AND ` + searchPathBoundary
const searchFTSCandidatesSQL = searchFTSCandidatesBaseSQL + ` LIMIT ?`
func searchBodyExcerptsSQL(count int) string {
// UUID is not indexed by FTS5. Restrict the MATCH/snippet work using the
// integer rowids collected from the same read snapshot, not payload UUIDs.
return `SELECT revision_uuid,substr(snippet(revision_search,3,'','',' … ',36),1,4096)
FROM revision_search WHERE revision_search MATCH ? AND rowid IN (` + strings.TrimRight(strings.Repeat("?,", count), ",") + `)`
}
// Excerpts are optional enrichment. Even small documents can have expensive
// prefix hit-position scans, so their time budget is separate from finding
// results. A spent excerpt budget omits snippets, not already-authorized hits.
func loadSearchBodyExcerpts(parent context.Context, tx *sql.Tx, expression string, ids []any, budget time.Duration) (excerpts map[string]string, err error) {
ctx, cancel := context.WithTimeout(parent, budget)
defer cancel()
defer func() {
if err != nil && ctx.Err() != nil {
if parent.Err() != nil {
err = parent.Err()
} else {
excerpts, err = nil, nil
}
}
}()
args := append([]any{"body : (" + expression + ")"}, ids...)
rows, err := tx.QueryContext(ctx, searchBodyExcerptsSQL(len(ids)), args...)
if err != nil {
return nil, err
}
defer rows.Close()
excerpts = make(map[string]string)
for rows.Next() {
var id, excerpt string
if err := rows.Scan(&id, &excerpt); err != nil {
return nil, err
}
excerpts[id] = excerpt
}
return excerpts, rows.Err()
}
type SearchResult struct {
Path string
ContentType string
Filename string
DocumentHash string
RevisionUUID string
Size int64 // Kept for API compatibility; search does not stat Document files.
UpdatedAt int64
PathMatch bool
FilenameMatch bool
BodyMatch bool
Excerpt string // Untrusted source text, never HTML-safe markup.
ExcerptOmitted bool // A body match remains visible when snippet work exceeds its budget.
}
type PathSuggestion struct {
SearchResult
Title string
}
// SearchPathsInDomainFiltered bounds SQL candidates before materialization and
// authorizes each Path before loading an excerpt. It never computes a global hit
// count or stats Document files. A narrow scope improves both recall and cost.
// allowed runs inside a read snapshot. Prefer a preloaded request-local policy;
// a callback doing DB reads requires spare pool capacity (not a one-connection
// pool). Keeping policy I/O outside this snapshot is separate follow-up work.
func (s *SQLite) SearchPathsInDomainFiltered(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]SearchResult, error) {
return s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{})
}
// SearchPathsInDomainWithOptions also permits filter-only searches. File/date
// predicates apply before SQL candidate caps, while ACL checks stay ahead of all
// snippet enrichment. Zero options preserve the original indexed query plans.
func (s *SQLite) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, searchErr error) {
return s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options, true)
}
// SearchPathSuggestionsInDomain shares the current-Revision FTS and ACL path
// with full search, but deliberately skips snippet generation. Typeahead calls
// are frequent and only need a bounded display label, never a body excerpt.
func (s *SQLite) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) {
results, err := s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}, false)
if err != nil || len(results) == 0 {
return nil, err
}
ids := make([]any, 0, len(results))
seen := make(map[string]bool, len(results))
for _, result := range results {
if result.RevisionUUID != "" && strings.HasPrefix(strings.ToLower(result.ContentType), "text/html") && !seen[result.RevisionUUID] {
ids = append(ids, result.RevisionUUID)
seen[result.RevisionUUID] = true
}
}
headings := make(map[string]string, len(ids))
if len(ids) > 0 {
rows, queryErr := s.db.QueryContext(ctx, `SELECT uuid,substr(CAST(body AS TEXT),1,65536) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
if queryErr != nil {
return nil, queryErr
}
for rows.Next() {
var revision, source string
if queryErr = rows.Scan(&revision, &source); queryErr != nil {
rows.Close()
return nil, queryErr
}
headings[revision] = FirstHTMLHeading(source)
}
queryErr = rows.Err()
rows.Close()
if queryErr != nil {
return nil, queryErr
}
}
out := make([]PathSuggestion, 0, len(results))
for _, result := range results {
out = append(out, PathSuggestion{SearchResult: result, Title: headings[result.RevisionUUID]})
}
return out, nil
}
func (s *SQLite) searchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions, loadExcerpts bool) (found []SearchResult, searchErr error) {
if err := options.Validate(); err != nil {
return nil, err
}
scope, query = normalizePath(scope), strings.TrimSpace(query)
if query == "" && !options.HasFilters() || allowed == nil {
return nil, nil
}
if len(query) > 512 || len(strings.Fields(query)) > 16 {
return nil, errors.New("search query is too long")
}
if limit <= 0 || limit > 200 {
limit = 100
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
defer func() {
// SQLite can report an interrupted statement instead of wrapping the
// context error. Preserve cancellation/deadline semantics for HTTP.
if ctx.Err() != nil {
found = nil
searchErr = ctx.Err()
}
}()
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return nil, err
}
defer tx.Rollback()
prefix := strings.TrimRight(scope, "/") + "/"
upper := strings.TrimRight(scope, "/") + "0"
results := make(map[string]SearchResult)
ftsRows := make(map[string]int64)
collect := func(statement string, body, filename bool, args ...any) error {
rows, err := tx.QueryContext(ctx, statement, args...)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var result SearchResult
var rowID int64
if err := rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt, &rowID); err != nil {
return err
}
if !allowed(result.Path) {
continue
}
if rowID != 0 {
ftsRows[result.RevisionUUID] = rowID
}
result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query))
result.FilenameMatch = filename || results[result.Path].FilenameMatch || query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query))
result.BodyMatch = body || results[result.Path].BodyMatch
results[result.Path] = result
}
return rows.Err()
}
filterSQL, filterArgs := searchOptionsPredicate(options)
pathSQL := searchPathCandidatesBaseSQL
pathArgs := []any{domainUUID, scope, prefix, upper}
if query != "" {
pathSQL += searchPathQueryPredicate
pathArgs = append(pathArgs, query, query)
}
pathSQL += filterSQL + ` ORDER BY p.path LIMIT ?`
pathArgs = append(pathArgs, filterArgs...)
pathArgs = append(pathArgs, SearchCandidateLimit)
if err := collect(pathSQL, false, false, pathArgs...); err != nil {
return nil, err
}
expression := ftsPrefixExpression(query)
if expression != "" {
// No global ORDER BY or unbounded IN materialization: stop the FTS scan
// after at most this many current, in-Domain, in-scope Path candidates.
for _, field := range []string{"body", "filename"} {
args := []any{field + " : (" + expression + ")", domainUUID, scope, prefix, upper}
args = append(args, filterArgs...)
args = append(args, SearchCandidateLimit)
if err := collect(searchFTSCandidatesBaseSQL+filterSQL+` LIMIT ?`, field == "body", field == "filename", args...); err != nil {
return nil, err
}
}
}
out := make([]SearchResult, 0, len(results))
for _, result := range results {
out = append(out, result)
}
sort.Slice(out, func(i, j int) bool {
// Direct name/Path matches precede body-only hits; all ties are stable.
iName, jName := out[i].PathMatch || out[i].FilenameMatch, out[j].PathMatch || out[j].FilenameMatch
if iName != jName {
return iName
}
if len(out[i].Path) != len(out[j].Path) {
return len(out[i].Path) < len(out[j].Path)
}
return out[i].Path < out[j].Path
})
if len(out) > limit {
out = out[:limit]
}
if !loadExcerpts {
return out, nil
}
// Only authorized, displayed results receive text excerpts. Batch the FTS
// lookup so a broad query is not re-scanned once for every visible result.
ids := make([]any, 0, len(out))
seen := make(map[string]bool)
for _, result := range out {
if result.BodyMatch && !seen[result.RevisionUUID] {
ids = append(ids, ftsRows[result.RevisionUUID])
seen[result.RevisionUUID] = true
}
}
if len(ids) != 0 {
// snippet's output token/character limit does not bound its input work:
// FTS5 may tokenize the entire matched document. Read byte lengths only
// for authorized displayed rows before spending the snippet budget.
rows, err := tx.QueryContext(ctx, `SELECT rowid,octet_length(body)
FROM revision_search WHERE rowid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
if err != nil {
return nil, err
}
sizes := make(map[int64]int64)
for rows.Next() {
var rowID, size int64
if err := rows.Scan(&rowID, &size); err != nil {
rows.Close()
return nil, err
}
sizes[rowID] = size
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
ids = nil
selected := make(map[string]bool)
remaining := int64(SearchExcerptTotalBytes)
for i := range out {
if !out[i].BodyMatch {
continue
}
revision := out[i].RevisionUUID
include, decided := selected[revision]
if !decided {
rowID := ftsRows[revision]
size, exists := sizes[rowID]
include = exists && size >= 0 && size <= SearchExcerptDocumentBytes && size <= remaining
if include {
ids = append(ids, rowID)
remaining -= size
}
selected[revision] = include
}
out[i].ExcerptOmitted = !include
}
}
if len(ids) != 0 {
excerpts, err := loadSearchBodyExcerpts(ctx, tx, expression, ids, searchExcerptTimeout)
if err != nil {
return nil, err
}
for i := range out {
if out[i].BodyMatch {
var present bool
out[i].Excerpt, present = excerpts[out[i].RevisionUUID]
out[i].ExcerptOmitted = !present
}
}
}
// A filename/Path-only hit still benefits from a short Page summary. Read
// only a bounded prefix from displayed textual revisions, in one batch.
ids, seen = nil, make(map[string]bool)
for _, result := range out {
media := strings.ToLower(result.ContentType)
textual := strings.HasPrefix(media, "text/") || strings.HasPrefix(media, "application/json") || strings.HasPrefix(media, "application/xml") || strings.HasPrefix(media, "application/javascript") || strings.HasPrefix(media, "application/yaml") || strings.HasPrefix(media, "application/x-yaml")
if !result.BodyMatch && textual && result.RevisionUUID != "" && !seen[result.RevisionUUID] {
ids = append(ids, result.RevisionUUID)
seen[result.RevisionUUID] = true
}
}
if len(ids) != 0 {
rows, err := tx.QueryContext(ctx, `SELECT uuid,substr(body,1,1024) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
if err != nil {
return nil, err
}
excerpts := make(map[string]string)
for rows.Next() {
var id, excerpt string
if err := rows.Scan(&id, &excerpt); err != nil {
rows.Close()
return nil, err
}
excerpts[id] = excerpt
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
for i := range out {
if !out[i].BodyMatch {
out[i].Excerpt = excerpts[out[i].RevisionUUID]
}
}
}
return out, nil
}
func ftsPrefixExpression(query string) string {
words := strings.Fields(query)
terms := make([]string, 0, len(words))
for _, word := range words {
word = strings.ReplaceAll(word, `"`, `""`)
if word != "" {
terms = append(terms, `"`+word+`"*`)
}
}
return strings.Join(terms, " AND ")
}