Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts. Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation. Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
378 lines
14 KiB
Go
378 lines
14 KiB
Go
package mestionora
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const SearchCandidateLimit = 1000
|
|
const SearchExcerptDocumentBytes = 64 << 10
|
|
const SearchExcerptTotalBytes = 1 << 20
|
|
const searchExcerptTimeout = 400 * time.Millisecond
|
|
|
|
const searchResultColumns = `p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.revision_uuid,''),COALESCE(r.created_at_ms/1000,0)`
|
|
const searchPathBoundary = `(p.path=? OR (p.path>=? AND p.path<?))`
|
|
const searchPathCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,0
|
|
FROM paths p LEFT JOIN latest_revision_index r ON r.object_uuid=p.object_uuid
|
|
WHERE p.domain_uuid=? AND ` + searchPathBoundary
|
|
const searchPathQueryPredicate = ` AND (instr(lower(p.path),lower(?))>0 OR instr(lower(COALESCE(r.filename,'')),lower(?))>0)`
|
|
const searchPathCandidatesSQL = searchPathCandidatesBaseSQL + searchPathQueryPredicate + ` ORDER BY p.path LIMIT ?`
|
|
|
|
// CROSS JOIN is intentional: revision_uuid is an UNINDEXED FTS payload column.
|
|
// A Path-first plan re-runs the complete MATCH scan for every Path. Start with
|
|
// MATCH exactly once, then seek current Revision metadata and Domain bindings
|
|
// through their B-tree indexes. Keep scope filtering before the candidate cap.
|
|
const searchFTSCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,revision_search.rowid
|
|
FROM revision_search
|
|
CROSS JOIN latest_revision_index r ON r.revision_uuid=revision_search.revision_uuid
|
|
CROSS JOIN paths p ON p.object_uuid=r.object_uuid
|
|
WHERE revision_search MATCH ? AND p.domain_uuid=? AND ` + searchPathBoundary
|
|
const searchFTSCandidatesSQL = searchFTSCandidatesBaseSQL + ` LIMIT ?`
|
|
|
|
func searchBodyExcerptsSQL(count int) string {
|
|
// UUID is not indexed by FTS5. Restrict the MATCH/snippet work using the
|
|
// integer rowids collected from the same read snapshot, not payload UUIDs.
|
|
return `SELECT revision_uuid,substr(snippet(revision_search,3,'','',' … ',36),1,4096)
|
|
FROM revision_search WHERE revision_search MATCH ? AND rowid IN (` + strings.TrimRight(strings.Repeat("?,", count), ",") + `)`
|
|
}
|
|
|
|
// Excerpts are optional enrichment. Even small documents can have expensive
|
|
// prefix hit-position scans, so their time budget is separate from finding
|
|
// results. A spent excerpt budget omits snippets, not already-authorized hits.
|
|
func loadSearchBodyExcerpts(parent context.Context, tx *sql.Tx, expression string, ids []any, budget time.Duration) (excerpts map[string]string, err error) {
|
|
ctx, cancel := context.WithTimeout(parent, budget)
|
|
defer cancel()
|
|
defer func() {
|
|
if err != nil && ctx.Err() != nil {
|
|
if parent.Err() != nil {
|
|
err = parent.Err()
|
|
} else {
|
|
excerpts, err = nil, nil
|
|
}
|
|
}
|
|
}()
|
|
args := append([]any{"body : (" + expression + ")"}, ids...)
|
|
rows, err := tx.QueryContext(ctx, searchBodyExcerptsSQL(len(ids)), args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
excerpts = make(map[string]string)
|
|
for rows.Next() {
|
|
var id, excerpt string
|
|
if err := rows.Scan(&id, &excerpt); err != nil {
|
|
return nil, err
|
|
}
|
|
excerpts[id] = excerpt
|
|
}
|
|
return excerpts, rows.Err()
|
|
}
|
|
|
|
type SearchResult struct {
|
|
Path string
|
|
ContentType string
|
|
Filename string
|
|
DocumentHash string
|
|
RevisionUUID string
|
|
Size int64 // Kept for API compatibility; search does not stat Document files.
|
|
UpdatedAt int64
|
|
PathMatch bool
|
|
FilenameMatch bool
|
|
BodyMatch bool
|
|
Excerpt string // Untrusted source text, never HTML-safe markup.
|
|
ExcerptOmitted bool // A body match remains visible when snippet work exceeds its budget.
|
|
}
|
|
|
|
type PathSuggestion struct {
|
|
SearchResult
|
|
Title string
|
|
}
|
|
|
|
// SearchPathsInDomainFiltered bounds SQL candidates before materialization and
|
|
// authorizes each Path before loading an excerpt. It never computes a global hit
|
|
// count or stats Document files. A narrow scope improves both recall and cost.
|
|
// allowed runs inside a read snapshot. Prefer a preloaded request-local policy;
|
|
// a callback doing DB reads requires spare pool capacity (not a one-connection
|
|
// pool). Keeping policy I/O outside this snapshot is separate follow-up work.
|
|
func (s *SQLite) SearchPathsInDomainFiltered(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]SearchResult, error) {
|
|
return s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{})
|
|
}
|
|
|
|
// SearchPathsInDomainWithOptions also permits filter-only searches. File/date
|
|
// predicates apply before SQL candidate caps, while ACL checks stay ahead of all
|
|
// snippet enrichment. Zero options preserve the original indexed query plans.
|
|
func (s *SQLite) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, searchErr error) {
|
|
return s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options, true)
|
|
}
|
|
|
|
// SearchPathSuggestionsInDomain shares the current-Revision FTS and ACL path
|
|
// with full search, but deliberately skips snippet generation. Typeahead calls
|
|
// are frequent and only need a bounded display label, never a body excerpt.
|
|
func (s *SQLite) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) {
|
|
results, err := s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}, false)
|
|
if err != nil || len(results) == 0 {
|
|
return nil, err
|
|
}
|
|
ids := make([]any, 0, len(results))
|
|
seen := make(map[string]bool, len(results))
|
|
for _, result := range results {
|
|
if result.RevisionUUID != "" && strings.HasPrefix(strings.ToLower(result.ContentType), "text/html") && !seen[result.RevisionUUID] {
|
|
ids = append(ids, result.RevisionUUID)
|
|
seen[result.RevisionUUID] = true
|
|
}
|
|
}
|
|
headings := make(map[string]string, len(ids))
|
|
if len(ids) > 0 {
|
|
rows, queryErr := s.db.QueryContext(ctx, `SELECT uuid,substr(CAST(body AS TEXT),1,65536) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
|
|
if queryErr != nil {
|
|
return nil, queryErr
|
|
}
|
|
for rows.Next() {
|
|
var revision, source string
|
|
if queryErr = rows.Scan(&revision, &source); queryErr != nil {
|
|
rows.Close()
|
|
return nil, queryErr
|
|
}
|
|
headings[revision] = FirstHTMLHeading(source)
|
|
}
|
|
queryErr = rows.Err()
|
|
rows.Close()
|
|
if queryErr != nil {
|
|
return nil, queryErr
|
|
}
|
|
}
|
|
out := make([]PathSuggestion, 0, len(results))
|
|
for _, result := range results {
|
|
out = append(out, PathSuggestion{SearchResult: result, Title: headings[result.RevisionUUID]})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (s *SQLite) searchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions, loadExcerpts bool) (found []SearchResult, searchErr error) {
|
|
if err := options.Validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
scope, query = normalizePath(scope), strings.TrimSpace(query)
|
|
if query == "" && !options.HasFilters() || allowed == nil {
|
|
return nil, nil
|
|
}
|
|
if len(query) > 512 || len(strings.Fields(query)) > 16 {
|
|
return nil, errors.New("search query is too long")
|
|
}
|
|
if limit <= 0 || limit > 200 {
|
|
limit = 100
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
|
defer cancel()
|
|
defer func() {
|
|
// SQLite can report an interrupted statement instead of wrapping the
|
|
// context error. Preserve cancellation/deadline semantics for HTTP.
|
|
if ctx.Err() != nil {
|
|
found = nil
|
|
searchErr = ctx.Err()
|
|
}
|
|
}()
|
|
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer tx.Rollback()
|
|
prefix := strings.TrimRight(scope, "/") + "/"
|
|
upper := strings.TrimRight(scope, "/") + "0"
|
|
results := make(map[string]SearchResult)
|
|
ftsRows := make(map[string]int64)
|
|
collect := func(statement string, body, filename bool, args ...any) error {
|
|
rows, err := tx.QueryContext(ctx, statement, args...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var result SearchResult
|
|
var rowID int64
|
|
if err := rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt, &rowID); err != nil {
|
|
return err
|
|
}
|
|
if !allowed(result.Path) {
|
|
continue
|
|
}
|
|
if rowID != 0 {
|
|
ftsRows[result.RevisionUUID] = rowID
|
|
}
|
|
result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query))
|
|
result.FilenameMatch = filename || results[result.Path].FilenameMatch || query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query))
|
|
result.BodyMatch = body || results[result.Path].BodyMatch
|
|
results[result.Path] = result
|
|
}
|
|
return rows.Err()
|
|
}
|
|
filterSQL, filterArgs := searchOptionsPredicate(options)
|
|
pathSQL := searchPathCandidatesBaseSQL
|
|
pathArgs := []any{domainUUID, scope, prefix, upper}
|
|
if query != "" {
|
|
pathSQL += searchPathQueryPredicate
|
|
pathArgs = append(pathArgs, query, query)
|
|
}
|
|
pathSQL += filterSQL + ` ORDER BY p.path LIMIT ?`
|
|
pathArgs = append(pathArgs, filterArgs...)
|
|
pathArgs = append(pathArgs, SearchCandidateLimit)
|
|
if err := collect(pathSQL, false, false, pathArgs...); err != nil {
|
|
return nil, err
|
|
}
|
|
expression := ftsPrefixExpression(query)
|
|
if expression != "" {
|
|
// No global ORDER BY or unbounded IN materialization: stop the FTS scan
|
|
// after at most this many current, in-Domain, in-scope Path candidates.
|
|
for _, field := range []string{"body", "filename"} {
|
|
args := []any{field + " : (" + expression + ")", domainUUID, scope, prefix, upper}
|
|
args = append(args, filterArgs...)
|
|
args = append(args, SearchCandidateLimit)
|
|
if err := collect(searchFTSCandidatesBaseSQL+filterSQL+` LIMIT ?`, field == "body", field == "filename", args...); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
out := make([]SearchResult, 0, len(results))
|
|
for _, result := range results {
|
|
out = append(out, result)
|
|
}
|
|
sort.Slice(out, func(i, j int) bool {
|
|
// Direct name/Path matches precede body-only hits; all ties are stable.
|
|
iName, jName := out[i].PathMatch || out[i].FilenameMatch, out[j].PathMatch || out[j].FilenameMatch
|
|
if iName != jName {
|
|
return iName
|
|
}
|
|
if len(out[i].Path) != len(out[j].Path) {
|
|
return len(out[i].Path) < len(out[j].Path)
|
|
}
|
|
return out[i].Path < out[j].Path
|
|
})
|
|
if len(out) > limit {
|
|
out = out[:limit]
|
|
}
|
|
if !loadExcerpts {
|
|
return out, nil
|
|
}
|
|
// Only authorized, displayed results receive text excerpts. Batch the FTS
|
|
// lookup so a broad query is not re-scanned once for every visible result.
|
|
ids := make([]any, 0, len(out))
|
|
seen := make(map[string]bool)
|
|
for _, result := range out {
|
|
if result.BodyMatch && !seen[result.RevisionUUID] {
|
|
ids = append(ids, ftsRows[result.RevisionUUID])
|
|
seen[result.RevisionUUID] = true
|
|
}
|
|
}
|
|
if len(ids) != 0 {
|
|
// snippet's output token/character limit does not bound its input work:
|
|
// FTS5 may tokenize the entire matched document. Read byte lengths only
|
|
// for authorized displayed rows before spending the snippet budget.
|
|
rows, err := tx.QueryContext(ctx, `SELECT rowid,octet_length(body)
|
|
FROM revision_search WHERE rowid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sizes := make(map[int64]int64)
|
|
for rows.Next() {
|
|
var rowID, size int64
|
|
if err := rows.Scan(&rowID, &size); err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
sizes[rowID] = size
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
ids = nil
|
|
selected := make(map[string]bool)
|
|
remaining := int64(SearchExcerptTotalBytes)
|
|
for i := range out {
|
|
if !out[i].BodyMatch {
|
|
continue
|
|
}
|
|
revision := out[i].RevisionUUID
|
|
include, decided := selected[revision]
|
|
if !decided {
|
|
rowID := ftsRows[revision]
|
|
size, exists := sizes[rowID]
|
|
include = exists && size >= 0 && size <= SearchExcerptDocumentBytes && size <= remaining
|
|
if include {
|
|
ids = append(ids, rowID)
|
|
remaining -= size
|
|
}
|
|
selected[revision] = include
|
|
}
|
|
out[i].ExcerptOmitted = !include
|
|
}
|
|
}
|
|
if len(ids) != 0 {
|
|
excerpts, err := loadSearchBodyExcerpts(ctx, tx, expression, ids, searchExcerptTimeout)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for i := range out {
|
|
if out[i].BodyMatch {
|
|
var present bool
|
|
out[i].Excerpt, present = excerpts[out[i].RevisionUUID]
|
|
out[i].ExcerptOmitted = !present
|
|
}
|
|
}
|
|
}
|
|
// A filename/Path-only hit still benefits from a short Page summary. Read
|
|
// only a bounded prefix from displayed textual revisions, in one batch.
|
|
ids, seen = nil, make(map[string]bool)
|
|
for _, result := range out {
|
|
media := strings.ToLower(result.ContentType)
|
|
textual := strings.HasPrefix(media, "text/") || strings.HasPrefix(media, "application/json") || strings.HasPrefix(media, "application/xml") || strings.HasPrefix(media, "application/javascript") || strings.HasPrefix(media, "application/yaml") || strings.HasPrefix(media, "application/x-yaml")
|
|
if !result.BodyMatch && textual && result.RevisionUUID != "" && !seen[result.RevisionUUID] {
|
|
ids = append(ids, result.RevisionUUID)
|
|
seen[result.RevisionUUID] = true
|
|
}
|
|
}
|
|
if len(ids) != 0 {
|
|
rows, err := tx.QueryContext(ctx, `SELECT uuid,substr(body,1,1024) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
excerpts := make(map[string]string)
|
|
for rows.Next() {
|
|
var id, excerpt string
|
|
if err := rows.Scan(&id, &excerpt); err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
excerpts[id] = excerpt
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for i := range out {
|
|
if !out[i].BodyMatch {
|
|
out[i].Excerpt = excerpts[out[i].RevisionUUID]
|
|
}
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func ftsPrefixExpression(query string) string {
|
|
words := strings.Fields(query)
|
|
terms := make([]string, 0, len(words))
|
|
for _, word := range words {
|
|
word = strings.ReplaceAll(word, `"`, `""`)
|
|
if word != "" {
|
|
terms = append(terms, `"`+word+`"*`)
|
|
}
|
|
}
|
|
return strings.Join(terms, " AND ")
|
|
}
|