package mestionora import ( "context" "database/sql" "errors" "sort" "strings" "time" ) const SearchCandidateLimit = 1000 const SearchExcerptDocumentBytes = 64 << 10 const SearchExcerptTotalBytes = 1 << 20 const searchExcerptTimeout = 400 * time.Millisecond const searchResultColumns = `p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.revision_uuid,''),COALESCE(r.created_at_ms/1000,0)` const searchPathBoundary = `(p.path=? OR (p.path>=? AND p.path0 OR instr(lower(COALESCE(r.filename,'')),lower(?))>0)` const searchPathCandidatesSQL = searchPathCandidatesBaseSQL + searchPathQueryPredicate + ` ORDER BY p.path LIMIT ?` // CROSS JOIN is intentional: revision_uuid is an UNINDEXED FTS payload column. // A Path-first plan re-runs the complete MATCH scan for every Path. Start with // MATCH exactly once, then seek current Revision metadata and Domain bindings // through their B-tree indexes. Keep scope filtering before the candidate cap. const searchFTSCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,revision_search.rowid FROM revision_search CROSS JOIN latest_revision_index r ON r.revision_uuid=revision_search.revision_uuid CROSS JOIN paths p ON p.object_uuid=r.object_uuid WHERE revision_search MATCH ? AND p.domain_uuid=? AND ` + searchPathBoundary const searchFTSCandidatesSQL = searchFTSCandidatesBaseSQL + ` LIMIT ?` func searchBodyExcerptsSQL(count int) string { // UUID is not indexed by FTS5. Restrict the MATCH/snippet work using the // integer rowids collected from the same read snapshot, not payload UUIDs. return `SELECT revision_uuid,substr(snippet(revision_search,3,'','',' … ',36),1,4096) FROM revision_search WHERE revision_search MATCH ? AND rowid IN (` + strings.TrimRight(strings.Repeat("?,", count), ",") + `)` } // Excerpts are optional enrichment. Even small documents can have expensive // prefix hit-position scans, so their time budget is separate from finding // results. A spent excerpt budget omits snippets, not already-authorized hits. func loadSearchBodyExcerpts(parent context.Context, tx *sql.Tx, expression string, ids []any, budget time.Duration) (excerpts map[string]string, err error) { ctx, cancel := context.WithTimeout(parent, budget) defer cancel() defer func() { if err != nil && ctx.Err() != nil { if parent.Err() != nil { err = parent.Err() } else { excerpts, err = nil, nil } } }() args := append([]any{"body : (" + expression + ")"}, ids...) rows, err := tx.QueryContext(ctx, searchBodyExcerptsSQL(len(ids)), args...) if err != nil { return nil, err } defer rows.Close() excerpts = make(map[string]string) for rows.Next() { var id, excerpt string if err := rows.Scan(&id, &excerpt); err != nil { return nil, err } excerpts[id] = excerpt } return excerpts, rows.Err() } type SearchResult struct { Path string ContentType string Filename string DocumentHash string RevisionUUID string Size int64 // Kept for API compatibility; search does not stat Document files. UpdatedAt int64 PathMatch bool FilenameMatch bool BodyMatch bool Excerpt string // Untrusted source text, never HTML-safe markup. ExcerptOmitted bool // A body match remains visible when snippet work exceeds its budget. } type PathSuggestion struct { SearchResult Title string } // SearchPathsInDomainFiltered bounds SQL candidates before materialization and // authorizes each Path before loading an excerpt. It never computes a global hit // count or stats Document files. A narrow scope improves both recall and cost. // allowed runs inside a read snapshot. Prefer a preloaded request-local policy; // a callback doing DB reads requires spare pool capacity (not a one-connection // pool). Keeping policy I/O outside this snapshot is separate follow-up work. func (s *SQLite) SearchPathsInDomainFiltered(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]SearchResult, error) { return s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}) } // SearchPathsInDomainWithOptions also permits filter-only searches. File/date // predicates apply before SQL candidate caps, while ACL checks stay ahead of all // snippet enrichment. Zero options preserve the original indexed query plans. func (s *SQLite) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, searchErr error) { return s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options, true) } // SearchPathSuggestionsInDomain shares the current-Revision FTS and ACL path // with full search, but deliberately skips snippet generation. Typeahead calls // are frequent and only need a bounded display label, never a body excerpt. func (s *SQLite) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) { results, err := s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}, false) if err != nil || len(results) == 0 { return nil, err } ids := make([]any, 0, len(results)) seen := make(map[string]bool, len(results)) for _, result := range results { if result.RevisionUUID != "" && strings.HasPrefix(strings.ToLower(result.ContentType), "text/html") && !seen[result.RevisionUUID] { ids = append(ids, result.RevisionUUID) seen[result.RevisionUUID] = true } } headings := make(map[string]string, len(ids)) if len(ids) > 0 { rows, queryErr := s.db.QueryContext(ctx, `SELECT uuid,substr(CAST(body AS TEXT),1,65536) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...) if queryErr != nil { return nil, queryErr } for rows.Next() { var revision, source string if queryErr = rows.Scan(&revision, &source); queryErr != nil { rows.Close() return nil, queryErr } headings[revision] = FirstHTMLHeading(source) } queryErr = rows.Err() rows.Close() if queryErr != nil { return nil, queryErr } } out := make([]PathSuggestion, 0, len(results)) for _, result := range results { out = append(out, PathSuggestion{SearchResult: result, Title: headings[result.RevisionUUID]}) } return out, nil } func (s *SQLite) searchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions, loadExcerpts bool) (found []SearchResult, searchErr error) { if err := options.Validate(); err != nil { return nil, err } scope, query = normalizePath(scope), strings.TrimSpace(query) if query == "" && !options.HasFilters() || allowed == nil { return nil, nil } if len(query) > 512 || len(strings.Fields(query)) > 16 { return nil, errors.New("search query is too long") } if limit <= 0 || limit > 200 { limit = 100 } ctx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() defer func() { // SQLite can report an interrupted statement instead of wrapping the // context error. Preserve cancellation/deadline semantics for HTTP. if ctx.Err() != nil { found = nil searchErr = ctx.Err() } }() tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) if err != nil { return nil, err } defer tx.Rollback() prefix := strings.TrimRight(scope, "/") + "/" upper := strings.TrimRight(scope, "/") + "0" results := make(map[string]SearchResult) ftsRows := make(map[string]int64) collect := func(statement string, body, filename bool, args ...any) error { rows, err := tx.QueryContext(ctx, statement, args...) if err != nil { return err } defer rows.Close() for rows.Next() { var result SearchResult var rowID int64 if err := rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt, &rowID); err != nil { return err } if !allowed(result.Path) { continue } if rowID != 0 { ftsRows[result.RevisionUUID] = rowID } result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query)) result.FilenameMatch = filename || results[result.Path].FilenameMatch || query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query)) result.BodyMatch = body || results[result.Path].BodyMatch results[result.Path] = result } return rows.Err() } filterSQL, filterArgs := searchOptionsPredicate(options) pathSQL := searchPathCandidatesBaseSQL pathArgs := []any{domainUUID, scope, prefix, upper} if query != "" { pathSQL += searchPathQueryPredicate pathArgs = append(pathArgs, query, query) } pathSQL += filterSQL + ` ORDER BY p.path LIMIT ?` pathArgs = append(pathArgs, filterArgs...) pathArgs = append(pathArgs, SearchCandidateLimit) if err := collect(pathSQL, false, false, pathArgs...); err != nil { return nil, err } expression := ftsPrefixExpression(query) if expression != "" { // No global ORDER BY or unbounded IN materialization: stop the FTS scan // after at most this many current, in-Domain, in-scope Path candidates. for _, field := range []string{"body", "filename"} { args := []any{field + " : (" + expression + ")", domainUUID, scope, prefix, upper} args = append(args, filterArgs...) args = append(args, SearchCandidateLimit) if err := collect(searchFTSCandidatesBaseSQL+filterSQL+` LIMIT ?`, field == "body", field == "filename", args...); err != nil { return nil, err } } } out := make([]SearchResult, 0, len(results)) for _, result := range results { out = append(out, result) } sort.Slice(out, func(i, j int) bool { // Direct name/Path matches precede body-only hits; all ties are stable. iName, jName := out[i].PathMatch || out[i].FilenameMatch, out[j].PathMatch || out[j].FilenameMatch if iName != jName { return iName } if len(out[i].Path) != len(out[j].Path) { return len(out[i].Path) < len(out[j].Path) } return out[i].Path < out[j].Path }) if len(out) > limit { out = out[:limit] } if !loadExcerpts { return out, nil } // Only authorized, displayed results receive text excerpts. Batch the FTS // lookup so a broad query is not re-scanned once for every visible result. ids := make([]any, 0, len(out)) seen := make(map[string]bool) for _, result := range out { if result.BodyMatch && !seen[result.RevisionUUID] { ids = append(ids, ftsRows[result.RevisionUUID]) seen[result.RevisionUUID] = true } } if len(ids) != 0 { // snippet's output token/character limit does not bound its input work: // FTS5 may tokenize the entire matched document. Read byte lengths only // for authorized displayed rows before spending the snippet budget. rows, err := tx.QueryContext(ctx, `SELECT rowid,octet_length(body) FROM revision_search WHERE rowid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...) if err != nil { return nil, err } sizes := make(map[int64]int64) for rows.Next() { var rowID, size int64 if err := rows.Scan(&rowID, &size); err != nil { rows.Close() return nil, err } sizes[rowID] = size } err = rows.Err() rows.Close() if err != nil { return nil, err } ids = nil selected := make(map[string]bool) remaining := int64(SearchExcerptTotalBytes) for i := range out { if !out[i].BodyMatch { continue } revision := out[i].RevisionUUID include, decided := selected[revision] if !decided { rowID := ftsRows[revision] size, exists := sizes[rowID] include = exists && size >= 0 && size <= SearchExcerptDocumentBytes && size <= remaining if include { ids = append(ids, rowID) remaining -= size } selected[revision] = include } out[i].ExcerptOmitted = !include } } if len(ids) != 0 { excerpts, err := loadSearchBodyExcerpts(ctx, tx, expression, ids, searchExcerptTimeout) if err != nil { return nil, err } for i := range out { if out[i].BodyMatch { var present bool out[i].Excerpt, present = excerpts[out[i].RevisionUUID] out[i].ExcerptOmitted = !present } } } // A filename/Path-only hit still benefits from a short Page summary. Read // only a bounded prefix from displayed textual revisions, in one batch. ids, seen = nil, make(map[string]bool) for _, result := range out { media := strings.ToLower(result.ContentType) textual := strings.HasPrefix(media, "text/") || strings.HasPrefix(media, "application/json") || strings.HasPrefix(media, "application/xml") || strings.HasPrefix(media, "application/javascript") || strings.HasPrefix(media, "application/yaml") || strings.HasPrefix(media, "application/x-yaml") if !result.BodyMatch && textual && result.RevisionUUID != "" && !seen[result.RevisionUUID] { ids = append(ids, result.RevisionUUID) seen[result.RevisionUUID] = true } } if len(ids) != 0 { rows, err := tx.QueryContext(ctx, `SELECT uuid,substr(body,1,1024) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...) if err != nil { return nil, err } excerpts := make(map[string]string) for rows.Next() { var id, excerpt string if err := rows.Scan(&id, &excerpt); err != nil { rows.Close() return nil, err } excerpts[id] = excerpt } err = rows.Err() rows.Close() if err != nil { return nil, err } for i := range out { if !out[i].BodyMatch { out[i].Excerpt = excerpts[out[i].RevisionUUID] } } } return out, nil } func ftsPrefixExpression(query string) string { words := strings.Fields(query) terms := make([]string, 0, len(words)) for _, word := range words { word = strings.ReplaceAll(word, `"`, `""`) if word != "" { terms = append(terms, `"`+word+`"*`) } } return strings.Join(terms, " AND ") }