Capture accumulated application, UI, CLI, regression tests and design documentation changes, including task project mapping and default CalDAV list naming. Keep deployment data, backups, local credentials and build artifacts out of Git. Move the Compose enrollment token to an ignored local environment file.
380 lines
14 KiB
Go
380 lines
14 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const SearchCandidateLimit = 1000
|
|
const SearchExcerptDocumentBytes = 64 << 10
|
|
const SearchExcerptTotalBytes = 1 << 20
|
|
const searchExcerptTimeout = 400 * time.Millisecond
|
|
|
|
const searchResultColumns = `p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.revision_uuid,''),COALESCE(r.created_at_ms/1000,0)`
|
|
const searchPathBoundary = `(p.path=? OR (p.path>=? AND p.path<?))`
|
|
const searchPathCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,0
|
|
FROM paths p LEFT JOIN latest_revision_index r ON r.object_uuid=p.object_uuid
|
|
WHERE p.domain_uuid=? AND ` + searchPathBoundary
|
|
const searchPathQueryPredicate = ` AND (instr(lower(p.path),lower(?))>0 OR instr(lower(COALESCE(r.filename,'')),lower(?))>0)`
|
|
const searchPathCandidatesSQL = searchPathCandidatesBaseSQL + searchPathQueryPredicate + ` ORDER BY p.path LIMIT ?`
|
|
|
|
// CROSS JOIN is intentional: revision_uuid is an UNINDEXED FTS payload column.
|
|
// A Path-first plan re-runs the complete MATCH scan for every Path. Start with
|
|
// MATCH exactly once, then seek current Revision metadata and Domain bindings
|
|
// through their B-tree indexes. Keep scope filtering before the candidate cap.
|
|
const searchFTSCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,revision_search.rowid
|
|
FROM revision_search
|
|
CROSS JOIN latest_revision_index r ON r.revision_uuid=revision_search.revision_uuid
|
|
CROSS JOIN paths p ON p.object_uuid=r.object_uuid
|
|
WHERE revision_search MATCH ? AND p.domain_uuid=? AND ` + searchPathBoundary
|
|
const searchFTSCandidatesSQL = searchFTSCandidatesBaseSQL + ` LIMIT ?`
|
|
|
|
func searchBodyExcerptsSQL(count int) string {
|
|
// UUID is not indexed by FTS5. Restrict the MATCH/snippet work using the
|
|
// integer rowids collected from the same read snapshot, not payload UUIDs.
|
|
return `SELECT revision_uuid,substr(snippet(revision_search,3,'','',' … ',36),1,4096)
|
|
FROM revision_search WHERE revision_search MATCH ? AND rowid IN (` + strings.TrimRight(strings.Repeat("?,", count), ",") + `)`
|
|
}
|
|
|
|
// Excerpts are optional enrichment. Even small documents can have expensive
|
|
// prefix hit-position scans, so their time budget is separate from finding
|
|
// results. A spent excerpt budget omits snippets, not already-authorized hits.
|
|
func loadSearchBodyExcerpts(parent context.Context, tx *sql.Tx, expression string, ids []any, budget time.Duration) (excerpts map[string]string, err error) {
|
|
ctx, cancel := context.WithTimeout(parent, budget)
|
|
defer cancel()
|
|
defer func() {
|
|
if err != nil && ctx.Err() != nil {
|
|
if parent.Err() != nil {
|
|
err = parent.Err()
|
|
} else {
|
|
excerpts, err = nil, nil
|
|
}
|
|
}
|
|
}()
|
|
args := append([]any{"body : (" + expression + ")"}, ids...)
|
|
rows, err := tx.QueryContext(ctx, searchBodyExcerptsSQL(len(ids)), args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
excerpts = make(map[string]string)
|
|
for rows.Next() {
|
|
var id, excerpt string
|
|
if err := rows.Scan(&id, &excerpt); err != nil {
|
|
return nil, err
|
|
}
|
|
excerpts[id] = excerpt
|
|
}
|
|
return excerpts, rows.Err()
|
|
}
|
|
|
|
type SearchResult struct {
|
|
Path string
|
|
ContentType string
|
|
Filename string
|
|
DocumentHash string
|
|
RevisionUUID string
|
|
Size int64 // Kept for API compatibility; search does not stat Document files.
|
|
UpdatedAt int64
|
|
PathMatch bool
|
|
FilenameMatch bool
|
|
BodyMatch bool
|
|
Excerpt string // Untrusted source text, never HTML-safe markup.
|
|
ExcerptOmitted bool // A body match remains visible when snippet work exceeds its budget.
|
|
}
|
|
|
|
type PathSuggestion struct {
|
|
SearchResult
|
|
Title string
|
|
}
|
|
|
|
// SearchPathsInDomain preserves the store API for trusted callers. HTTP callers
|
|
// must use SearchPathsInDomainFiltered with a request-scoped authorization filter.
|
|
func (s *Store) SearchPathsInDomain(domainUUID, scope, query string, limit int) ([]SearchResult, error) {
|
|
return s.SearchPathsInDomainFiltered(context.Background(), domainUUID, scope, query, limit, func(string) bool { return true })
|
|
}
|
|
|
|
// SearchPathsInDomainFiltered bounds SQL candidates before materialization and
|
|
// authorizes each Path before loading an excerpt. It never computes a global hit
|
|
// count or stats Document files. A narrow scope improves both recall and cost.
|
|
func (s *Store) SearchPathsInDomainFiltered(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]SearchResult, error) {
|
|
return s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{})
|
|
}
|
|
|
|
// SearchPathsInDomainWithOptions also permits filter-only searches. File/date
|
|
// predicates apply before SQL candidate caps, while ACL checks stay ahead of all
|
|
// snippet enrichment. Zero options preserve the original indexed query plans.
|
|
func (s *Store) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, searchErr error) {
|
|
return s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options, true)
|
|
}
|
|
|
|
// SearchPathSuggestionsInDomain shares the current-Revision FTS and ACL path
|
|
// with full search, but deliberately skips snippet generation. Typeahead calls
|
|
// are frequent and only need a bounded display label, never a body excerpt.
|
|
func (s *Store) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) {
|
|
results, err := s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}, false)
|
|
if err != nil || len(results) == 0 {
|
|
return nil, err
|
|
}
|
|
ids := make([]any, 0, len(results))
|
|
seen := make(map[string]bool, len(results))
|
|
for _, result := range results {
|
|
if result.RevisionUUID != "" && strings.HasPrefix(strings.ToLower(result.ContentType), "text/html") && !seen[result.RevisionUUID] {
|
|
ids = append(ids, result.RevisionUUID)
|
|
seen[result.RevisionUUID] = true
|
|
}
|
|
}
|
|
headings := make(map[string]string, len(ids))
|
|
if len(ids) > 0 {
|
|
rows, queryErr := s.DB.QueryContext(ctx, `SELECT uuid,substr(CAST(body AS TEXT),1,65536) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
|
|
if queryErr != nil {
|
|
return nil, queryErr
|
|
}
|
|
for rows.Next() {
|
|
var revision, source string
|
|
if queryErr = rows.Scan(&revision, &source); queryErr != nil {
|
|
rows.Close()
|
|
return nil, queryErr
|
|
}
|
|
headings[revision] = firstHTMLHeading(source)
|
|
}
|
|
queryErr = rows.Err()
|
|
rows.Close()
|
|
if queryErr != nil {
|
|
return nil, queryErr
|
|
}
|
|
}
|
|
out := make([]PathSuggestion, 0, len(results))
|
|
for _, result := range results {
|
|
out = append(out, PathSuggestion{SearchResult: result, Title: headings[result.RevisionUUID]})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (s *Store) searchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions, loadExcerpts bool) (found []SearchResult, searchErr error) {
|
|
if err := options.Validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
scope, query = normalizePath(scope), strings.TrimSpace(query)
|
|
if query == "" && !options.HasFilters() || allowed == nil {
|
|
return nil, nil
|
|
}
|
|
if len(query) > 512 || len(strings.Fields(query)) > 16 {
|
|
return nil, errors.New("search query is too long")
|
|
}
|
|
if limit <= 0 || limit > 200 {
|
|
limit = 100
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
|
defer cancel()
|
|
defer func() {
|
|
// SQLite can report an interrupted statement instead of wrapping the
|
|
// context error. Preserve cancellation/deadline semantics for HTTP.
|
|
if searchErr != nil && ctx.Err() != nil {
|
|
searchErr = ctx.Err()
|
|
}
|
|
}()
|
|
tx, err := s.DB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer tx.Rollback()
|
|
prefix := strings.TrimRight(scope, "/") + "/"
|
|
upper := strings.TrimRight(scope, "/") + "0"
|
|
results := make(map[string]SearchResult)
|
|
ftsRows := make(map[string]int64)
|
|
collect := func(statement string, body, filename bool, args ...any) error {
|
|
rows, err := tx.QueryContext(ctx, statement, args...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var result SearchResult
|
|
var rowID int64
|
|
if err := rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt, &rowID); err != nil {
|
|
return err
|
|
}
|
|
if !allowed(result.Path) {
|
|
continue
|
|
}
|
|
if rowID != 0 {
|
|
ftsRows[result.RevisionUUID] = rowID
|
|
}
|
|
result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query))
|
|
result.FilenameMatch = filename || results[result.Path].FilenameMatch || query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query))
|
|
result.BodyMatch = body || results[result.Path].BodyMatch
|
|
results[result.Path] = result
|
|
}
|
|
return rows.Err()
|
|
}
|
|
filterSQL, filterArgs := searchOptionsPredicate(options)
|
|
pathSQL := searchPathCandidatesBaseSQL
|
|
pathArgs := []any{domainUUID, scope, prefix, upper}
|
|
if query != "" {
|
|
pathSQL += searchPathQueryPredicate
|
|
pathArgs = append(pathArgs, query, query)
|
|
}
|
|
pathSQL += filterSQL + ` ORDER BY p.path LIMIT ?`
|
|
pathArgs = append(pathArgs, filterArgs...)
|
|
pathArgs = append(pathArgs, SearchCandidateLimit)
|
|
if err := collect(pathSQL, false, false, pathArgs...); err != nil {
|
|
return nil, err
|
|
}
|
|
expression := ftsPrefixExpression(query)
|
|
if expression != "" {
|
|
// No global ORDER BY or unbounded IN materialization: stop the FTS scan
|
|
// after at most this many current, in-Domain, in-scope Path candidates.
|
|
for _, field := range []string{"body", "filename"} {
|
|
args := []any{field + " : (" + expression + ")", domainUUID, scope, prefix, upper}
|
|
args = append(args, filterArgs...)
|
|
args = append(args, SearchCandidateLimit)
|
|
if err := collect(searchFTSCandidatesBaseSQL+filterSQL+` LIMIT ?`, field == "body", field == "filename", args...); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
out := make([]SearchResult, 0, len(results))
|
|
for _, result := range results {
|
|
out = append(out, result)
|
|
}
|
|
sort.Slice(out, func(i, j int) bool {
|
|
// Direct name/Path matches precede body-only hits; all ties are stable.
|
|
iName, jName := out[i].PathMatch || out[i].FilenameMatch, out[j].PathMatch || out[j].FilenameMatch
|
|
if iName != jName {
|
|
return iName
|
|
}
|
|
if len(out[i].Path) != len(out[j].Path) {
|
|
return len(out[i].Path) < len(out[j].Path)
|
|
}
|
|
return out[i].Path < out[j].Path
|
|
})
|
|
if len(out) > limit {
|
|
out = out[:limit]
|
|
}
|
|
if !loadExcerpts {
|
|
return out, nil
|
|
}
|
|
// Only authorized, displayed results receive text excerpts. Batch the FTS
|
|
// lookup so a broad query is not re-scanned once for every visible result.
|
|
ids := make([]any, 0, len(out))
|
|
seen := make(map[string]bool)
|
|
for _, result := range out {
|
|
if result.BodyMatch && !seen[result.RevisionUUID] {
|
|
ids = append(ids, ftsRows[result.RevisionUUID])
|
|
seen[result.RevisionUUID] = true
|
|
}
|
|
}
|
|
if len(ids) != 0 {
|
|
// snippet's output token/character limit does not bound its input work:
|
|
// FTS5 may tokenize the entire matched document. Read byte lengths only
|
|
// for authorized displayed rows before spending the snippet budget.
|
|
rows, err := tx.QueryContext(ctx, `SELECT rowid,octet_length(body)
|
|
FROM revision_search WHERE rowid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sizes := make(map[int64]int64)
|
|
for rows.Next() {
|
|
var rowID, size int64
|
|
if err := rows.Scan(&rowID, &size); err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
sizes[rowID] = size
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
ids = nil
|
|
selected := make(map[string]bool)
|
|
remaining := int64(SearchExcerptTotalBytes)
|
|
for i := range out {
|
|
if !out[i].BodyMatch {
|
|
continue
|
|
}
|
|
revision := out[i].RevisionUUID
|
|
include, decided := selected[revision]
|
|
if !decided {
|
|
rowID := ftsRows[revision]
|
|
size, exists := sizes[rowID]
|
|
include = exists && size >= 0 && size <= SearchExcerptDocumentBytes && size <= remaining
|
|
if include {
|
|
ids = append(ids, rowID)
|
|
remaining -= size
|
|
}
|
|
selected[revision] = include
|
|
}
|
|
out[i].ExcerptOmitted = !include
|
|
}
|
|
}
|
|
if len(ids) != 0 {
|
|
excerpts, err := loadSearchBodyExcerpts(ctx, tx, expression, ids, searchExcerptTimeout)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for i := range out {
|
|
if out[i].BodyMatch {
|
|
var present bool
|
|
out[i].Excerpt, present = excerpts[out[i].RevisionUUID]
|
|
out[i].ExcerptOmitted = !present
|
|
}
|
|
}
|
|
}
|
|
// A filename/Path-only hit still benefits from a short Page summary. Read
|
|
// only a bounded prefix from displayed textual revisions, in one batch.
|
|
ids, seen = nil, make(map[string]bool)
|
|
for _, result := range out {
|
|
media := strings.ToLower(result.ContentType)
|
|
textual := strings.HasPrefix(media, "text/") || strings.HasPrefix(media, "application/json") || strings.HasPrefix(media, "application/xml") || strings.HasPrefix(media, "application/javascript") || strings.HasPrefix(media, "application/yaml") || strings.HasPrefix(media, "application/x-yaml")
|
|
if !result.BodyMatch && textual && result.RevisionUUID != "" && !seen[result.RevisionUUID] {
|
|
ids = append(ids, result.RevisionUUID)
|
|
seen[result.RevisionUUID] = true
|
|
}
|
|
}
|
|
if len(ids) != 0 {
|
|
rows, err := tx.QueryContext(ctx, `SELECT uuid,substr(body,1,1024) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
excerpts := make(map[string]string)
|
|
for rows.Next() {
|
|
var id, excerpt string
|
|
if err := rows.Scan(&id, &excerpt); err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
excerpts[id] = excerpt
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for i := range out {
|
|
if !out[i].BodyMatch {
|
|
out[i].Excerpt = excerpts[out[i].RevisionUUID]
|
|
}
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func ftsPrefixExpression(query string) string {
|
|
words := strings.Fields(query)
|
|
terms := make([]string, 0, len(words))
|
|
for _, word := range words {
|
|
word = strings.ReplaceAll(word, `"`, `""`)
|
|
if word != "" {
|
|
terms = append(terms, `"`+word+`"*`)
|
|
}
|
|
}
|
|
return strings.Join(terms, " AND ")
|
|
}
|