Files
Anya Tokugawa c1175d389f Implement Knowledge City platform, organizer and DAV integration
Capture accumulated application, UI, CLI, regression tests and design documentation changes, including task project mapping and default CalDAV list naming.

Keep deployment data, backups, local credentials and build artifacts out of Git. Move the Compose enrollment token to an ignored local environment file.
2026-09-27 14:44:43 +09:00

380 lines
14 KiB
Go

package store
import (
"context"
"database/sql"
"errors"
"sort"
"strings"
"time"
)
const SearchCandidateLimit = 1000
const SearchExcerptDocumentBytes = 64 << 10
const SearchExcerptTotalBytes = 1 << 20
const searchExcerptTimeout = 400 * time.Millisecond
const searchResultColumns = `p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.revision_uuid,''),COALESCE(r.created_at_ms/1000,0)`
const searchPathBoundary = `(p.path=? OR (p.path>=? AND p.path<?))`
const searchPathCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,0
FROM paths p LEFT JOIN latest_revision_index r ON r.object_uuid=p.object_uuid
WHERE p.domain_uuid=? AND ` + searchPathBoundary
const searchPathQueryPredicate = ` AND (instr(lower(p.path),lower(?))>0 OR instr(lower(COALESCE(r.filename,'')),lower(?))>0)`
const searchPathCandidatesSQL = searchPathCandidatesBaseSQL + searchPathQueryPredicate + ` ORDER BY p.path LIMIT ?`
// CROSS JOIN is intentional: revision_uuid is an UNINDEXED FTS payload column.
// A Path-first plan re-runs the complete MATCH scan for every Path. Start with
// MATCH exactly once, then seek current Revision metadata and Domain bindings
// through their B-tree indexes. Keep scope filtering before the candidate cap.
const searchFTSCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,revision_search.rowid
FROM revision_search
CROSS JOIN latest_revision_index r ON r.revision_uuid=revision_search.revision_uuid
CROSS JOIN paths p ON p.object_uuid=r.object_uuid
WHERE revision_search MATCH ? AND p.domain_uuid=? AND ` + searchPathBoundary
const searchFTSCandidatesSQL = searchFTSCandidatesBaseSQL + ` LIMIT ?`
func searchBodyExcerptsSQL(count int) string {
// UUID is not indexed by FTS5. Restrict the MATCH/snippet work using the
// integer rowids collected from the same read snapshot, not payload UUIDs.
return `SELECT revision_uuid,substr(snippet(revision_search,3,'','',' … ',36),1,4096)
FROM revision_search WHERE revision_search MATCH ? AND rowid IN (` + strings.TrimRight(strings.Repeat("?,", count), ",") + `)`
}
// Excerpts are optional enrichment. Even small documents can have expensive
// prefix hit-position scans, so their time budget is separate from finding
// results. A spent excerpt budget omits snippets, not already-authorized hits.
func loadSearchBodyExcerpts(parent context.Context, tx *sql.Tx, expression string, ids []any, budget time.Duration) (excerpts map[string]string, err error) {
ctx, cancel := context.WithTimeout(parent, budget)
defer cancel()
defer func() {
if err != nil && ctx.Err() != nil {
if parent.Err() != nil {
err = parent.Err()
} else {
excerpts, err = nil, nil
}
}
}()
args := append([]any{"body : (" + expression + ")"}, ids...)
rows, err := tx.QueryContext(ctx, searchBodyExcerptsSQL(len(ids)), args...)
if err != nil {
return nil, err
}
defer rows.Close()
excerpts = make(map[string]string)
for rows.Next() {
var id, excerpt string
if err := rows.Scan(&id, &excerpt); err != nil {
return nil, err
}
excerpts[id] = excerpt
}
return excerpts, rows.Err()
}
type SearchResult struct {
Path string
ContentType string
Filename string
DocumentHash string
RevisionUUID string
Size int64 // Kept for API compatibility; search does not stat Document files.
UpdatedAt int64
PathMatch bool
FilenameMatch bool
BodyMatch bool
Excerpt string // Untrusted source text, never HTML-safe markup.
ExcerptOmitted bool // A body match remains visible when snippet work exceeds its budget.
}
type PathSuggestion struct {
SearchResult
Title string
}
// SearchPathsInDomain preserves the store API for trusted callers. HTTP callers
// must use SearchPathsInDomainFiltered with a request-scoped authorization filter.
func (s *Store) SearchPathsInDomain(domainUUID, scope, query string, limit int) ([]SearchResult, error) {
return s.SearchPathsInDomainFiltered(context.Background(), domainUUID, scope, query, limit, func(string) bool { return true })
}
// SearchPathsInDomainFiltered bounds SQL candidates before materialization and
// authorizes each Path before loading an excerpt. It never computes a global hit
// count or stats Document files. A narrow scope improves both recall and cost.
func (s *Store) SearchPathsInDomainFiltered(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]SearchResult, error) {
return s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{})
}
// SearchPathsInDomainWithOptions also permits filter-only searches. File/date
// predicates apply before SQL candidate caps, while ACL checks stay ahead of all
// snippet enrichment. Zero options preserve the original indexed query plans.
func (s *Store) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, searchErr error) {
return s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options, true)
}
// SearchPathSuggestionsInDomain shares the current-Revision FTS and ACL path
// with full search, but deliberately skips snippet generation. Typeahead calls
// are frequent and only need a bounded display label, never a body excerpt.
func (s *Store) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) {
results, err := s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}, false)
if err != nil || len(results) == 0 {
return nil, err
}
ids := make([]any, 0, len(results))
seen := make(map[string]bool, len(results))
for _, result := range results {
if result.RevisionUUID != "" && strings.HasPrefix(strings.ToLower(result.ContentType), "text/html") && !seen[result.RevisionUUID] {
ids = append(ids, result.RevisionUUID)
seen[result.RevisionUUID] = true
}
}
headings := make(map[string]string, len(ids))
if len(ids) > 0 {
rows, queryErr := s.DB.QueryContext(ctx, `SELECT uuid,substr(CAST(body AS TEXT),1,65536) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
if queryErr != nil {
return nil, queryErr
}
for rows.Next() {
var revision, source string
if queryErr = rows.Scan(&revision, &source); queryErr != nil {
rows.Close()
return nil, queryErr
}
headings[revision] = firstHTMLHeading(source)
}
queryErr = rows.Err()
rows.Close()
if queryErr != nil {
return nil, queryErr
}
}
out := make([]PathSuggestion, 0, len(results))
for _, result := range results {
out = append(out, PathSuggestion{SearchResult: result, Title: headings[result.RevisionUUID]})
}
return out, nil
}
func (s *Store) searchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions, loadExcerpts bool) (found []SearchResult, searchErr error) {
if err := options.Validate(); err != nil {
return nil, err
}
scope, query = normalizePath(scope), strings.TrimSpace(query)
if query == "" && !options.HasFilters() || allowed == nil {
return nil, nil
}
if len(query) > 512 || len(strings.Fields(query)) > 16 {
return nil, errors.New("search query is too long")
}
if limit <= 0 || limit > 200 {
limit = 100
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
defer func() {
// SQLite can report an interrupted statement instead of wrapping the
// context error. Preserve cancellation/deadline semantics for HTTP.
if searchErr != nil && ctx.Err() != nil {
searchErr = ctx.Err()
}
}()
tx, err := s.DB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return nil, err
}
defer tx.Rollback()
prefix := strings.TrimRight(scope, "/") + "/"
upper := strings.TrimRight(scope, "/") + "0"
results := make(map[string]SearchResult)
ftsRows := make(map[string]int64)
collect := func(statement string, body, filename bool, args ...any) error {
rows, err := tx.QueryContext(ctx, statement, args...)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var result SearchResult
var rowID int64
if err := rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt, &rowID); err != nil {
return err
}
if !allowed(result.Path) {
continue
}
if rowID != 0 {
ftsRows[result.RevisionUUID] = rowID
}
result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query))
result.FilenameMatch = filename || results[result.Path].FilenameMatch || query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query))
result.BodyMatch = body || results[result.Path].BodyMatch
results[result.Path] = result
}
return rows.Err()
}
filterSQL, filterArgs := searchOptionsPredicate(options)
pathSQL := searchPathCandidatesBaseSQL
pathArgs := []any{domainUUID, scope, prefix, upper}
if query != "" {
pathSQL += searchPathQueryPredicate
pathArgs = append(pathArgs, query, query)
}
pathSQL += filterSQL + ` ORDER BY p.path LIMIT ?`
pathArgs = append(pathArgs, filterArgs...)
pathArgs = append(pathArgs, SearchCandidateLimit)
if err := collect(pathSQL, false, false, pathArgs...); err != nil {
return nil, err
}
expression := ftsPrefixExpression(query)
if expression != "" {
// No global ORDER BY or unbounded IN materialization: stop the FTS scan
// after at most this many current, in-Domain, in-scope Path candidates.
for _, field := range []string{"body", "filename"} {
args := []any{field + " : (" + expression + ")", domainUUID, scope, prefix, upper}
args = append(args, filterArgs...)
args = append(args, SearchCandidateLimit)
if err := collect(searchFTSCandidatesBaseSQL+filterSQL+` LIMIT ?`, field == "body", field == "filename", args...); err != nil {
return nil, err
}
}
}
out := make([]SearchResult, 0, len(results))
for _, result := range results {
out = append(out, result)
}
sort.Slice(out, func(i, j int) bool {
// Direct name/Path matches precede body-only hits; all ties are stable.
iName, jName := out[i].PathMatch || out[i].FilenameMatch, out[j].PathMatch || out[j].FilenameMatch
if iName != jName {
return iName
}
if len(out[i].Path) != len(out[j].Path) {
return len(out[i].Path) < len(out[j].Path)
}
return out[i].Path < out[j].Path
})
if len(out) > limit {
out = out[:limit]
}
if !loadExcerpts {
return out, nil
}
// Only authorized, displayed results receive text excerpts. Batch the FTS
// lookup so a broad query is not re-scanned once for every visible result.
ids := make([]any, 0, len(out))
seen := make(map[string]bool)
for _, result := range out {
if result.BodyMatch && !seen[result.RevisionUUID] {
ids = append(ids, ftsRows[result.RevisionUUID])
seen[result.RevisionUUID] = true
}
}
if len(ids) != 0 {
// snippet's output token/character limit does not bound its input work:
// FTS5 may tokenize the entire matched document. Read byte lengths only
// for authorized displayed rows before spending the snippet budget.
rows, err := tx.QueryContext(ctx, `SELECT rowid,octet_length(body)
FROM revision_search WHERE rowid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
if err != nil {
return nil, err
}
sizes := make(map[int64]int64)
for rows.Next() {
var rowID, size int64
if err := rows.Scan(&rowID, &size); err != nil {
rows.Close()
return nil, err
}
sizes[rowID] = size
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
ids = nil
selected := make(map[string]bool)
remaining := int64(SearchExcerptTotalBytes)
for i := range out {
if !out[i].BodyMatch {
continue
}
revision := out[i].RevisionUUID
include, decided := selected[revision]
if !decided {
rowID := ftsRows[revision]
size, exists := sizes[rowID]
include = exists && size >= 0 && size <= SearchExcerptDocumentBytes && size <= remaining
if include {
ids = append(ids, rowID)
remaining -= size
}
selected[revision] = include
}
out[i].ExcerptOmitted = !include
}
}
if len(ids) != 0 {
excerpts, err := loadSearchBodyExcerpts(ctx, tx, expression, ids, searchExcerptTimeout)
if err != nil {
return nil, err
}
for i := range out {
if out[i].BodyMatch {
var present bool
out[i].Excerpt, present = excerpts[out[i].RevisionUUID]
out[i].ExcerptOmitted = !present
}
}
}
// A filename/Path-only hit still benefits from a short Page summary. Read
// only a bounded prefix from displayed textual revisions, in one batch.
ids, seen = nil, make(map[string]bool)
for _, result := range out {
media := strings.ToLower(result.ContentType)
textual := strings.HasPrefix(media, "text/") || strings.HasPrefix(media, "application/json") || strings.HasPrefix(media, "application/xml") || strings.HasPrefix(media, "application/javascript") || strings.HasPrefix(media, "application/yaml") || strings.HasPrefix(media, "application/x-yaml")
if !result.BodyMatch && textual && result.RevisionUUID != "" && !seen[result.RevisionUUID] {
ids = append(ids, result.RevisionUUID)
seen[result.RevisionUUID] = true
}
}
if len(ids) != 0 {
rows, err := tx.QueryContext(ctx, `SELECT uuid,substr(body,1,1024) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
if err != nil {
return nil, err
}
excerpts := make(map[string]string)
for rows.Next() {
var id, excerpt string
if err := rows.Scan(&id, &excerpt); err != nil {
rows.Close()
return nil, err
}
excerpts[id] = excerpt
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
for i := range out {
if !out[i].BodyMatch {
out[i].Excerpt = excerpts[out[i].RevisionUUID]
}
}
}
return out, nil
}
func ftsPrefixExpression(query string) string {
words := strings.Fields(query)
terms := make([]string, 0, len(words))
for _, word := range words {
word = strings.ReplaceAll(word, `"`, `""`)
if word != "" {
terms = append(terms, `"`+word+`"*`)
}
}
return strings.Join(terms, " AND ")
}