feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s
Build and publish container images / images (push) Failing after 12m29s
This commit is contained in:
1 parent
96065d1e89
commit
0ba1dcfef0
243 files changed
+21441
-3090
No files matched your search
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -36,6 +37,14 @@ type citySearchCandidate struct {
|
||||
// Scope selects a locality's own boundary, not an ancestor of the scope. Every
|
||||
// boundary in its parent chain must be readable before the result is exposed.
|
||||
func (s *SQLite) SearchCityInDomain(parent context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []CitySearchResult, searchErr error) {
|
||||
return searchCity(s.db, false, parent, domainUUID, scope, query, limit, allowed, options)
|
||||
}
|
||||
|
||||
func (s *PostgreSQL) SearchCityInDomain(parent context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) ([]CitySearchResult, error) {
|
||||
return searchCity(s.db, true, parent, domainUUID, scope, query, limit, allowed, options)
|
||||
}
|
||||
|
||||
func searchCity(database *sql.DB, native bool, parent context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []CitySearchResult, searchErr error) {
|
||||
if err := options.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -58,7 +67,7 @@ func (s *SQLite) SearchCityInDomain(parent context.Context, domainUUID, scope, q
|
||||
searchErr = ctx.Err()
|
||||
}
|
||||
}()
|
||||
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
|
||||
tx, err := database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -70,22 +79,35 @@ func (s *SQLite) SearchCityInDomain(parent context.Context, domainUUID, scope, q
|
||||
sql.Named("prefix", prefix), sql.Named("upper", upper),
|
||||
sql.Named("query", query), sql.Named("limit", CitySearchCandidateLimit),
|
||||
}
|
||||
if native {
|
||||
args = []any{domainUUID, scope, prefix, upper, query, CitySearchCandidateLimit}
|
||||
}
|
||||
dateColumn := "n.updated_at_ms"
|
||||
if options.DateField == "created" {
|
||||
dateColumn = "n.created_at_ms"
|
||||
}
|
||||
dateWhere := ""
|
||||
if options.FromMS != nil {
|
||||
dateWhere += " AND " + dateColumn + ">=:from_ms"
|
||||
args = append(args, sql.Named("from_ms", *options.FromMS))
|
||||
if native {
|
||||
args = append(args, *options.FromMS)
|
||||
dateWhere += " AND " + dateColumn + ">=" + fmt.Sprintf("$%d", len(args))
|
||||
} else {
|
||||
dateWhere += " AND " + dateColumn + ">=:from_ms"
|
||||
args = append(args, sql.Named("from_ms", *options.FromMS))
|
||||
}
|
||||
}
|
||||
if options.BeforeMS != nil {
|
||||
dateWhere += " AND " + dateColumn + "<:before_ms"
|
||||
args = append(args, sql.Named("before_ms", *options.BeforeMS))
|
||||
if native {
|
||||
args = append(args, *options.BeforeMS)
|
||||
dateWhere += " AND " + dateColumn + "<" + fmt.Sprintf("$%d", len(args))
|
||||
} else {
|
||||
dateWhere += " AND " + dateColumn + "<:before_ms"
|
||||
args = append(args, sql.Named("before_ms", *options.BeforeMS))
|
||||
}
|
||||
}
|
||||
candidates := make([]citySearchCandidate, 0)
|
||||
for _, kind := range []string{"quartier", "street", "spot"} {
|
||||
rows, err := tx.QueryContext(ctx, citySearchSQL(kind, dateWhere), args...)
|
||||
rows, err := tx.QueryContext(ctx, citySearchDialectSQL(kind, dateWhere, native), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -173,6 +195,18 @@ func citySearchKindOrder(kind string) int {
|
||||
// All interpolated SQL fragments are package-owned constants. User input is
|
||||
// bound, so %, _, quotes and backslashes are literal substring characters.
|
||||
func citySearchSQL(kind, dateWhere string) string {
|
||||
return citySearchDialectSQL(kind, dateWhere, false)
|
||||
}
|
||||
|
||||
func citySearchDialectSQL(kind, dateWhere string, native bool) string {
|
||||
domain, scope, prefix, upper, query, limit := ":domain", ":scope", ":prefix", ":upper", ":query", ":limit"
|
||||
collation, position, greatest := " COLLATE BINARY", "instr", "max"
|
||||
absolute := citySearchAbsolutePathSQL
|
||||
if native {
|
||||
domain, scope, prefix, upper, query, limit = "$1", "$2", "$3", "$4", "$5", "$6"
|
||||
collation, position, greatest = ` COLLATE "C"`, "strpos", "greatest"
|
||||
absolute = func(column string) string { return `(substr(` + column + `,1,1)='/' AND length(` + column + `)<=4096)` }
|
||||
}
|
||||
from, description := "zones n", "n.description"
|
||||
navigationPath := "n.path_prefix"
|
||||
parentColumns := "'','','','',''"
|
||||
@@ -181,28 +215,32 @@ func citySearchSQL(kind, dateWhere string) string {
|
||||
case "street":
|
||||
from += ` JOIN zones q ON q.uuid=n.parent_uuid AND q.domain_uuid=n.domain_uuid AND q.kind='quartier' AND q.state!='archived'`
|
||||
parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),'',q.path_prefix,'',q.name"
|
||||
where = "n.kind='street' AND " + citySearchAbsolutePathSQL("q.path_prefix")
|
||||
where = "n.kind='street' AND " + absolute("q.path_prefix")
|
||||
case "spot":
|
||||
navigationPath = `CASE WHEN substr(n.path_prefix,-11)='/_undefined' THEN st.path_prefix ELSE n.path_prefix END`
|
||||
if native {
|
||||
navigationPath = `CASE WHEN right(n.path_prefix,11)='/_undefined' THEN st.path_prefix ELSE n.path_prefix END`
|
||||
}
|
||||
from = `spots n JOIN street_spots ss ON ss.spot_uuid=n.uuid
|
||||
JOIN zones st ON st.uuid=ss.street_uuid AND st.domain_uuid=n.domain_uuid AND st.kind='street' AND st.state!='archived'
|
||||
JOIN zones q ON q.uuid=st.parent_uuid AND q.domain_uuid=st.domain_uuid AND q.kind='quartier' AND q.state!='archived'`
|
||||
description = "n.summary"
|
||||
parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),substr(COALESCE(NULLIF(st.title,''),st.name),1,256),q.path_prefix,st.path_prefix,q.name"
|
||||
where = citySearchAbsolutePathSQL("q.path_prefix") + " AND " + citySearchAbsolutePathSQL("st.path_prefix")
|
||||
where = absolute("q.path_prefix") + " AND " + absolute("st.path_prefix")
|
||||
}
|
||||
nameMatch := `(:query!='' AND (instr(lower(n.name),lower(:query))>0 OR instr(lower(n.title),lower(:query))>0))`
|
||||
pathMatch := `(:query!='' AND instr(lower(n.path_prefix),lower(:query))>0)`
|
||||
descriptionMatch := `(:query!='' AND instr(lower(` + description + `),lower(:query))>0)`
|
||||
contains := func(column string) string { return position + "(lower(" + column + "),lower(" + query + "))" }
|
||||
nameMatch := `(` + query + `!='' AND (` + contains("n.name") + `>0 OR ` + contains("n.title") + `>0))`
|
||||
pathMatch := `(` + query + `!='' AND ` + contains("n.path_prefix") + `>0)`
|
||||
descriptionMatch := `(` + query + `!='' AND ` + contains(description) + `>0)`
|
||||
return `SELECT n.uuid,substr(n.name,1,256),substr(n.title,1,256),
|
||||
substr(` + description + `,max(1,instr(lower(` + description + `),lower(:query))-160),512),
|
||||
substr(` + description + `,` + greatest + `(1,` + contains(description) + `-160),512),
|
||||
n.path_prefix,n.created_at_ms,n.updated_at_ms,` + nameMatch + `,` + pathMatch + `,` + descriptionMatch + `,` + parentColumns + `,` + navigationPath + `,
|
||||
EXISTS(SELECT 1 FROM paths cp WHERE cp.domain_uuid=n.domain_uuid AND cp.path=(` + navigationPath + `))
|
||||
FROM ` + from + ` WHERE n.domain_uuid=:domain AND n.state!='archived' AND ` + where + `
|
||||
AND ` + citySearchAbsolutePathSQL("n.path_prefix") + `
|
||||
AND (n.path_prefix COLLATE BINARY=:scope OR (n.path_prefix COLLATE BINARY>=:prefix AND n.path_prefix COLLATE BINARY<:upper))
|
||||
AND (:query='' OR ` + nameMatch + ` OR ` + pathMatch + ` OR ` + descriptionMatch + `)` + dateWhere + `
|
||||
ORDER BY ` + nameMatch + ` DESC,` + pathMatch + ` DESC,n.path_prefix COLLATE BINARY,n.uuid LIMIT :limit`
|
||||
FROM ` + from + ` WHERE n.domain_uuid=` + domain + ` AND n.state!='archived' AND ` + where + `
|
||||
AND ` + absolute("n.path_prefix") + `
|
||||
AND (n.path_prefix` + collation + `=` + scope + ` OR (n.path_prefix` + collation + `>=` + prefix + ` AND n.path_prefix` + collation + `<` + upper + `))
|
||||
AND (` + query + `='' OR ` + nameMatch + ` OR ` + pathMatch + ` OR ` + descriptionMatch + `)` + dateWhere + `
|
||||
ORDER BY ` + nameMatch + ` DESC,` + pathMatch + ` DESC,n.path_prefix` + collation + `,n.uuid LIMIT ` + limit
|
||||
}
|
||||
|
||||
func citySearchAbsolutePathSQL(column string) string {
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package mestionora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PostgreSQL's initial search profile is metadata-only. It deliberately does
|
||||
// not scan canonical files or the legacy body column to emulate SQLite FTS.
|
||||
// The caller owns the pool and must provision the native metadata schema.
|
||||
type PostgreSQL struct{ db *sql.DB }
|
||||
|
||||
func NewPostgreSQL(db *sql.DB) *PostgreSQL { return &PostgreSQL{db: db} }
|
||||
|
||||
const pgSearchMIME = `lower(trim(split_part(COALESCE(r.content_type,''),';',1)))`
|
||||
|
||||
var pgSearchFileType = buildSearchFileTypeSQL(pgSearchMIME, func(suffixes ...string) string {
|
||||
clauses := make([]string, 0, len(suffixes))
|
||||
for _, suffix := range suffixes {
|
||||
clauses = append(clauses, searchFilenameSQL+fmt.Sprintf(` LIKE '%%.%s'`, suffix))
|
||||
}
|
||||
return "(" + strings.Join(clauses, " OR ") + ")"
|
||||
}, func(pattern string) string {
|
||||
// Only the package-owned MIME glob constants above reach this function.
|
||||
return pgSearchMIME + ` LIKE '` + strings.ReplaceAll(pattern, "*", "%") + `'`
|
||||
})
|
||||
|
||||
func (s *PostgreSQL) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, err error) {
|
||||
if err := options.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scope, query = normalizePath(scope), strings.TrimSpace(query)
|
||||
if allowed == nil || query == "" && !options.HasFilters() {
|
||||
return nil, nil
|
||||
}
|
||||
if len(query) > 512 || len(strings.Fields(query)) > 16 {
|
||||
return nil, errors.New("search query is too long")
|
||||
}
|
||||
if limit <= 0 || limit > 200 {
|
||||
limit = 100
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
defer func() {
|
||||
if ctx.Err() != nil {
|
||||
found, err = nil, ctx.Err()
|
||||
}
|
||||
}()
|
||||
var args []any
|
||||
bind := func(v any) string { args = append(args, v); return fmt.Sprintf("$%d", len(args)) }
|
||||
prefix := strings.TrimRight(scope, "/") + "/"
|
||||
upper := prefix[:len(prefix)-1] + "0"
|
||||
predicate := "p.domain_uuid=" + bind(domainUUID) + " AND (p.path=" + bind(scope) + " OR (p.path>=" + bind(prefix) + " AND p.path<" + bind(upper) + "))"
|
||||
if query != "" {
|
||||
q := bind(query)
|
||||
predicate += " AND (strpos(lower(p.path),lower(" + q + "))>0 OR strpos(lower(COALESCE(r.filename,'')),lower(" + q + "))>0)"
|
||||
}
|
||||
date := "r.created_at_ms"
|
||||
if options.DateField == "created" {
|
||||
date = "(SELECT o.created_at*1000 FROM objects o WHERE o.uuid=p.object_uuid)"
|
||||
}
|
||||
if options.FromMS != nil {
|
||||
predicate += " AND " + date + ">=" + bind(*options.FromMS)
|
||||
}
|
||||
if options.BeforeMS != nil {
|
||||
predicate += " AND " + date + "<" + bind(*options.BeforeMS)
|
||||
}
|
||||
if options.Extension != "" {
|
||||
ext := bind("." + options.Extension)
|
||||
predicate += " AND right(" + searchFilenameSQL + ",length(" + ext + "))=" + ext
|
||||
}
|
||||
if options.FileType != "" {
|
||||
predicate += " AND (" + pgSearchFileType + ")=" + bind(options.FileType)
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.uuid,''),COALESCE(r.created_at_ms/1000,0)
|
||||
FROM paths p LEFT JOIN LATERAL (SELECT uuid,content_type,filename,document_hash,created_at_ms FROM revisions WHERE object_uuid=p.object_uuid ORDER BY created_at_ms DESC,uuid DESC LIMIT 1) r ON true
|
||||
WHERE `+predicate+` ORDER BY p.path LIMIT `+bind(SearchCandidateLimit), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
candidates := make([]SearchResult, 0)
|
||||
for rows.Next() {
|
||||
var result SearchResult
|
||||
if err = rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query))
|
||||
result.FilenameMatch = query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query))
|
||||
candidates = append(candidates, result)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Release the cursor before policy callbacks: authorization may need the
|
||||
// same single-connection pool. Never return a denied Path or load its body.
|
||||
sort.Slice(candidates, func(i, j int) bool {
|
||||
if len(candidates[i].Path) != len(candidates[j].Path) {
|
||||
return len(candidates[i].Path) < len(candidates[j].Path)
|
||||
}
|
||||
return candidates[i].Path < candidates[j].Path
|
||||
})
|
||||
for _, candidate := range candidates {
|
||||
if err = ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if allowed(candidate.Path) {
|
||||
found = append(found, candidate)
|
||||
if len(found) == limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
func (s *PostgreSQL) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) {
|
||||
results, err := s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]PathSuggestion, 0, len(results))
|
||||
for _, result := range results {
|
||||
out = append(out, PathSuggestion{SearchResult: result})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -71,18 +71,25 @@ func searchSuffixesSQL(suffixes ...string) string {
|
||||
// known suffix (e.g. a Python source uploaded as text/plain); unrecognized,
|
||||
// specific MIME types stay Other instead of pretending to be a different type.
|
||||
// This uses metadata only and never hydrates a Document to sniff its bytes.
|
||||
var searchFileTypeSQL = `CASE
|
||||
var searchFileTypeSQL = buildSearchFileTypeSQL(searchMIMESQL, searchSuffixesSQL, func(pattern string) string {
|
||||
return searchMIMESQL + ` GLOB '` + pattern + `'`
|
||||
})
|
||||
|
||||
// All patterns are package-owned constants. The dialect-specific builders do
|
||||
// not translate incoming SQL or interpolate user input.
|
||||
func buildSearchFileTypeSQL(searchMIMESQL string, searchSuffixesSQL func(...string) string, mimeGlob func(string) string) string {
|
||||
return `CASE
|
||||
WHEN ` + searchMIMESQL + ` IN ('text/html','application/xhtml+xml') THEN 'html'
|
||||
WHEN ` + searchMIMESQL + `='application/pdf' THEN 'pdf'
|
||||
WHEN ` + searchMIMESQL + ` LIKE 'image/%' THEN 'image'
|
||||
WHEN ` + searchMIMESQL + ` LIKE 'audio/%' THEN 'audio'
|
||||
WHEN ` + searchMIMESQL + ` LIKE 'video/%' THEN 'video'
|
||||
WHEN ` + searchMIMESQL + ` IN ('text/csv','application/csv','text/tab-separated-values') THEN 'csv'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/json','text/json') OR ` + searchMIMESQL + ` GLOB '*+json' THEN 'json'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/json','text/json') OR ` + mimeGlob("*+json") + ` THEN 'json'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/zip','application/x-zip-compressed','application/x-7z-compressed','application/vnd.rar','application/x-rar','application/x-rar-compressed','application/x-tar','application/gzip','application/x-gzip','application/x-xz','application/x-bzip2','application/zstd','application/x-zstd','application/x-compress','application/x-cpio') THEN 'archive'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/msword','application/vnd.ms-excel','application/vnd.ms-powerpoint','application/rtf','text/rtf') OR ` + searchMIMESQL + ` GLOB 'application/vnd.openxmlformats-officedocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.oasis.opendocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.ms-*.macroenabled.*' THEN 'office'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/msword','application/vnd.ms-excel','application/vnd.ms-powerpoint','application/rtf','text/rtf') OR ` + mimeGlob("application/vnd.openxmlformats-officedocument.*") + ` OR ` + mimeGlob("application/vnd.oasis.opendocument.*") + ` OR ` + mimeGlob("application/vnd.ms-*.macroenabled.*") + ` THEN 'office'
|
||||
WHEN ` + searchMIMESQL + ` IN ('text/markdown','text/x-markdown') THEN 'text'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/javascript','application/x-javascript','text/javascript','application/typescript','text/typescript','application/xml','text/xml','application/yaml','application/x-yaml','text/yaml','application/toml','application/x-sh','application/x-shellscript','application/x-python','application/x-python-code','application/x-httpd-php','application/sql','text/css') OR ` + searchMIMESQL + ` GLOB 'text/x-*' OR ` + searchMIMESQL + ` GLOB '*+xml' THEN 'code'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/javascript','application/x-javascript','text/javascript','application/typescript','text/typescript','application/xml','text/xml','application/yaml','application/x-yaml','text/yaml','application/toml','application/x-sh','application/x-shellscript','application/x-python','application/x-python-code','application/x-httpd-php','application/sql','text/css') OR ` + mimeGlob("text/x-*") + ` OR ` + mimeGlob("*+xml") + ` THEN 'code'
|
||||
WHEN ` + searchMIMESQL + ` NOT IN ('','text/plain','application/octet-stream','binary/octet-stream','application/x-empty') THEN CASE WHEN ` + searchMIMESQL + ` LIKE 'text/%' THEN 'text' ELSE 'other' END
|
||||
WHEN ` + searchSuffixesSQL("html", "htm", "xhtml") + ` THEN 'html'
|
||||
WHEN ` + searchSuffixesSQL("pdf") + ` THEN 'pdf'
|
||||
@@ -96,6 +103,7 @@ var searchFileTypeSQL = `CASE
|
||||
WHEN ` + searchSuffixesSQL("py", "pyw", "sh", "bash", "zsh", "fish", "ps1", "bat", "cmd", "js", "mjs", "cjs", "jsx", "ts", "tsx", "go", "rs", "c", "h", "cpp", "hpp", "cc", "cs", "java", "kt", "kts", "swift", "rb", "php", "pl", "pm", "lua", "r", "sql", "css", "scss", "sass", "less", "xml", "xsl", "yaml", "yml", "toml", "ini", "conf", "vue", "svelte", "dart", "ex", "exs", "erl", "hs", "scala", "clj", "lisp", "el", "vim", "wasm") + ` THEN 'code'
|
||||
WHEN ` + searchMIMESQL + `='text/plain' OR ` + searchSuffixesSQL("txt", "text", "log", "md", "markdown", "rst", "adoc", "org") + ` THEN 'text'
|
||||
ELSE 'other' END`
|
||||
}
|
||||
|
||||
// The returned predicate is inserted before each candidate stream's LIMIT,
|
||||
// including FTS-first streams. Filtering after their cap silently loses hits.
|
||||
|
||||
Reference in new issue
Block a user