feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s

This commit is contained in:
Anya Tokugawa committed 2026-11-15 05:47:14 +09:00
1 parent 96065d1e89
commit 0ba1dcfef0
243 files changed
+21441 -3090

No files matched your search

+55 -17
View File
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"sort"
"strings"
"time"
@@ -36,6 +37,14 @@ type citySearchCandidate struct {
// Scope selects a locality's own boundary, not an ancestor of the scope. Every
// boundary in its parent chain must be readable before the result is exposed.
func (s *SQLite) SearchCityInDomain(parent context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []CitySearchResult, searchErr error) {
return searchCity(s.db, false, parent, domainUUID, scope, query, limit, allowed, options)
}
func (s *PostgreSQL) SearchCityInDomain(parent context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) ([]CitySearchResult, error) {
return searchCity(s.db, true, parent, domainUUID, scope, query, limit, allowed, options)
}
func searchCity(database *sql.DB, native bool, parent context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []CitySearchResult, searchErr error) {
if err := options.Validate(); err != nil {
return nil, err
}
@@ -58,7 +67,7 @@ func (s *SQLite) SearchCityInDomain(parent context.Context, domainUUID, scope, q
searchErr = ctx.Err()
}
}()
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
tx, err := database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelRepeatableRead})
if err != nil {
return nil, err
}
@@ -70,22 +79,35 @@ func (s *SQLite) SearchCityInDomain(parent context.Context, domainUUID, scope, q
sql.Named("prefix", prefix), sql.Named("upper", upper),
sql.Named("query", query), sql.Named("limit", CitySearchCandidateLimit),
}
if native {
args = []any{domainUUID, scope, prefix, upper, query, CitySearchCandidateLimit}
}
dateColumn := "n.updated_at_ms"
if options.DateField == "created" {
dateColumn = "n.created_at_ms"
}
dateWhere := ""
if options.FromMS != nil {
dateWhere += " AND " + dateColumn + ">=:from_ms"
args = append(args, sql.Named("from_ms", *options.FromMS))
if native {
args = append(args, *options.FromMS)
dateWhere += " AND " + dateColumn + ">=" + fmt.Sprintf("$%d", len(args))
} else {
dateWhere += " AND " + dateColumn + ">=:from_ms"
args = append(args, sql.Named("from_ms", *options.FromMS))
}
}
if options.BeforeMS != nil {
dateWhere += " AND " + dateColumn + "<:before_ms"
args = append(args, sql.Named("before_ms", *options.BeforeMS))
if native {
args = append(args, *options.BeforeMS)
dateWhere += " AND " + dateColumn + "<" + fmt.Sprintf("$%d", len(args))
} else {
dateWhere += " AND " + dateColumn + "<:before_ms"
args = append(args, sql.Named("before_ms", *options.BeforeMS))
}
}
candidates := make([]citySearchCandidate, 0)
for _, kind := range []string{"quartier", "street", "spot"} {
rows, err := tx.QueryContext(ctx, citySearchSQL(kind, dateWhere), args...)
rows, err := tx.QueryContext(ctx, citySearchDialectSQL(kind, dateWhere, native), args...)
if err != nil {
return nil, err
}
@@ -173,6 +195,18 @@ func citySearchKindOrder(kind string) int {
// All interpolated SQL fragments are package-owned constants. User input is
// bound, so %, _, quotes and backslashes are literal substring characters.
func citySearchSQL(kind, dateWhere string) string {
return citySearchDialectSQL(kind, dateWhere, false)
}
func citySearchDialectSQL(kind, dateWhere string, native bool) string {
domain, scope, prefix, upper, query, limit := ":domain", ":scope", ":prefix", ":upper", ":query", ":limit"
collation, position, greatest := " COLLATE BINARY", "instr", "max"
absolute := citySearchAbsolutePathSQL
if native {
domain, scope, prefix, upper, query, limit = "$1", "$2", "$3", "$4", "$5", "$6"
collation, position, greatest = ` COLLATE "C"`, "strpos", "greatest"
absolute = func(column string) string { return `(substr(` + column + `,1,1)='/' AND length(` + column + `)<=4096)` }
}
from, description := "zones n", "n.description"
navigationPath := "n.path_prefix"
parentColumns := "'','','','',''"
@@ -181,28 +215,32 @@ func citySearchSQL(kind, dateWhere string) string {
case "street":
from += ` JOIN zones q ON q.uuid=n.parent_uuid AND q.domain_uuid=n.domain_uuid AND q.kind='quartier' AND q.state!='archived'`
parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),'',q.path_prefix,'',q.name"
where = "n.kind='street' AND " + citySearchAbsolutePathSQL("q.path_prefix")
where = "n.kind='street' AND " + absolute("q.path_prefix")
case "spot":
navigationPath = `CASE WHEN substr(n.path_prefix,-11)='/_undefined' THEN st.path_prefix ELSE n.path_prefix END`
if native {
navigationPath = `CASE WHEN right(n.path_prefix,11)='/_undefined' THEN st.path_prefix ELSE n.path_prefix END`
}
from = `spots n JOIN street_spots ss ON ss.spot_uuid=n.uuid
JOIN zones st ON st.uuid=ss.street_uuid AND st.domain_uuid=n.domain_uuid AND st.kind='street' AND st.state!='archived'
JOIN zones q ON q.uuid=st.parent_uuid AND q.domain_uuid=st.domain_uuid AND q.kind='quartier' AND q.state!='archived'`
description = "n.summary"
parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),substr(COALESCE(NULLIF(st.title,''),st.name),1,256),q.path_prefix,st.path_prefix,q.name"
where = citySearchAbsolutePathSQL("q.path_prefix") + " AND " + citySearchAbsolutePathSQL("st.path_prefix")
where = absolute("q.path_prefix") + " AND " + absolute("st.path_prefix")
}
nameMatch := `(:query!='' AND (instr(lower(n.name),lower(:query))>0 OR instr(lower(n.title),lower(:query))>0))`
pathMatch := `(:query!='' AND instr(lower(n.path_prefix),lower(:query))>0)`
descriptionMatch := `(:query!='' AND instr(lower(` + description + `),lower(:query))>0)`
contains := func(column string) string { return position + "(lower(" + column + "),lower(" + query + "))" }
nameMatch := `(` + query + `!='' AND (` + contains("n.name") + `>0 OR ` + contains("n.title") + `>0))`
pathMatch := `(` + query + `!='' AND ` + contains("n.path_prefix") + `>0)`
descriptionMatch := `(` + query + `!='' AND ` + contains(description) + `>0)`
return `SELECT n.uuid,substr(n.name,1,256),substr(n.title,1,256),
substr(` + description + `,max(1,instr(lower(` + description + `),lower(:query))-160),512),
substr(` + description + `,` + greatest + `(1,` + contains(description) + `-160),512),
n.path_prefix,n.created_at_ms,n.updated_at_ms,` + nameMatch + `,` + pathMatch + `,` + descriptionMatch + `,` + parentColumns + `,` + navigationPath + `,
EXISTS(SELECT 1 FROM paths cp WHERE cp.domain_uuid=n.domain_uuid AND cp.path=(` + navigationPath + `))
FROM ` + from + ` WHERE n.domain_uuid=:domain AND n.state!='archived' AND ` + where + `
AND ` + citySearchAbsolutePathSQL("n.path_prefix") + `
AND (n.path_prefix COLLATE BINARY=:scope OR (n.path_prefix COLLATE BINARY>=:prefix AND n.path_prefix COLLATE BINARY<:upper))
AND (:query='' OR ` + nameMatch + ` OR ` + pathMatch + ` OR ` + descriptionMatch + `)` + dateWhere + `
ORDER BY ` + nameMatch + ` DESC,` + pathMatch + ` DESC,n.path_prefix COLLATE BINARY,n.uuid LIMIT :limit`
FROM ` + from + ` WHERE n.domain_uuid=` + domain + ` AND n.state!='archived' AND ` + where + `
AND ` + absolute("n.path_prefix") + `
AND (n.path_prefix` + collation + `=` + scope + ` OR (n.path_prefix` + collation + `>=` + prefix + ` AND n.path_prefix` + collation + `<` + upper + `))
AND (` + query + `='' OR ` + nameMatch + ` OR ` + pathMatch + ` OR ` + descriptionMatch + `)` + dateWhere + `
ORDER BY ` + nameMatch + ` DESC,` + pathMatch + ` DESC,n.path_prefix` + collation + `,n.uuid LIMIT ` + limit
}
func citySearchAbsolutePathSQL(column string) string {
+134
View File
@@ -0,0 +1,134 @@
package mestionora
import (
"context"
"database/sql"
"errors"
"fmt"
"sort"
"strings"
"time"
)
// PostgreSQL's initial search profile is metadata-only. It deliberately does
// not scan canonical files or the legacy body column to emulate SQLite FTS.
// The caller owns the pool and must provision the native metadata schema.
type PostgreSQL struct{ db *sql.DB }
func NewPostgreSQL(db *sql.DB) *PostgreSQL { return &PostgreSQL{db: db} }
const pgSearchMIME = `lower(trim(split_part(COALESCE(r.content_type,''),';',1)))`
var pgSearchFileType = buildSearchFileTypeSQL(pgSearchMIME, func(suffixes ...string) string {
clauses := make([]string, 0, len(suffixes))
for _, suffix := range suffixes {
clauses = append(clauses, searchFilenameSQL+fmt.Sprintf(` LIKE '%%.%s'`, suffix))
}
return "(" + strings.Join(clauses, " OR ") + ")"
}, func(pattern string) string {
// Only the package-owned MIME glob constants above reach this function.
return pgSearchMIME + ` LIKE '` + strings.ReplaceAll(pattern, "*", "%") + `'`
})
func (s *PostgreSQL) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, err error) {
if err := options.Validate(); err != nil {
return nil, err
}
scope, query = normalizePath(scope), strings.TrimSpace(query)
if allowed == nil || query == "" && !options.HasFilters() {
return nil, nil
}
if len(query) > 512 || len(strings.Fields(query)) > 16 {
return nil, errors.New("search query is too long")
}
if limit <= 0 || limit > 200 {
limit = 100
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
defer func() {
if ctx.Err() != nil {
found, err = nil, ctx.Err()
}
}()
var args []any
bind := func(v any) string { args = append(args, v); return fmt.Sprintf("$%d", len(args)) }
prefix := strings.TrimRight(scope, "/") + "/"
upper := prefix[:len(prefix)-1] + "0"
predicate := "p.domain_uuid=" + bind(domainUUID) + " AND (p.path=" + bind(scope) + " OR (p.path>=" + bind(prefix) + " AND p.path<" + bind(upper) + "))"
if query != "" {
q := bind(query)
predicate += " AND (strpos(lower(p.path),lower(" + q + "))>0 OR strpos(lower(COALESCE(r.filename,'')),lower(" + q + "))>0)"
}
date := "r.created_at_ms"
if options.DateField == "created" {
date = "(SELECT o.created_at*1000 FROM objects o WHERE o.uuid=p.object_uuid)"
}
if options.FromMS != nil {
predicate += " AND " + date + ">=" + bind(*options.FromMS)
}
if options.BeforeMS != nil {
predicate += " AND " + date + "<" + bind(*options.BeforeMS)
}
if options.Extension != "" {
ext := bind("." + options.Extension)
predicate += " AND right(" + searchFilenameSQL + ",length(" + ext + "))=" + ext
}
if options.FileType != "" {
predicate += " AND (" + pgSearchFileType + ")=" + bind(options.FileType)
}
rows, err := s.db.QueryContext(ctx, `SELECT p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.uuid,''),COALESCE(r.created_at_ms/1000,0)
FROM paths p LEFT JOIN LATERAL (SELECT uuid,content_type,filename,document_hash,created_at_ms FROM revisions WHERE object_uuid=p.object_uuid ORDER BY created_at_ms DESC,uuid DESC LIMIT 1) r ON true
WHERE `+predicate+` ORDER BY p.path LIMIT `+bind(SearchCandidateLimit), args...)
if err != nil {
return nil, err
}
candidates := make([]SearchResult, 0)
for rows.Next() {
var result SearchResult
if err = rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt); err != nil {
rows.Close()
return nil, err
}
result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query))
result.FilenameMatch = query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query))
candidates = append(candidates, result)
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
// Release the cursor before policy callbacks: authorization may need the
// same single-connection pool. Never return a denied Path or load its body.
sort.Slice(candidates, func(i, j int) bool {
if len(candidates[i].Path) != len(candidates[j].Path) {
return len(candidates[i].Path) < len(candidates[j].Path)
}
return candidates[i].Path < candidates[j].Path
})
for _, candidate := range candidates {
if err = ctx.Err(); err != nil {
return nil, err
}
if allowed(candidate.Path) {
found = append(found, candidate)
if len(found) == limit {
break
}
}
}
return found, nil
}
func (s *PostgreSQL) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) {
results, err := s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{})
if err != nil {
return nil, err
}
out := make([]PathSuggestion, 0, len(results))
for _, result := range results {
out = append(out, PathSuggestion{SearchResult: result})
}
return out, nil
}
+12 -4
View File
@@ -71,18 +71,25 @@ func searchSuffixesSQL(suffixes ...string) string {
// known suffix (e.g. a Python source uploaded as text/plain); unrecognized,
// specific MIME types stay Other instead of pretending to be a different type.
// This uses metadata only and never hydrates a Document to sniff its bytes.
var searchFileTypeSQL = `CASE
var searchFileTypeSQL = buildSearchFileTypeSQL(searchMIMESQL, searchSuffixesSQL, func(pattern string) string {
return searchMIMESQL + ` GLOB '` + pattern + `'`
})
// All patterns are package-owned constants. The dialect-specific builders do
// not translate incoming SQL or interpolate user input.
func buildSearchFileTypeSQL(searchMIMESQL string, searchSuffixesSQL func(...string) string, mimeGlob func(string) string) string {
return `CASE
WHEN ` + searchMIMESQL + ` IN ('text/html','application/xhtml+xml') THEN 'html'
WHEN ` + searchMIMESQL + `='application/pdf' THEN 'pdf'
WHEN ` + searchMIMESQL + ` LIKE 'image/%' THEN 'image'
WHEN ` + searchMIMESQL + ` LIKE 'audio/%' THEN 'audio'
WHEN ` + searchMIMESQL + ` LIKE 'video/%' THEN 'video'
WHEN ` + searchMIMESQL + ` IN ('text/csv','application/csv','text/tab-separated-values') THEN 'csv'
WHEN ` + searchMIMESQL + ` IN ('application/json','text/json') OR ` + searchMIMESQL + ` GLOB '*+json' THEN 'json'
WHEN ` + searchMIMESQL + ` IN ('application/json','text/json') OR ` + mimeGlob("*+json") + ` THEN 'json'
WHEN ` + searchMIMESQL + ` IN ('application/zip','application/x-zip-compressed','application/x-7z-compressed','application/vnd.rar','application/x-rar','application/x-rar-compressed','application/x-tar','application/gzip','application/x-gzip','application/x-xz','application/x-bzip2','application/zstd','application/x-zstd','application/x-compress','application/x-cpio') THEN 'archive'
WHEN ` + searchMIMESQL + ` IN ('application/msword','application/vnd.ms-excel','application/vnd.ms-powerpoint','application/rtf','text/rtf') OR ` + searchMIMESQL + ` GLOB 'application/vnd.openxmlformats-officedocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.oasis.opendocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.ms-*.macroenabled.*' THEN 'office'
WHEN ` + searchMIMESQL + ` IN ('application/msword','application/vnd.ms-excel','application/vnd.ms-powerpoint','application/rtf','text/rtf') OR ` + mimeGlob("application/vnd.openxmlformats-officedocument.*") + ` OR ` + mimeGlob("application/vnd.oasis.opendocument.*") + ` OR ` + mimeGlob("application/vnd.ms-*.macroenabled.*") + ` THEN 'office'
WHEN ` + searchMIMESQL + ` IN ('text/markdown','text/x-markdown') THEN 'text'
WHEN ` + searchMIMESQL + ` IN ('application/javascript','application/x-javascript','text/javascript','application/typescript','text/typescript','application/xml','text/xml','application/yaml','application/x-yaml','text/yaml','application/toml','application/x-sh','application/x-shellscript','application/x-python','application/x-python-code','application/x-httpd-php','application/sql','text/css') OR ` + searchMIMESQL + ` GLOB 'text/x-*' OR ` + searchMIMESQL + ` GLOB '*+xml' THEN 'code'
WHEN ` + searchMIMESQL + ` IN ('application/javascript','application/x-javascript','text/javascript','application/typescript','text/typescript','application/xml','text/xml','application/yaml','application/x-yaml','text/yaml','application/toml','application/x-sh','application/x-shellscript','application/x-python','application/x-python-code','application/x-httpd-php','application/sql','text/css') OR ` + mimeGlob("text/x-*") + ` OR ` + mimeGlob("*+xml") + ` THEN 'code'
WHEN ` + searchMIMESQL + ` NOT IN ('','text/plain','application/octet-stream','binary/octet-stream','application/x-empty') THEN CASE WHEN ` + searchMIMESQL + ` LIKE 'text/%' THEN 'text' ELSE 'other' END
WHEN ` + searchSuffixesSQL("html", "htm", "xhtml") + ` THEN 'html'
WHEN ` + searchSuffixesSQL("pdf") + ` THEN 'pdf'
@@ -96,6 +103,7 @@ var searchFileTypeSQL = `CASE
WHEN ` + searchSuffixesSQL("py", "pyw", "sh", "bash", "zsh", "fish", "ps1", "bat", "cmd", "js", "mjs", "cjs", "jsx", "ts", "tsx", "go", "rs", "c", "h", "cpp", "hpp", "cc", "cs", "java", "kt", "kts", "swift", "rb", "php", "pl", "pm", "lua", "r", "sql", "css", "scss", "sass", "less", "xml", "xsl", "yaml", "yml", "toml", "ini", "conf", "vue", "svelte", "dart", "ex", "exs", "erl", "hs", "scala", "clj", "lisp", "el", "vim", "wasm") + ` THEN 'code'
WHEN ` + searchMIMESQL + `='text/plain' OR ` + searchSuffixesSQL("txt", "text", "log", "md", "markdown", "rst", "adoc", "org") + ` THEN 'text'
ELSE 'other' END`
}
// The returned predicate is inserted before each candidate stream's LIMIT,
// including FTS-first streams. Filtering after their cap silently loses hits.