Refactor architecture into eight components and consolidate reliability work
Build and publish container images / images (push) Successful in 9m34s
Release container images / release (push) Successful in 36s

Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts.

Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation.

Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
This commit is contained in:
Anya Tokugawa committed 2026-11-01 15:23:31 +09:00
1 parent 9dd425f765
commit d5d21b7841
302 files changed
+20037 -3492

No files matched your search

+1 -16
View File
@@ -38,22 +38,7 @@ func adapterSecret(prefix string) (string, string, error) {
}
func (s *Store) EnsureAdapterSessionsSchema() error {
_, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS adapter_sessions (
ticket_hash TEXT PRIMARY KEY, session_hash TEXT NOT NULL DEFAULT '',
domain_uuid TEXT NOT NULL, plugin_id TEXT NOT NULL, object_uuid TEXT NOT NULL,
principal_uuid TEXT NOT NULL DEFAULT '',
base_revision_uuid TEXT NOT NULL, ticket_expires_at_ms INTEGER NOT NULL,
session_expires_at_ms INTEGER NOT NULL DEFAULT 0, exchanged_at_ms INTEGER NOT NULL DEFAULT 0,
created_at_ms INTEGER NOT NULL
)`)
if err != nil {
return err
}
_, _ = s.DB.Exec(`ALTER TABLE adapter_sessions ADD COLUMN root_path TEXT NOT NULL DEFAULT ''`)
_, _ = s.DB.Exec(`ALTER TABLE adapter_sessions ADD COLUMN principal_uuid TEXT NOT NULL DEFAULT ''`)
_, err = s.DB.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_adapter_session_hash
ON adapter_sessions(session_hash) WHERE session_hash != ''`)
return err
return s.ensureSchemaComponent("adapter-sessions")
}
func (s *Store) IssueAdapterTicket(domainUUID, pluginID, objectUUID, revisionUUID, rootPath string) (string, error) {
+2 -2
View File
@@ -177,7 +177,7 @@ func (s *Store) createTypedAuthSession(domainUUID, principalUUID string, mfaLeve
result, err := s.DB.Exec(`INSERT INTO auth_sessions(uuid,token_hash,principal_uuid,domain_uuid,auth_time_ms,expires_at_ms,last_seen_at_ms,mfa_level)
SELECT ?,?,p.uuid,m.domain_uuid,?,?,?,? FROM principals p JOIN domain_memberships m ON m.principal_uuid=p.uuid
WHERE p.uuid=? AND p.state IN ('active','readonly') AND m.domain_uuid=? AND m.state='active'`,
uuid.NewString(), hash, now, now+int64((24*time.Hour)/time.Millisecond), now, mfaLevel, principalUUID, domainUUID)
uuid.NewString(), hash, now, now+s.AuthSessionLifetime(domainUUID).Milliseconds(), now, mfaLevel, principalUUID, domainUUID)
if err != nil {
return "", err
}
@@ -472,7 +472,7 @@ func (s *Store) Authorize(request AuthorizationRequest) error {
if err != nil {
return err
}
if p.decide(request.TargetID, request.Capability, base == nil) {
if p.Decide(request.TargetID, request.Capability, base == nil) {
return nil
}
return ErrAccessDenied
+32
View File
@@ -0,0 +1,32 @@
package store
import (
"errors"
"strconv"
"time"
)
const DefaultAuthSessionMinutes = 24 * 60
const MaxAuthSessionMinutes = 30 * 24 * 60
// Absolute lifetime, fixed at issuance. Changing the policy cannot silently
// revive, extend or invalidate existing sessions. App passwords are separate.
func (s *Store) AuthSessionLifetimeMinutes(domain string) int {
raw, err := s.GetConfigInDomain(domain, "auth_session_minutes")
value, parseErr := strconv.Atoi(raw)
if err != nil || parseErr != nil || value < 5 || value > MaxAuthSessionMinutes {
return DefaultAuthSessionMinutes
}
return value
}
func (s *Store) SetAuthSessionLifetimeMinutes(domain string, minutes int) error {
if minutes < 5 || minutes > MaxAuthSessionMinutes {
return errors.New("session lifetime must be between 5 and 43200 minutes")
}
return s.SetConfigInDomain(domain, "auth_session_minutes", strconv.Itoa(minutes))
}
func (s *Store) AuthSessionLifetime(domain string) time.Duration {
return time.Duration(s.AuthSessionLifetimeMinutes(domain)) * time.Minute
}
@@ -0,0 +1,84 @@
package store
import (
"database/sql"
"testing"
"time"
"pageswiki/internal/db"
)
func TestAuthSessionLifetimeIsDomainScopedAndFixedAtIssuance(t *testing.T) {
database, err := sql.Open("sqlite", ":memory:")
if err != nil {
t.Fatal(err)
}
database.SetMaxOpenConns(1)
t.Cleanup(func() { database.Close() })
if err = db.Migrate(database); err != nil {
t.Fatal(err)
}
s := NewWithDocumentRoot(database, t.TempDir()+"/documents")
if err = s.EnsureAuthSchema(); err != nil {
t.Fatal(err)
}
if err = s.EnsureConfigTable(); err != nil {
t.Fatal(err)
}
p, err := s.BootstrapOwner("domain", "admin", "a long test password", "Admin", "")
if err != nil {
t.Fatal(err)
}
if s.AuthSessionLifetimeMinutes("domain") != 1440 {
t.Fatal("default changed")
}
for _, minutes := range []int{-1, 0, 4, 43201} {
if s.SetAuthSessionLifetimeMinutes("domain", minutes) == nil {
t.Fatalf("accepted %d", minutes)
}
}
if err = s.SetAuthSessionLifetimeMinutes("domain", 60); err != nil {
t.Fatal(err)
}
if s.AuthSessionLifetimeMinutes("other") != 1440 {
t.Fatal("policy leaked across Domains")
}
for _, browser := range []bool{false, true} {
var token string
if browser {
token, err = s.CreateBrowserAuthSession("domain", p.UUID, 0)
} else {
token, err = s.CreateAuthSession("domain", p.UUID)
}
if err != nil {
t.Fatal(err)
}
var duration int64
if err = database.QueryRow(`SELECT expires_at_ms-auth_time_ms FROM auth_sessions ORDER BY rowid DESC LIMIT 1`).Scan(&duration); err != nil {
t.Fatal(err)
}
if duration != time.Hour.Milliseconds() {
t.Fatalf("lifetime %d", duration)
}
if _, err = s.PrincipalForSession("domain", token); err != nil {
t.Fatal(err)
}
}
if err = s.SetAuthSessionLifetimeMinutes("domain", 5); err != nil {
t.Fatal(err)
}
var changed int
if err = database.QueryRow(`SELECT count(*) FROM auth_sessions WHERE expires_at_ms-auth_time_ms<>?`, time.Hour.Milliseconds()).Scan(&changed); err != nil || changed != 0 {
t.Fatalf("existing sessions changed: %d %v", changed, err)
}
token, err := s.CreateAuthSession("domain", p.UUID)
if err != nil {
t.Fatal(err)
}
if _, err = database.Exec(`UPDATE auth_sessions SET expires_at_ms=?`, time.Now().Add(-time.Second).UnixMilli()); err != nil {
t.Fatal(err)
}
if _, err = s.PrincipalForSession("domain", token); err == nil {
t.Fatal("expired token authenticated")
}
}
+1 -6
View File
@@ -21,12 +21,7 @@ type Broadcast struct {
}
func (s *Store) EnsureBroadcastSchema() error {
_, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS broadcasts (uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,title TEXT NOT NULL,body TEXT NOT NULL,severity TEXT NOT NULL DEFAULT 'info' CHECK(severity IN ('info','success','warning','critical')),starts_at_ms INTEGER NOT NULL DEFAULT 0,ends_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`)
if err != nil {
return err
}
_, err = s.DB.Exec(`CREATE INDEX IF NOT EXISTS idx_broadcast_active ON broadcasts(domain_uuid,starts_at_ms,ends_at_ms,updated_at_ms DESC)`)
return err
return s.ensureSchemaComponent("broadcasts")
}
const broadcastColumns = `uuid,domain_uuid,title,body,severity,starts_at_ms,ends_at_ms,created_at_ms,updated_at_ms`
+4 -33
View File
@@ -1,11 +1,9 @@
package store
import (
"io"
"os"
"strings"
"golang.org/x/net/html"
"pageswiki/internal/mestionora"
)
type BrowserEntry struct {
@@ -23,32 +21,7 @@ type BrowserEntry struct {
Title string
}
func firstHTMLHeading(source string) string {
t := html.NewTokenizer(strings.NewReader(source))
in := false
var out strings.Builder
for {
kind := t.Next()
if kind == html.ErrorToken {
if t.Err() == io.EOF {
break
}
break
}
tok := t.Token()
if kind == html.StartTagToken && strings.EqualFold(tok.Data, "h1") {
in = true
continue
}
if kind == html.EndTagToken && strings.EqualFold(tok.Data, "h1") {
break
}
if in && kind == html.TextToken {
out.WriteString(tok.Data)
}
}
return strings.TrimSpace(out.String())
}
func firstHTMLHeading(source string) string { return mestionora.FirstHTMLHeading(source) }
func normalizeBrowserPath(p string) string {
p = strings.TrimSpace(p)
@@ -117,10 +90,8 @@ func (s *Store) ListBrowserEntriesInDomain(domainUUID, parent string) ([]Browser
}
entry.Path = prefix + entry.Name
if !entry.IsDir && s.DocumentRoot != "" && entry.DocumentHash != "" {
if filename, err := s.DocumentPath(entry.DocumentHash); err == nil {
if info, err := os.Stat(filename); err == nil {
entry.Size = int(info.Size())
}
if size, err := s.DocumentSize(entry.DocumentHash); err == nil {
entry.Size = int(size)
}
}
out = append(out, entry)
+1 -19
View File
@@ -23,25 +23,7 @@ type BrowserNavigationLink struct {
}
func (s *Store) EnsureBrowserNavigationSchema() error {
_, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS browser_navigation_links (
uuid TEXT PRIMARY KEY,
domain_uuid TEXT NOT NULL,
principal_uuid TEXT NOT NULL,
label TEXT NOT NULL,
path TEXT NOT NULL,
is_directory INTEGER NOT NULL DEFAULT 0,
position INTEGER NOT NULL DEFAULT 0,
created_at_ms INTEGER NOT NULL,
updated_at_ms INTEGER NOT NULL,
UNIQUE(domain_uuid, principal_uuid, path)
)`)
if err != nil {
return err
}
_, _ = s.DB.Exec(`ALTER TABLE browser_navigation_links ADD COLUMN is_directory INTEGER NOT NULL DEFAULT 0`)
_, err = s.DB.Exec(`CREATE INDEX IF NOT EXISTS idx_browser_navigation_owner
ON browser_navigation_links(domain_uuid, principal_uuid, position, created_at_ms)`)
return err
return s.ensureSchemaComponent("browser-navigation")
}
func (s *Store) ListBrowserNavigationLinks(domainUUID, principalUUID string) ([]BrowserNavigationLink, error) {
+10 -3
View File
@@ -13,6 +13,7 @@ import (
var cityNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{1,63}$`)
type CitySpot struct {
DisplayTitle string `json:"display_title,omitempty"`
DetailHref string `json:"detail_href,omitempty"`
UUID string `json:"uuid"`
Name string `json:"name"`
@@ -74,6 +75,7 @@ type CityPagePlacement struct {
type CitySpotAppearance struct {
SpotUUID, SpotName, SpotTitle, EdgeType, Label, Source, PathPrefix string
DisplayTitle, StreetPath, QuartierPath string
}
type CityNodeDetail struct {
@@ -538,13 +540,16 @@ func listSpotPages(db cityQuerier, domainUUID, spotUUID string, limit int) ([]Ci
}
func (s *Store) ListPageSpotAppearances(domainUUID, pageUUID string) ([]CitySpotAppearance, error) {
rows, err := s.DB.Query(`SELECT uuid,name,title,edge_type,label,source,path_prefix FROM (
rows, err := s.DB.Query(`SELECT a.uuid,a.name,a.title,a.edge_type,a.label,a.source,a.path_prefix,COALESCE(NULLIF(st.title,''),st.name),st.path_prefix,q.path_prefix FROM (
SELECT s.uuid,s.name,s.title,sp.edge_type,sp.label,sp.source,s.path_prefix FROM spot_pages sp JOIN spots s ON s.uuid=sp.spot_uuid AND s.domain_uuid=? AND s.state!='archived' WHERE sp.page_uuid=?
UNION ALL
SELECT s.uuid,s.name,s.title,'path','','path',s.path_prefix FROM spots s JOIN paths p ON p.domain_uuid=s.domain_uuid AND p.object_uuid=? AND `+cityPathScopeSQL("s.path_prefix", "p.path")+`
WHERE s.domain_uuid=? AND s.state!='archived' AND NOT EXISTS(SELECT 1 FROM spot_pages explicit WHERE explicit.spot_uuid=s.uuid AND explicit.page_uuid=?)
AND NOT EXISTS(SELECT 1 FROM city_page_overrides ov JOIN spots dest ON dest.uuid=ov.spot_uuid AND dest.domain_uuid=ov.domain_uuid AND dest.state!='archived' WHERE ov.domain_uuid=p.domain_uuid AND ov.page_uuid=p.object_uuid AND ov.spot_uuid!=s.uuid)
) ORDER BY lower(title),edge_type`, domainUUID, pageUUID, pageUUID, domainUUID, pageUUID)
) a JOIN street_spots ss ON ss.spot_uuid=a.uuid
JOIN zones st ON st.uuid=ss.street_uuid AND st.domain_uuid=? AND st.kind='street' AND st.state!='archived'
JOIN zones q ON q.uuid=st.parent_uuid AND q.domain_uuid=st.domain_uuid AND q.kind='quartier' AND q.state!='archived'
ORDER BY lower(a.title),a.edge_type`, domainUUID, pageUUID, pageUUID, domainUUID, pageUUID, domainUUID)
if err != nil {
return nil, err
}
@@ -552,9 +557,11 @@ func (s *Store) ListPageSpotAppearances(domainUUID, pageUUID string) ([]CitySpot
var out []CitySpotAppearance
for rows.Next() {
var item CitySpotAppearance
if err := rows.Scan(&item.SpotUUID, &item.SpotName, &item.SpotTitle, &item.EdgeType, &item.Label, &item.Source, &item.PathPrefix); err != nil {
var parentTitle string
if err := rows.Scan(&item.SpotUUID, &item.SpotName, &item.SpotTitle, &item.EdgeType, &item.Label, &item.Source, &item.PathPrefix, &parentTitle, &item.StreetPath, &item.QuartierPath); err != nil {
return nil, err
}
item.DisplayTitle = CitySpotDisplayTitle(item.SpotName, item.SpotTitle, item.PathPrefix, parentTitle)
out = append(out, item)
}
return out, rows.Err()
+28
View File
@@ -0,0 +1,28 @@
package store
import "pageswiki/internal/mestionora"
func IsUnclassifiedCitySpot(name, boundary string) bool {
return mestionora.IsUnclassifiedCitySpot(name, boundary)
}
func CitySpotDisplayTitle(name, title, boundary, parentTitle string) string {
return mestionora.CitySpotDisplayTitle(name, title, boundary, parentTitle)
}
func (n CityNodeDetail) DisplayTitle() string {
if n.Kind == "spot" {
parent := ""
if n.Street != nil {
parent = n.Street.Title
if parent == "" {
parent = n.Street.Name
}
}
return CitySpotDisplayTitle(n.Name, n.Title, n.PathPrefix, parent)
}
if n.Title != "" {
return n.Title
}
return n.Name
}
+50
View File
@@ -0,0 +1,50 @@
package store
import (
"context"
"testing"
)
func TestCitySpotDisplayTitlePreservesIdentity(t *testing.T) {
for _, test := range []struct{ name, title, boundary, parent, want string }{
{"home-undefined", "Undefined", "/life/home/_undefined", "Home & Garden", "Home & Garden"},
{"home-undefined", "home-undefined", "/life/home/_undefined", "Home", "Home"},
{"home-undefined", "My notes", "/life/home/_undefined", "Home", "My notes"},
{"manual", "Undefined", "/life/home/manual", "Home", "Undefined"},
{"home-undefined", "", "/life/home/_undefined", "", "home"},
} {
if got := CitySpotDisplayTitle(test.name, test.title, test.boundary, test.parent); got != test.want {
t.Fatalf("%+v: %q", test, got)
}
}
}
func TestCitySearchContentPathMetadata(t *testing.T) {
st := cityReadFixture(t)
citySearchInsertZone(t, st, "q", "d", "quartier", "", "campus", "Campus", "needle", "/life")
citySearchInsertZone(t, st, "s", "d", "street", "q", "home", "Home", "needle", "/life/home")
citySearchInsertSpot(t, st, "p", "d", "s", "home-undefined", "Undefined", "needle", "/life/home/_undefined")
// Same Path in another Domain must not be mistaken for a local portal.
cityTestExec(t, st, `INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('other','/life','other-page'),('d','/life/home','home-page')`)
results, err := st.SearchCityInDomain(context.Background(), "d", "/", "needle", 30, citySearchAllowAll, SearchOptions{})
if err != nil || len(results) != 3 {
t.Fatalf("results: %+v %v", results, err)
}
if results[0].HasPage || !results[1].HasPage || !results[2].HasPage || results[2].ContentPath() != "/life/home" || results[2].DisplayTitle() != "Home" {
t.Fatalf("projection: %+v", results)
}
node, err := st.CityNode("d", "spot", "home-undefined")
if err != nil || node.DisplayTitle() != "Home" || node.Title != "Undefined" || node.Name != "home-undefined" || node.PathPrefix != "/life/home/_undefined" {
t.Fatalf("raw identity mutated: %+v %v", node, err)
}
cityTestExec(t, st, `INSERT INTO spot_pages(spot_uuid,page_uuid,edge_type,label,source,created_at_ms) VALUES('p','home-page','primary','','local',1)`)
appearances, err := st.ListPageSpotAppearances("d", "home-page")
if err != nil || len(appearances) != 1 || appearances[0].DisplayTitle != "Home" || appearances[0].SpotTitle != "Undefined" || appearances[0].StreetPath != "/life/home" || appearances[0].QuartierPath != "/life" {
t.Fatalf("appearance projection: %+v %v", appearances, err)
}
// Both parent and boundary ACLs still gate the result.
results, err = st.SearchCityInDomain(context.Background(), "d", "/", "needle", 30, func(p string) bool { return p != "/life/home" }, SearchOptions{})
if err != nil || len(results) != 1 {
t.Fatalf("parent ACL: %+v %v", results, err)
}
}
+4 -23
View File
@@ -7,6 +7,8 @@ import (
"errors"
"strings"
"time"
"pageswiki/internal/schutzaria"
)
var (
@@ -16,11 +18,7 @@ var (
ErrGovernanceConfirm = errors.New("city.governance.confirm_error")
)
type LocalityPolicy struct {
Kind string `json:"kind"`
UUID string `json:"uuid"`
Policy string `json:"policy"`
}
type LocalityPolicy = schutzaria.LocalityPolicy
type LocalityOwner struct {
MembershipUUID string `json:"membership_uuid"`
@@ -72,24 +70,7 @@ func (s *Store) ensureCityGovernanceSchema() error {
// EvaluateLocalityPublication combines ceilings with intent. A public Domain
// merely allows publication: it never publishes its root or arbitrary Pages.
func EvaluateLocalityPublication(chain []LocalityPolicy) (allows, published bool) {
allows = true
for i, p := range chain {
if i == 0 && p.Policy == "inherit" {
return false, false
}
switch p.Policy {
case "authenticated_acl":
allows = false
case "public":
if i > 0 {
published = true
}
case "inherit":
default:
return false, false
}
}
return allows, allows && published
return schutzaria.EvaluateLocalityPublication(chain)
}
// All authority, parentage and policy reads use this one snapshot. Mutation
+8 -196
View File
@@ -2,205 +2,17 @@ package store
import (
"context"
"database/sql"
"errors"
"sort"
"strings"
"time"
"pageswiki/internal/mestionora"
)
type CitySearchResult = mestionora.CitySearchResult
const (
CitySearchCandidateLimit = 1000 // Per locality kind, before authorization.
CitySearchResultLimit = 30
citySearchTimeout = 2 * time.Second
CitySearchCandidateLimit = mestionora.CitySearchCandidateLimit
CitySearchResultLimit = mestionora.CitySearchResultLimit
)
// CitySearchResult describes a locality, not its contained Pages. Description
// is a bounded plain-text excerpt around the match and is never trusted HTML.
type CitySearchResult struct {
UUID, Kind, Name, Title, Description, PathPrefix string
QuartierTitle, StreetTitle, QuartierName string
CreatedAtMS, UpdatedAtMS int64
NameMatch, PathMatch, DescriptionMatch bool
}
type citySearchCandidate struct {
CitySearchResult
quartierPath, streetPath string
}
// SearchCityInDomain searches locality metadata only. It does not construct a
// CityMap, expand descendants, read Page bodies, or count hidden localities.
// Scope selects a locality's own boundary, not an ancestor of the scope. Every
// boundary in its parent chain must be readable before the result is exposed.
func (s *Store) SearchCityInDomain(parent context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []CitySearchResult, searchErr error) {
if err := options.Validate(); err != nil {
return nil, err
}
query = strings.TrimSpace(query)
if len(query) > 512 || len(strings.Fields(query)) > 16 {
return nil, errors.New("search query is too long")
}
// File filters are intentionally not applied to locality metadata: a Spot
// containing a PDF is not itself a PDF and must not claim to match one.
if allowed == nil || options.FileType != "" || options.Extension != "" || (query == "" && !options.HasFilters()) {
return nil, nil
}
if limit <= 0 || limit > CitySearchResultLimit {
limit = CitySearchResultLimit
}
ctx, cancel := context.WithTimeout(parent, citySearchTimeout)
defer cancel()
defer func() {
if searchErr != nil && ctx.Err() != nil {
searchErr = ctx.Err()
}
}()
tx, err := s.DB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return nil, err
}
defer tx.Rollback()
scope = normalizePath(scope)
prefix, upper := strings.TrimRight(scope, "/")+"/", strings.TrimRight(scope, "/")+"0"
args := []any{
sql.Named("domain", domainUUID), sql.Named("scope", scope),
sql.Named("prefix", prefix), sql.Named("upper", upper),
sql.Named("query", query), sql.Named("limit", CitySearchCandidateLimit),
}
dateColumn := "n.updated_at_ms"
if options.DateField == "created" {
dateColumn = "n.created_at_ms"
}
dateWhere := ""
if options.FromMS != nil {
dateWhere += " AND " + dateColumn + ">=:from_ms"
args = append(args, sql.Named("from_ms", *options.FromMS))
}
if options.BeforeMS != nil {
dateWhere += " AND " + dateColumn + "<:before_ms"
args = append(args, sql.Named("before_ms", *options.BeforeMS))
}
candidates := make([]citySearchCandidate, 0)
for _, kind := range []string{"quartier", "street", "spot"} {
rows, err := tx.QueryContext(ctx, citySearchSQL(kind, dateWhere), args...)
if err != nil {
return nil, err
}
for rows.Next() {
var candidate citySearchCandidate
candidate.Kind = kind
if err := rows.Scan(&candidate.UUID, &candidate.Name, &candidate.Title, &candidate.Description,
&candidate.PathPrefix, &candidate.CreatedAtMS, &candidate.UpdatedAtMS,
&candidate.NameMatch, &candidate.PathMatch, &candidate.DescriptionMatch,
&candidate.QuartierTitle, &candidate.StreetTitle, &candidate.quartierPath, &candidate.streetPath, &candidate.QuartierName); err != nil {
rows.Close()
return nil, err
}
candidates = append(candidates, candidate)
}
err = rows.Err()
closeErr := rows.Close()
if err != nil {
return nil, err
}
if closeErr != nil {
return nil, closeErr
}
}
// Release the sole read connection before calling authorization, which may
// itself need a database connection. HTTP normally passes its cached policy.
if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) {
return nil, err
}
sort.SliceStable(candidates, func(i, j int) bool {
a, b := candidates[i], candidates[j]
if a.Kind != b.Kind {
return citySearchKindOrder(a.Kind) < citySearchKindOrder(b.Kind)
}
if a.NameMatch != b.NameMatch {
return a.NameMatch
}
if a.PathMatch != b.PathMatch {
return a.PathMatch
}
if a.PathPrefix != b.PathPrefix {
return a.PathPrefix < b.PathPrefix
}
return a.UUID < b.UUID
})
readable := make(map[string]bool)
canRead := func(path string) bool {
if value, exists := readable[path]; exists {
return value
}
value := allowed(path)
readable[path] = value
return value
}
for _, candidate := range candidates {
if err := ctx.Err(); err != nil {
return nil, err
}
if !canRead(candidate.PathPrefix) || (candidate.quartierPath != "" && !canRead(candidate.quartierPath)) || (candidate.streetPath != "" && !canRead(candidate.streetPath)) {
continue
}
found = append(found, candidate.CitySearchResult)
if len(found) == limit {
break
}
}
if err := ctx.Err(); err != nil {
return nil, err
}
return found, nil
}
func citySearchKindOrder(kind string) int {
switch kind {
case "quartier":
return 0
case "street":
return 1
default:
return 2
}
}
// All interpolated SQL fragments are package-owned constants. User input is
// bound, so %, _, quotes and backslashes are literal substring characters.
func citySearchSQL(kind, dateWhere string) string {
from, description := "zones n", "n.description"
parentColumns := "'','','','',''"
where := "n.kind='quartier'"
switch kind {
case "street":
from += ` JOIN zones q ON q.uuid=n.parent_uuid AND q.domain_uuid=n.domain_uuid AND q.kind='quartier' AND q.state!='archived'`
parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),'',q.path_prefix,'',q.name"
where = "n.kind='street' AND " + citySearchAbsolutePathSQL("q.path_prefix")
case "spot":
from = `spots n JOIN street_spots ss ON ss.spot_uuid=n.uuid
JOIN zones st ON st.uuid=ss.street_uuid AND st.domain_uuid=n.domain_uuid AND st.kind='street' AND st.state!='archived'
JOIN zones q ON q.uuid=st.parent_uuid AND q.domain_uuid=st.domain_uuid AND q.kind='quartier' AND q.state!='archived'`
description = "n.summary"
parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),substr(COALESCE(NULLIF(st.title,''),st.name),1,256),q.path_prefix,st.path_prefix,q.name"
where = citySearchAbsolutePathSQL("q.path_prefix") + " AND " + citySearchAbsolutePathSQL("st.path_prefix")
}
nameMatch := `(:query!='' AND (instr(lower(n.name),lower(:query))>0 OR instr(lower(n.title),lower(:query))>0))`
pathMatch := `(:query!='' AND instr(lower(n.path_prefix),lower(:query))>0)`
descriptionMatch := `(:query!='' AND instr(lower(` + description + `),lower(:query))>0)`
return `SELECT n.uuid,substr(n.name,1,256),substr(n.title,1,256),
substr(` + description + `,max(1,instr(lower(` + description + `),lower(:query))-160),512),
n.path_prefix,n.created_at_ms,n.updated_at_ms,` + nameMatch + `,` + pathMatch + `,` + descriptionMatch + `,` + parentColumns + `
FROM ` + from + ` WHERE n.domain_uuid=:domain AND n.state!='archived' AND ` + where + `
AND ` + citySearchAbsolutePathSQL("n.path_prefix") + `
AND (n.path_prefix COLLATE BINARY=:scope OR (n.path_prefix COLLATE BINARY>=:prefix AND n.path_prefix COLLATE BINARY<:upper))
AND (:query='' OR ` + nameMatch + ` OR ` + pathMatch + ` OR ` + descriptionMatch + `)` + dateWhere + `
ORDER BY ` + nameMatch + ` DESC,` + pathMatch + ` DESC,n.path_prefix COLLATE BINARY,n.uuid LIMIT :limit`
}
func citySearchAbsolutePathSQL(column string) string {
// Keep authorization paths intact rather than truncating them. Malformed
// legacy boundaries and impractically large metadata are not navigable hits.
return `(substr(` + column + `,1,1)='/' AND length(` + column + `)<=4096 AND instr(` + column + `,char(0))=0)`
func (s *Store) SearchCityInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) ([]CitySearchResult, error) {
return mestionora.NewSQLite(s.DB).SearchCityInDomain(ctx, domainUUID, scope, query, limit, allowed, options)
}
+2 -2
View File
@@ -33,8 +33,8 @@ func citySearchIDs(results []CitySearchResult) []string {
func TestCitySearchMetadataRankingAndMatchSources(t *testing.T) {
st := cityReadFixture(t)
// No Page table remains: this search must not expand a CityMap or bodies.
cityTestExec(t, st, `DROP TABLE paths`)
// The fixture has only Path metadata, no document/revision bodies. Search
// may test boundary existence but must not expand a CityMap or read bodies.
citySearchInsertZone(t, st, "q-description", "d", "quartier", "", "description", "Description", "needle in charter", "/a-description")
citySearchInsertZone(t, st, "q-path", "d", "quartier", "", "path", "Path", "", "/b-needle")
citySearchInsertZone(t, st, "q-title", "d", "quartier", "", "title", "NeEdLe quartier", "", "/c-title")
+228
View File
@@ -0,0 +1,228 @@
package store
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"syscall"
"time"
"pageswiki/internal/db"
)
var documentBackfillMigrations = []db.SQLMigration{{Version: 1, Name: "document_backfill_progress", Statements: []string{
`CREATE TABLE document_backfills (
store_id TEXT PRIMARY KEY, last_uuid TEXT NOT NULL DEFAULT '',
processed INTEGER NOT NULL DEFAULT 0, completed INTEGER NOT NULL DEFAULT 0,
updated_at_ms INTEGER NOT NULL)`,
}}, {Version: 2, Name: "verify_backfill_contract", Assertions: []string{
schemaColumns("document_backfills", "store_id,last_uuid,processed,completed,updated_at_ms"),
schemaUnique("document_backfills", "store_id"),
}}}
type DocumentBackfillProgress struct {
StoreID string
LastUUID string
Processed int64
Completed bool
}
// BackfillDocuments performs one restartable, bounded batch. It is a maintenance
// operation: legacy DB-only writers must be stopped. Normal application saves
// persist canonical bytes before inserting revisions and need no backfill.
// The UUID cursor is stable across SQLite VACUUM (rowid is not).
func (s *Store) BackfillDocuments(ctx context.Context, batchSize int) (DocumentBackfillProgress, error) {
var progress DocumentBackfillProgress
if s.ReadOnly {
return progress, ErrReadOnly
}
if err := ctx.Err(); err != nil {
return progress, err
}
if s.DocumentRoot == "" {
progress.Completed = true
return progress, nil
}
if batchSize <= 0 || batchSize > 128 {
batchSize = 128
}
if err := os.MkdirAll(filepath.Join(s.DocumentRoot, "sha256"), 0755); err != nil {
return progress, err
}
root, err := os.OpenRoot(s.DocumentRoot)
if err != nil {
return progress, err
}
defer root.Close()
unlock, err := lockDocumentBackfill(ctx, root)
if err != nil {
return progress, err
}
defer unlock()
progress.StoreID, err = documentStoreIdentity(root)
if err != nil {
return progress, err
}
if err := db.ApplySQLiteMigrations(ctx, s.DB, "document-store", documentBackfillMigrations); err != nil {
return progress, err
}
if _, err := s.DB.ExecContext(ctx, `INSERT INTO document_backfills(store_id,updated_at_ms) VALUES(?,?) ON CONFLICT(store_id) DO NOTHING`, progress.StoreID, time.Now().UnixMilli()); err != nil {
return progress, err
}
if err := s.DB.QueryRowContext(ctx, `SELECT last_uuid,processed,completed FROM document_backfills WHERE store_id=?`, progress.StoreID).Scan(&progress.LastUUID, &progress.Processed, &progress.Completed); err != nil {
return progress, err
}
if progress.Completed {
return progress, nil
}
rows, err := s.DB.QueryContext(ctx, `SELECT uuid,document_hash FROM revisions WHERE uuid>? AND document_hash IS NOT NULL AND document_hash!='' ORDER BY uuid LIMIT ?`, progress.LastUUID, batchSize)
if err != nil {
return progress, err
}
type ref struct{ uuid, hash string }
var refs []ref
for rows.Next() {
var r ref
if err := rows.Scan(&r.uuid, &r.hash); err != nil {
rows.Close()
return progress, err
}
refs = append(refs, r)
}
if err := rows.Err(); err != nil {
rows.Close()
return progress, err
}
if err := rows.Close(); err != nil {
return progress, err
}
backend, err := s.documentStorage()
if err != nil {
return progress, err
}
for _, ref := range refs {
// Existing canonical bytes outrank a stale/missing compatibility body.
// Never overwrite a corrupt canonical file with a cache implicitly.
err := backend.Verify(ctx, ref.hash)
if errors.Is(err, os.ErrNotExist) {
var body string
if err = s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE uuid=? AND document_hash=?`, ref.uuid, ref.hash).Scan(&body); err == nil {
err = backend.PutIfAbsent(ctx, ref.hash, int64(len(body)), strings.NewReader(body))
}
}
if err != nil {
return progress, fmt.Errorf("Document backfill revision=%s: %w", ref.uuid, err)
}
}
next := progress
if len(refs) != 0 {
next.LastUUID = refs[len(refs)-1].uuid
}
next.Processed += int64(len(refs))
next.Completed = len(refs) < batchSize
if _, err := s.DB.ExecContext(ctx, `UPDATE document_backfills SET last_uuid=?,processed=?,completed=?,updated_at_ms=? WHERE store_id=?`, next.LastUUID, next.Processed, next.Completed, time.Now().UnixMilli(), next.StoreID); err != nil {
return progress, err
}
return next, nil
}
func lockDocumentBackfill(ctx context.Context, root *os.Root) (func(), error) {
file, err := root.OpenFile(".document-backfill.lock", os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return nil, err
}
for {
if err := ctx.Err(); err != nil {
file.Close()
return nil, err
}
err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
if err == nil {
return func() { _ = syscall.Flock(int(file.Fd()), syscall.LOCK_UN); file.Close() }, nil
}
if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) {
file.Close()
return nil, err
}
timer := time.NewTimer(20 * time.Millisecond)
select {
case <-ctx.Done():
timer.Stop()
file.Close()
return nil, ctx.Err()
case <-timer.C:
}
}
}
// Identity follows the physical store, not its pathname. A replaced/empty root
// gets a new identity and cannot inherit the DB's previous completion marker.
func documentStoreIdentity(root *os.Root) (string, error) {
file, err := root.Open(".document-store-id")
if err == nil {
defer file.Close()
info, err := file.Stat()
if err != nil {
return "", err
}
if !info.Mode().IsRegular() || info.Size() > 64 {
return "", errors.New("invalid Document store identity file")
}
body, err := io.ReadAll(io.LimitReader(file, 128))
if err != nil {
return "", err
}
id := strings.TrimSpace(string(body))
if decoded, err := hex.DecodeString(id); err != nil || len(decoded) != 16 {
return "", errors.New("invalid Document store identity")
}
// Also sync on resume: the previous process may have published the
// identity then crashed before syncing its directory.
return id, syncDocumentRoot(root)
}
if !errors.Is(err, os.ErrNotExist) {
return "", err
}
var nonce [16]byte
if _, err := rand.Read(nonce[:]); err != nil {
return "", err
}
id := hex.EncodeToString(nonce[:])
tmp := ".document-store-id-" + id
file, err = root.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return "", err
}
defer root.Remove(tmp)
_, err = file.WriteString(id + "\n")
if err == nil {
err = file.Sync()
}
closeErr := file.Close()
if err == nil {
err = closeErr
}
if err != nil {
return "", err
}
if err := root.Link(tmp, ".document-store-id"); err != nil {
return "", err
}
return id, syncDocumentRoot(root)
}
func syncDocumentRoot(root *os.Root) error {
dir, err := root.Open(".")
if err != nil {
return err
}
err = dir.Sync()
dir.Close()
return err
}
+200
View File
@@ -0,0 +1,200 @@
package store_test
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"os"
"path/filepath"
"sync"
"syscall"
"testing"
"time"
"pageswiki/internal/store"
)
func backfillFixture(t *testing.T, st *store.Store, id, body string) string {
t.Helper()
sum := sha256.Sum256([]byte(body))
hash := hex.EncodeToString(sum[:])
_, err := st.DB.Exec(`INSERT INTO revisions(uuid,object_uuid,body,document_hash,lang,content_type,filename,message,created_at_ms) VALUES(?,'backfill-object',?,?,'und','text/plain','','',0)`, id, body, hash)
if err != nil {
t.Fatal(err)
}
return hash
}
func TestDocumentBackfillResumesAndDoesNotRescanCompletedStore(t *testing.T) {
st := newTopologyStore(t)
hashA := backfillFixture(t, st, "a", "first")
hashB := backfillFixture(t, st, "b", "second")
st.DocumentRoot = t.TempDir()
first, err := st.BackfillDocuments(context.Background(), 1)
if err != nil || first.LastUUID != "a" || first.Processed != 1 || first.Completed {
t.Fatalf("first=%+v err=%v", first, err)
}
if _, err := st.DB.Exec(`UPDATE revisions SET body='broken cache' WHERE uuid='a'`); err != nil {
t.Fatal(err)
}
if _, err := st.DB.Exec(`VACUUM`); err != nil {
t.Fatal(err)
}
// A new Store instance resumes the durable UUID cursor, not in-memory state.
restarted := &store.Store{DB: st.DB, DocumentRoot: st.DocumentRoot}
second, err := restarted.BackfillDocuments(context.Background(), 1)
if err != nil || second.LastUUID != "b" || second.Processed != 2 {
t.Fatalf("second=%+v err=%v", second, err)
}
final, err := restarted.BackfillDocuments(context.Background(), 1)
if err != nil || !final.Completed {
t.Fatalf("final=%+v err=%v", final, err)
}
for _, hash := range []string{hashA, hashB} {
if _, err := os.Stat(filepath.Join(st.DocumentRoot, "sha256", hash[:2], hash)); err != nil {
t.Fatal(err)
}
}
// Renaming the revisions table proves the completed path does not even
// query it. This is synthetic data in a temporary DB, never production.
if _, err := st.DB.Exec(`ALTER TABLE revisions RENAME TO hidden_revisions`); err != nil {
t.Fatal(err)
}
again, err := restarted.BackfillDocuments(context.Background(), 128)
if err != nil || again != final {
t.Fatalf("completed progress=%+v err=%v", again, err)
}
}
func TestDocumentBackfillRootReplacementAndFailedBatch(t *testing.T) {
st := newTopologyStore(t)
hash := backfillFixture(t, st, "a", "original")
st.DocumentRoot = filepath.Join(t.TempDir(), "documents")
first, err := st.BackfillDocuments(context.Background(), 128)
if err != nil || !first.Completed {
t.Fatal(err)
}
// Keep exactly the same pathname, but replace its storage. Path-based
// completion markers would incorrectly skip this empty replacement.
if err := os.Rename(st.DocumentRoot, st.DocumentRoot+"-old"); err != nil {
t.Fatal(err)
}
if _, err := st.DB.Exec(`UPDATE revisions SET body='wrong' WHERE uuid='a'`); err != nil {
t.Fatal(err)
}
failed, err := st.BackfillDocuments(context.Background(), 128)
if !errors.Is(err, store.ErrDocumentHashMismatch) || failed.Processed != 0 || failed.Completed || failed.StoreID == first.StoreID {
t.Fatalf("failed=%+v err=%v", failed, err)
}
if _, err := st.DB.Exec(`UPDATE revisions SET body='original' WHERE uuid='a'`); err != nil {
t.Fatal(err)
}
retried, err := st.BackfillDocuments(context.Background(), 128)
if err != nil || !retried.Completed || retried.Processed != 1 {
t.Fatalf("retry=%+v err=%v", retried, err)
}
if _, err := os.Stat(filepath.Join(st.DocumentRoot, "sha256", hash[:2], hash)); err != nil {
t.Fatal(err)
}
}
func TestDocumentBackfillPrefersCanonicalAndNeverRepairsCorruptionImplicitly(t *testing.T) {
st := newTopologyStore(t)
hash := backfillFixture(t, st, "a", "canonical")
st.DocumentRoot = t.TempDir()
dir := filepath.Join(st.DocumentRoot, "sha256", hash[:2])
if err := os.MkdirAll(dir, 0755); err != nil {
t.Fatal(err)
}
filename := filepath.Join(dir, hash)
if err := os.WriteFile(filename, []byte("canonical"), 0644); err != nil {
t.Fatal(err)
}
if _, err := st.DB.Exec(`UPDATE revisions SET body='' WHERE uuid='a'`); err != nil {
t.Fatal(err)
}
progress, err := st.BackfillDocuments(context.Background(), 128)
if err != nil || !progress.Completed {
t.Fatalf("canonical rejected: %v", err)
}
if _, err := st.DB.Exec(`DELETE FROM document_backfills`); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte("broken"), 0644); err != nil {
t.Fatal(err)
}
if _, err := st.BackfillDocuments(context.Background(), 128); !errors.Is(err, store.ErrDocumentHashMismatch) {
t.Fatalf("corrupt canonical: %v", err)
}
data, _ := os.ReadFile(filename)
if string(data) != "broken" {
t.Fatal("corruption overwritten")
}
}
func TestDocumentBackfillCancellationAndReadOnlyDoNotCreateFiles(t *testing.T) {
st := newTopologyStore(t)
st.DocumentRoot = filepath.Join(t.TempDir(), "new-root")
ctx, cancel := context.WithCancel(context.Background())
cancel()
if err := st.EnsureDocumentStoreContext(ctx); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
st.ReadOnly = true
if err := st.EnsureDocumentStore(); !errors.Is(err, store.ErrReadOnly) {
t.Fatal(err)
}
if _, err := os.Stat(st.DocumentRoot); !os.IsNotExist(err) {
t.Fatal("cancelled/standby operation created root")
}
}
func TestDocumentBackfillSerializesAndLockWaitHonorsCancellation(t *testing.T) {
st := newTopologyStore(t)
backfillFixture(t, st, "a", "first")
backfillFixture(t, st, "b", "second")
st.DocumentRoot = t.TempDir()
lock, err := os.OpenFile(filepath.Join(st.DocumentRoot, ".document-backfill.lock"), os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
t.Fatal(err)
}
defer lock.Close()
if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
if _, err := st.BackfillDocuments(ctx, 1); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("lock cancellation: %v", err)
}
if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_UN); err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
errs := make(chan error, 2)
for i := 0; i < 2; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for {
progress, err := st.BackfillDocuments(context.Background(), 1)
if err != nil || progress.Completed {
errs <- err
return
}
}
}()
}
wg.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
progress, err := st.BackfillDocuments(context.Background(), 1)
if err != nil || !progress.Completed || progress.Processed != 2 {
t.Fatalf("double-counted migration: %+v %v", progress, err)
}
}
+33
View File
@@ -0,0 +1,33 @@
package store
import (
"context"
"time"
"pageswiki/internal/geduldh"
)
// MaterializeDocument supplies a verified node-local, read-only scratch copy
// for tools needing a pathname. Callers must authorize the reference first and
// close the lease after all subprocesses finish. DB-only legacy stores work too.
// TMPDIR is the operator-controlled local scratch root, not DocumentRoot.
func (s *Store) MaterializeDocument(ctx context.Context, hash string, maxBytes int64) (*geduldh.Lease, error) {
s.documentMaterializerOnce.Do(func() {
// These fixed safe defaults precede configurable deployment profiles.
s.documentMaterializer, _ = geduldh.NewMaterializer(documentSource{s}, geduldh.MaterializeOptions{
MaxBytes: 2 << 30, Concurrent: 2, Timeout: 2 * time.Minute,
})
})
return s.documentMaterializer.Acquire(ctx, hash, maxBytes)
}
type documentSource struct{ store *Store }
func (source documentSource) Stat(ctx context.Context, hash string) (geduldh.Info, error) {
size, err := source.store.DocumentSizeContext(ctx, hash)
return geduldh.Info{Hash: hash, Size: size}, err
}
func (source documentSource) Open(ctx context.Context, hash string) (geduldh.Reader, error) {
return source.store.OpenDocumentContext(ctx, hash)
}
@@ -0,0 +1,74 @@
package store_test
import (
"context"
"errors"
"io"
"os"
"testing"
)
func TestDocumentMaterializeContextAndDBByteSize(t *testing.T) {
for _, disk := range []bool{false, true} {
name := "database"
if disk {
name = "local"
}
t.Run(name, func(t *testing.T) {
st := newTopologyStore(t)
if disk {
st.DocumentRoot = t.TempDir()
if err := st.EnsureDocumentStore(); err != nil {
t.Fatal(err)
}
}
object, err := st.CreatePageAtPathInDomain(mustDefaultDomainUUID(t, st), "/materialize.txt")
if err != nil {
t.Fatal(err)
}
body := []byte("日本語\x00\xff\n")
revision, err := st.SaveRevisionContent(object, body, "test", "und", "application/octet-stream", "materialize.txt")
if err != nil {
t.Fatal(err)
}
size, err := st.DocumentSizeContext(context.Background(), revision.DocumentHash)
if err != nil || size != int64(len(body)) {
t.Fatalf("size=%d err=%v", size, err)
}
lease, err := st.MaterializeDocument(context.Background(), revision.DocumentHash, 1024)
if err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(lease.Path)
if err != nil || string(data) != string(body) {
t.Fatalf("data=%q err=%v", data, err)
}
lease.Close()
if _, err := os.Stat(lease.Path); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("cleanup: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
reader, err := st.OpenDocumentContext(ctx, revision.DocumentHash)
if err != nil {
t.Fatal(err)
}
defer reader.Close()
cancel()
if _, err := reader.Read(make([]byte, 1)); !errors.Is(err, context.Canceled) {
t.Fatalf("read: %v", err)
}
if _, err := reader.ReadAt(make([]byte, 1), 0); !errors.Is(err, context.Canceled) {
t.Fatalf("readat: %v", err)
}
if _, err := reader.Seek(0, io.SeekStart); !errors.Is(err, context.Canceled) {
t.Fatalf("seek: %v", err)
}
if _, err := st.ReadDocumentWindowContext(ctx, revision.DocumentHash, 0, 10); !errors.Is(err, context.Canceled) {
t.Fatalf("window: %v", err)
}
if _, err := st.DocumentSizeContext(ctx, revision.DocumentHash); !errors.Is(err, context.Canceled) {
t.Fatalf("stat: %v", err)
}
})
}
}
+79 -171
View File
@@ -2,28 +2,50 @@ package store
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"pageswiki/internal/geduldh"
)
type memoryDocumentReader struct {
*bytes.Reader
reader *bytes.Reader
ctx context.Context
}
func (memoryDocumentReader) Close() error { return nil }
func (r memoryDocumentReader) Read(p []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.reader.Read(p)
}
func (r memoryDocumentReader) ReadAt(p []byte, offset int64) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.reader.ReadAt(p, offset)
}
func (r memoryDocumentReader) Seek(offset int64, whence int) (int64, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.reader.Seek(offset, whence)
}
var (
ErrInvalidDocumentHash = errors.New("invalid Document SHA-256")
ErrDocumentHashMismatch = errors.New(
"Document payload does not match SHA-256",
)
ErrInvalidDocumentHash = geduldh.ErrInvalidHash
ErrDocumentHashMismatch = geduldh.ErrHashMismatch
)
func MigrateLegacyDocumentRoot(
@@ -131,18 +153,38 @@ func (s *Store) DocumentPath(hash string) (string, error) {
// OpenDocument opens an immutable content-addressed Document. Callers must
// perform authorization through a referencing object before invoking it.
func (s *Store) OpenDocument(hash string) (io.ReadSeekCloser, error) {
path, err := s.documentPath(hash)
if err != nil {
return s.OpenDocumentContext(context.Background(), hash)
}
// OpenDocumentContext supports bounded ReaderAt access without hydrating a
// revision body. A request/job context must be used by new storage consumers.
func (s *Store) OpenDocumentContext(ctx context.Context, hash string) (geduldh.Reader, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if path != "" {
return os.Open(path)
if _, err := geduldh.Key(hash); err != nil {
return nil, err
}
if s.DocumentRoot != "" {
backend, err := s.documentStorage()
if err != nil {
return nil, err
}
return backend.Open(ctx, hash)
}
var body string
if err := s.DB.QueryRow(`SELECT COALESCE(body,'') FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, strings.ToLower(strings.TrimSpace(hash))).Scan(&body); err != nil {
if err := s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, strings.ToLower(strings.TrimSpace(hash))).Scan(&body); err != nil {
return nil, err
}
return memoryDocumentReader{bytes.NewReader([]byte(body))}, nil
return memoryDocumentReader{reader: bytes.NewReader([]byte(body)), ctx: ctx}, nil
}
// documentStorage is the payload-only backend boundary. DocumentRoot still
// anchors legacy calendars and instance paths; it must not become an S3 URL.
// Remote selection is intentionally not exposed until canonical state, CLI,
// deployment profiles and backend contract tests have been migrated.
func (s *Store) documentStorage() (geduldh.Store, error) {
return geduldh.NewLocal(s.DocumentRoot)
}
type DocumentReference struct {
@@ -205,191 +247,57 @@ func (s *Store) DocumentReferencesInDomain(domainUUID, hash string) ([]DocumentR
return references, nil
}
func (s *Store) persistDocument(
expectedHash string,
body string,
) error {
func (s *Store) persistDocument(expectedHash, body string) error {
if s.ReadOnly {
return ErrReadOnly
}
if s.DocumentRoot == "" {
return nil
}
actualHash :=
revisionHash(
body,
)
if !strings.EqualFold(expectedHash, actualHash) {
return fmt.Errorf(
"%w: expected %s, got %s",
ErrDocumentHashMismatch,
expectedHash,
actualHash,
)
}
path, err :=
s.documentPath(
actualHash,
)
backend, err := s.documentStorage()
if err != nil {
return err
}
if existing, err := os.ReadFile(path); err == nil {
if revisionHash(string(existing)) != actualHash {
return fmt.Errorf(
"%w: corrupt existing file %s",
ErrDocumentHashMismatch,
path,
)
}
return nil
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
dir :=
filepath.Dir(
path,
)
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
temp, err :=
os.CreateTemp(
dir,
".document-*",
)
if err != nil {
return err
}
tempPath :=
temp.Name()
defer os.Remove(tempPath)
if err := temp.Chmod(0644); err != nil {
temp.Close()
return err
}
if _, err := temp.WriteString(body); err != nil {
temp.Close()
return err
}
if err := temp.Sync(); err != nil {
temp.Close()
return err
}
if err := temp.Close(); err != nil {
return err
}
if err := os.Rename(tempPath, path); err != nil {
return err
}
return nil
return backend.PutIfAbsent(context.Background(), expectedHash, int64(len(body)), strings.NewReader(body))
}
func (s *Store) readDocument(
hash string,
) (string, error) {
func (s *Store) readDocument(hash string) (string, error) {
if s.DocumentRoot == "" {
return "", os.ErrNotExist
}
path, err :=
s.documentPath(
hash,
)
backend, err := s.documentStorage()
if err != nil {
return "", err
}
payload, err :=
os.ReadFile(
path,
)
reader, err := backend.Open(context.Background(), hash)
if err != nil {
return "", err
}
body :=
string(
payload,
)
if revisionHash(body) != strings.ToLower(hash) {
return "", fmt.Errorf(
"%w: corrupt file %s",
ErrDocumentHashMismatch,
path,
)
defer reader.Close()
payload, err := io.ReadAll(reader)
if err != nil {
return "", err
}
return body, nil
if revisionHash(string(payload)) != strings.ToLower(strings.TrimSpace(hash)) {
return "", ErrDocumentHashMismatch
}
return string(payload), nil
}
func (s *Store) EnsureDocumentStore() error {
if s.DocumentRoot == "" {
return nil
}
return s.EnsureDocumentStoreContext(context.Background())
}
if err := os.MkdirAll(
filepath.Join(
s.DocumentRoot,
"sha256",
),
0755,
); err != nil {
return err
}
rows, err := s.DB.Query(`
SELECT
document_hash,
COALESCE(body, '')
FROM revisions
WHERE document_hash IS NOT NULL
AND document_hash != ''
`)
if err != nil {
return err
}
for rows.Next() {
var hash string
var body string
if err := rows.Scan(&hash, &body); err != nil {
rows.Close()
func (s *Store) EnsureDocumentStoreContext(ctx context.Context) error {
for {
progress, err := s.BackfillDocuments(ctx, 128)
if err != nil {
return err
}
if err := s.persistDocument(hash, body); err != nil {
rows.Close()
return err
if progress.Completed {
return nil
}
}
if err := rows.Err(); err != nil {
rows.Close()
return err
}
return rows.Close()
}
func (s *Store) hydrateRevision(
+27 -13
View File
@@ -1,9 +1,9 @@
package store
import (
"context"
"errors"
"io"
"os"
)
const MaxDocumentWindowBytes int64 = 256 << 10
@@ -17,6 +17,13 @@ type DocumentWindow struct {
}
func (s *Store) ReadDocumentWindow(documentHash string, offset, limit int64) (DocumentWindow, error) {
return s.ReadDocumentWindowContext(context.Background(), documentHash, offset, limit)
}
func (s *Store) ReadDocumentWindowContext(ctx context.Context, documentHash string, offset, limit int64) (DocumentWindow, error) {
if err := ctx.Err(); err != nil {
return DocumentWindow{}, err
}
if offset < 0 {
return DocumentWindow{}, errors.New("negative Document offset")
}
@@ -27,32 +34,32 @@ func (s *Store) ReadDocumentWindow(documentHash string, offset, limit int64) (Do
limit = MaxDocumentWindowBytes
}
if s.DocumentRoot != "" {
path, err := s.DocumentPath(documentHash)
backend, err := s.documentStorage()
if err != nil {
return DocumentWindow{}, err
}
file, err := os.Open(path)
file, err := backend.Open(ctx, documentHash)
if err != nil {
return DocumentWindow{}, err
}
defer file.Close()
info, err := file.Stat()
size, err := file.Seek(0, io.SeekEnd)
if err != nil {
return DocumentWindow{}, err
}
if offset > info.Size() {
offset = info.Size()
if offset > size {
offset = size
}
data := make([]byte, minInt64(limit, info.Size()-offset))
data := make([]byte, minInt64(limit, size-offset))
n, err := file.ReadAt(data, offset)
if err != nil && !errors.Is(err, io.EOF) {
return DocumentWindow{}, err
}
data = data[:n]
return DocumentWindow{Data: data, Offset: offset, NextOffset: offset + int64(n), TotalBytes: info.Size(), Truncated: offset+int64(n) < info.Size()}, nil
return DocumentWindow{Data: data, Offset: offset, NextOffset: offset + int64(n), TotalBytes: size, Truncated: offset+int64(n) < size}, nil
}
var body string
if err := s.DB.QueryRow(`SELECT COALESCE(body,'') FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, documentHash).Scan(&body); err != nil {
if err := s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, documentHash).Scan(&body); err != nil {
return DocumentWindow{}, err
}
total := int64(len(body))
@@ -79,18 +86,25 @@ func (s *Store) GetRevisionMetadata(revisionUUID string) (*Revision, error) {
}
func (s *Store) DocumentSize(documentHash string) (int64, error) {
return s.DocumentSizeContext(context.Background(), documentHash)
}
func (s *Store) DocumentSizeContext(ctx context.Context, documentHash string) (int64, error) {
if err := ctx.Err(); err != nil {
return 0, err
}
if s.DocumentRoot != "" {
path, err := s.DocumentPath(documentHash)
backend, err := s.documentStorage()
if err != nil {
return 0, err
}
info, err := os.Stat(path)
info, err := backend.Stat(ctx, documentHash)
if err != nil {
return 0, err
}
return info.Size(), nil
return info.Size, nil
}
var size int64
err := s.DB.QueryRow(`SELECT length(COALESCE(body,'')) FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, documentHash).Scan(&size)
err := s.DB.QueryRowContext(ctx, `SELECT length(CAST(COALESCE(body,'') AS BLOB)) FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, documentHash).Scan(&size)
return size, err
}
+104
View File
@@ -0,0 +1,104 @@
package store
import (
"context"
"database/sql"
"errors"
"fmt"
"testing"
"time"
"pageswiki/internal/dregarnuhr"
)
func TestDregarnuhrPersistedLifecycle(t *testing.T) {
for _, terminal := range []string{"succeeded", "failed", "skipped", "cancelled"} {
t.Run(terminal, func(t *testing.T) {
s := newSchemaTestStore(t)
if err := s.EnsureEventJobsSchema(); err != nil {
t.Fatal(err)
}
settings, err := s.GetQueueSettings()
if err != nil {
t.Fatal(err)
}
settings.RetryBaseSeconds = 30
if err := s.SetQueueSettings(settings); err != nil {
t.Fatal(err)
}
at := time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)
if _, err := s.EnqueueEventWithOptions("", "test.lifecycle", terminal, "{}", at, 7, 3); err != nil {
t.Fatal(err)
}
ctx := context.Background()
if _, err := s.ClaimDueEventAt(ctx, at.Add(-time.Millisecond)); !errors.Is(err, sql.ErrNoRows) {
t.Fatalf("early claim: %v", err)
}
for attempt := 1; attempt <= 3; attempt++ {
job, err := s.ClaimDueEventAt(ctx, at)
if err != nil {
t.Fatal(err)
}
if job.Attempts != attempt || job.Status != "running" || job.StartedAtMS != at.UnixMilli() {
t.Fatalf("claim: %+v", job)
}
finished := at.Add(time.Second)
runErr := errors.New("synthetic failure")
if attempt == 3 {
switch terminal {
case "succeeded":
runErr = nil
case "skipped":
runErr = fmt.Errorf("inapplicable: %w", ErrEventSkipped)
case "cancelled":
if err := s.CancelEventJob(job.UUID); err != nil {
t.Fatal(err)
}
}
}
if err := s.FinishEventAt(ctx, job.UUID, runErr, finished); err != nil {
t.Fatal(err)
}
persisted, err := scanEventJob(s.DB.QueryRow(`SELECT `+eventJobColumns+` FROM event_jobs WHERE uuid=?`, job.UUID))
if err != nil {
t.Fatal(err)
}
if persisted.UpdatedAt != finished.UnixMilli() {
t.Fatalf("completion time: %+v", persisted)
}
if attempt < 3 {
at = finished.Add(time.Duration(30*(1<<(attempt-1))) * time.Second)
if persisted.Status != "queued" || persisted.FinishedAtMS != 0 || persisted.RunAtMS != at.UnixMilli() {
t.Fatalf("retry: %+v", persisted)
}
if _, err := s.ClaimDueEventAt(ctx, at.Add(-time.Millisecond)); !errors.Is(err, sql.ErrNoRows) {
t.Fatalf("early retry: %v", err)
}
} else if persisted.Status != terminal || persisted.FinishedAtMS != finished.UnixMilli() {
t.Fatalf("terminal: %+v", persisted)
}
}
if _, err := s.ClaimDueEventAt(ctx, at.Add(48*time.Hour)); !errors.Is(err, sql.ErrNoRows) {
t.Fatalf("terminal claimed again: %v", err)
}
})
}
if ErrEventSkipped != dregarnuhr.ErrSkipped {
t.Fatal("legacy skip identity changed")
}
}
func TestDregarnuhrCancellationErrorsPropagate(t *testing.T) {
s := newSchemaTestStore(t)
if err := s.EnsureEventJobsSchema(); err != nil {
t.Fatal(err)
}
if _, err := s.EventCancellationState(context.Background(), "missing"); !errors.Is(err, sql.ErrNoRows) {
t.Fatalf("missing job: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := s.ClaimDueEventAt(ctx, time.Now()); !errors.Is(err, context.Canceled) {
t.Fatalf("cancelled claim: %v", err)
}
}
+84
View File
@@ -0,0 +1,84 @@
package store_test
import (
"errors"
"pageswiki/internal/db"
"pageswiki/internal/store"
"sync"
"testing"
)
func TestEditorRevisionCAS(t *testing.T) {
file := t.TempDir() + "/editor.db"
database, err := db.Open(file)
if err != nil {
t.Fatal(err)
}
defer database.Close()
if err = db.Migrate(database); err != nil {
t.Fatal(err)
}
root := t.TempDir() + "/documents"
s := store.NewWithDocumentRoot(database, root)
object, err := s.CreatePageAtPath("/editor")
if err != nil {
t.Fatal(err)
}
base, err := s.SaveRevisionContentIfHead(object, []byte("base"), "", "und", "text/html", "", "")
if err != nil {
t.Fatal(err)
}
if _, err = s.SaveRevisionContentIfHead(object, []byte("stale"), "", "und", "text/html", "", ""); !errors.Is(err, store.ErrRevisionConflict) {
t.Fatalf("stale empty head: %v", err)
}
en, err := s.GetRevisionForExactLanguage(object, "en")
if err != nil || en.UUID != "" || en.Body != "" {
t.Fatalf("language fallback: %+v %v", en, err)
}
if _, err = s.SaveRevisionContentIfHead(object, []byte("english"), "", "en", "text/html", "", ""); err != nil {
t.Fatal(err)
}
// Independent Store and connection: mutex alone cannot enforce this CAS.
database2, err := db.Open(file)
if err != nil {
t.Fatal(err)
}
defer database2.Close()
s2 := store.NewWithDocumentRoot(database2, root)
start := make(chan struct{})
results := make(chan error, 2)
var wg sync.WaitGroup
for _, st := range []*store.Store{s, s2} {
wg.Add(1)
go func(st *store.Store) {
defer wg.Done()
<-start
_, e := st.SaveRevisionContentIfHead(object, []byte("next"), "", "und", "text/html", "", base.UUID)
results <- e
}(st)
}
close(start)
wg.Wait()
close(results)
won, lost := 0, 0
for err := range results {
if err == nil {
won++
} else if errors.Is(err, store.ErrRevisionConflict) {
lost++
} else {
t.Fatal(err)
}
}
if won != 1 || lost != 1 {
t.Fatalf("won=%d lost=%d", won, lost)
}
old, err := s.GetRevision(base.UUID)
if err != nil || old.Body != "base" {
t.Fatal("immutable base overwritten", err)
}
head, err := s.GetRevisionMetadataForExactLanguage(object, "und")
if err != nil || head.UUID == base.UUID || head.Body != "" {
t.Fatalf("head metadata: %+v %v", head, err)
}
}
+124
View File
@@ -0,0 +1,124 @@
package store
import (
"database/sql"
"errors"
"fmt"
"path/filepath"
"sync"
"testing"
"time"
"pageswiki/internal/db"
)
func TestClaimDueEventPriorityAndReturnedState(t *testing.T) {
s := newSchemaTestStore(t)
if err := s.EnsureEventJobsSchema(); err != nil {
t.Fatal(err)
}
for _, item := range []struct {
key string
priority int
due time.Time
}{
{"low", 1, time.Now().Add(-time.Hour)}, {"high", 9, time.Now().Add(-time.Minute)}, {"future", 99, time.Now().Add(time.Hour)},
} {
if _, err := s.EnqueueEventWithOptions("", "test", item.key, "opaque", item.due, item.priority, 3); err != nil {
t.Fatal(err)
}
}
for _, key := range []string{"high", "low"} {
job, err := s.ClaimDueEvent()
if err != nil {
t.Fatal(err)
}
if job.Key != key || job.Status != "running" || job.Attempts != 1 || job.StartedAtMS == 0 || job.UpdatedAt != job.StartedAtMS {
t.Fatalf("wrong claimed state: %+v", job)
}
}
if _, err := s.ClaimDueEvent(); !errors.Is(err, sql.ErrNoRows) {
t.Fatalf("future/running claimed: %v", err)
}
s.ReadOnly = true
if _, err := s.ClaimDueEvent(); !errors.Is(err, ErrReadOnly) {
t.Fatalf("standby claim: %v", err)
}
}
func TestClaimDueEventConcurrentWriters(t *testing.T) {
path := filepath.Join(t.TempDir(), "queue.db")
database, err := db.Open(path)
if err != nil {
t.Fatal(err)
}
defer database.Close()
s := &Store{DB: database}
if err := s.EnsureEventJobsSchema(); err != nil {
t.Fatal(err)
}
const total = 64
for n := 0; n < total; n++ {
if _, err := s.EnqueueEvent("test", fmt.Sprint(n), "opaque", time.Now().Add(-time.Second)); err != nil {
t.Fatal(err)
}
}
var wg sync.WaitGroup
jobs := make(chan string, total)
errs := make(chan error, 9)
// Independent DB handles exercise file-level contention, not an in-process
// mutex or a single sql.DB connection serializing the test for us.
for range 8 {
other, err := db.Open(path)
if err != nil {
t.Fatal(err)
}
wg.Add(1)
go func() {
defer wg.Done()
defer other.Close()
worker := &Store{DB: other}
for {
job, err := worker.ClaimDueEvent()
if errors.Is(err, sql.ErrNoRows) {
return
}
if err != nil {
errs <- err
return
}
jobs <- job.UUID
}
}()
}
wg.Add(1)
go func() {
defer wg.Done()
for n := 0; n < 100; n++ {
if _, err := database.Exec(`UPDATE event_queue_settings SET max_depth=max_depth+1 WHERE id=1`); err != nil {
errs <- err
return
}
}
}()
wg.Wait()
close(jobs)
close(errs)
for err := range errs {
t.Error(err)
}
seen := map[string]bool{}
for id := range jobs {
if seen[id] {
t.Error("duplicate claim", id)
}
seen[id] = true
}
if len(seen) != total {
t.Fatalf("claimed=%d want=%d", len(seen), total)
}
var once int
if err := database.QueryRow(`SELECT count(*) FROM event_jobs WHERE status='running' AND attempts=1 AND started_at_ms=updated_at_ms`).Scan(&once); err != nil || once != total {
t.Fatalf("persisted claims=%d err=%v", once, err)
}
}
+69 -181
View File
@@ -9,26 +9,11 @@ import (
"time"
"github.com/google/uuid"
"pageswiki/internal/dregarnuhr"
)
type EventJob struct {
UUID string `json:"uuid"`
EventUUID string `json:"event_uuid,omitempty"`
Type string `json:"event_type"`
Key string `json:"event_key"`
Payload string `json:"payload,omitempty"`
Status string `json:"status"`
Priority int `json:"priority"`
Attempts int `json:"attempts"`
MaxAttempts int `json:"max_attempts"`
RunAtMS int64 `json:"run_at_ms"`
StartedAtMS int64 `json:"started_at_ms,omitempty"`
FinishedAtMS int64 `json:"finished_at_ms,omitempty"`
CancelRequested bool `json:"cancel_requested"`
Error string `json:"error,omitempty"`
CreatedAt int64 `json:"created_at_ms"`
UpdatedAt int64 `json:"updated_at_ms"`
}
// EventJob retains the existing public JSON shape and source compatibility.
type EventJob = dregarnuhr.Job
type EventDefinition struct {
UUID string `json:"uuid"`
@@ -74,13 +59,7 @@ type EventLogTrend struct {
SampleMessage string `json:"sample_message,omitempty"`
}
type QueueSettings struct {
Paused bool `json:"paused"`
Concurrency int `json:"concurrency"`
MaxDepth int `json:"max_depth"`
RetryBaseSeconds int `json:"retry_base_seconds"`
RetentionHours int `json:"retention_hours"`
}
type QueueSettings = dregarnuhr.Settings
type EventJobStats struct {
Queued int `json:"queued"`
@@ -91,97 +70,18 @@ type EventJobStats struct {
Cancelled int `json:"cancelled"`
}
var ErrEventSkipped = errors.New("event is not applicable")
var ErrEventSkipped = dregarnuhr.ErrSkipped
func (s *Store) EnsureEventJobsSchema() error {
if _, err := s.DB.Exec(`
CREATE TABLE IF NOT EXISTS event_jobs (
uuid TEXT PRIMARY KEY,
event_type TEXT NOT NULL,
event_key TEXT NOT NULL,
payload TEXT NOT NULL,
status TEXT NOT NULL,
attempts INTEGER NOT NULL DEFAULT 0,
run_at_ms INTEGER NOT NULL,
error TEXT NOT NULL DEFAULT '',
created_at_ms INTEGER NOT NULL,
updated_at_ms INTEGER NOT NULL
)
`); err != nil {
if err := s.ensureSchemaComponent("event-jobs"); err != nil {
return err
}
for _, statement := range []string{
`ALTER TABLE event_jobs ADD COLUMN event_uuid TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE event_jobs ADD COLUMN priority INTEGER NOT NULL DEFAULT 0`,
`ALTER TABLE event_jobs ADD COLUMN max_attempts INTEGER NOT NULL DEFAULT 3`,
`ALTER TABLE event_jobs ADD COLUMN started_at_ms INTEGER NOT NULL DEFAULT 0`,
`ALTER TABLE event_jobs ADD COLUMN finished_at_ms INTEGER NOT NULL DEFAULT 0`,
`ALTER TABLE event_jobs ADD COLUMN cancel_requested INTEGER NOT NULL DEFAULT 0`,
} {
_, _ = s.DB.Exec(statement)
}
statements := []string{
`CREATE INDEX IF NOT EXISTS idx_event_jobs_listing ON event_jobs (created_at_ms DESC, uuid DESC)`,
`CREATE INDEX IF NOT EXISTS idx_event_jobs_status_listing ON event_jobs (status, created_at_ms DESC, uuid DESC)`,
`CREATE INDEX IF NOT EXISTS idx_event_jobs_history ON event_jobs (created_at_ms DESC, uuid DESC) WHERE status IN ('succeeded','skipped','failed','cancelled')`,
`CREATE INDEX IF NOT EXISTS idx_event_jobs_due
ON event_jobs (status, priority DESC, run_at_ms, created_at_ms)`,
`CREATE TABLE IF NOT EXISTS event_definitions (
uuid TEXT PRIMARY KEY, name TEXT NOT NULL, event_type TEXT NOT NULL,
event_key TEXT NOT NULL, payload TEXT NOT NULL DEFAULT '',
schedule_kind TEXT NOT NULL DEFAULT 'manual',
interval_seconds INTEGER NOT NULL DEFAULT 0,
next_run_at_ms INTEGER NOT NULL DEFAULT 0,
enabled INTEGER NOT NULL DEFAULT 1, priority INTEGER NOT NULL DEFAULT 0,
max_attempts INTEGER NOT NULL DEFAULT 3,
created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL
)`,
`CREATE INDEX IF NOT EXISTS idx_event_definitions_due
ON event_definitions (enabled, next_run_at_ms)`,
`CREATE TABLE IF NOT EXISTS event_logs (
id INTEGER PRIMARY KEY AUTOINCREMENT, job_uuid TEXT NOT NULL,
event_uuid TEXT NOT NULL DEFAULT '', level TEXT NOT NULL,
message TEXT NOT NULL, created_at_ms INTEGER NOT NULL
)`,
`CREATE INDEX IF NOT EXISTS idx_event_logs_job
ON event_logs (job_uuid, created_at_ms DESC)`,
`CREATE INDEX IF NOT EXISTS idx_event_logs_job_id ON event_logs (job_uuid, id DESC)`,
`CREATE TABLE IF NOT EXISTS event_log_policies (
event_type TEXT PRIMARY KEY,
retention_hours INTEGER NOT NULL DEFAULT 168,
aggregate_after_hours INTEGER NOT NULL DEFAULT 24,
bucket_seconds INTEGER NOT NULL DEFAULT 60,
max_rows INTEGER NOT NULL DEFAULT 5000
)`,
`INSERT OR IGNORE INTO event_log_policies(event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows)
VALUES('*',168,24,60,5000)`,
`CREATE TABLE IF NOT EXISTS event_log_buckets (
event_type TEXT NOT NULL,
bucket_start_ms INTEGER NOT NULL,
level TEXT NOT NULL,
count INTEGER NOT NULL DEFAULT 0,
first_at_ms INTEGER NOT NULL,
last_at_ms INTEGER NOT NULL,
sample_message TEXT NOT NULL DEFAULT '',
PRIMARY KEY(event_type,bucket_start_ms,level)
)`,
`CREATE INDEX IF NOT EXISTS idx_event_log_buckets_time ON event_log_buckets(event_type,bucket_start_ms DESC)`,
`CREATE TABLE IF NOT EXISTS event_queue_settings (
id INTEGER PRIMARY KEY CHECK (id = 1), paused INTEGER NOT NULL DEFAULT 0,
concurrency INTEGER NOT NULL DEFAULT 1,
max_depth INTEGER NOT NULL DEFAULT 10000,
retry_base_seconds INTEGER NOT NULL DEFAULT 30,
retention_hours INTEGER NOT NULL DEFAULT 168
)`,
`INSERT OR IGNORE INTO event_queue_settings
(id, paused, concurrency, max_depth, retry_base_seconds, retention_hours)
VALUES (1, 0, 1, 10000, 30, 168)`,
}
for _, statement := range statements {
if _, err := s.DB.Exec(statement); err != nil {
return err
}
}
return s.recoverInterruptedEventJobs()
}
// Startup recovery mutates runtime state, not schema. Never call it from
// status/preflight, read-only standby or a migration rehearsal.
func (s *Store) recoverInterruptedEventJobs() error {
now := time.Now().UnixMilli()
if _, err := s.DB.Exec(`UPDATE event_jobs SET status='queued',
error=CASE WHEN error='' THEN 'execution interrupted by process restart' ELSE error END,
@@ -250,86 +150,67 @@ const eventJobColumns = `uuid, event_uuid, event_type, event_key, payload,
finished_at_ms, cancel_requested, error, created_at_ms, updated_at_ms`
func (s *Store) ClaimDueEvent() (*EventJob, error) {
tx, err := s.DB.Begin()
if err != nil {
return nil, err
return s.ClaimDueEventAt(context.Background(), time.Now())
}
// ClaimDueEventAt keeps selection and claim in one statement; injected time
// allows scheduler tests without real sleeps or production data.
func (s *Store) ClaimDueEventAt(ctx context.Context, at time.Time) (*EventJob, error) {
if s.ReadOnly {
return nil, ErrReadOnly
}
defer tx.Rollback()
job, err := scanEventJob(tx.QueryRow(`
SELECT `+eventJobColumns+` FROM event_jobs
WHERE status = 'queued' AND run_at_ms <= ?
ORDER BY priority DESC, run_at_ms, created_at_ms LIMIT 1`,
time.Now().UnixMilli()))
if err != nil {
return nil, err
}
now := time.Now().UnixMilli()
result, err := tx.Exec(`
now := at.UnixMilli()
// A deferred SELECT transaction cannot always upgrade its WAL snapshot to a
// writer (SQLITE_BUSY[_SNAPSHOT]), even with busy_timeout. Select and claim in
// one write statement instead. SQLite serializes concurrent claimers; RETURNING
// gives the committed attempt/status/timestamps, not the pre-claim snapshot.
return scanEventJob(s.DB.QueryRowContext(ctx, `
UPDATE event_jobs SET status='running', attempts=attempts+1,
started_at_ms=?, updated_at_ms=? WHERE uuid=? AND status='queued'`,
now, now, job.UUID)
if err != nil {
return nil, err
}
count, err := result.RowsAffected()
if err != nil || count != 1 {
return nil, sql.ErrNoRows
}
if err := tx.Commit(); err != nil {
return nil, err
}
job.Status, job.Attempts, job.StartedAtMS = "running", job.Attempts+1, now
return job, nil
started_at_ms=?, updated_at_ms=?
WHERE uuid=(SELECT uuid FROM event_jobs
WHERE status='queued' AND run_at_ms<=?
ORDER BY priority DESC,run_at_ms,created_at_ms,uuid LIMIT 1)
AND status='queued'
RETURNING `+eventJobColumns, now, now, now))
}
func (s *Store) FinishEvent(uuid string, runErr error) error {
job, err := scanEventJob(s.DB.QueryRow(`SELECT `+eventJobColumns+` FROM event_jobs WHERE uuid=?`, uuid))
return s.FinishEventAt(context.Background(), uuid, runErr, time.Now())
}
func (s *Store) FinishEventAt(ctx context.Context, uuid string, runErr error, at time.Time) error {
job, err := scanEventJob(s.DB.QueryRowContext(ctx, `SELECT `+eventJobColumns+` FROM event_jobs WHERE uuid=?`, uuid))
if err != nil {
return err
}
status, message := "succeeded", ""
if runErr != nil {
message = runErr.Error()
if errors.Is(runErr, ErrEventSkipped) {
status = "skipped"
} else if job.CancelRequested {
status = "cancelled"
} else if job.Attempts < job.MaxAttempts {
settings, settingsErr := s.GetQueueSettings()
if settingsErr != nil {
return settingsErr
}
delay := time.Duration(settings.RetryBaseSeconds) * time.Second
for i := 1; i < job.Attempts; i++ {
delay *= 2
if delay >= 24*time.Hour {
delay = 24 * time.Hour
break
}
}
_, err = s.DB.Exec(`UPDATE event_jobs SET status='queued', error=?,
run_at_ms=?, finished_at_ms=0, updated_at_ms=? WHERE uuid=?`,
message, time.Now().Add(delay).UnixMilli(), time.Now().UnixMilli(), uuid)
_ = s.AddEventLog(uuid, job.EventUUID, "warning", "execution failed; retry scheduled: "+message)
base := 1
if runErr != nil && !errors.Is(runErr, ErrEventSkipped) && !job.CancelRequested && job.Attempts < job.MaxAttempts {
settings, err := s.GetQueueSettings()
if err != nil {
return err
} else {
status = "failed"
}
base = settings.RetryBaseSeconds
}
now := time.Now().UnixMilli()
_, err = s.DB.Exec(`UPDATE event_jobs SET status=?, error=?,
outcome := dregarnuhr.Complete(*job, runErr, base, at)
if outcome.Status == "queued" {
_, err = s.DB.ExecContext(ctx, `UPDATE event_jobs SET status='queued', error=?,
run_at_ms=?, finished_at_ms=0, updated_at_ms=? WHERE uuid=?`,
outcome.Message, outcome.RetryAt.UnixMilli(), at.UnixMilli(), uuid)
if err == nil {
_ = s.AddEventLog(uuid, job.EventUUID, outcome.Level, "execution failed; retry scheduled: "+outcome.Message)
}
return err
}
_, err = s.DB.ExecContext(ctx, `UPDATE event_jobs SET status=?, error=?,
finished_at_ms=?, updated_at_ms=? WHERE uuid=?`,
status, message, now, now, uuid)
level := "info"
if status == "failed" {
level = "error"
outcome.Status, outcome.Message, at.UnixMilli(), at.UnixMilli(), uuid)
message := outcome.Status
if outcome.Message != "" {
message += ": " + outcome.Message
}
if err == nil {
_ = s.AddEventLog(uuid, job.EventUUID, outcome.Level, message)
}
_ = s.AddEventLog(uuid, job.EventUUID, level, status+func() string {
if message != "" {
return ": " + message
}
return ""
}())
return err
}
@@ -442,9 +323,16 @@ func (s *Store) RetryEventJob(jobUUID string) error {
}
func (s *Store) EventCancellationRequested(jobUUID string) bool {
requested, err := s.EventCancellationState(context.Background(), jobUUID)
return err == nil && requested
}
// EventCancellationState lets the runner distinguish a read failure from a
// confirmed absence of cancellation. The boolean compatibility API stays.
func (s *Store) EventCancellationState(ctx context.Context, jobUUID string) (bool, error) {
var requested int
err := s.DB.QueryRow(`SELECT cancel_requested FROM event_jobs WHERE uuid=?`, jobUUID).Scan(&requested)
return err == nil && requested != 0
err := s.DB.QueryRowContext(ctx, `SELECT cancel_requested FROM event_jobs WHERE uuid=?`, jobUUID).Scan(&requested)
return requested != 0, err
}
func (s *Store) PurgeEventHistory(retentionHours int) error {
+99
View File
@@ -0,0 +1,99 @@
package store_test
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"os"
"path/filepath"
"testing"
"pageswiki/internal/db"
"pageswiki/internal/geduldh"
"pageswiki/internal/store"
)
func TestGeduldhPublicationPrecedesRevisionCommit(t *testing.T) {
for _, mode := range []string{"missing-root", "corrupt-existing", "metadata-conflict"} {
t.Run(mode, func(t *testing.T) {
root := t.TempDir()
database, err := db.Open(filepath.Join(root, "pages.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
if err := db.Migrate(database); err != nil {
t.Fatal(err)
}
documents := filepath.Join(root, "documents")
st := store.NewWithDocumentRoot(database, documents)
page, err := st.CreatePageAtPath("/geduldh-contract")
if err != nil {
t.Fatal(err)
}
base, err := st.SaveRevisionContent(page, []byte("base"), "base", "en", "text/plain", "base.txt")
if err != nil {
t.Fatal(err)
}
payload := []byte("new immutable payload")
sum := sha256.Sum256(payload)
hash := hex.EncodeToString(sum[:])
filename := filepath.Join(documents, "sha256", hash[:2], hash)
switch mode {
case "missing-root":
st.DocumentRoot = filepath.Join(root, "unavailable")
case "corrupt-existing":
if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte("damaged"), 0644); err != nil {
t.Fatal(err)
}
}
expected := base.UUID
if mode == "metadata-conflict" {
expected = "not-the-current-head"
}
_, err = st.SaveRevisionContentIfHead(page, payload, "must not commit", "en", "text/plain", "new.txt", expected)
if err == nil {
t.Fatal("unsafe revision committed")
}
if mode == "corrupt-existing" && (!errors.Is(err, geduldh.ErrHashMismatch) || !errors.Is(err, store.ErrDocumentHashMismatch)) {
t.Fatalf("storage error identity changed: %v", err)
}
if mode == "metadata-conflict" && !errors.Is(err, store.ErrRevisionConflict) {
t.Fatalf("conflict changed: %v", err)
}
var count int
if err := database.QueryRow(`SELECT count(*) FROM revisions WHERE object_uuid=?`, page).Scan(&count); err != nil {
t.Fatal(err)
}
if count != 1 {
t.Fatalf("failed save added revision: count=%d", count)
}
st.DocumentRoot = documents
loaded, err := st.GetRevision(base.UUID)
if err != nil || loaded.Body != "base" {
t.Fatalf("previous revision lost: %v", err)
}
switch mode {
case "corrupt-existing":
data, err := os.ReadFile(filename)
if err != nil || string(data) != "damaged" {
t.Fatalf("corruption silently repaired: %v", err)
}
case "metadata-conflict":
// A failed metadata transaction may leave a valid unreferenced blob;
// the payload layer must not delete it without reference/retention proof.
backend, err := geduldh.NewLocal(documents)
if err != nil {
t.Fatal(err)
}
if err := backend.Verify(context.Background(), hash); err != nil {
t.Fatal(err)
}
}
})
}
}
+4 -14
View File
@@ -7,6 +7,8 @@ import (
"time"
"github.com/google/uuid"
"pageswiki/internal/schutzaria"
)
var groupNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{1,63}$`)
@@ -25,19 +27,8 @@ type GroupPermission struct {
UUID, GroupUUID, ScopePath, Permission string
}
var groupPermissions = map[string]map[string]bool{
"read": {"read": true},
"read_write_once": {"read": true, "create": true, "upload": true},
"read_write": {"read": true, "create": true, "write": true, "upload": true, "rename": true, "link": true},
"upload_only": {"upload": true},
}
func NormalizeGroupPermission(value string) (string, error) {
value = strings.ToLower(strings.TrimSpace(value))
if _, ok := groupPermissions[value]; !ok {
return "", errors.New("invalid group permission")
}
return value, nil
return schutzaria.NormalizeGroupPermission(value)
}
func normalizeGroupName(name string) (string, error) {
@@ -375,8 +366,7 @@ func groupPermissionAllowsFrom(q pathQueryer, domainUUID, principalUUID, capabil
if err := rows.Scan(&scope, &permission); err != nil {
return false, err
}
inside := targetPath == scope || scope == "/" || strings.HasPrefix(targetPath, strings.TrimRight(scope, "/")+"/")
if inside && groupPermissions[permission][capability] {
if schutzaria.GroupPermissionAllows(scope, permission, capability, targetPath) {
return true, nil
}
}
+1 -42
View File
@@ -1,7 +1,6 @@
package store
import (
"database/sql"
"errors"
"strings"
"time"
@@ -30,47 +29,7 @@ type ImportItem struct {
}
func (s *Store) EnsureImportSchema() error {
statements := []string{
`CREATE TABLE IF NOT EXISTS import_sessions(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,actor_uuid TEXT NOT NULL DEFAULT '',archive_name TEXT NOT NULL,archive_path TEXT NOT NULL,target_prefix TEXT NOT NULL DEFAULT '/',status TEXT NOT NULL DEFAULT 'queued',total_files INTEGER NOT NULL DEFAULT 0,markdown_files INTEGER NOT NULL DEFAULT 0,error TEXT NOT NULL DEFAULT '',created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`,
`CREATE TABLE IF NOT EXISTS import_items(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,source_path TEXT NOT NULL,target_path TEXT NOT NULL,size INTEGER NOT NULL DEFAULT 0,content_type TEXT NOT NULL DEFAULT '',title TEXT NOT NULL DEFAULT '',suggested_quartier TEXT NOT NULL DEFAULT '',suggested_street TEXT NOT NULL DEFAULT '',suggested_spot TEXT NOT NULL DEFAULT '',confidence REAL NOT NULL DEFAULT 0,selected INTEGER NOT NULL DEFAULT 1,PRIMARY KEY(session_uuid,sequence))`,
`CREATE TABLE IF NOT EXISTS import_journal(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,path TEXT NOT NULL,object_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,sequence))`,
`CREATE TABLE IF NOT EXISTS import_city_journal(session_uuid TEXT NOT NULL,kind TEXT NOT NULL,node_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,kind,node_uuid))`,
`CREATE TABLE IF NOT EXISTS import_city_edges(session_uuid TEXT NOT NULL,spot_uuid TEXT NOT NULL,page_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,spot_uuid,page_uuid))`,
`CREATE INDEX IF NOT EXISTS idx_import_sessions_domain ON import_sessions(domain_uuid,created_at_ms DESC)`,
}
for _, statement := range statements {
if _, err := s.DB.Exec(statement); err != nil {
return err
}
}
_ = ensureImportColumn(s.DB, "import_journal", "completed", "INTEGER NOT NULL DEFAULT 0")
return nil
}
func ensureImportColumn(db *sql.DB, table, column, declaration string) error {
rows, err := db.Query(`PRAGMA table_info(` + table + `)`)
if err != nil {
return err
}
found := false
for rows.Next() {
var cid, notnull, pk int
var name, kind string
var defaultValue any
if err := rows.Scan(&cid, &name, &kind, &notnull, &defaultValue, &pk); err != nil {
rows.Close()
return err
}
if name == column {
found = true
}
}
rows.Close()
if found {
return nil
}
_, err = db.Exec(`ALTER TABLE ` + table + ` ADD COLUMN ` + column + ` ` + declaration)
return err
return s.ensureSchemaComponent("imports")
}
func (s *Store) CreateImportSession(domainUUID, actor, archiveName, archivePath, targetPrefix string) (string, error) {
+2 -5
View File
@@ -2,7 +2,6 @@ package store
import (
"fmt"
"os"
"strings"
)
@@ -246,10 +245,8 @@ func (s *Store) ListSelectedChildEntriesInDomain(domainUUID, parent string, page
entry.Title = firstHTMLHeading(sourcePrefix)
}
if documentHash != "" && s.DocumentRoot != "" {
if path, pathErr := s.documentPath(documentHash); pathErr == nil {
if info, statErr := os.Stat(path); statErr == nil {
entry.Size = int(info.Size())
}
if size, err := s.DocumentSize(documentHash); err == nil {
entry.Size = int(size)
}
}
entries = append(entries, entry)
+16 -49
View File
@@ -7,6 +7,8 @@ import (
"errors"
"strings"
"time"
"pageswiki/internal/schutzaria"
)
var ErrLocalityAccessEnforcing = errors.New("city.access.enforcing_error")
@@ -41,52 +43,14 @@ func (s *Store) ensureLocalityAccessSchema() error {
return nil
}
type localityAccessLayer struct{ read, write, grant bool }
type localityAccessSnapshot struct {
admin bool
paths map[string][]localityAccessLayer
}
func (p *localityAccessSnapshot) decide(path, capability string, basis bool) bool {
if !strings.HasPrefix(path, "/") {
return false
}
if p == nil || p.admin {
return basis
}
grant := false
for at := path; ; {
for _, layer := range p.paths[at] {
allowed := layer.write
if capability == "read" {
allowed = layer.read
}
if !allowed {
return false
}
if layer.grant && (capability == "read" || capability == "write" || capability == "create" || capability == "upload") {
grant = true
}
}
if at == "/" {
break
}
i := strings.LastIndex(at, "/")
if i <= 0 {
at = "/"
} else {
at = at[:i]
}
}
return basis || grant
}
type localityAccessSnapshot = schutzaria.LocalityAccess
func (s *Store) CheckLocalityAccessCaps(domain, principal, capability, path string) error {
p, err := loadLocalityAccess(s.DB, domain, principal)
if err != nil {
return err
}
if !p.decide(path, capability, true) {
if !p.Decide(path, capability, true) {
return ErrAccessDenied
}
return nil
@@ -112,7 +76,7 @@ func (s *Store) CheckObjectLocalityAccessCaps(domain, principal, capability, obj
if err != nil {
return err
}
if p.admin || len(p.paths) == 0 {
if !p.RequiresPathChecks() {
return nil
}
rows, err := s.DB.Query(`SELECT path FROM paths WHERE domain_uuid=? AND object_uuid=? LIMIT 10001`, domain, object)
@@ -130,7 +94,7 @@ func (s *Store) CheckObjectLocalityAccessCaps(domain, principal, capability, obj
if err = rows.Scan(&path); err != nil {
return err
}
if !p.decide(path, capability, true) {
if !p.Decide(path, capability, true) {
return ErrAccessDenied
}
}
@@ -176,7 +140,8 @@ func installLocalityAccessGuards(ctx context.Context, tx *sql.Tx) error {
}
func loadLocalityAccess(q pathQueryer, domain, principal string) (*localityAccessSnapshot, error) {
p := &localityAccessSnapshot{paths: map[string][]localityAccessLayer{}}
var layers []schutzaria.AccessLayer
var admin bool
rows, err := q.Query(`SELECT a.scope_path,a.grants_access,
EXISTS(SELECT 1 FROM locality_access_entries e JOIN domain_memberships m ON m.domain_uuid=e.domain_uuid AND m.principal_uuid=? AND m.state='active' JOIN principals u ON u.uuid=m.principal_uuid AND u.state IN ('active','readonly') WHERE e.domain_uuid=a.domain_uuid AND e.node_kind=a.node_kind AND e.node_uuid=a.node_uuid AND ((e.subject_kind='user' AND e.subject_uuid=m.uuid) OR (e.subject_kind='group' AND EXISTS(SELECT 1 FROM groups g JOIN group_members gm ON gm.group_uuid=g.uuid AND gm.membership_uuid=m.uuid WHERE g.uuid=e.subject_uuid AND g.domain_uuid=m.domain_uuid AND g.state='active')))),
EXISTS(SELECT 1 FROM locality_access_entries e JOIN domain_memberships m ON m.domain_uuid=e.domain_uuid AND m.principal_uuid=? AND m.state='active' JOIN principals u ON u.uuid=m.principal_uuid AND u.state='active' WHERE e.domain_uuid=a.domain_uuid AND e.node_kind=a.node_kind AND e.node_uuid=a.node_uuid AND e.permission='read_write' AND ((e.subject_kind='user' AND e.subject_uuid=m.uuid) OR (e.subject_kind='group' AND EXISTS(SELECT 1 FROM groups g JOIN group_members gm ON gm.group_uuid=g.uuid AND gm.membership_uuid=m.uuid WHERE g.uuid=e.subject_uuid AND g.domain_uuid=m.domain_uuid AND g.state='active')))),
@@ -193,15 +158,17 @@ func loadLocalityAccess(q pathQueryer, domain, principal string) (*localityAcces
return nil, ErrGovernanceInvalid
}
var path string
var layer localityAccessLayer
var admin bool
if err = rows.Scan(&path, &layer.grant, &layer.read, &layer.write, &admin); err != nil {
var layer schutzaria.AccessLayer
if err = rows.Scan(&path, &layer.Grant, &layer.Read, &layer.Write, &admin); err != nil {
return nil, err
}
p.admin = admin
p.paths[path] = append(p.paths[path], layer)
layer.Path = path
layers = append(layers, layer)
}
return p, rows.Err()
if err := rows.Err(); err != nil {
return nil, err
}
return schutzaria.NewLocalityAccess(admin, layers), nil
}
func readLocalityAccess(ctx context.Context, tx *sql.Tx, domain, kind, id string, g CityGovernance) (LocalityAccess, error) {
+3 -14
View File
@@ -11,6 +11,8 @@ import (
"sort"
"strings"
"time"
"pageswiki/internal/schutzaria"
)
var ErrPublicationImpact = errors.New("city.publication.impact_error")
@@ -313,20 +315,7 @@ func (s *Store) LocalityPublication(ctx context.Context, domain, actor, kind, id
// Explicitly exclude every application/user/group/editor namespace. Public
// projections must never turn an arbitrary reserved handler into a document.
func PublicSemanticPath(path string) bool {
if !strings.HasPrefix(path, "/") || strings.ContainsAny(path, "\\\x00\r\n") || strings.Contains(path, "//") {
return false
}
for _, segment := range strings.Split(path, "/") {
if segment == "." || segment == ".." {
return false
}
}
first := strings.Split(strings.TrimPrefix(path, "/"), "/")[0]
switch first {
case "_", "-", ":", "o", "r", "v", "l", "e", "s", "u", "g", "static", "health", "new":
return false
}
return true
return schutzaria.PublicSemanticPath(path)
}
// Lookup is metadata-only and Path-indexed. The manifest is fenced by current
@@ -0,0 +1,61 @@
package store_test
import (
"context"
"reflect"
"strings"
"testing"
"pageswiki/internal/mestionora"
"pageswiki/internal/store"
)
func TestMestionoraStoreCompatibilityAndRequestIsolation(t *testing.T) {
st := searchStoreFixture(t)
domain := defaultDomainUUID(t, st)
object := insertSearchFilterPage(t, st, domain, "/docs/needle", "text/html", "needle.html", "<h1>Needle Guide</h1><p>needle body</p>", 1, 2000)
insertSearchFilterPage(t, st, domain, "/private/needle", "text/html", "secret.html", "<h1>Secret Needle</h1>", 1, 2000)
if _, err := st.DB.Exec(`INSERT INTO paths(domain_uuid,path,object_uuid,created_at) VALUES(?,'/private/alias',?,1)`, domain, object); err != nil {
t.Fatal(err)
}
engine := mestionora.NewSQLite(st.DB)
allow := func(p string) bool { return strings.HasPrefix(p, "/docs/") }
ctx := context.Background()
for _, options := range []store.SearchOptions{{}, {FileType: "html"}, {Extension: "html"}, {FromMS: searchFilterTime(1000), BeforeMS: searchFilterTime(3000)}} {
legacy, err := st.SearchPathsInDomainWithOptions(ctx, domain, "/", "needle", 20, allow, options)
if err != nil {
t.Fatal(err)
}
current, err := engine.SearchPathsInDomainWithOptions(ctx, domain, "/", "needle", 20, allow, options)
if err != nil || !reflect.DeepEqual(legacy, current) || len(current) != 1 || current[0].Path != "/docs/needle" {
t.Fatalf("incompatible/exposed results: %v %v", current, err)
}
}
legacy, err := st.SearchPathSuggestionsInDomain(ctx, domain, "/", "needle", 8, allow)
if err != nil {
t.Fatal(err)
}
current, err := engine.SearchPathSuggestionsInDomain(ctx, domain, "/", "needle", 8, allow)
if err != nil || !reflect.DeepEqual(legacy, current) || len(current) != 1 || current[0].Title != "Needle Guide" {
t.Fatalf("suggestion compatibility: %v %v", current, err)
}
for _, request := range []struct {
domain string
allowed func(string) bool
}{
{domain, nil}, {domain, func(string) bool { return false }}, {"other-domain", allow},
} {
results, err := engine.SearchPathsInDomainFiltered(ctx, request.domain, "/", "needle", 20, request.allowed)
if err != nil || len(results) != 0 {
t.Fatalf("prior request leaked: %v %v", results, err)
}
}
if store.ErrInvalidSearchOptions != mestionora.ErrInvalidSearchOptions || store.SearchCandidateLimit != mestionora.SearchCandidateLimit {
t.Fatal("compatibility identity changed")
}
// Alias method sets preserve City display/navigation projections too.
var city store.CitySearchResult = mestionora.CitySearchResult{Kind: "spot", Name: "home-undefined", Title: "Undefined", StreetTitle: "Home", PathPrefix: "/life/home/_undefined", NavigationPath: "/life/home"}
if city.DisplayTitle() != "Home" || city.ContentPath() != "/life/home" {
t.Fatal("City projection changed")
}
}
+34 -11
View File
@@ -8,9 +8,11 @@ import (
"errors"
"strings"
"time"
"pageswiki/internal/ordoschnellic"
)
const NotificationPushEventType = "notification.push.deliver"
const NotificationPushEventType = ordoschnellic.PushEventType
func (s *Store) ensureNotificationPushSchema() error {
_, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS notification_push_keys(id INTEGER PRIMARY KEY CHECK(id=1),private_key TEXT NOT NULL,public_key TEXT NOT NULL);
@@ -81,11 +83,7 @@ func (s *Store) DeleteNotificationPush(ctx context.Context, domain, owner, endpo
return err
}
type NotificationPushDelivery struct {
ID, Notification, Domain, Owner, Subscription string
Focused bool
Attempt int
}
type NotificationPushDelivery = ordoschnellic.PushDelivery
func (s *Store) AcknowledgeNotificationPush(ctx context.Context, domain, owner, endpoint string, ids []string) error {
if len(ids) == 0 {
@@ -104,12 +102,20 @@ func (s *Store) AcknowledgeNotificationPush(ctx context.Context, domain, owner,
return err
}
func (s *Store) DeferFocusedNotificationPush(ctx context.Context, d *NotificationPushDelivery) error {
_, err := s.DB.ExecContext(ctx, `UPDATE notification_push_queue SET attempts=attempts-1,lease_until_ms=0,next_at_ms=? WHERE subscription_id=? AND notification_uuid=? AND attempts=?`, time.Now().Add(time.Minute).UnixMilli(), d.ID, d.Notification, d.Attempt)
return s.DeferFocusedNotificationPushAt(ctx, d, time.Now())
}
func (s *Store) DeferFocusedNotificationPushAt(ctx context.Context, d *NotificationPushDelivery, at time.Time) error {
_, err := s.DB.ExecContext(ctx, `UPDATE notification_push_queue SET attempts=attempts-1,lease_until_ms=0,next_at_ms=? WHERE subscription_id=? AND notification_uuid=? AND attempts=?`, at.Add(ordoschnellic.PushFocusDelay).UnixMilli(), d.ID, d.Notification, d.Attempt)
return err
}
func (s *Store) ClaimNotificationPush(ctx context.Context) (*NotificationPushDelivery, error) {
now := time.Now().UnixMilli()
return s.ClaimNotificationPushAt(ctx, time.Now())
}
func (s *Store) ClaimNotificationPushAt(ctx context.Context, at time.Time) (*NotificationPushDelivery, error) {
now := at.UnixMilli()
tx, err := s.DB.BeginTx(ctx, nil)
if err != nil {
return nil, err
@@ -117,7 +123,7 @@ func (s *Store) ClaimNotificationPush(ctx context.Context) (*NotificationPushDel
defer tx.Rollback()
// Atomic claim across scheduler workers/processes. Outbound I/O is never inside this transaction.
var d NotificationPushDelivery
err = tx.QueryRowContext(ctx, `UPDATE notification_push_queue SET lease_until_ms=?,attempts=attempts+1 WHERE (subscription_id,notification_uuid)=(SELECT subscription_id,notification_uuid FROM notification_push_queue WHERE next_at_ms<=? AND lease_until_ms<=? ORDER BY next_at_ms LIMIT 1) RETURNING subscription_id,notification_uuid,attempts`, now+60000, now, now).Scan(&d.ID, &d.Notification, &d.Attempt)
err = tx.QueryRowContext(ctx, `UPDATE notification_push_queue SET lease_until_ms=?,attempts=attempts+1 WHERE (subscription_id,notification_uuid)=(SELECT subscription_id,notification_uuid FROM notification_push_queue WHERE next_at_ms<=? AND lease_until_ms<=? ORDER BY next_at_ms LIMIT 1) RETURNING subscription_id,notification_uuid,attempts`, at.Add(ordoschnellic.PushLeaseDuration).UnixMilli(), now, now).Scan(&d.ID, &d.Notification, &d.Attempt)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
@@ -141,15 +147,32 @@ func (s *Store) ClaimNotificationPush(ctx context.Context) (*NotificationPushDel
}
func (s *Store) FinishNotificationPush(ctx context.Context, d *NotificationPushDelivery, retry, expired bool) error {
return s.FinishNotificationPushAt(ctx, d, retry, expired, time.Now())
}
func (s *Store) FinishNotificationPushAt(ctx context.Context, d *NotificationPushDelivery, retry, expired bool, at time.Time) error {
if expired {
if _, err := s.DB.ExecContext(ctx, `DELETE FROM notification_push_subscriptions WHERE id=?`, d.ID); err != nil {
return err
}
}
if retry && d.Attempt < 5 {
_, err := s.DB.ExecContext(ctx, `UPDATE notification_push_queue SET next_at_ms=?,lease_until_ms=0 WHERE subscription_id=? AND notification_uuid=? AND attempts=?`, time.Now().Add(time.Duration(1<<d.Attempt)*time.Minute).UnixMilli(), d.ID, d.Notification, d.Attempt)
if next, again := ordoschnellic.RetryAt(d.Attempt, at); retry && again {
_, err := s.DB.ExecContext(ctx, `UPDATE notification_push_queue SET next_at_ms=?,lease_until_ms=0 WHERE subscription_id=? AND notification_uuid=? AND attempts=?`, next.UnixMilli(), d.ID, d.Notification, d.Attempt)
return err
}
_, err := s.DB.ExecContext(ctx, `DELETE FROM notification_push_queue WHERE subscription_id=? AND notification_uuid=? AND attempts=?`, d.ID, d.Notification, d.Attempt)
return err
}
// CleanupNotificationPush only removes bounded stale delivery state, not Inbox.
func (s *Store) CleanupNotificationPush(ctx context.Context, at time.Time) error {
if s.ReadOnly {
return ErrReadOnly
}
_, err := s.DB.ExecContext(ctx, `DELETE FROM notification_push_subscriptions WHERE id IN (SELECT id FROM notification_push_subscriptions WHERE expires_at_ms<? LIMIT 100)`, at.UnixMilli())
if err != nil {
return err
}
_, err = s.DB.ExecContext(ctx, `DELETE FROM notification_push_queue WHERE rowid IN (SELECT rowid FROM notification_push_queue WHERE created_at_ms<? LIMIT 500)`, at.Add(-ordoschnellic.PushRetention).UnixMilli())
return err
}
+3 -14
View File
@@ -8,23 +8,12 @@ import (
"time"
"github.com/google/uuid"
"pageswiki/internal/ordoschnellic"
)
type Notification struct {
UUID string `json:"uuid"`
DomainUUID string `json:"domain_uuid"`
RecipientUUID string `json:"recipient_uuid"`
EventUUID string `json:"event_uuid,omitempty"`
Kind string `json:"kind"`
Status string `json:"status"`
RequestUUID string `json:"request_uuid,omitempty"`
Href string `json:"href,omitempty"`
Message string `json:"message,omitempty"`
CreatedAtMS int64 `json:"created_at_ms"`
ReadAtMS int64 `json:"read_at_ms,omitempty"`
}
type Notification = ordoschnellic.Notification
var ErrNotificationInput = errors.New("invalid notification")
var ErrNotificationInput = ordoschnellic.ErrNotificationInput
func (s *Store) EnsureNotificationSchema() error {
for _, statement := range []string{
@@ -0,0 +1,21 @@
package store_test
import (
"encoding/json"
"testing"
"pageswiki/internal/ordoschnellic"
"pageswiki/internal/store"
)
func TestOrdoschnellicStoreCompatibility(t *testing.T) {
var notification store.Notification = ordoschnellic.Notification{UUID: "notification", DomainUUID: "domain", RecipientUUID: "owner", Kind: "task.due", Status: "unread", CreatedAtMS: 1234}
var delivery store.NotificationPushDelivery = ordoschnellic.PushDelivery{Notification: notification.UUID, Attempt: 1}
if delivery.Notification != notification.UUID || store.ErrNotificationInput != ordoschnellic.ErrNotificationInput || store.NotificationPushEventType != ordoschnellic.PushEventType {
t.Fatal("legacy identities changed")
}
data, err := json.Marshal(notification)
if err != nil || string(data) != `{"uuid":"notification","domain_uuid":"domain","recipient_uuid":"owner","kind":"task.due","status":"unread","created_at_ms":1234}` {
t.Fatalf("notification wire contract changed: %s %v", data, err)
}
}
+41
View File
@@ -0,0 +1,41 @@
package store
import (
"context"
"strings"
)
// Completion needs names, never Revision bodies, titles, thumbnails or counts.
// Cap the candidate budget before the caller applies its request's ACL filter.
func (s *Store) ListPathCompletionNames(ctx context.Context, domain, parent, prefix string) ([]BrowserEntry, bool, error) {
base := strings.TrimRight(normalizeBrowserPath(parent), "/") + "/"
rows, err := s.DB.QueryContext(ctx, `WITH relative AS (
SELECT substr(path,length(?)+1) AS rest FROM paths
WHERE domain_uuid=? AND path>=? AND path<? AND path!=?
), children AS (
SELECT CASE WHEN instr(rest,'/')=0 THEN rest ELSE substr(rest,1,instr(rest,'/')-1) END AS name,
MAX(instr(rest,'/')>0) AS is_dir FROM relative WHERE rest!='' GROUP BY 1
) SELECT name,is_dir FROM children
WHERE substr(lower(name),1,length(?))=lower(?) ORDER BY name LIMIT 1001`, base, domain, base, strings.TrimSuffix(base, "/")+"0", parent, prefix, prefix)
if err != nil {
return nil, false, err
}
defer rows.Close()
entries := make([]BrowserEntry, 0)
for rows.Next() {
var entry BrowserEntry
if err := rows.Scan(&entry.Name, &entry.IsDir); err != nil {
return nil, false, err
}
entry.Path = base + entry.Name
entries = append(entries, entry)
}
if err := rows.Err(); err != nil {
return nil, false, err
}
truncated := len(entries) > 1000
if truncated {
entries = entries[:1000]
}
return entries, truncated, nil
}
+38
View File
@@ -0,0 +1,38 @@
package store
import (
"context"
"testing"
)
func TestPathCompletionNamesNoRevisionsAndBounded(t *testing.T) {
s := newSchemaTestStore(t)
// Deliberately omit revisions and documents: completion must never hydrate them.
_, err := s.DB.Exec(`CREATE TABLE paths(domain_uuid TEXT,path TEXT,object_uuid TEXT);
INSERT INTO paths VALUES('d','/docs/日本語 #?/child','o'),('d','/docs/a%b','o'),('d','/docs/a_b','o'),('other','/docs/secret','o'),('d','/docs-other/no','o');`)
if err != nil {
t.Fatal(err)
}
for _, prefix := range []string{"日本", "a%", "a_"} {
entries, more, err := s.ListPathCompletionNames(context.Background(), "d", "/docs", prefix)
if err != nil || more || len(entries) != 1 {
t.Fatalf("%q: %+v more=%v err=%v", prefix, entries, more, err)
}
if prefix == "日本" && (!entries[0].IsDir || entries[0].Path != "/docs/日本語 #?") {
t.Fatal(entries)
}
}
_, err = s.DB.Exec(`WITH RECURSIVE n(v) AS (VALUES(1) UNION ALL SELECT v+1 FROM n WHERE v<1200) INSERT INTO paths SELECT 'd','/many/'||printf('%04d',v),'o' FROM n`)
if err != nil {
t.Fatal(err)
}
entries, more, err := s.ListPathCompletionNames(context.Background(), "d", "/many", "")
if err != nil || !more || len(entries) != 1000 {
t.Fatalf("budget: %d %v %v", len(entries), more, err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, _, err = s.ListPathCompletionNames(ctx, "d", "/many", ""); err == nil {
t.Fatal("ignored cancellation")
}
}
+1 -1
View File
@@ -252,7 +252,7 @@ func (s *Store) changePeriodical(ctx context.Context, domain, actor, kind string
if err != nil {
return err
}
if !caps.decide(e.Path, capability, basis == nil) {
if !caps.Decide(e.Path, capability, basis == nil) {
return ErrAccessDenied
}
}
+185
View File
@@ -0,0 +1,185 @@
package store
import (
"context"
"database/sql"
"errors"
"strings"
"time"
"pageswiki/internal/db"
)
var profileContactMigrations = []db.SQLMigration{{Version: 1, Name: "private_profile_phone", Statements: []string{
`CREATE TABLE profile_contacts (
domain_uuid TEXT NOT NULL, principal_uuid TEXT NOT NULL,
phone_display TEXT NOT NULL DEFAULT '', phone_number TEXT NOT NULL DEFAULT '',
visibility TEXT NOT NULL DEFAULT 'private' CHECK(visibility IN ('private','domain','public')),
version INTEGER NOT NULL DEFAULT 1, updated_at_ms INTEGER NOT NULL,
PRIMARY KEY(domain_uuid,principal_uuid))`,
}, Assertions: []string{
schemaColumns("profile_contacts", "domain_uuid,principal_uuid,phone_display,phone_number,visibility,version,updated_at_ms"),
schemaUnique("profile_contacts", "domain_uuid,principal_uuid"),
}}}
// Kept outside Principal: account lists, identity federation and auth responses
// must not accidentally export personal contact details.
type ProfileContact struct {
PhoneDisplay string
PhoneNumber string
Visibility string
Version int64
}
var ErrContactPhone = errors.New("contact.invalid_phone")
// Initial scope: ordinary local/international numbers only. No service codes,
// extensions, pauses, URI parameters, or control characters are dialled.
func NormalizeContactPhone(value string) (string, string, error) {
if len(value) > 64 {
return "", "", ErrContactPhone
}
for _, c := range value {
if !(c >= '0' && c <= '9') && c != '+' && c != ' ' && c != '-' && c != '(' && c != ')' {
return "", "", ErrContactPhone
}
}
display := strings.TrimSpace(value)
if display == "" {
return "", "", nil
}
var digits strings.Builder
for i, c := range display {
if c >= '0' && c <= '9' || c == '+' && i == 0 {
digits.WriteRune(c)
} else if c == '+' {
return "", "", ErrContactPhone
}
}
number := digits.String()
n := len(strings.TrimPrefix(number, "+"))
if n < 3 || n > 15 {
return "", "", ErrContactPhone
}
return display, number, nil
}
func loadProfileContact(ctx context.Context, q interface {
QueryRowContext(context.Context, string, ...any) *sql.Row
}, domain, owner string) (ProfileContact, error) {
c := ProfileContact{Visibility: "private"}
err := q.QueryRowContext(ctx, `SELECT phone_display,phone_number,visibility,version FROM profile_contacts WHERE domain_uuid=? AND principal_uuid=?`, domain, owner).Scan(&c.PhoneDisplay, &c.PhoneNumber, &c.Visibility, &c.Version)
if errors.Is(err, sql.ErrNoRows) {
err = nil
}
return c, err
}
// ProfileContactForViewer returns no private data, including visibility/version,
// to unauthorized readers. Administrator status is not a contact consent grant.
func (s *Store) ProfileContactForViewer(ctx context.Context, domain, owner, viewer string) (ProfileContact, error) {
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
tx, err := s.DB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return ProfileContact{}, err
}
defer tx.Rollback()
var username string
err = tx.QueryRowContext(ctx, `SELECT i.subject FROM principals p JOIN domain_memberships m ON m.principal_uuid=p.uuid AND m.domain_uuid=? AND m.state='active' JOIN principal_identities i ON i.principal_uuid=p.uuid AND i.provider_type='password' WHERE p.uuid=? AND p.state IN ('active','readonly')`, domain, owner).Scan(&username)
if err != nil {
return ProfileContact{}, err
}
if viewer != "" {
var member bool
if err = tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM domain_memberships m JOIN principals p ON p.uuid=m.principal_uuid WHERE m.domain_uuid=? AND m.principal_uuid=? AND m.state='active' AND p.state IN ('active','readonly'))`, domain, viewer).Scan(&member); err != nil {
return ProfileContact{}, err
}
if !member {
return ProfileContact{}, nil
}
}
c, err := loadProfileContact(ctx, tx, domain, owner)
if err != nil {
return ProfileContact{}, err
}
if viewer != owner {
policy, err := loadUserSpacePolicy(ctx, tx, domain, owner, username)
if err != nil {
return ProfileContact{}, err
}
if viewer != "" {
if userSpaceVisibilityRank(c.Visibility) < 1 || userSpaceVisibilityRank(policy.ProfileVisibility) < 1 {
return ProfileContact{}, nil
}
} else {
allowed, err := s.domainAllowsUserSpacePublic(ctx, tx, domain)
if err != nil {
return ProfileContact{}, err
}
if !allowed || c.Visibility != "public" || policy.ProfileVisibility != "public" {
return ProfileContact{}, nil
}
}
}
if err = tx.Commit(); err != nil {
return ProfileContact{}, err
}
return c, nil
}
// The actor is the owner; callers cannot pass a separate target principal.
func (s *Store) SaveProfileContact(ctx context.Context, domain, actor string, proposed ProfileContact, confirmPublic bool) error {
if s.ReadOnly {
return ErrReadOnly
}
display, number, err := NormalizeContactPhone(proposed.PhoneDisplay)
if err != nil {
return err
}
if userSpaceVisibilityRank(proposed.Visibility) < 0 || proposed.Version < 0 {
return ErrGovernanceInvalid
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
tx, err := s.DB.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
var username string
if err = tx.QueryRowContext(ctx, `SELECT i.subject FROM principals p JOIN domain_memberships m ON m.principal_uuid=p.uuid AND m.domain_uuid=? AND m.state='active' JOIN principal_identities i ON i.principal_uuid=p.uuid AND i.provider_type='password' WHERE p.uuid=? AND p.state='active'`, domain, actor).Scan(&username); err != nil {
return err
}
current, err := loadProfileContact(ctx, tx, domain, actor)
if err != nil {
return err
}
if current.Version != proposed.Version {
return ErrGovernanceConflict
}
if number == "" {
proposed.Visibility = "private"
}
if proposed.Visibility == "public" {
policy, err := loadUserSpacePolicy(ctx, tx, domain, actor, username)
if err != nil {
return err
}
allowed, err := s.domainAllowsUserSpacePublic(ctx, tx, domain)
if err != nil {
return err
}
if !allowed || policy.ProfileVisibility != "public" {
return ErrGovernanceCeiling
}
if (current.Visibility != "public" || current.PhoneNumber != number) && !confirmPublic {
return ErrGovernanceInvalid
}
}
_, err = tx.ExecContext(ctx, `INSERT INTO profile_contacts(domain_uuid,principal_uuid,phone_display,phone_number,visibility,version,updated_at_ms) VALUES(?,?,?,?,?,1,?) ON CONFLICT(domain_uuid,principal_uuid) DO UPDATE SET phone_display=excluded.phone_display,phone_number=excluded.phone_number,visibility=excluded.visibility,version=profile_contacts.version+1,updated_at_ms=excluded.updated_at_ms`, domain, actor, display, number, proposed.Visibility, time.Now().UnixMilli())
if err != nil {
return err
}
return tx.Commit()
}
+19
View File
@@ -0,0 +1,19 @@
package store
import "testing"
func TestNormalizeContactPhone(t *testing.T) {
for _, tc := range []struct{ input, want string }{
{"", ""}, {" 03-1234-5678 ", "0312345678"}, {"+81 (90) 1234-5678", "+819012345678"},
} {
_, got, err := NormalizeContactPhone(tc.input)
if err != nil || got != tc.want {
t.Fatalf("%q: %q %v", tc.input, got, err)
}
}
for _, input := range []string{"*21*123#", "tel:1234", "javascript:alert(1)", "123;ext=4", "123,456", "123\r\n456", "\t123", "123", "12+34", "+", "12", "1234567890123456", "(+)123", "123\x1b"} {
if _, _, err := NormalizeContactPhone(input); err == nil {
t.Errorf("accepted %q", input)
}
}
}
+1 -14
View File
@@ -27,20 +27,7 @@ type ProjectionMetadata struct {
}
func (s *Store) EnsureProjectionMetadataSchema() error {
for _, statement := range []string{
`CREATE TABLE IF NOT EXISTS projection_metadata (
document_hash TEXT NOT NULL,kind TEXT NOT NULL,version INTEGER NOT NULL,
payload TEXT NOT NULL DEFAULT '{}',error TEXT NOT NULL DEFAULT '',
updated_at_ms INTEGER NOT NULL,retry_at_ms INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY(document_hash,kind,version))`,
`CREATE INDEX IF NOT EXISTS projection_metadata_eviction ON projection_metadata(updated_at_ms DESC)`,
`CREATE INDEX IF NOT EXISTS idx_event_jobs_identity_status ON event_jobs(event_type,event_key,status)`,
} {
if _, err := s.DB.Exec(statement); err != nil {
return err
}
}
return nil
return s.ensureSchemaComponent("projection-metadata")
}
func (s *Store) GetProjectionMetadata(hash, kind string, version int) (*ProjectionMetadata, error) {
+6 -8
View File
@@ -11,9 +11,10 @@ import (
"time"
"github.com/google/uuid"
"pageswiki/internal/schutzaria"
)
var ErrApprovalTransition = errors.New("invalid approval request transition")
var ErrApprovalTransition = schutzaria.ErrApprovalTransition
type ApprovalRequest struct {
UUID string `json:"uuid"`
@@ -127,7 +128,7 @@ func (s *Store) DecideApprovalRequestScoped(ctx context.Context, domainUUID, req
}
func (s *Store) decideApprovalRequest(ctx context.Context, domainUUID, requestUUID, reviewerUUID, decision, justification string, generation int64) error {
if requestUUID == "" || reviewerUUID == "" || generation < 1 || (decision != "approved" && decision != "rejected" && decision != "changes_requested") {
if requestUUID == "" || reviewerUUID == "" || generation < 1 || !schutzaria.ValidApprovalDecision(decision) {
return ErrApprovalTransition
}
tx, err := s.DB.BeginTx(ctx, nil)
@@ -146,17 +147,14 @@ func (s *Store) decideApprovalRequest(ctx context.Context, domainUUID, requestUU
if err := tx.QueryRowContext(ctx, query, args...).Scan(&state, &currentGeneration); err != nil {
return err
}
if state != "pending" || currentGeneration != generation {
return ErrApprovalTransition
newState, err := schutzaria.ApprovalNextState(state, currentGeneration, generation, decision)
if err != nil {
return err
}
now := time.Now().UnixMilli()
if _, err := tx.ExecContext(ctx, `INSERT INTO publication_decisions(uuid,request_uuid,reviewer_uuid,generation,decision,justification,created_at_ms) VALUES(?,?,?,?,?,?,?)`, uuid.NewString(), requestUUID, reviewerUUID, generation, decision, justification, now); err != nil {
return err
}
newState := decision
if decision == "changes_requested" {
newState = "rejected"
}
if _, err := tx.ExecContext(ctx, `UPDATE publication_requests SET state=?,updated_at_ms=? WHERE uuid=? AND state='pending' AND generation=?`, newState, now, requestUUID, generation); err != nil {
return err
}
+17 -39
View File
@@ -3,38 +3,12 @@ package store
import (
"errors"
"strings"
"pageswiki/internal/schutzaria"
)
// ReadPathPolicy is an immutable, request-scoped snapshot of positive Path
// read grants. Never share it across requests or use it to authorize writes.
type ReadPathPolicy struct {
access *localityAccessSnapshot
all bool
exact map[string]bool
subtrees map[string]bool
}
func (p *ReadPathPolicy) Allows(path string) bool {
if p == nil {
return false
}
return p.access.decide(path, "read", p.allowsBase(path))
}
func (p *ReadPathPolicy) allowsBase(path string) bool {
if p == nil || !strings.HasPrefix(path, "/") {
return false
}
if p.all || p.exact[path] {
return true
}
for at := strings.LastIndex(path, "/"); at > 0; at = strings.LastIndex(path[:at], "/") {
if p.subtrees[path[:at]] {
return true
}
}
return false
}
// ReadPathPolicy is a request-scoped, read-only policy snapshot.
type ReadPathPolicy = schutzaria.ReadPathPolicy
// LoadReadPathPolicy uses one bounded query, independent of the number of
// directory entries. It deliberately excludes Object grants: an Object grant
@@ -44,9 +18,10 @@ func (s *Store) LoadReadPathPolicy(domain, principal string) (*ReadPathPolicy, e
}
func loadReadPathPolicy(q pathQueryer, domain, principal string) (*ReadPathPolicy, error) {
p := &ReadPathPolicy{exact: map[string]bool{}, subtrees: map[string]bool{}}
var all bool
var exact, subtrees []string
if domain == "" || principal == "" {
return p, ErrAccessDenied
return &ReadPathPolicy{}, ErrAccessDenied
}
rows, err := q.Query(`
SELECT 'role',g.target_type,g.target_id FROM principals p
@@ -79,15 +54,15 @@ func loadReadPathPolicy(q pathQueryer, domain, principal string) (*ReadPathPolic
}
if source == "role" {
if kind == "domain" && target == domain || kind == "path" && target == "*" {
p.all = true
all = true
} else if kind == "path" {
p.exact[target] = true
exact = append(exact, target)
}
} else if target == "/" {
p.all = true
all = true
} else if strings.HasPrefix(target, "/") {
p.exact[target] = true
p.subtrees[strings.TrimRight(target, "/")] = true
exact = append(exact, target)
subtrees = append(subtrees, strings.TrimRight(target, "/"))
}
}
if err = rows.Err(); err != nil {
@@ -96,6 +71,9 @@ func loadReadPathPolicy(q pathQueryer, domain, principal string) (*ReadPathPolic
if err = rows.Close(); err != nil {
return nil, err
}
p.access, err = loadLocalityAccess(q, domain, principal)
return p, err
access, err := loadLocalityAccess(q, domain, principal)
if err != nil {
return nil, err
}
return schutzaria.NewReadPathPolicy(all, exact, subtrees, access), nil
}
+38
View File
@@ -0,0 +1,38 @@
package store
import (
"crypto/sha256"
"database/sql"
"errors"
"os"
"path/filepath"
"strings"
)
var ErrReadOnly = errors.New("DR standby is read-only")
// No migrations, schema repair, key generation or file creation on standby.
func NewReadOnlyWithDocumentRoot(database *sql.DB, root string) (*Store, error) {
var readOnly int
if err := database.QueryRow("PRAGMA query_only").Scan(&readOnly); err != nil {
return nil, err
}
if readOnly != 1 {
return nil, errors.New("standby requires a read-only database connection")
}
st := &Store{DB: database, DocumentRoot: root, childCache: map[string][]string{}, ReadOnly: true}
if encoded := strings.TrimSpace(os.Getenv("PAGESWIKI_AUTH_KEY")); encoded != "" {
sum := sha256.Sum256([]byte(encoded))
st.authKey = sum[:]
} else {
key, err := os.ReadFile(filepath.Join(filepath.Dir(root), "auth.key"))
if err != nil {
return nil, err
}
if len(key) != 32 {
return nil, errors.New("standby requires the original 32-byte auth.key")
}
st.authKey = key
}
return st, nil
}
+3 -5
View File
@@ -1,7 +1,5 @@
package store
import "os"
type RecentObject struct {
UUID string
Path string
@@ -117,9 +115,9 @@ func (s *Store) ListRecentObjectsInDomain(
if out[i].documentHash == "" {
continue
}
if path, err := s.documentPath(out[i].documentHash); err == nil && path != "" {
if info, err := os.Stat(path); err == nil {
out[i].Size = int(info.Size())
if s.DocumentRoot != "" {
if size, err := s.DocumentSize(out[i].documentHash); err == nil {
out[i].Size = int(size)
}
}
}
+18 -23
View File
@@ -4,17 +4,15 @@ import (
"context"
"encoding/json"
"time"
"pageswiki/internal/seheweit"
)
const ResourceEventID = "builtin.resource.sample"
const ResourceEventType = "resource.sample"
const ResourceEventID = seheweit.ResourceEventID
const ResourceEventType = seheweit.ResourceEventType
func (s *Store) EnsureResourceHistorySchema() error {
_, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS resource_samples (
id INTEGER PRIMARY KEY, job_uuid TEXT NOT NULL UNIQUE,
timestamp_ms INTEGER NOT NULL, payload TEXT NOT NULL
); CREATE INDEX IF NOT EXISTS idx_resource_samples_time ON resource_samples(timestamp_ms)`)
return err
return s.ensureSchemaComponent("resource-history")
}
func (s *Store) EnsureBuiltinResourceEvent() error {
@@ -26,37 +24,34 @@ func (s *Store) EnsureBuiltinResourceEvent() error {
}
func (s *Store) SaveResourceSample(job string, at int64, payload []byte) error {
tx, err := s.DB.Begin()
return s.SaveResourceSampleContext(context.Background(), job, at, payload)
}
func (s *Store) SaveResourceSampleContext(ctx context.Context, job string, at int64, payload []byte) error {
if s.ReadOnly {
return ErrReadOnly
}
tx, err := s.DB.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if _, err = tx.Exec(`INSERT OR IGNORE INTO resource_samples(job_uuid,timestamp_ms,payload) VALUES(?,?,?)`, job, at, string(payload)); err != nil {
if _, err = tx.ExecContext(ctx, `INSERT OR IGNORE INTO resource_samples(job_uuid,timestamp_ms,payload) VALUES(?,?,?)`, job, at, string(payload)); err != nil {
return err
}
if _, err = tx.Exec(`DELETE FROM resource_samples WHERE timestamp_ms < ?`, at-int64(7*24*time.Hour/time.Millisecond)); err != nil {
if _, err = tx.ExecContext(ctx, `DELETE FROM resource_samples WHERE timestamp_ms < ?`, seheweit.RetentionCutoff(at)); err != nil {
return err
}
return tx.Commit()
}
// Return the last sample in each time bucket, not averages disguised as peaks.
// Return the last inserted sample in each time bucket, not averages or peaks.
// At most 361 points and an indexed seven-day scan; no request-side collection.
func (s *Store) ResourceHistory(ctx context.Context, hours int, now time.Time) ([]json.RawMessage, error) {
if hours < 1 {
hours = 1
}
if hours > 168 {
hours = 168
}
span := int64(hours) * 3600000
bucket := span / 360
if bucket < 60000 {
bucket = 60000
}
window := seheweit.Window(hours, now)
rows, err := s.DB.QueryContext(ctx, `SELECT payload FROM resource_samples WHERE id IN
(SELECT MAX(id) FROM resource_samples WHERE timestamp_ms>=? AND timestamp_ms<=? GROUP BY timestamp_ms / ?)
ORDER BY timestamp_ms LIMIT 361`, now.UnixMilli()-span, now.UnixMilli(), bucket)
ORDER BY timestamp_ms LIMIT ?`, window.FromMS, window.ToMS, window.BucketMS, seheweit.MaxHistoryPoints)
if err != nil {
return nil, err
}
+66
View File
@@ -107,6 +107,19 @@ func (s *Store) saveRevisionContentUnlocked(
contentType string,
filename string,
) (*Revision, error) {
return s.saveRevisionContentExpected(objectUUID, payload, message, rawLanguage, contentType, filename, nil)
}
// SaveRevisionContentIfHead compares the exact language head and appends within
// one SQLite write transaction, including writers outside this Store process.
// An empty expected UUID asserts that this language has no revision yet.
func (s *Store) SaveRevisionContentIfHead(objectUUID string, payload []byte, message, lang, contentType, filename, expected string) (*Revision, error) {
s.revisionWriteMu.Lock()
defer s.revisionWriteMu.Unlock()
return s.saveRevisionContentExpected(objectUUID, payload, message, lang, contentType, filename, &expected)
}
func (s *Store) saveRevisionContentExpected(objectUUID string, payload []byte, message, rawLanguage, contentType, filename string, expected *string) (*Revision, error) {
lang, err :=
normalizeRevisionLanguage(
rawLanguage,
@@ -165,6 +178,29 @@ func (s *Store) saveRevisionContentUnlocked(
return nil, err
}
defer tx.Rollback()
if expected != nil {
// Acquire a write lock before reading the head. Avoid a read-to-write
// upgrade race with CLI/another process writing this same database.
if _, err = tx.Exec(`UPDATE objects SET uuid=uuid WHERE uuid=?`, objectUUID); err != nil {
return nil, err
}
var current string
var currentMS int64
err = tx.QueryRow(`SELECT uuid,COALESCE(created_at_ms,created_at*1000) FROM revisions WHERE object_uuid=? AND COALESCE(NULLIF(lang,''),'und')=? ORDER BY COALESCE(created_at_ms,created_at*1000) DESC,rowid DESC LIMIT 1`, objectUUID, lang).Scan(&current, &currentMS)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return nil, err
}
if current != *expected {
return nil, ErrRevisionConflict
}
// Hashing/persisting may wait behind another process. The revision we
// accepted must sort after its base even if the wall clock moved back.
revision.CreatedAtMS = time.Now().UnixMilli()
if revision.CreatedAtMS <= currentMS {
revision.CreatedAtMS = currentMS + 1
}
revision.CreatedAt = revision.CreatedAtMS / 1000
}
if err = insertRevision(tx, revision, embeddedHashes); err != nil {
return nil, err
}
@@ -178,6 +214,36 @@ func (s *Store) saveRevisionContentUnlocked(
return revision, nil
}
// GetRevisionForExactLanguage deliberately has no viewer-language fallback.
func (s *Store) GetRevisionForExactLanguage(objectUUID, rawLanguage string) (*Revision, error) {
return s.revisionForExactLanguage(objectUUID, rawLanguage, false)
}
// GetRevisionMetadataForExactLanguage does not read the document file. Polling
// an editor head must not hydrate a potentially large body on every request.
func (s *Store) GetRevisionMetadataForExactLanguage(objectUUID, rawLanguage string) (*Revision, error) {
return s.revisionForExactLanguage(objectUUID, rawLanguage, true)
}
func (s *Store) revisionForExactLanguage(objectUUID, rawLanguage string, metadata bool) (*Revision, error) {
lang, err := normalizeRevisionLanguage(rawLanguage)
if err != nil {
return nil, err
}
var id string
err = s.DB.QueryRow(`SELECT uuid FROM revisions WHERE object_uuid=? AND COALESCE(NULLIF(lang,''),'und')=? ORDER BY COALESCE(created_at_ms,created_at*1000) DESC,rowid DESC LIMIT 1`, objectUUID, lang).Scan(&id)
if errors.Is(err, sql.ErrNoRows) {
return &Revision{ObjectUUID: objectUUID, Language: lang, ContentType: "text/html; charset=utf-8"}, nil
}
if err != nil {
return nil, err
}
if metadata {
return s.GetRevisionMetadata(id)
}
return s.GetRevision(id)
}
func insertRevision(tx *sql.Tx, revision *Revision, embeddedHashes []string) error {
result, err := tx.Exec(`
INSERT INTO revisions (
+29
View File
@@ -0,0 +1,29 @@
package store
import (
"context"
"database/sql"
"pageswiki/internal/db"
)
type SchemaInspection struct {
Coverage string `json:"coverage"`
Components []db.MigrationInspection `json:"components"`
LegacyAreas []string `json:"legacy_areas"`
}
// InspectSchema deliberately does not construct a Store: constructors can
// migrate, backfill, generate keys, seed defaults and requeue interrupted jobs.
func InspectSchema(ctx context.Context, database *sql.DB) (SchemaInspection, error) {
plans := make(map[string][]db.SQLMigration, len(managedStoreSchemas)+1)
for name, steps := range managedStoreSchemas {
plans[name] = steps
}
plans["document-store"] = documentBackfillMigrations
components, err := db.InspectSQLiteMigrations(ctx, database, plans)
return SchemaInspection{Coverage: "managed-components-only", Components: components, LegacyAreas: []string{
"core-bootstrap", "campuses/domains/federation", "auth/ACL/user-space", "outbox/notifications/publication-approval",
"City", "tasks/calendars", "periodicals", "search/object-listing/assets/GC",
}}, err
}
+83
View File
@@ -0,0 +1,83 @@
package store
import (
"context"
"testing"
"pageswiki/internal/db"
)
func TestSchemaInspectionDoesNotRecoverJobsOrBackfill(t *testing.T) {
st := newSchemaTestStore(t)
for name := range managedStoreSchemas {
if err := st.ensureSchemaComponent(name); err != nil {
t.Fatal(err)
}
}
if err := db.ApplySQLiteMigrations(context.Background(), st.DB, "document-store", documentBackfillMigrations); err != nil {
t.Fatal(err)
}
if _, err := st.DB.Exec(`INSERT INTO event_jobs(uuid,event_type,event_key,payload,status,run_at_ms,created_at_ms,updated_at_ms) VALUES('running','test','key','','running',1,1,1); INSERT INTO document_backfills VALUES('store','cursor',7,0,1)`); err != nil {
t.Fatal(err)
}
report, err := InspectSchema(context.Background(), st.DB)
if err != nil {
t.Fatal(err)
}
for _, component := range report.Components {
if component.State != "current" {
t.Fatalf("component=%+v", component)
}
}
var status string
if err := st.DB.QueryRow(`SELECT status FROM event_jobs WHERE uuid='running'`).Scan(&status); err != nil || status != "running" {
t.Fatalf("job modified: %q %v", status, err)
}
var processed, complete int
if err := st.DB.QueryRow(`SELECT processed,completed FROM document_backfills WHERE store_id='store'`).Scan(&processed, &complete); err != nil || processed != 7 || complete != 0 {
t.Fatalf("backfill modified: %d %d %v", processed, complete, err)
}
if err := st.EnsureEventJobsSchema(); err != nil {
t.Fatal(err)
}
if err := st.DB.QueryRow(`SELECT status FROM event_jobs WHERE uuid='running'`).Scan(&status); err != nil || status != "queued" {
t.Fatalf("startup recovery broken: %q %v", status, err)
}
}
func TestEventJobMigrationAdoptsLegacyAndPreservesSettings(t *testing.T) {
st := newSchemaTestStore(t)
if _, err := st.DB.Exec(eventJobMigrations[0].Statements[0]); err != nil {
t.Fatal(err)
}
if _, err := st.DB.Exec(`INSERT INTO event_jobs(uuid,event_type,event_key,payload,status,run_at_ms,created_at_ms,updated_at_ms) VALUES('old','test','key','keep','running',1,1,1)`); err != nil {
t.Fatal(err)
}
if err := st.ensureSchemaComponent("event-jobs"); err != nil {
t.Fatal(err)
}
var payload, status string
var attempts int
if err := st.DB.QueryRow(`SELECT payload,status,max_attempts FROM event_jobs WHERE uuid='old'`).Scan(&payload, &status, &attempts); err != nil || payload != "keep" || status != "running" || attempts != 3 {
t.Fatalf("legacy job=%q %q %d %v", payload, status, attempts, err)
}
if _, err := st.DB.Exec(`UPDATE event_queue_settings SET paused=1,concurrency=3; UPDATE event_log_policies SET max_rows=42 WHERE event_type='*'`); err != nil {
t.Fatal(err)
}
if err := st.ensureSchemaComponent("event-jobs"); err != nil {
t.Fatal(err)
}
var paused, workers, rows int
if err := st.DB.QueryRow(`SELECT paused,concurrency FROM event_queue_settings WHERE id=1`).Scan(&paused, &workers); err != nil || paused != 1 || workers != 3 {
t.Fatalf("settings overwritten %d %d %v", paused, workers, err)
}
if err := st.DB.QueryRow(`SELECT max_rows FROM event_log_policies WHERE event_type='*'`).Scan(&rows); err != nil || rows != 42 {
t.Fatalf("policy overwritten %d %v", rows, err)
}
if _, err := st.DB.Exec(`DROP INDEX idx_event_jobs_due`); err != nil {
t.Fatal(err)
}
if err := st.ensureSchemaComponent("event-jobs"); err == nil {
t.Fatal("missing queue index accepted")
}
}
+120
View File
@@ -0,0 +1,120 @@
package store
import "pageswiki/internal/db"
// Version 1 repairs only the known legacy columns; version 2 can then safely
// create indexes using them. Recovery of running jobs is intentionally separate.
var eventJobMigrations = []db.SQLMigration{
{Version: 1, Name: "adopt_event_job_columns", Statements: []string{`
CREATE TABLE IF NOT EXISTS event_jobs (
uuid TEXT PRIMARY KEY,
event_type TEXT NOT NULL,
event_key TEXT NOT NULL,
payload TEXT NOT NULL,
status TEXT NOT NULL,
attempts INTEGER NOT NULL DEFAULT 0,
run_at_ms INTEGER NOT NULL,
error TEXT NOT NULL DEFAULT '',
created_at_ms INTEGER NOT NULL,
updated_at_ms INTEGER NOT NULL
)
`}, AddColumns: []db.SQLiteColumn{
{Table: "event_jobs", Name: "event_uuid", Declaration: "TEXT NOT NULL DEFAULT ''"},
{Table: "event_jobs", Name: "priority", Declaration: "INTEGER NOT NULL DEFAULT 0"},
{Table: "event_jobs", Name: "max_attempts", Declaration: "INTEGER NOT NULL DEFAULT 3"},
{Table: "event_jobs", Name: "started_at_ms", Declaration: "INTEGER NOT NULL DEFAULT 0"},
{Table: "event_jobs", Name: "finished_at_ms", Declaration: "INTEGER NOT NULL DEFAULT 0"},
{Table: "event_jobs", Name: "cancel_requested", Declaration: "INTEGER NOT NULL DEFAULT 0"},
}},
{Version: 2, Name: "adopt_event_queue_and_logs", Statements: []string{
`CREATE INDEX IF NOT EXISTS idx_event_jobs_identity_status ON event_jobs(event_type,event_key,status)`,
`CREATE INDEX IF NOT EXISTS idx_event_jobs_listing ON event_jobs (created_at_ms DESC, uuid DESC)`,
`CREATE INDEX IF NOT EXISTS idx_event_jobs_status_listing ON event_jobs (status, created_at_ms DESC, uuid DESC)`,
`CREATE INDEX IF NOT EXISTS idx_event_jobs_history ON event_jobs (created_at_ms DESC, uuid DESC) WHERE status IN ('succeeded','skipped','failed','cancelled')`,
// Legacy installations already have this name without priority. IF NOT
// EXISTS would retain that definition and fail the contract below. This
// non-unique scheduling index is rebuilt once, in the adoption transaction;
// neither jobs nor their status/attempts are changed.
`DROP INDEX IF EXISTS idx_event_jobs_due`,
`CREATE INDEX IF NOT EXISTS idx_event_jobs_due
ON event_jobs (status, priority DESC, run_at_ms, created_at_ms)`,
`CREATE TABLE IF NOT EXISTS event_definitions (
uuid TEXT PRIMARY KEY, name TEXT NOT NULL, event_type TEXT NOT NULL,
event_key TEXT NOT NULL, payload TEXT NOT NULL DEFAULT '',
schedule_kind TEXT NOT NULL DEFAULT 'manual',
interval_seconds INTEGER NOT NULL DEFAULT 0,
next_run_at_ms INTEGER NOT NULL DEFAULT 0,
enabled INTEGER NOT NULL DEFAULT 1, priority INTEGER NOT NULL DEFAULT 0,
max_attempts INTEGER NOT NULL DEFAULT 3,
created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL
)`,
`CREATE INDEX IF NOT EXISTS idx_event_definitions_due
ON event_definitions (enabled, next_run_at_ms)`,
`CREATE TABLE IF NOT EXISTS event_logs (
id INTEGER PRIMARY KEY AUTOINCREMENT, job_uuid TEXT NOT NULL,
event_uuid TEXT NOT NULL DEFAULT '', level TEXT NOT NULL,
message TEXT NOT NULL, created_at_ms INTEGER NOT NULL
)`,
`CREATE INDEX IF NOT EXISTS idx_event_logs_job
ON event_logs (job_uuid, created_at_ms DESC)`,
`CREATE INDEX IF NOT EXISTS idx_event_logs_job_id ON event_logs (job_uuid, id DESC)`,
`CREATE TABLE IF NOT EXISTS event_log_policies (
event_type TEXT PRIMARY KEY,
retention_hours INTEGER NOT NULL DEFAULT 168,
aggregate_after_hours INTEGER NOT NULL DEFAULT 24,
bucket_seconds INTEGER NOT NULL DEFAULT 60,
max_rows INTEGER NOT NULL DEFAULT 5000
)`,
`INSERT OR IGNORE INTO event_log_policies(event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows)
VALUES('*',168,24,60,5000)`,
`CREATE TABLE IF NOT EXISTS event_log_buckets (
event_type TEXT NOT NULL,
bucket_start_ms INTEGER NOT NULL,
level TEXT NOT NULL,
count INTEGER NOT NULL DEFAULT 0,
first_at_ms INTEGER NOT NULL,
last_at_ms INTEGER NOT NULL,
sample_message TEXT NOT NULL DEFAULT '',
PRIMARY KEY(event_type,bucket_start_ms,level)
)`,
`CREATE INDEX IF NOT EXISTS idx_event_log_buckets_time ON event_log_buckets(event_type,bucket_start_ms DESC)`,
`CREATE TABLE IF NOT EXISTS event_queue_settings (
id INTEGER PRIMARY KEY CHECK (id = 1), paused INTEGER NOT NULL DEFAULT 0,
concurrency INTEGER NOT NULL DEFAULT 1,
max_depth INTEGER NOT NULL DEFAULT 10000,
retry_base_seconds INTEGER NOT NULL DEFAULT 30,
retention_hours INTEGER NOT NULL DEFAULT 168
)`,
`INSERT OR IGNORE INTO event_queue_settings
(id, paused, concurrency, max_depth, retry_base_seconds, retention_hours)
VALUES (1, 0, 1, 10000, 30, 168)`,
}, Assertions: []string{
schemaColumns("event_jobs", "uuid,event_type,event_key,payload,status,attempts,run_at_ms,error,created_at_ms,updated_at_ms,event_uuid,priority,max_attempts,started_at_ms,finished_at_ms,cancel_requested"),
schemaColumns("event_definitions", "uuid,name,event_type,event_key,payload,schedule_kind,interval_seconds,next_run_at_ms,enabled,priority,max_attempts,created_at_ms,updated_at_ms"),
schemaColumns("event_logs", "id,job_uuid,event_uuid,level,message,created_at_ms"),
schemaColumns("event_log_policies", "event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows"),
schemaColumns("event_log_buckets", "event_type,bucket_start_ms,level,count,first_at_ms,last_at_ms,sample_message"),
schemaColumns("event_queue_settings", "id,paused,concurrency,max_depth,retry_base_seconds,retention_hours"),
schemaUnique("event_jobs", "uuid"), schemaUnique("event_definitions", "uuid"),
schemaUnique("event_log_policies", "event_type"), schemaUnique("event_log_buckets", "event_type,bucket_start_ms,level"),
schemaIndex("event_jobs", "idx_event_jobs_identity_status", "event_type,event_key,status"),
schemaIndex("event_jobs", "idx_event_jobs_due", "status,priority,run_at_ms,created_at_ms"),
schemaIndex("event_jobs", "idx_event_jobs_listing", "created_at_ms,uuid"),
schemaIndex("event_jobs", "idx_event_jobs_status_listing", "status,created_at_ms,uuid"),
schemaIndex("event_jobs", "idx_event_jobs_history", "created_at_ms,uuid"),
schemaIndexPredicate("idx_event_jobs_history", "status IN ('succeeded','skipped','failed','cancelled')"),
schemaIndex("event_definitions", "idx_event_definitions_due", "enabled,next_run_at_ms"),
schemaIndex("event_logs", "idx_event_logs_job", "job_uuid,created_at_ms"),
schemaIndex("event_logs", "idx_event_logs_job_id", "job_uuid,id"),
schemaIndex("event_log_buckets", "idx_event_log_buckets_time", "event_type,bucket_start_ms"),
schemaAddedColumn("event_jobs", "priority", "INTEGER", "0"),
schemaAddedColumn("event_jobs", "max_attempts", "INTEGER", "3"),
schemaAddedColumn("event_jobs", "event_uuid", "TEXT", "''"),
schemaAddedColumn("event_jobs", "started_at_ms", "INTEGER", "0"),
schemaAddedColumn("event_jobs", "finished_at_ms", "INTEGER", "0"),
schemaAddedColumn("event_jobs", "cancel_requested", "INTEGER", "0"),
`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_table_info('event_logs') WHERE name='id' AND type='INTEGER' AND pk=1)`,
`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_table_info('event_queue_settings') WHERE name='id' AND type='INTEGER' AND pk=1)`,
`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM event_queue_settings WHERE id=1)`,
}},
}
+141
View File
@@ -0,0 +1,141 @@
package store
import (
"context"
"fmt"
"strings"
"pageswiki/internal/db"
)
// These definitions adopt known legacy tables without replacing their rows.
// Once released, add a new version; never edit an applied definition.
// Auth, City, tasks and calendars remain in their existing migration paths.
var managedStoreSchemas = map[string][]db.SQLMigration{
"profile-contacts": profileContactMigrations,
"event-jobs": eventJobMigrations,
"projection-metadata": {{Version: 1, Name: "adopt_projection_metadata", Statements: []string{
`CREATE TABLE IF NOT EXISTS projection_metadata (
document_hash TEXT NOT NULL,kind TEXT NOT NULL,version INTEGER NOT NULL,
payload TEXT NOT NULL DEFAULT '{}',error TEXT NOT NULL DEFAULT '',
updated_at_ms INTEGER NOT NULL,retry_at_ms INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY(document_hash,kind,version))`,
`CREATE INDEX IF NOT EXISTS projection_metadata_eviction ON projection_metadata(updated_at_ms DESC)`,
}, Assertions: []string{
schemaColumns("projection_metadata", "document_hash,kind,version,payload,error,updated_at_ms,retry_at_ms"),
schemaUnique("projection_metadata", "document_hash,kind,version"),
schemaIndex("projection_metadata", "projection_metadata_eviction", "updated_at_ms"),
}}},
"resource-history": {{Version: 1, Name: "adopt_resource_history", Statements: []string{
`CREATE TABLE IF NOT EXISTS resource_samples (
id INTEGER PRIMARY KEY, job_uuid TEXT NOT NULL UNIQUE,
timestamp_ms INTEGER NOT NULL, payload TEXT NOT NULL)`,
`CREATE INDEX IF NOT EXISTS idx_resource_samples_time ON resource_samples(timestamp_ms)`,
}, Assertions: []string{
schemaColumns("resource_samples", "id,job_uuid,timestamp_ms,payload"),
schemaUnique("resource_samples", "job_uuid"),
`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_table_info('resource_samples') WHERE name='id' AND type='INTEGER' AND pk=1)`,
schemaIndex("resource_samples", "idx_resource_samples_time", "timestamp_ms"),
}}},
"broadcasts": {{Version: 1, Name: "adopt_broadcasts", Statements: []string{
`CREATE TABLE IF NOT EXISTS broadcasts (uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,title TEXT NOT NULL,body TEXT NOT NULL,severity TEXT NOT NULL DEFAULT 'info' CHECK(severity IN ('info','success','warning','critical')),starts_at_ms INTEGER NOT NULL DEFAULT 0,ends_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`,
`CREATE INDEX IF NOT EXISTS idx_broadcast_active ON broadcasts(domain_uuid,starts_at_ms,ends_at_ms,updated_at_ms DESC)`,
}, Assertions: []string{
schemaColumns("broadcasts", "uuid,domain_uuid,title,body,severity,starts_at_ms,ends_at_ms,created_at_ms,updated_at_ms"),
schemaUnique("broadcasts", "uuid"),
schemaIndex("broadcasts", "idx_broadcast_active", "domain_uuid,starts_at_ms,ends_at_ms,updated_at_ms"),
}}},
"browser-navigation": {{Version: 1, Name: "adopt_browser_navigation", Statements: []string{
`CREATE TABLE IF NOT EXISTS browser_navigation_links (
uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,principal_uuid TEXT NOT NULL,
label TEXT NOT NULL,path TEXT NOT NULL,is_directory INTEGER NOT NULL DEFAULT 0,
position INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL,
UNIQUE(domain_uuid, principal_uuid, path))`,
`CREATE INDEX IF NOT EXISTS idx_browser_navigation_owner ON browser_navigation_links(domain_uuid, principal_uuid, position, created_at_ms)`,
}, AddColumns: []db.SQLiteColumn{{Table: "browser_navigation_links", Name: "is_directory", Declaration: "INTEGER NOT NULL DEFAULT 0"}}, Assertions: []string{
schemaColumns("browser_navigation_links", "uuid,domain_uuid,principal_uuid,label,path,is_directory,position,created_at_ms,updated_at_ms"),
schemaUnique("browser_navigation_links", "uuid"),
schemaUnique("browser_navigation_links", "domain_uuid,principal_uuid,path"),
schemaAddedColumn("browser_navigation_links", "is_directory", "INTEGER", "0"),
schemaIndex("browser_navigation_links", "idx_browser_navigation_owner", "domain_uuid,principal_uuid,position,created_at_ms"),
}}},
"adapter-sessions": {{Version: 1, Name: "adopt_adapter_sessions", Statements: []string{
`CREATE TABLE IF NOT EXISTS adapter_sessions (
ticket_hash TEXT PRIMARY KEY,session_hash TEXT NOT NULL DEFAULT '',
domain_uuid TEXT NOT NULL,plugin_id TEXT NOT NULL,object_uuid TEXT NOT NULL,
principal_uuid TEXT NOT NULL DEFAULT '',base_revision_uuid TEXT NOT NULL,
ticket_expires_at_ms INTEGER NOT NULL,session_expires_at_ms INTEGER NOT NULL DEFAULT 0,
exchanged_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL)`,
`CREATE UNIQUE INDEX IF NOT EXISTS idx_adapter_session_hash ON adapter_sessions(session_hash) WHERE session_hash != ''`,
}, AddColumns: []db.SQLiteColumn{
{Table: "adapter_sessions", Name: "root_path", Declaration: "TEXT NOT NULL DEFAULT ''"},
{Table: "adapter_sessions", Name: "principal_uuid", Declaration: "TEXT NOT NULL DEFAULT ''"},
}, Assertions: []string{
schemaColumns("adapter_sessions", "ticket_hash,session_hash,domain_uuid,plugin_id,object_uuid,principal_uuid,base_revision_uuid,ticket_expires_at_ms,session_expires_at_ms,exchanged_at_ms,created_at_ms,root_path"),
schemaUnique("adapter_sessions", "ticket_hash"),
schemaAddedColumn("adapter_sessions", "root_path", "TEXT", "''"),
schemaAddedColumn("adapter_sessions", "principal_uuid", "TEXT", "''"),
schemaIndex("adapter_sessions", "idx_adapter_session_hash", "session_hash"),
`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_index_list('adapter_sessions') WHERE name='idx_adapter_session_hash' AND "unique"=1 AND partial=1)`,
schemaIndexPredicate("idx_adapter_session_hash", "session_hash != ''"),
}}},
"imports": {{Version: 1, Name: "adopt_imports", Statements: []string{
`CREATE TABLE IF NOT EXISTS import_sessions(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,actor_uuid TEXT NOT NULL DEFAULT '',archive_name TEXT NOT NULL,archive_path TEXT NOT NULL,target_prefix TEXT NOT NULL DEFAULT '/',status TEXT NOT NULL DEFAULT 'queued',total_files INTEGER NOT NULL DEFAULT 0,markdown_files INTEGER NOT NULL DEFAULT 0,error TEXT NOT NULL DEFAULT '',created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`,
`CREATE TABLE IF NOT EXISTS import_items(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,source_path TEXT NOT NULL,target_path TEXT NOT NULL,size INTEGER NOT NULL DEFAULT 0,content_type TEXT NOT NULL DEFAULT '',title TEXT NOT NULL DEFAULT '',suggested_quartier TEXT NOT NULL DEFAULT '',suggested_street TEXT NOT NULL DEFAULT '',suggested_spot TEXT NOT NULL DEFAULT '',confidence REAL NOT NULL DEFAULT 0,selected INTEGER NOT NULL DEFAULT 1,PRIMARY KEY(session_uuid,sequence))`,
`CREATE TABLE IF NOT EXISTS import_journal(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,path TEXT NOT NULL,object_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,sequence))`,
`CREATE TABLE IF NOT EXISTS import_city_journal(session_uuid TEXT NOT NULL,kind TEXT NOT NULL,node_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,kind,node_uuid))`,
`CREATE TABLE IF NOT EXISTS import_city_edges(session_uuid TEXT NOT NULL,spot_uuid TEXT NOT NULL,page_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,spot_uuid,page_uuid))`,
`CREATE INDEX IF NOT EXISTS idx_import_sessions_domain ON import_sessions(domain_uuid,created_at_ms DESC)`,
}, AddColumns: []db.SQLiteColumn{{Table: "import_journal", Name: "completed", Declaration: "INTEGER NOT NULL DEFAULT 0"}}, Assertions: []string{
schemaColumns("import_sessions", "uuid,domain_uuid,actor_uuid,archive_name,archive_path,target_prefix,status,total_files,markdown_files,error,created_at_ms,updated_at_ms"),
schemaColumns("import_items", "session_uuid,sequence,source_path,target_path,size,content_type,title,suggested_quartier,suggested_street,suggested_spot,confidence,selected"),
schemaColumns("import_journal", "session_uuid,sequence,path,object_uuid,created_at_ms,completed"),
schemaColumns("import_city_journal", "session_uuid,kind,node_uuid,created_at_ms"),
schemaColumns("import_city_edges", "session_uuid,spot_uuid,page_uuid,created_at_ms"),
schemaUnique("import_sessions", "uuid"),
schemaUnique("import_items", "session_uuid,sequence"),
schemaUnique("import_journal", "session_uuid,sequence"),
schemaUnique("import_city_journal", "session_uuid,kind,node_uuid"),
schemaUnique("import_city_edges", "session_uuid,spot_uuid,page_uuid"),
schemaAddedColumn("import_journal", "completed", "INTEGER", "0"),
schemaIndex("import_sessions", "idx_import_sessions_domain", "domain_uuid,created_at_ms"),
}}},
}
func (s *Store) ensureSchemaComponent(component string) error {
if s.ReadOnly {
return ErrReadOnly
}
steps, ok := managedStoreSchemas[component]
if !ok {
return fmt.Errorf("unknown schema component %q", component)
}
return db.ApplySQLiteMigrations(context.Background(), s.DB, component, steps)
}
// These helpers consume only static definitions above, never request input.
func schemaColumns(table, columns string) string {
return fmt.Sprintf("SELECT %s FROM %q LIMIT 0", columns, table)
}
func schemaUnique(table, columns string) string {
return fmt.Sprintf(`SELECT 1 WHERE NOT EXISTS (
SELECT 1 FROM pragma_index_list('%s') AS i WHERE i."unique"=1 AND i.partial=0
AND (SELECT group_concat(name,',') FROM (SELECT name FROM pragma_index_info(i.name) ORDER BY seqno))='%s')`, table, columns)
}
func schemaIndex(table, name, columns string) string {
return fmt.Sprintf(`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_index_list('%s') WHERE name='%s')
OR (SELECT group_concat(name,',') FROM (SELECT name FROM pragma_index_info('%s') ORDER BY seqno)) IS NOT '%s'`, table, name, name, strings.TrimSpace(columns))
}
func schemaAddedColumn(table, column, kind, defaultSQL string) string {
return fmt.Sprintf(`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_table_info('%s') WHERE name='%s' AND upper(type)='%s' AND "notnull"=1 AND dflt_value='%s')`, table, column, kind, strings.ReplaceAll(defaultSQL, "'", "''"))
}
func schemaIndexPredicate(name, predicate string) string {
normalized := strings.ToLower(strings.Join(strings.Fields(predicate), ""))
return fmt.Sprintf(`SELECT 1 WHERE NOT EXISTS (
SELECT 1 FROM (SELECT lower(replace(replace(replace(replace(sql,' ',''),char(9),''),char(10),''),char(13),'')) AS normalized FROM sqlite_master WHERE type='index' AND name='%s')
WHERE substr(normalized,instr(normalized,'where')+5)='%s')`, name, strings.ReplaceAll(normalized, "'", "''"))
}
+143
View File
@@ -0,0 +1,143 @@
package store
import (
"errors"
"path/filepath"
"testing"
"pageswiki/internal/db"
)
func newSchemaTestStore(t *testing.T) *Store {
t.Helper()
database, err := db.Open(filepath.Join(t.TempDir(), "schema.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { database.Close() })
return &Store{DB: database}
}
func TestManagedSchemasFreshRepeatAndReadOnly(t *testing.T) {
for component := range managedStoreSchemas {
t.Run(component, func(t *testing.T) {
st := newSchemaTestStore(t)
if err := st.ensureSchemaComponent(component); err != nil {
t.Fatal(err)
}
if err := st.ensureSchemaComponent(component); err != nil {
t.Fatal(err)
}
var count int
if err := st.DB.QueryRow(`SELECT count(*) FROM schema_migration_steps WHERE component=?`, component).Scan(&count); err != nil || count != len(managedStoreSchemas[component]) {
t.Fatalf("ledger=%d err=%v", count, err)
}
st.ReadOnly = true
if err := st.ensureSchemaComponent(component); !errors.Is(err, ErrReadOnly) {
t.Fatalf("standby migration: %v", err)
}
})
}
}
func TestManagedSchemasAdoptLegacyRowsWithoutRecreatingTables(t *testing.T) {
for _, tc := range []struct{ component, ddl, insert, query, want string }{
{"browser-navigation", `CREATE TABLE browser_navigation_links(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,principal_uuid TEXT NOT NULL,label TEXT NOT NULL,path TEXT NOT NULL,position INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL,UNIQUE(domain_uuid,principal_uuid,path))`,
`INSERT INTO browser_navigation_links VALUES('u','d','p','keep','/page',3,10,20)`, `SELECT label||':'||position||':'||is_directory FROM browser_navigation_links WHERE uuid='u'`, "keep:3:0"},
{"adapter-sessions", `CREATE TABLE adapter_sessions(ticket_hash TEXT PRIMARY KEY,session_hash TEXT NOT NULL DEFAULT '',domain_uuid TEXT NOT NULL,plugin_id TEXT NOT NULL,object_uuid TEXT NOT NULL,base_revision_uuid TEXT NOT NULL,ticket_expires_at_ms INTEGER NOT NULL,session_expires_at_ms INTEGER NOT NULL DEFAULT 0,exchanged_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL)`,
`INSERT INTO adapter_sessions VALUES('ticket','session','d','code','o','r',100,200,50,10)`, `SELECT session_hash||':'||root_path||':'||principal_uuid FROM adapter_sessions WHERE ticket_hash='ticket'`, "session::"},
{"imports", `CREATE TABLE import_journal(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,path TEXT NOT NULL,object_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,sequence))`,
`INSERT INTO import_journal VALUES('s',2,'/keep','o',20)`, `SELECT path||':'||completed FROM import_journal WHERE session_uuid='s' AND sequence=2`, "/keep:0"},
{"resource-history", `CREATE TABLE resource_samples(id INTEGER PRIMARY KEY,job_uuid TEXT NOT NULL UNIQUE,timestamp_ms INTEGER NOT NULL,payload TEXT NOT NULL)`,
`INSERT INTO resource_samples VALUES(9,'j',100,'keep')`, `SELECT id||':'||payload FROM resource_samples WHERE job_uuid='j'`, "9:keep"},
{"projection-metadata", `CREATE TABLE projection_metadata(document_hash TEXT NOT NULL,kind TEXT NOT NULL,version INTEGER NOT NULL,payload TEXT NOT NULL DEFAULT '{}',error TEXT NOT NULL DEFAULT '',updated_at_ms INTEGER NOT NULL,retry_at_ms INTEGER NOT NULL DEFAULT 0,PRIMARY KEY(document_hash,kind,version))`,
`INSERT INTO projection_metadata VALUES('h','pdf',1,'keep','',10,0)`, `SELECT payload FROM projection_metadata WHERE document_hash='h'`, "keep"},
{"broadcasts", `CREATE TABLE broadcasts(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,title TEXT NOT NULL,body TEXT NOT NULL,severity TEXT NOT NULL DEFAULT 'info' CHECK(severity IN ('info','success','warning','critical')),starts_at_ms INTEGER NOT NULL DEFAULT 0,ends_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`,
`INSERT INTO broadcasts VALUES('u','d','title','keep','info',0,0,10,20)`, `SELECT body FROM broadcasts WHERE uuid='u'`, "keep"},
} {
t.Run(tc.component, func(t *testing.T) {
st := newSchemaTestStore(t)
for _, statement := range []string{tc.ddl, tc.insert} {
if _, err := st.DB.Exec(statement); err != nil {
t.Fatal(err)
}
}
if err := st.ensureSchemaComponent(tc.component); err != nil {
t.Fatal(err)
}
var value string
if err := st.DB.QueryRow(tc.query).Scan(&value); err != nil || value != tc.want {
t.Fatalf("row=%q want=%q err=%v", value, tc.want, err)
}
if err := st.ensureSchemaComponent(tc.component); err != nil {
t.Fatal(err)
}
})
}
}
func TestManagedSchemasRejectBrokenAdoptionAndLaterDrift(t *testing.T) {
st := newSchemaTestStore(t)
// Matching columns without the conflict-target uniqueness is not sufficient.
if _, err := st.DB.Exec(`CREATE TABLE resource_samples(id INTEGER PRIMARY KEY,job_uuid TEXT NOT NULL,timestamp_ms INTEGER NOT NULL,payload TEXT NOT NULL); INSERT INTO resource_samples VALUES(1,'j',10,'keep')`); err != nil {
t.Fatal(err)
}
if err := st.EnsureResourceHistorySchema(); err == nil {
t.Fatal("missing unique constraint accepted")
}
var payload string
if err := st.DB.QueryRow(`SELECT payload FROM resource_samples WHERE id=1`).Scan(&payload); err != nil || payload != "keep" {
t.Fatal("failed adoption modified data", err)
}
var count int
if err := st.DB.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='idx_resource_samples_time'`).Scan(&count); err != nil || count != 0 {
t.Fatal("failed adoption left partial DDL", err)
}
if _, err := st.DB.Exec(`CREATE UNIQUE INDEX restored_resource_unique ON resource_samples(job_uuid)`); err != nil {
t.Fatal(err)
}
if err := st.EnsureResourceHistorySchema(); err != nil {
t.Fatal(err)
}
if _, err := st.DB.Exec(`DROP INDEX idx_resource_samples_time`); err != nil {
t.Fatal(err)
}
if err := st.EnsureResourceHistorySchema(); err == nil {
t.Fatal("ledger masked missing index")
}
if err := st.EnsureAdapterSessionsSchema(); err != nil {
t.Fatal(err)
}
if _, err := st.DB.Exec(`DROP INDEX idx_adapter_session_hash; CREATE UNIQUE INDEX idx_adapter_session_hash ON adapter_sessions(session_hash) WHERE session_hash = ''`); err != nil {
t.Fatal(err)
}
if err := st.EnsureAdapterSessionsSchema(); err == nil {
t.Fatal("incorrect partial index predicate accepted")
}
}
func TestEventJobsAdoptLegacyDueIndex(t *testing.T) {
st := newSchemaTestStore(t)
// The deployed pre-ledger scheduler used the same index name without priority.
for _, statement := range []string{
eventJobMigrations[0].Statements[0],
`CREATE INDEX idx_event_jobs_due ON event_jobs(status,run_at_ms,created_at_ms)`,
`INSERT INTO event_jobs(uuid,event_type,event_key,payload,status,attempts,run_at_ms,created_at_ms,updated_at_ms) VALUES('keep','test','key','unchanged','running',2,10,20,30)`,
} {
if _, err := st.DB.Exec(statement); err != nil {
t.Fatal(err)
}
}
for range 2 {
if err := st.ensureSchemaComponent("event-jobs"); err != nil {
t.Fatal(err)
}
}
var columns, row string
if err := st.DB.QueryRow(`SELECT group_concat(name,',') FROM (SELECT name FROM pragma_index_info('idx_event_jobs_due') ORDER BY seqno)`).Scan(&columns); err != nil || columns != "status,priority,run_at_ms,created_at_ms" {
t.Fatalf("index=%q err=%v", columns, err)
}
if err := st.DB.QueryRow(`SELECT payload||':'||status||':'||attempts||':'||run_at_ms||':'||created_at_ms||':'||updated_at_ms FROM event_jobs WHERE uuid='keep'`).Scan(&row); err != nil || row != "unchanged:running:2:10:20:30" {
t.Fatalf("adoption changed job: %q err=%v", row, err)
}
}
+91
View File
@@ -0,0 +1,91 @@
package store
import (
"database/sql"
"encoding/json"
"testing"
"pageswiki/internal/schutzaria"
)
func schutzariaStoreFixture(t *testing.T) *Store {
t.Helper()
db, err := sql.Open("sqlite", ":memory:")
if err != nil {
t.Fatal(err)
}
db.SetMaxOpenConns(1)
t.Cleanup(func() { db.Close() })
s := &Store{DB: db}
if err = s.EnsureAuthSchema(); err != nil {
t.Fatal(err)
}
for _, stmt := range []string{
`INSERT INTO principals(uuid,state,created_at_ms,updated_at_ms) VALUES('p','active',1,1)`,
`INSERT INTO domain_memberships(uuid,domain_uuid,principal_uuid,state,created_at_ms) VALUES('m','d','p','active',1)`,
`INSERT INTO auth_roles(uuid,domain_uuid,name,created_at_ms) VALUES('r','d','reader',1)`,
`INSERT INTO auth_role_bindings(membership_uuid,role_uuid,created_at_ms) VALUES('m','r',1)`,
} {
if _, err = db.Exec(stmt); err != nil {
t.Fatal(err)
}
}
return s
}
func TestSchutzariaReadLoaderBudgetsAndErrors(t *testing.T) {
for _, kind := range []string{"grant", "locality", "db-error"} {
t.Run(kind, func(t *testing.T) {
s := schutzariaStoreFixture(t)
if _, err := s.DB.Exec(`INSERT INTO auth_capability_grants(uuid,role_uuid,capability,target_type,target_id,created_at_ms) VALUES('all','r','read','domain','d',1)`); err != nil {
t.Fatal(err)
}
switch kind {
case "grant":
_, err := s.DB.Exec(`WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<4096) INSERT INTO auth_capability_grants(uuid,role_uuid,capability,target_type,target_id,created_at_ms) SELECT 'grant-'||x,'r','read','path','/page/'||x,1 FROM n`)
if err != nil {
t.Fatal(err)
}
case "locality":
_, err := s.DB.Exec(`WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<4097) INSERT INTO locality_access_policies(domain_uuid,node_kind,node_uuid,scope_path,grants_access,actor_uuid,authority_rank) SELECT 'd','spot','spot-'||x,'/scope/'||x,0,'p',3 FROM n`)
if err != nil {
t.Fatal(err)
}
case "db-error":
if _, err := s.DB.Exec(`DROP TABLE locality_access_policies`); err != nil {
t.Fatal(err)
}
}
policy, err := s.LoadReadPathPolicy("d", "p")
if err == nil || policy != nil || policy.Allows("/would-be-allowed") {
t.Fatal("incomplete ACL snapshot escaped", policy, err)
}
})
}
}
func TestSchutzariaStoreCompatibility(t *testing.T) {
var policy *ReadPathPolicy = schutzaria.NewReadPathPolicy(false, []string{"/exact"}, nil, nil)
if !policy.Allows("/exact") || policy.Allows("/exact/child") {
t.Fatal("alias mismatch")
}
var locality LocalityPolicy = schutzaria.LocalityPolicy{Kind: "spot", UUID: "id", Policy: "public"}
data, err := json.Marshal(locality)
if err != nil || string(data) != `{"kind":"spot","uuid":"id","policy":"public"}` {
t.Fatal(string(data), err)
}
var user UserSpacePolicy = schutzaria.UserSpacePolicy{Username: "alice", RootVisibility: "private", DescendantVisibility: "domain"}
for _, p := range []string{"/u/alice", "/u/alice/page", "/u/bob"} {
if EffectiveUserSpaceVisibility(user, p) != schutzaria.EffectiveUserSpaceVisibility(user, p) {
t.Fatal("user policy wrapper mismatch")
}
}
if ErrApprovalTransition != schutzaria.ErrApprovalTransition {
t.Fatal("error identity changed")
}
for _, p := range []string{"/public", "/u/private", "/v/tree"} {
if PublicSemanticPath(p) != schutzaria.PublicSemanticPath(p) {
t.Fatal("publication wrapper mismatch")
}
}
}
+16 -360
View File
@@ -2,378 +2,34 @@ package store
import (
"context"
"database/sql"
"errors"
"sort"
"strings"
"time"
"pageswiki/internal/mestionora"
)
const SearchCandidateLimit = 1000
const SearchExcerptDocumentBytes = 64 << 10
const SearchExcerptTotalBytes = 1 << 20
const searchExcerptTimeout = 400 * time.Millisecond
type SearchResult = mestionora.SearchResult
type PathSuggestion = mestionora.PathSuggestion
const searchResultColumns = `p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.revision_uuid,''),COALESCE(r.created_at_ms/1000,0)`
const searchPathBoundary = `(p.path=? OR (p.path>=? AND p.path<?))`
const searchPathCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,0
FROM paths p LEFT JOIN latest_revision_index r ON r.object_uuid=p.object_uuid
WHERE p.domain_uuid=? AND ` + searchPathBoundary
const searchPathQueryPredicate = ` AND (instr(lower(p.path),lower(?))>0 OR instr(lower(COALESCE(r.filename,'')),lower(?))>0)`
const searchPathCandidatesSQL = searchPathCandidatesBaseSQL + searchPathQueryPredicate + ` ORDER BY p.path LIMIT ?`
const (
SearchCandidateLimit = mestionora.SearchCandidateLimit
SearchExcerptDocumentBytes = mestionora.SearchExcerptDocumentBytes
SearchExcerptTotalBytes = mestionora.SearchExcerptTotalBytes
)
// CROSS JOIN is intentional: revision_uuid is an UNINDEXED FTS payload column.
// A Path-first plan re-runs the complete MATCH scan for every Path. Start with
// MATCH exactly once, then seek current Revision metadata and Domain bindings
// through their B-tree indexes. Keep scope filtering before the candidate cap.
const searchFTSCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,revision_search.rowid
FROM revision_search
CROSS JOIN latest_revision_index r ON r.revision_uuid=revision_search.revision_uuid
CROSS JOIN paths p ON p.object_uuid=r.object_uuid
WHERE revision_search MATCH ? AND p.domain_uuid=? AND ` + searchPathBoundary
const searchFTSCandidatesSQL = searchFTSCandidatesBaseSQL + ` LIMIT ?`
func searchBodyExcerptsSQL(count int) string {
// UUID is not indexed by FTS5. Restrict the MATCH/snippet work using the
// integer rowids collected from the same read snapshot, not payload UUIDs.
return `SELECT revision_uuid,substr(snippet(revision_search,3,'','',' … ',36),1,4096)
FROM revision_search WHERE revision_search MATCH ? AND rowid IN (` + strings.TrimRight(strings.Repeat("?,", count), ",") + `)`
}
// Excerpts are optional enrichment. Even small documents can have expensive
// prefix hit-position scans, so their time budget is separate from finding
// results. A spent excerpt budget omits snippets, not already-authorized hits.
func loadSearchBodyExcerpts(parent context.Context, tx *sql.Tx, expression string, ids []any, budget time.Duration) (excerpts map[string]string, err error) {
ctx, cancel := context.WithTimeout(parent, budget)
defer cancel()
defer func() {
if err != nil && ctx.Err() != nil {
if parent.Err() != nil {
err = parent.Err()
} else {
excerpts, err = nil, nil
}
}
}()
args := append([]any{"body : (" + expression + ")"}, ids...)
rows, err := tx.QueryContext(ctx, searchBodyExcerptsSQL(len(ids)), args...)
if err != nil {
return nil, err
}
defer rows.Close()
excerpts = make(map[string]string)
for rows.Next() {
var id, excerpt string
if err := rows.Scan(&id, &excerpt); err != nil {
return nil, err
}
excerpts[id] = excerpt
}
return excerpts, rows.Err()
}
type SearchResult struct {
Path string
ContentType string
Filename string
DocumentHash string
RevisionUUID string
Size int64 // Kept for API compatibility; search does not stat Document files.
UpdatedAt int64
PathMatch bool
FilenameMatch bool
BodyMatch bool
Excerpt string // Untrusted source text, never HTML-safe markup.
ExcerptOmitted bool // A body match remains visible when snippet work exceeds its budget.
}
type PathSuggestion struct {
SearchResult
Title string
}
// SearchPathsInDomain preserves the store API for trusted callers. HTTP callers
// must use SearchPathsInDomainFiltered with a request-scoped authorization filter.
// SearchPathsInDomain is the legacy trusted-caller entry point. HTTP callers
// must pass their request policy to the filtered APIs; there is no implicit
// allow-all entry point in Mestionora itself.
func (s *Store) SearchPathsInDomain(domainUUID, scope, query string, limit int) ([]SearchResult, error) {
return s.SearchPathsInDomainFiltered(context.Background(), domainUUID, scope, query, limit, func(string) bool { return true })
}
// SearchPathsInDomainFiltered bounds SQL candidates before materialization and
// authorizes each Path before loading an excerpt. It never computes a global hit
// count or stats Document files. A narrow scope improves both recall and cost.
func (s *Store) SearchPathsInDomainFiltered(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]SearchResult, error) {
return s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{})
return mestionora.NewSQLite(s.DB).SearchPathsInDomainFiltered(ctx, domainUUID, scope, query, limit, allowed)
}
// SearchPathsInDomainWithOptions also permits filter-only searches. File/date
// predicates apply before SQL candidate caps, while ACL checks stay ahead of all
// snippet enrichment. Zero options preserve the original indexed query plans.
func (s *Store) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, searchErr error) {
return s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options, true)
func (s *Store) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) ([]SearchResult, error) {
return mestionora.NewSQLite(s.DB).SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options)
}
// SearchPathSuggestionsInDomain shares the current-Revision FTS and ACL path
// with full search, but deliberately skips snippet generation. Typeahead calls
// are frequent and only need a bounded display label, never a body excerpt.
func (s *Store) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) {
results, err := s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}, false)
if err != nil || len(results) == 0 {
return nil, err
}
ids := make([]any, 0, len(results))
seen := make(map[string]bool, len(results))
for _, result := range results {
if result.RevisionUUID != "" && strings.HasPrefix(strings.ToLower(result.ContentType), "text/html") && !seen[result.RevisionUUID] {
ids = append(ids, result.RevisionUUID)
seen[result.RevisionUUID] = true
}
}
headings := make(map[string]string, len(ids))
if len(ids) > 0 {
rows, queryErr := s.DB.QueryContext(ctx, `SELECT uuid,substr(CAST(body AS TEXT),1,65536) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
if queryErr != nil {
return nil, queryErr
}
for rows.Next() {
var revision, source string
if queryErr = rows.Scan(&revision, &source); queryErr != nil {
rows.Close()
return nil, queryErr
}
headings[revision] = firstHTMLHeading(source)
}
queryErr = rows.Err()
rows.Close()
if queryErr != nil {
return nil, queryErr
}
}
out := make([]PathSuggestion, 0, len(results))
for _, result := range results {
out = append(out, PathSuggestion{SearchResult: result, Title: headings[result.RevisionUUID]})
}
return out, nil
}
func (s *Store) searchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions, loadExcerpts bool) (found []SearchResult, searchErr error) {
if err := options.Validate(); err != nil {
return nil, err
}
scope, query = normalizePath(scope), strings.TrimSpace(query)
if query == "" && !options.HasFilters() || allowed == nil {
return nil, nil
}
if len(query) > 512 || len(strings.Fields(query)) > 16 {
return nil, errors.New("search query is too long")
}
if limit <= 0 || limit > 200 {
limit = 100
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
defer func() {
// SQLite can report an interrupted statement instead of wrapping the
// context error. Preserve cancellation/deadline semantics for HTTP.
if searchErr != nil && ctx.Err() != nil {
searchErr = ctx.Err()
}
}()
tx, err := s.DB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return nil, err
}
defer tx.Rollback()
prefix := strings.TrimRight(scope, "/") + "/"
upper := strings.TrimRight(scope, "/") + "0"
results := make(map[string]SearchResult)
ftsRows := make(map[string]int64)
collect := func(statement string, body, filename bool, args ...any) error {
rows, err := tx.QueryContext(ctx, statement, args...)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var result SearchResult
var rowID int64
if err := rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt, &rowID); err != nil {
return err
}
if !allowed(result.Path) {
continue
}
if rowID != 0 {
ftsRows[result.RevisionUUID] = rowID
}
result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query))
result.FilenameMatch = filename || results[result.Path].FilenameMatch || query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query))
result.BodyMatch = body || results[result.Path].BodyMatch
results[result.Path] = result
}
return rows.Err()
}
filterSQL, filterArgs := searchOptionsPredicate(options)
pathSQL := searchPathCandidatesBaseSQL
pathArgs := []any{domainUUID, scope, prefix, upper}
if query != "" {
pathSQL += searchPathQueryPredicate
pathArgs = append(pathArgs, query, query)
}
pathSQL += filterSQL + ` ORDER BY p.path LIMIT ?`
pathArgs = append(pathArgs, filterArgs...)
pathArgs = append(pathArgs, SearchCandidateLimit)
if err := collect(pathSQL, false, false, pathArgs...); err != nil {
return nil, err
}
expression := ftsPrefixExpression(query)
if expression != "" {
// No global ORDER BY or unbounded IN materialization: stop the FTS scan
// after at most this many current, in-Domain, in-scope Path candidates.
for _, field := range []string{"body", "filename"} {
args := []any{field + " : (" + expression + ")", domainUUID, scope, prefix, upper}
args = append(args, filterArgs...)
args = append(args, SearchCandidateLimit)
if err := collect(searchFTSCandidatesBaseSQL+filterSQL+` LIMIT ?`, field == "body", field == "filename", args...); err != nil {
return nil, err
}
}
}
out := make([]SearchResult, 0, len(results))
for _, result := range results {
out = append(out, result)
}
sort.Slice(out, func(i, j int) bool {
// Direct name/Path matches precede body-only hits; all ties are stable.
iName, jName := out[i].PathMatch || out[i].FilenameMatch, out[j].PathMatch || out[j].FilenameMatch
if iName != jName {
return iName
}
if len(out[i].Path) != len(out[j].Path) {
return len(out[i].Path) < len(out[j].Path)
}
return out[i].Path < out[j].Path
})
if len(out) > limit {
out = out[:limit]
}
if !loadExcerpts {
return out, nil
}
// Only authorized, displayed results receive text excerpts. Batch the FTS
// lookup so a broad query is not re-scanned once for every visible result.
ids := make([]any, 0, len(out))
seen := make(map[string]bool)
for _, result := range out {
if result.BodyMatch && !seen[result.RevisionUUID] {
ids = append(ids, ftsRows[result.RevisionUUID])
seen[result.RevisionUUID] = true
}
}
if len(ids) != 0 {
// snippet's output token/character limit does not bound its input work:
// FTS5 may tokenize the entire matched document. Read byte lengths only
// for authorized displayed rows before spending the snippet budget.
rows, err := tx.QueryContext(ctx, `SELECT rowid,octet_length(body)
FROM revision_search WHERE rowid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
if err != nil {
return nil, err
}
sizes := make(map[int64]int64)
for rows.Next() {
var rowID, size int64
if err := rows.Scan(&rowID, &size); err != nil {
rows.Close()
return nil, err
}
sizes[rowID] = size
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
ids = nil
selected := make(map[string]bool)
remaining := int64(SearchExcerptTotalBytes)
for i := range out {
if !out[i].BodyMatch {
continue
}
revision := out[i].RevisionUUID
include, decided := selected[revision]
if !decided {
rowID := ftsRows[revision]
size, exists := sizes[rowID]
include = exists && size >= 0 && size <= SearchExcerptDocumentBytes && size <= remaining
if include {
ids = append(ids, rowID)
remaining -= size
}
selected[revision] = include
}
out[i].ExcerptOmitted = !include
}
}
if len(ids) != 0 {
excerpts, err := loadSearchBodyExcerpts(ctx, tx, expression, ids, searchExcerptTimeout)
if err != nil {
return nil, err
}
for i := range out {
if out[i].BodyMatch {
var present bool
out[i].Excerpt, present = excerpts[out[i].RevisionUUID]
out[i].ExcerptOmitted = !present
}
}
}
// A filename/Path-only hit still benefits from a short Page summary. Read
// only a bounded prefix from displayed textual revisions, in one batch.
ids, seen = nil, make(map[string]bool)
for _, result := range out {
media := strings.ToLower(result.ContentType)
textual := strings.HasPrefix(media, "text/") || strings.HasPrefix(media, "application/json") || strings.HasPrefix(media, "application/xml") || strings.HasPrefix(media, "application/javascript") || strings.HasPrefix(media, "application/yaml") || strings.HasPrefix(media, "application/x-yaml")
if !result.BodyMatch && textual && result.RevisionUUID != "" && !seen[result.RevisionUUID] {
ids = append(ids, result.RevisionUUID)
seen[result.RevisionUUID] = true
}
}
if len(ids) != 0 {
rows, err := tx.QueryContext(ctx, `SELECT uuid,substr(body,1,1024) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
if err != nil {
return nil, err
}
excerpts := make(map[string]string)
for rows.Next() {
var id, excerpt string
if err := rows.Scan(&id, &excerpt); err != nil {
rows.Close()
return nil, err
}
excerpts[id] = excerpt
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
for i := range out {
if !out[i].BodyMatch {
out[i].Excerpt = excerpts[out[i].RevisionUUID]
}
}
}
return out, nil
}
func ftsPrefixExpression(query string) string {
words := strings.Fields(query)
terms := make([]string, 0, len(words))
for _, word := range words {
word = strings.ReplaceAll(word, `"`, `""`)
if word != "" {
terms = append(terms, `"`+word+`"*`)
}
}
return strings.Join(terms, " AND ")
return mestionora.NewSQLite(s.DB).SearchPathSuggestionsInDomain(ctx, domainUUID, scope, query, limit, allowed)
}
+3 -131
View File
@@ -1,135 +1,7 @@
package store
import (
"errors"
"fmt"
"strings"
)
import "pageswiki/internal/mestionora"
var ErrInvalidSearchOptions = errors.New("invalid search options")
type SearchOptions = mestionora.SearchOptions
// SearchOptions applies to current Page metadata, not historical Revisions.
// Dates are Unix milliseconds: FromMS is inclusive and BeforeMS is exclusive.
// The created field refers to Object creation, not creation of a Path alias.
// FileType and Extension are normalized lower-case values; Extension has no dot
// prefix and may contain a compound suffix such as tar.gz.
type SearchOptions struct {
FileType string
Extension string
DateField string
FromMS *int64
BeforeMS *int64
}
func (o SearchOptions) HasFilters() bool {
return o.FileType != "" || o.Extension != "" || o.FromMS != nil || o.BeforeMS != nil
}
func (o SearchOptions) Validate() error {
switch o.FileType {
case "", "html", "text", "image", "audio", "video", "pdf", "archive", "csv", "json", "code", "office", "other":
default:
return fmt.Errorf("%w: unknown file type", ErrInvalidSearchOptions)
}
switch o.DateField {
case "", "updated", "created":
default:
return fmt.Errorf("%w: unknown date field", ErrInvalidSearchOptions)
}
if len(o.Extension) > 32 {
return fmt.Errorf("%w: extension is too long", ErrInvalidSearchOptions)
}
for i, c := range o.Extension {
alnum := c >= 'a' && c <= 'z' || c >= '0' && c <= '9'
if !alnum && (i == 0 || !strings.ContainsRune("._+-", c)) {
return fmt.Errorf("%w: invalid extension", ErrInvalidSearchOptions)
}
}
if o.FromMS != nil && o.BeforeMS != nil && *o.FromMS >= *o.BeforeMS {
return fmt.Errorf("%w: empty or reversed date range", ErrInvalidSearchOptions)
}
return nil
}
const searchMIMESQL = `lower(trim(substr(COALESCE(r.content_type,''),1,instr(COALESCE(r.content_type,'')||';',';')-1)))`
// The original filename wins over the Path suffix when present. Names remain
// data: even extension patterns are bound arguments and do not become SQL.
const searchFilenameSQL = `lower(CASE WHEN COALESCE(r.filename,'')<>'' THEN r.filename ELSE p.path END)`
func searchSuffixesSQL(suffixes ...string) string {
clauses := make([]string, 0, len(suffixes))
for _, suffix := range suffixes {
// Only package-owned ASCII constants reach this helper. GLOB has no
// escaping ambiguity for underscores, unlike LIKE.
clauses = append(clauses, searchFilenameSQL+` GLOB '*.`+suffix+`'`)
}
return "(" + strings.Join(clauses, " OR ") + ")"
}
// Specific MIME types are authoritative. Generic text/binary types may use a
// known suffix (e.g. a Python source uploaded as text/plain); unrecognized,
// specific MIME types stay Other instead of pretending to be a different type.
// This uses metadata only and never hydrates a Document to sniff its bytes.
var searchFileTypeSQL = `CASE
WHEN ` + searchMIMESQL + ` IN ('text/html','application/xhtml+xml') THEN 'html'
WHEN ` + searchMIMESQL + `='application/pdf' THEN 'pdf'
WHEN ` + searchMIMESQL + ` LIKE 'image/%' THEN 'image'
WHEN ` + searchMIMESQL + ` LIKE 'audio/%' THEN 'audio'
WHEN ` + searchMIMESQL + ` LIKE 'video/%' THEN 'video'
WHEN ` + searchMIMESQL + ` IN ('text/csv','application/csv','text/tab-separated-values') THEN 'csv'
WHEN ` + searchMIMESQL + ` IN ('application/json','text/json') OR ` + searchMIMESQL + ` GLOB '*+json' THEN 'json'
WHEN ` + searchMIMESQL + ` IN ('application/zip','application/x-zip-compressed','application/x-7z-compressed','application/vnd.rar','application/x-rar','application/x-rar-compressed','application/x-tar','application/gzip','application/x-gzip','application/x-xz','application/x-bzip2','application/zstd','application/x-zstd','application/x-compress','application/x-cpio') THEN 'archive'
WHEN ` + searchMIMESQL + ` IN ('application/msword','application/vnd.ms-excel','application/vnd.ms-powerpoint','application/rtf','text/rtf') OR ` + searchMIMESQL + ` GLOB 'application/vnd.openxmlformats-officedocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.oasis.opendocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.ms-*.macroenabled.*' THEN 'office'
WHEN ` + searchMIMESQL + ` IN ('text/markdown','text/x-markdown') THEN 'text'
WHEN ` + searchMIMESQL + ` IN ('application/javascript','application/x-javascript','text/javascript','application/typescript','text/typescript','application/xml','text/xml','application/yaml','application/x-yaml','text/yaml','application/toml','application/x-sh','application/x-shellscript','application/x-python','application/x-python-code','application/x-httpd-php','application/sql','text/css') OR ` + searchMIMESQL + ` GLOB 'text/x-*' OR ` + searchMIMESQL + ` GLOB '*+xml' THEN 'code'
WHEN ` + searchMIMESQL + ` NOT IN ('','text/plain','application/octet-stream','binary/octet-stream','application/x-empty') THEN CASE WHEN ` + searchMIMESQL + ` LIKE 'text/%' THEN 'text' ELSE 'other' END
WHEN ` + searchSuffixesSQL("html", "htm", "xhtml") + ` THEN 'html'
WHEN ` + searchSuffixesSQL("pdf") + ` THEN 'pdf'
WHEN ` + searchSuffixesSQL("png", "jpg", "jpeg", "gif", "webp", "avif", "bmp", "svg", "tif", "tiff", "ico", "heic", "heif") + ` THEN 'image'
WHEN ` + searchSuffixesSQL("mp3", "flac", "wav", "ogg", "oga", "opus", "m4a", "aac", "aiff", "alac", "mid", "midi") + ` THEN 'audio'
WHEN ` + searchSuffixesSQL("mp4", "webm", "mkv", "mov", "avi", "m4v", "ogv", "mpeg", "mpg") + ` THEN 'video'
WHEN ` + searchSuffixesSQL("zip", "7z", "rar", "tar", "gz", "tgz", "xz", "txz", "bz2", "tbz", "tbz2", "zst", "tzst", "cbz", "cbr", "cpio") + ` THEN 'archive'
WHEN ` + searchSuffixesSQL("csv", "tsv") + ` THEN 'csv'
WHEN ` + searchSuffixesSQL("json", "jsonl", "ndjson", "excalidraw", "ipynb") + ` THEN 'json'
WHEN ` + searchSuffixesSQL("doc", "docx", "docm", "xls", "xlsx", "xlsm", "ppt", "pptx", "pptm", "odt", "ods", "odp", "rtf") + ` THEN 'office'
WHEN ` + searchSuffixesSQL("py", "pyw", "sh", "bash", "zsh", "fish", "ps1", "bat", "cmd", "js", "mjs", "cjs", "jsx", "ts", "tsx", "go", "rs", "c", "h", "cpp", "hpp", "cc", "cs", "java", "kt", "kts", "swift", "rb", "php", "pl", "pm", "lua", "r", "sql", "css", "scss", "sass", "less", "xml", "xsl", "yaml", "yml", "toml", "ini", "conf", "vue", "svelte", "dart", "ex", "exs", "erl", "hs", "scala", "clj", "lisp", "el", "vim", "wasm") + ` THEN 'code'
WHEN ` + searchMIMESQL + `='text/plain' OR ` + searchSuffixesSQL("txt", "text", "log", "md", "markdown", "rst", "adoc", "org") + ` THEN 'text'
ELSE 'other' END`
// The returned predicate is inserted before each candidate stream's LIMIT,
// including FTS-first streams. Filtering after their cap silently loses hits.
func searchOptionsPredicate(options SearchOptions) (string, []any) {
var predicate strings.Builder
var args []any
if options.FromMS != nil || options.BeforeMS != nil {
dateSQL := "r.created_at_ms"
if options.DateField == "created" {
// Objects.uuid is a primary-key lookup. Avoid joining Objects in the
// normal search plan when Object creation is not being filtered.
predicate.WriteString(" AND EXISTS (SELECT 1 FROM objects o WHERE o.uuid=p.object_uuid")
dateSQL = "o.created_at*1000"
}
if options.FromMS != nil {
predicate.WriteString(" AND " + dateSQL + ">=?")
args = append(args, *options.FromMS)
}
if options.BeforeMS != nil {
predicate.WriteString(" AND " + dateSQL + "<?")
args = append(args, *options.BeforeMS)
}
if options.DateField == "created" {
predicate.WriteString(")")
}
}
// Evaluate inexpensive date/suffix exclusions before the metadata category.
if options.Extension != "" {
predicate.WriteString(" AND " + searchFilenameSQL + " GLOB ?")
args = append(args, "*."+options.Extension)
}
if options.FileType != "" {
predicate.WriteString(" AND (" + searchFileTypeSQL + ")=?")
args = append(args, options.FileType)
}
return predicate.String(), args
}
var ErrInvalidSearchOptions = mestionora.ErrInvalidSearchOptions
+2 -206
View File
@@ -1,211 +1,7 @@
package store
import "database/sql"
import "pageswiki/internal/mestionora"
// EnsureSearchIndexSchema installs an FTS5 index over immutable Revisions.
// Paths stay outside the index because links can be renamed independently; the
// search query joins matching latest Revisions to the current domain Paths.
func (s *Store) EnsureSearchIndexSchema() error {
var revisionsTable string
if err := s.DB.QueryRow(`
SELECT name FROM sqlite_master
WHERE type = 'table' AND name = 'revisions'
`).Scan(&revisionsTable); err != nil {
if err == sql.ErrNoRows {
return nil
}
return err
}
if _, err := s.DB.Exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS revision_search USING fts5(
revision_uuid UNINDEXED,
object_uuid UNINDEXED,
filename,
body,
tokenize = 'unicode61'
)
`); err != nil {
return err
}
if _, err := s.DB.Exec(`
CREATE TABLE IF NOT EXISTS revision_search_state (
revision_uuid TEXT PRIMARY KEY
)
`); err != nil {
return err
}
if _, err := s.DB.Exec(`
CREATE TABLE IF NOT EXISTS latest_revision_index (
object_uuid TEXT PRIMARY KEY,
revision_uuid TEXT NOT NULL,
content_type TEXT NOT NULL,
filename TEXT NOT NULL,
document_hash TEXT NOT NULL,
created_at_ms INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS search_schema_state (
name TEXT PRIMARY KEY
);
CREATE INDEX IF NOT EXISTS idx_latest_revision_search_uuid ON latest_revision_index(revision_uuid)
`); err != nil {
return err
}
var latestBackfilled int
if err := s.DB.QueryRow(`
SELECT EXISTS(
SELECT 1 FROM search_schema_state
WHERE name = 'latest_revision_index_v1'
)
`).Scan(&latestBackfilled); err != nil {
return err
}
if latestBackfilled == 0 {
if _, err := s.DB.Exec(`
INSERT OR REPLACE INTO latest_revision_index (
object_uuid, revision_uuid, content_type, filename,
document_hash, created_at_ms
)
SELECT
object_uuid, uuid, COALESCE(content_type, ''),
COALESCE(filename, ''), COALESCE(document_hash, ''),
COALESCE(created_at_ms, created_at * 1000, 0)
FROM (
SELECT
uuid, object_uuid, content_type, filename,
document_hash, created_at_ms, created_at,
ROW_NUMBER() OVER (
PARTITION BY object_uuid
ORDER BY COALESCE(created_at_ms, created_at * 1000) DESC,
rowid DESC
) AS rn
FROM revisions
)
WHERE rn = 1;
INSERT OR IGNORE INTO search_schema_state (name)
VALUES ('latest_revision_index_v1');
`); err != nil {
return err
}
}
// Populate the cheap membership table once for installations that already
// created revision_search before revision_search_state was introduced.
var stateExists int
if err := s.DB.QueryRow(`
SELECT EXISTS(SELECT 1 FROM revision_search_state LIMIT 1)
`).Scan(&stateExists); err != nil {
return err
}
if stateExists == 0 {
if _, err := s.DB.Exec(`
INSERT OR IGNORE INTO revision_search_state (revision_uuid)
SELECT revision_uuid FROM revision_search
`); err != nil {
return err
}
}
// Trigger definitions evolve with the projection schema; recreate them so
// existing installations receive the current maintenance logic.
if _, err := s.DB.Exec(`
DROP TRIGGER IF EXISTS revision_search_insert;
DROP TRIGGER IF EXISTS revision_search_delete;
`); err != nil {
return err
}
if _, err := s.DB.Exec(`
CREATE TRIGGER IF NOT EXISTS revision_search_insert
AFTER INSERT ON revisions BEGIN
INSERT INTO revision_search (
revision_uuid, object_uuid, filename, body
) VALUES (
new.uuid,
new.object_uuid,
COALESCE(new.filename, ''),
CASE
WHEN lower(COALESCE(new.content_type, '')) LIKE 'text/%'
OR lower(COALESCE(new.content_type, '')) LIKE 'application/json%'
OR lower(COALESCE(new.content_type, '')) LIKE 'application/xml%'
OR lower(COALESCE(new.content_type, '')) LIKE 'application/javascript%'
OR lower(COALESCE(new.content_type, '')) LIKE 'application/x-yaml%'
OR lower(COALESCE(new.content_type, '')) LIKE 'application/yaml%'
THEN COALESCE(new.body, '')
ELSE ''
END
);
INSERT OR IGNORE INTO revision_search_state (revision_uuid)
VALUES (new.uuid);
INSERT INTO latest_revision_index (
object_uuid, revision_uuid, content_type, filename,
document_hash, created_at_ms
) VALUES (
new.object_uuid, new.uuid, COALESCE(new.content_type, ''),
COALESCE(new.filename, ''), COALESCE(new.document_hash, ''),
COALESCE(new.created_at_ms, new.created_at * 1000, 0)
)
ON CONFLICT(object_uuid) DO UPDATE SET
revision_uuid = excluded.revision_uuid,
content_type = excluded.content_type,
filename = excluded.filename,
document_hash = excluded.document_hash,
created_at_ms = excluded.created_at_ms
WHERE excluded.created_at_ms >= latest_revision_index.created_at_ms;
END
`); err != nil {
return err
}
if _, err := s.DB.Exec(`
CREATE TRIGGER IF NOT EXISTS revision_search_delete
AFTER DELETE ON revisions BEGIN
DELETE FROM revision_search WHERE revision_uuid = old.uuid;
DELETE FROM revision_search_state WHERE revision_uuid = old.uuid;
DELETE FROM latest_revision_index
WHERE object_uuid = old.object_uuid
AND revision_uuid = old.uuid;
INSERT OR REPLACE INTO latest_revision_index (
object_uuid, revision_uuid, content_type, filename,
document_hash, created_at_ms
)
SELECT
object_uuid, uuid, COALESCE(content_type, ''),
COALESCE(filename, ''), COALESCE(document_hash, ''),
COALESCE(created_at_ms, created_at * 1000, 0)
FROM revisions
WHERE object_uuid = old.object_uuid
ORDER BY COALESCE(created_at_ms, created_at * 1000) DESC, rowid DESC
LIMIT 1;
END
`); err != nil {
return err
}
_, err := s.DB.Exec(`
INSERT INTO revision_search (
revision_uuid, object_uuid, filename, body
)
SELECT
r.uuid,
r.object_uuid,
COALESCE(r.filename, ''),
CASE
WHEN lower(COALESCE(r.content_type, '')) LIKE 'text/%'
OR lower(COALESCE(r.content_type, '')) LIKE 'application/json%'
OR lower(COALESCE(r.content_type, '')) LIKE 'application/xml%'
OR lower(COALESCE(r.content_type, '')) LIKE 'application/javascript%'
OR lower(COALESCE(r.content_type, '')) LIKE 'application/x-yaml%'
OR lower(COALESCE(r.content_type, '')) LIKE 'application/yaml%'
THEN COALESCE(r.body, '')
ELSE ''
END
FROM revisions r
WHERE NOT EXISTS (
SELECT 1 FROM revision_search_state state
WHERE state.revision_uuid = r.uuid
)
`)
if err != nil {
return err
}
_, err = s.DB.Exec(`
INSERT OR IGNORE INTO revision_search_state (revision_uuid)
SELECT uuid FROM revisions
`)
return err
return mestionora.NewSQLite(s.DB).EnsureSearchIndexSchema()
}
-132
View File
@@ -1,132 +0,0 @@
package store
import (
"context"
"database/sql"
"errors"
"strings"
"testing"
"time"
"pageswiki/internal/db"
)
func searchQueryPlan(t *testing.T, database *sql.DB, query string, args ...any) []string {
t.Helper()
rows, err := database.Query("EXPLAIN QUERY PLAN "+query, args...)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var plan []string
for rows.Next() {
var id, parent, unused int
var detail string
if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
t.Fatal(err)
}
plan = append(plan, detail)
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
return plan
}
func TestSearchExcerptDeadlineIsOptionalButParentCancellationIsNot(t *testing.T) {
database, err := db.Open(t.TempDir() + "/excerpt-deadline.db")
if err != nil {
t.Fatal(err)
}
defer database.Close()
if err := db.Migrate(database); err != nil {
t.Fatal(err)
}
New(database)
tx, err := database.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true})
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
excerpts, err := loadSearchBodyExcerpts(context.Background(), tx, `"needle"*`, []any{1}, -time.Second)
if err != nil || len(excerpts) != 0 {
t.Fatalf("optional excerpt deadline discarded search: excerpts=%v err=%v", excerpts, err)
}
var one int
if err := tx.QueryRow(`SELECT 1`).Scan(&one); err != nil || one != 1 {
t.Fatalf("excerpt timeout left search transaction unusable: %d %v", one, err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := loadSearchBodyExcerpts(ctx, tx, `"needle"*`, []any{1}, time.Second); !errors.Is(err, context.Canceled) {
t.Fatalf("parent cancellation was swallowed: %v", err)
}
if _, err := loadSearchBodyExcerpts(context.Background(), tx, `NEAR(`, []any{1}, time.Second); err == nil {
t.Fatal("non-timeout SQL failure was swallowed")
}
}
func TestSearchFTSPlanScansMatchesOnce(t *testing.T) {
database, err := db.Open(t.TempDir() + "/search-plan.db")
if err != nil {
t.Fatal(err)
}
defer database.Close()
if err := db.Migrate(database); err != nil {
t.Fatal(err)
}
New(database)
for _, field := range []string{"body", "filename"} {
plan := searchQueryPlan(t, database, searchFTSCandidatesSQL,
field+` : ("needle"*)`, "domain", "/", "/", "0", SearchCandidateLimit)
if len(plan) != 3 || !strings.Contains(plan[0], "SCAN revision_search VIRTUAL TABLE") ||
!strings.Contains(plan[1], "revision_uuid=?") || !strings.Contains(plan[2], "object_uuid=? AND domain_uuid=?") {
t.Fatalf("%s search must scan FTS once, then use indexed Revision/Path seeks:\n%s", field, strings.Join(plan, "\n"))
}
}
plan := searchQueryPlan(t, database, searchBodyExcerptsSQL(2), `body : ("needle"*)`, 1, 2)
if len(plan) != 1 || !strings.Contains(plan[0], "VIRTUAL TABLE INDEX 0:=M4") {
t.Fatalf("snippets must use both rowid and MATCH constraints:\n%s", strings.Join(plan, "\n"))
}
}
func TestSearchFilteredFTSPlanKeepsMatchFirstAndIndexedSeeks(t *testing.T) {
database, err := db.Open(t.TempDir() + "/search-filter-plan.db")
if err != nil {
t.Fatal(err)
}
defer database.Close()
if err := db.Migrate(database); err != nil {
t.Fatal(err)
}
New(database)
from, before := int64(1000), int64(2000)
for _, options := range []SearchOptions{
{FileType: "code"}, {Extension: "py"}, {FromMS: &from, BeforeMS: &before},
{DateField: "created", FromMS: &from, BeforeMS: &before, FileType: "code", Extension: "py"},
} {
predicate, filterArgs := searchOptionsPredicate(options)
args := []any{`body : ("needle"*)`, "domain", "/", "/", "0"}
args = append(args, filterArgs...)
args = append(args, SearchCandidateLimit)
plan := searchQueryPlan(t, database, searchFTSCandidatesBaseSQL+predicate+` LIMIT ?`, args...)
if len(plan) < 3 || !strings.Contains(plan[0], "SCAN revision_search VIRTUAL TABLE") ||
!strings.Contains(plan[1], "revision_uuid=?") || !strings.Contains(plan[2], "object_uuid=? AND domain_uuid=?") {
t.Fatalf("filtered FTS must scan MATCH once, then indexed Revision/Path seeks: %+v\n%s", options, strings.Join(plan, "\n"))
}
if options.DateField == "created" {
if len(plan) != 5 || !strings.Contains(plan[4], "SEARCH o USING INDEX") || !strings.Contains(plan[4], "uuid=?") {
t.Fatalf("Object date needs one indexed PK lookup, not a scan: %s", strings.Join(plan, "\n"))
}
}
args = []any{"domain", "/", "/", "0"}
args = append(args, filterArgs...)
args = append(args, SearchCandidateLimit)
plan = searchQueryPlan(t, database, searchPathCandidatesBaseSQL+predicate+` ORDER BY p.path LIMIT ?`, args...)
joinedPlan := strings.Join(plan, "\n")
if len(plan) < 2 || !strings.Contains(plan[0], "SEARCH p USING INDEX") || !strings.Contains(plan[0], "domain_uuid=?") ||
!strings.Contains(joinedPlan, "SEARCH r USING INDEX") || !strings.Contains(joinedPlan, "object_uuid=?") || strings.Contains(joinedPlan, "TEMP B-TREE") {
t.Fatalf("filter-only search must keep indexed Domain order and bounded candidate memory: %s", strings.Join(plan, "\n"))
}
}
}
+124
View File
@@ -0,0 +1,124 @@
package store
import (
"context"
"errors"
"fmt"
"testing"
"time"
"pageswiki/internal/seheweit"
)
func resourceTestStore(t *testing.T) *Store {
t.Helper()
s := newSchemaTestStore(t)
if err := s.EnsureResourceHistorySchema(); err != nil {
t.Fatal(err)
}
return s
}
func TestSeheweitHistoryBucketCompatibility(t *testing.T) {
s := resourceTestStore(t)
now := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC)
// Boundary-inclusive query. Later insertion, not later timestamp, wins a
// bucket, matching the old MAX(id) query even with out-of-order samples.
for i, tc := range []struct {
offset time.Duration
payload string
}{
{-time.Hour - time.Millisecond, `{"label":"too-old"}`},
{-time.Hour, `{"label":"start"}`},
{-10 * time.Second, `{"label":"later-time"}`},
{-20 * time.Second, `{"label":"last-insert"}`},
{0, `{"label":"end"}`},
{time.Millisecond, `{"label":"future"}`},
} {
if err := s.SaveResourceSample(fmt.Sprint(i), now.Add(tc.offset).UnixMilli(), []byte(tc.payload)); err != nil {
t.Fatal(err)
}
}
got, err := s.ResourceHistory(t.Context(), 1, now)
if err != nil {
t.Fatal(err)
}
want := []string{`{"label":"start"}`, `{"label":"last-insert"}`, `{"label":"end"}`}
if len(got) != len(want) {
t.Fatalf("samples %s", got)
}
for i := range want {
if string(got[i]) != want[i] {
t.Fatalf("sample %d: %s", i, got[i])
}
}
}
func TestSeheweitHistoryRetentionAndIdempotence(t *testing.T) {
s := resourceTestStore(t)
at := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC).UnixMilli()
cutoff := seheweit.RetentionCutoff(at)
for _, tc := range []struct {
job string
at int64
}{{"expired", cutoff - 1}, {"boundary", cutoff}, {"current", at}} {
if err := s.SaveResourceSample(tc.job, tc.at, []byte(`{"original":true}`)); err != nil {
t.Fatal(err)
}
}
if err := s.SaveResourceSample("current", at+1, []byte(`{"replacement":true}`)); err != nil {
t.Fatal(err)
}
var payload string
if err := s.DB.QueryRow(`SELECT payload FROM resource_samples WHERE job_uuid='current'`).Scan(&payload); err != nil || payload != `{"original":true}` {
t.Fatalf("retry replaced sample: %s %v", payload, err)
}
// The retry uses its collection time for pruning, so the old cutoff is now
// expired too. A distinct test covers exact retention-boundary preservation.
var count int
if err := s.DB.QueryRow(`SELECT COUNT(*) FROM resource_samples`).Scan(&count); err != nil || count != 1 {
t.Fatalf("retention count=%d %v", count, err)
}
s2 := resourceTestStore(t)
if err := s2.SaveResourceSample("boundary", cutoff, []byte(`{}`)); err != nil {
t.Fatal(err)
}
if err := s2.SaveResourceSample("current", at, []byte(`{}`)); err != nil {
t.Fatal(err)
}
if err := s2.DB.QueryRow(`SELECT COUNT(*) FROM resource_samples`).Scan(&count); err != nil || count != 2 {
t.Fatalf("inclusive cutoff count=%d %v", count, err)
}
}
func TestSeheweitHistoryCancelledReadOnlyAndRollback(t *testing.T) {
s := resourceTestStore(t)
at := time.Now().UnixMilli()
ctx, cancel := context.WithCancel(t.Context())
cancel()
if err := s.SaveResourceSampleContext(ctx, "cancelled", at, []byte(`{}`)); !errors.Is(err, context.Canceled) {
t.Fatalf("cancel: %v", err)
}
if _, err := s.ResourceHistory(ctx, 24, time.Now()); !errors.Is(err, context.Canceled) {
t.Fatalf("read cancel: %v", err)
}
s.ReadOnly = true
if err := s.SaveResourceSample("readonly", at, []byte(`{}`)); !errors.Is(err, ErrReadOnly) {
t.Fatalf("readonly: %v", err)
}
s.ReadOnly = false
if err := s.SaveResourceSample("old", seheweit.RetentionCutoff(at)-1, []byte(`{}`)); err != nil {
t.Fatal(err)
}
// Failure during expiration must roll back the insertion as well.
if _, err := s.DB.Exec(`CREATE TRIGGER prevent_resource_expire BEFORE DELETE ON resource_samples BEGIN SELECT RAISE(ABORT,'synthetic failure'); END`); err != nil {
t.Fatal(err)
}
if err := s.SaveResourceSample("rollback", at, []byte(`{}`)); err == nil {
t.Fatal("expiration failure ignored")
}
var count int
if err := s.DB.QueryRow(`SELECT COUNT(*) FROM resource_samples WHERE job_uuid<>'old'`).Scan(&count); err != nil || count != 0 {
t.Fatalf("partial transaction or cancelled write count=%d %v", count, err)
}
}
+18 -6
View File
@@ -8,15 +8,24 @@ import (
"path/filepath"
"strings"
"sync"
"time"
"pageswiki/internal/geduldh"
)
type Store struct {
DB *sql.DB
DocumentRoot string
pathCacheMu sync.RWMutex
revisionWriteMu sync.Mutex
childCache map[string][]string
authKey []byte
// Standby is immutable for the lifetime of this process. Role changes require restart.
ReadOnly bool
StandbyReadUntil time.Time
BackgroundJobsDisabled bool
DB *sql.DB
DocumentRoot string
documentMaterializerOnce sync.Once
documentMaterializer *geduldh.Materializer
pathCacheMu sync.RWMutex
revisionWriteMu sync.Mutex
childCache map[string][]string
authKey []byte
}
func New(
@@ -108,6 +117,9 @@ func NewWithDocumentRoot(
if err := st.EnsureUserSpaceSchema(); err != nil {
panic(err)
}
if err := st.ensureSchemaComponent("profile-contacts"); err != nil {
panic(err)
}
if err := st.EnsureBuiltinAdminGroups(); err != nil {
panic(err)
}
+7 -56
View File
@@ -8,26 +8,14 @@ import (
"sort"
"strings"
"time"
"pageswiki/internal/schutzaria"
)
const MaxUserSpaceOverrides = 256
type UserSpacePathPolicy struct {
Path string `json:"path"`
Visibility string `json:"visibility"`
}
type UserSpacePolicy struct {
PrincipalUUID string `json:"principal_uuid"`
Username string `json:"username"`
ProfileVisibility string `json:"profile_visibility"`
AvatarVisibility string `json:"avatar_visibility"`
RootVisibility string `json:"root_visibility"`
DescendantVisibility string `json:"descendant_visibility"`
Version int64 `json:"version"`
DomainAllowsPublic bool `json:"domain_allows_public"`
Overrides []UserSpacePathPolicy `json:"overrides"`
}
type UserSpacePathPolicy = schutzaria.UserSpacePathPolicy
type UserSpacePolicy = schutzaria.UserSpacePolicy
type UserSpacePage struct {
Path string
@@ -76,18 +64,7 @@ func defaultUserSpacePolicy(principal, username string) UserSpacePolicy {
return UserSpacePolicy{PrincipalUUID: principal, Username: username, ProfileVisibility: "domain", AvatarVisibility: "domain", RootVisibility: "domain", DescendantVisibility: "domain", Overrides: []UserSpacePathPolicy{}}
}
func userSpaceVisibilityRank(value string) int {
switch value {
case "private":
return 0
case "domain":
return 1
case "public":
return 2
default:
return -1
}
}
func userSpaceVisibilityRank(value string) int { return schutzaria.UserSpaceVisibilityRank(value) }
func userSpaceRoot(username string) string { return "/u/" + username }
@@ -98,18 +75,7 @@ func escapeUserSpaceLike(value string) string {
}
func userSpaceDocumentPath(username, value string, descendant bool) bool {
root := userSpaceRoot(username)
if value != path.Clean(value) || strings.ContainsAny(value, "\\\x00\r\n") {
return false
}
if descendant {
if !strings.HasPrefix(value, root+"/") {
return false
}
rest := strings.TrimPrefix(value, root+"/")
return rest != "" && !strings.HasPrefix(rest, "_/") && rest != "_"
}
return value == root
return schutzaria.UserSpaceDocumentPath(username, value, descendant)
}
func (s *Store) domainAllowsUserSpacePublic(ctx context.Context, q interface {
@@ -170,22 +136,7 @@ func (s *Store) UserSpacePolicy(ctx context.Context, domain, principal string) (
}
func userSpaceEffectiveVisibility(p UserSpacePolicy, target string) string {
root := userSpaceRoot(p.Username)
if target == root {
return p.RootVisibility
}
if !userSpaceDocumentPath(p.Username, target, true) {
return "private"
}
visibility := p.DescendantVisibility
for _, item := range p.Overrides {
if target == item.Path || strings.HasPrefix(target, strings.TrimRight(item.Path, "/")+"/") {
if userSpaceVisibilityRank(item.Visibility) < userSpaceVisibilityRank(visibility) {
visibility = item.Visibility
}
}
}
return visibility
return schutzaria.EffectiveUserSpaceVisibility(p, target)
}
func EffectiveUserSpaceVisibility(p UserSpacePolicy, target string) string {