Refactor architecture into eight components and consolidate reliability work
Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts. Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation. Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
This commit is contained in:
1 parent
9dd425f765
commit
d5d21b7841
302 files changed
+20037
-3492
No files matched your search
@@ -38,22 +38,7 @@ func adapterSecret(prefix string) (string, string, error) {
|
||||
}
|
||||
|
||||
func (s *Store) EnsureAdapterSessionsSchema() error {
|
||||
_, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS adapter_sessions (
|
||||
ticket_hash TEXT PRIMARY KEY, session_hash TEXT NOT NULL DEFAULT '',
|
||||
domain_uuid TEXT NOT NULL, plugin_id TEXT NOT NULL, object_uuid TEXT NOT NULL,
|
||||
principal_uuid TEXT NOT NULL DEFAULT '',
|
||||
base_revision_uuid TEXT NOT NULL, ticket_expires_at_ms INTEGER NOT NULL,
|
||||
session_expires_at_ms INTEGER NOT NULL DEFAULT 0, exchanged_at_ms INTEGER NOT NULL DEFAULT 0,
|
||||
created_at_ms INTEGER NOT NULL
|
||||
)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = s.DB.Exec(`ALTER TABLE adapter_sessions ADD COLUMN root_path TEXT NOT NULL DEFAULT ''`)
|
||||
_, _ = s.DB.Exec(`ALTER TABLE adapter_sessions ADD COLUMN principal_uuid TEXT NOT NULL DEFAULT ''`)
|
||||
_, err = s.DB.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_adapter_session_hash
|
||||
ON adapter_sessions(session_hash) WHERE session_hash != ''`)
|
||||
return err
|
||||
return s.ensureSchemaComponent("adapter-sessions")
|
||||
}
|
||||
|
||||
func (s *Store) IssueAdapterTicket(domainUUID, pluginID, objectUUID, revisionUUID, rootPath string) (string, error) {
|
||||
|
||||
@@ -177,7 +177,7 @@ func (s *Store) createTypedAuthSession(domainUUID, principalUUID string, mfaLeve
|
||||
result, err := s.DB.Exec(`INSERT INTO auth_sessions(uuid,token_hash,principal_uuid,domain_uuid,auth_time_ms,expires_at_ms,last_seen_at_ms,mfa_level)
|
||||
SELECT ?,?,p.uuid,m.domain_uuid,?,?,?,? FROM principals p JOIN domain_memberships m ON m.principal_uuid=p.uuid
|
||||
WHERE p.uuid=? AND p.state IN ('active','readonly') AND m.domain_uuid=? AND m.state='active'`,
|
||||
uuid.NewString(), hash, now, now+int64((24*time.Hour)/time.Millisecond), now, mfaLevel, principalUUID, domainUUID)
|
||||
uuid.NewString(), hash, now, now+s.AuthSessionLifetime(domainUUID).Milliseconds(), now, mfaLevel, principalUUID, domainUUID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -472,7 +472,7 @@ func (s *Store) Authorize(request AuthorizationRequest) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if p.decide(request.TargetID, request.Capability, base == nil) {
|
||||
if p.Decide(request.TargetID, request.Capability, base == nil) {
|
||||
return nil
|
||||
}
|
||||
return ErrAccessDenied
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
const DefaultAuthSessionMinutes = 24 * 60
|
||||
const MaxAuthSessionMinutes = 30 * 24 * 60
|
||||
|
||||
// Absolute lifetime, fixed at issuance. Changing the policy cannot silently
|
||||
// revive, extend or invalidate existing sessions. App passwords are separate.
|
||||
func (s *Store) AuthSessionLifetimeMinutes(domain string) int {
|
||||
raw, err := s.GetConfigInDomain(domain, "auth_session_minutes")
|
||||
value, parseErr := strconv.Atoi(raw)
|
||||
if err != nil || parseErr != nil || value < 5 || value > MaxAuthSessionMinutes {
|
||||
return DefaultAuthSessionMinutes
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func (s *Store) SetAuthSessionLifetimeMinutes(domain string, minutes int) error {
|
||||
if minutes < 5 || minutes > MaxAuthSessionMinutes {
|
||||
return errors.New("session lifetime must be between 5 and 43200 minutes")
|
||||
}
|
||||
return s.SetConfigInDomain(domain, "auth_session_minutes", strconv.Itoa(minutes))
|
||||
}
|
||||
|
||||
func (s *Store) AuthSessionLifetime(domain string) time.Duration {
|
||||
return time.Duration(s.AuthSessionLifetimeMinutes(domain)) * time.Minute
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
func TestAuthSessionLifetimeIsDomainScopedAndFixedAtIssuance(t *testing.T) {
|
||||
database, err := sql.Open("sqlite", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
database.SetMaxOpenConns(1)
|
||||
t.Cleanup(func() { database.Close() })
|
||||
if err = db.Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := NewWithDocumentRoot(database, t.TempDir()+"/documents")
|
||||
if err = s.EnsureAuthSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = s.EnsureConfigTable(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := s.BootstrapOwner("domain", "admin", "a long test password", "Admin", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.AuthSessionLifetimeMinutes("domain") != 1440 {
|
||||
t.Fatal("default changed")
|
||||
}
|
||||
for _, minutes := range []int{-1, 0, 4, 43201} {
|
||||
if s.SetAuthSessionLifetimeMinutes("domain", minutes) == nil {
|
||||
t.Fatalf("accepted %d", minutes)
|
||||
}
|
||||
}
|
||||
if err = s.SetAuthSessionLifetimeMinutes("domain", 60); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.AuthSessionLifetimeMinutes("other") != 1440 {
|
||||
t.Fatal("policy leaked across Domains")
|
||||
}
|
||||
for _, browser := range []bool{false, true} {
|
||||
var token string
|
||||
if browser {
|
||||
token, err = s.CreateBrowserAuthSession("domain", p.UUID, 0)
|
||||
} else {
|
||||
token, err = s.CreateAuthSession("domain", p.UUID)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var duration int64
|
||||
if err = database.QueryRow(`SELECT expires_at_ms-auth_time_ms FROM auth_sessions ORDER BY rowid DESC LIMIT 1`).Scan(&duration); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if duration != time.Hour.Milliseconds() {
|
||||
t.Fatalf("lifetime %d", duration)
|
||||
}
|
||||
if _, err = s.PrincipalForSession("domain", token); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err = s.SetAuthSessionLifetimeMinutes("domain", 5); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var changed int
|
||||
if err = database.QueryRow(`SELECT count(*) FROM auth_sessions WHERE expires_at_ms-auth_time_ms<>?`, time.Hour.Milliseconds()).Scan(&changed); err != nil || changed != 0 {
|
||||
t.Fatalf("existing sessions changed: %d %v", changed, err)
|
||||
}
|
||||
token, err := s.CreateAuthSession("domain", p.UUID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = database.Exec(`UPDATE auth_sessions SET expires_at_ms=?`, time.Now().Add(-time.Second).UnixMilli()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = s.PrincipalForSession("domain", token); err == nil {
|
||||
t.Fatal("expired token authenticated")
|
||||
}
|
||||
}
|
||||
@@ -21,12 +21,7 @@ type Broadcast struct {
|
||||
}
|
||||
|
||||
func (s *Store) EnsureBroadcastSchema() error {
|
||||
_, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS broadcasts (uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,title TEXT NOT NULL,body TEXT NOT NULL,severity TEXT NOT NULL DEFAULT 'info' CHECK(severity IN ('info','success','warning','critical')),starts_at_ms INTEGER NOT NULL DEFAULT 0,ends_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.DB.Exec(`CREATE INDEX IF NOT EXISTS idx_broadcast_active ON broadcasts(domain_uuid,starts_at_ms,ends_at_ms,updated_at_ms DESC)`)
|
||||
return err
|
||||
return s.ensureSchemaComponent("broadcasts")
|
||||
}
|
||||
|
||||
const broadcastColumns = `uuid,domain_uuid,title,body,severity,starts_at_ms,ends_at_ms,created_at_ms,updated_at_ms`
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
"pageswiki/internal/mestionora"
|
||||
)
|
||||
|
||||
type BrowserEntry struct {
|
||||
@@ -23,32 +21,7 @@ type BrowserEntry struct {
|
||||
Title string
|
||||
}
|
||||
|
||||
func firstHTMLHeading(source string) string {
|
||||
t := html.NewTokenizer(strings.NewReader(source))
|
||||
in := false
|
||||
var out strings.Builder
|
||||
for {
|
||||
kind := t.Next()
|
||||
if kind == html.ErrorToken {
|
||||
if t.Err() == io.EOF {
|
||||
break
|
||||
}
|
||||
break
|
||||
}
|
||||
tok := t.Token()
|
||||
if kind == html.StartTagToken && strings.EqualFold(tok.Data, "h1") {
|
||||
in = true
|
||||
continue
|
||||
}
|
||||
if kind == html.EndTagToken && strings.EqualFold(tok.Data, "h1") {
|
||||
break
|
||||
}
|
||||
if in && kind == html.TextToken {
|
||||
out.WriteString(tok.Data)
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(out.String())
|
||||
}
|
||||
func firstHTMLHeading(source string) string { return mestionora.FirstHTMLHeading(source) }
|
||||
|
||||
func normalizeBrowserPath(p string) string {
|
||||
p = strings.TrimSpace(p)
|
||||
@@ -117,10 +90,8 @@ func (s *Store) ListBrowserEntriesInDomain(domainUUID, parent string) ([]Browser
|
||||
}
|
||||
entry.Path = prefix + entry.Name
|
||||
if !entry.IsDir && s.DocumentRoot != "" && entry.DocumentHash != "" {
|
||||
if filename, err := s.DocumentPath(entry.DocumentHash); err == nil {
|
||||
if info, err := os.Stat(filename); err == nil {
|
||||
entry.Size = int(info.Size())
|
||||
}
|
||||
if size, err := s.DocumentSize(entry.DocumentHash); err == nil {
|
||||
entry.Size = int(size)
|
||||
}
|
||||
}
|
||||
out = append(out, entry)
|
||||
|
||||
@@ -23,25 +23,7 @@ type BrowserNavigationLink struct {
|
||||
}
|
||||
|
||||
func (s *Store) EnsureBrowserNavigationSchema() error {
|
||||
_, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS browser_navigation_links (
|
||||
uuid TEXT PRIMARY KEY,
|
||||
domain_uuid TEXT NOT NULL,
|
||||
principal_uuid TEXT NOT NULL,
|
||||
label TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
is_directory INTEGER NOT NULL DEFAULT 0,
|
||||
position INTEGER NOT NULL DEFAULT 0,
|
||||
created_at_ms INTEGER NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL,
|
||||
UNIQUE(domain_uuid, principal_uuid, path)
|
||||
)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = s.DB.Exec(`ALTER TABLE browser_navigation_links ADD COLUMN is_directory INTEGER NOT NULL DEFAULT 0`)
|
||||
_, err = s.DB.Exec(`CREATE INDEX IF NOT EXISTS idx_browser_navigation_owner
|
||||
ON browser_navigation_links(domain_uuid, principal_uuid, position, created_at_ms)`)
|
||||
return err
|
||||
return s.ensureSchemaComponent("browser-navigation")
|
||||
}
|
||||
|
||||
func (s *Store) ListBrowserNavigationLinks(domainUUID, principalUUID string) ([]BrowserNavigationLink, error) {
|
||||
|
||||
+10
-3
@@ -13,6 +13,7 @@ import (
|
||||
var cityNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{1,63}$`)
|
||||
|
||||
type CitySpot struct {
|
||||
DisplayTitle string `json:"display_title,omitempty"`
|
||||
DetailHref string `json:"detail_href,omitempty"`
|
||||
UUID string `json:"uuid"`
|
||||
Name string `json:"name"`
|
||||
@@ -74,6 +75,7 @@ type CityPagePlacement struct {
|
||||
|
||||
type CitySpotAppearance struct {
|
||||
SpotUUID, SpotName, SpotTitle, EdgeType, Label, Source, PathPrefix string
|
||||
DisplayTitle, StreetPath, QuartierPath string
|
||||
}
|
||||
|
||||
type CityNodeDetail struct {
|
||||
@@ -538,13 +540,16 @@ func listSpotPages(db cityQuerier, domainUUID, spotUUID string, limit int) ([]Ci
|
||||
}
|
||||
|
||||
func (s *Store) ListPageSpotAppearances(domainUUID, pageUUID string) ([]CitySpotAppearance, error) {
|
||||
rows, err := s.DB.Query(`SELECT uuid,name,title,edge_type,label,source,path_prefix FROM (
|
||||
rows, err := s.DB.Query(`SELECT a.uuid,a.name,a.title,a.edge_type,a.label,a.source,a.path_prefix,COALESCE(NULLIF(st.title,''),st.name),st.path_prefix,q.path_prefix FROM (
|
||||
SELECT s.uuid,s.name,s.title,sp.edge_type,sp.label,sp.source,s.path_prefix FROM spot_pages sp JOIN spots s ON s.uuid=sp.spot_uuid AND s.domain_uuid=? AND s.state!='archived' WHERE sp.page_uuid=?
|
||||
UNION ALL
|
||||
SELECT s.uuid,s.name,s.title,'path','','path',s.path_prefix FROM spots s JOIN paths p ON p.domain_uuid=s.domain_uuid AND p.object_uuid=? AND `+cityPathScopeSQL("s.path_prefix", "p.path")+`
|
||||
WHERE s.domain_uuid=? AND s.state!='archived' AND NOT EXISTS(SELECT 1 FROM spot_pages explicit WHERE explicit.spot_uuid=s.uuid AND explicit.page_uuid=?)
|
||||
AND NOT EXISTS(SELECT 1 FROM city_page_overrides ov JOIN spots dest ON dest.uuid=ov.spot_uuid AND dest.domain_uuid=ov.domain_uuid AND dest.state!='archived' WHERE ov.domain_uuid=p.domain_uuid AND ov.page_uuid=p.object_uuid AND ov.spot_uuid!=s.uuid)
|
||||
) ORDER BY lower(title),edge_type`, domainUUID, pageUUID, pageUUID, domainUUID, pageUUID)
|
||||
) a JOIN street_spots ss ON ss.spot_uuid=a.uuid
|
||||
JOIN zones st ON st.uuid=ss.street_uuid AND st.domain_uuid=? AND st.kind='street' AND st.state!='archived'
|
||||
JOIN zones q ON q.uuid=st.parent_uuid AND q.domain_uuid=st.domain_uuid AND q.kind='quartier' AND q.state!='archived'
|
||||
ORDER BY lower(a.title),a.edge_type`, domainUUID, pageUUID, pageUUID, domainUUID, pageUUID, domainUUID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -552,9 +557,11 @@ func (s *Store) ListPageSpotAppearances(domainUUID, pageUUID string) ([]CitySpot
|
||||
var out []CitySpotAppearance
|
||||
for rows.Next() {
|
||||
var item CitySpotAppearance
|
||||
if err := rows.Scan(&item.SpotUUID, &item.SpotName, &item.SpotTitle, &item.EdgeType, &item.Label, &item.Source, &item.PathPrefix); err != nil {
|
||||
var parentTitle string
|
||||
if err := rows.Scan(&item.SpotUUID, &item.SpotName, &item.SpotTitle, &item.EdgeType, &item.Label, &item.Source, &item.PathPrefix, &parentTitle, &item.StreetPath, &item.QuartierPath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.DisplayTitle = CitySpotDisplayTitle(item.SpotName, item.SpotTitle, item.PathPrefix, parentTitle)
|
||||
out = append(out, item)
|
||||
}
|
||||
return out, rows.Err()
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package store
|
||||
|
||||
import "pageswiki/internal/mestionora"
|
||||
|
||||
func IsUnclassifiedCitySpot(name, boundary string) bool {
|
||||
return mestionora.IsUnclassifiedCitySpot(name, boundary)
|
||||
}
|
||||
|
||||
func CitySpotDisplayTitle(name, title, boundary, parentTitle string) string {
|
||||
return mestionora.CitySpotDisplayTitle(name, title, boundary, parentTitle)
|
||||
}
|
||||
|
||||
func (n CityNodeDetail) DisplayTitle() string {
|
||||
if n.Kind == "spot" {
|
||||
parent := ""
|
||||
if n.Street != nil {
|
||||
parent = n.Street.Title
|
||||
if parent == "" {
|
||||
parent = n.Street.Name
|
||||
}
|
||||
}
|
||||
return CitySpotDisplayTitle(n.Name, n.Title, n.PathPrefix, parent)
|
||||
}
|
||||
if n.Title != "" {
|
||||
return n.Title
|
||||
}
|
||||
return n.Name
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCitySpotDisplayTitlePreservesIdentity(t *testing.T) {
|
||||
for _, test := range []struct{ name, title, boundary, parent, want string }{
|
||||
{"home-undefined", "Undefined", "/life/home/_undefined", "Home & Garden", "Home & Garden"},
|
||||
{"home-undefined", "home-undefined", "/life/home/_undefined", "Home", "Home"},
|
||||
{"home-undefined", "My notes", "/life/home/_undefined", "Home", "My notes"},
|
||||
{"manual", "Undefined", "/life/home/manual", "Home", "Undefined"},
|
||||
{"home-undefined", "", "/life/home/_undefined", "", "home"},
|
||||
} {
|
||||
if got := CitySpotDisplayTitle(test.name, test.title, test.boundary, test.parent); got != test.want {
|
||||
t.Fatalf("%+v: %q", test, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCitySearchContentPathMetadata(t *testing.T) {
|
||||
st := cityReadFixture(t)
|
||||
citySearchInsertZone(t, st, "q", "d", "quartier", "", "campus", "Campus", "needle", "/life")
|
||||
citySearchInsertZone(t, st, "s", "d", "street", "q", "home", "Home", "needle", "/life/home")
|
||||
citySearchInsertSpot(t, st, "p", "d", "s", "home-undefined", "Undefined", "needle", "/life/home/_undefined")
|
||||
// Same Path in another Domain must not be mistaken for a local portal.
|
||||
cityTestExec(t, st, `INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('other','/life','other-page'),('d','/life/home','home-page')`)
|
||||
results, err := st.SearchCityInDomain(context.Background(), "d", "/", "needle", 30, citySearchAllowAll, SearchOptions{})
|
||||
if err != nil || len(results) != 3 {
|
||||
t.Fatalf("results: %+v %v", results, err)
|
||||
}
|
||||
if results[0].HasPage || !results[1].HasPage || !results[2].HasPage || results[2].ContentPath() != "/life/home" || results[2].DisplayTitle() != "Home" {
|
||||
t.Fatalf("projection: %+v", results)
|
||||
}
|
||||
node, err := st.CityNode("d", "spot", "home-undefined")
|
||||
if err != nil || node.DisplayTitle() != "Home" || node.Title != "Undefined" || node.Name != "home-undefined" || node.PathPrefix != "/life/home/_undefined" {
|
||||
t.Fatalf("raw identity mutated: %+v %v", node, err)
|
||||
}
|
||||
cityTestExec(t, st, `INSERT INTO spot_pages(spot_uuid,page_uuid,edge_type,label,source,created_at_ms) VALUES('p','home-page','primary','','local',1)`)
|
||||
appearances, err := st.ListPageSpotAppearances("d", "home-page")
|
||||
if err != nil || len(appearances) != 1 || appearances[0].DisplayTitle != "Home" || appearances[0].SpotTitle != "Undefined" || appearances[0].StreetPath != "/life/home" || appearances[0].QuartierPath != "/life" {
|
||||
t.Fatalf("appearance projection: %+v %v", appearances, err)
|
||||
}
|
||||
// Both parent and boundary ACLs still gate the result.
|
||||
results, err = st.SearchCityInDomain(context.Background(), "d", "/", "needle", 30, func(p string) bool { return p != "/life/home" }, SearchOptions{})
|
||||
if err != nil || len(results) != 1 {
|
||||
t.Fatalf("parent ACL: %+v %v", results, err)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/schutzaria"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -16,11 +18,7 @@ var (
|
||||
ErrGovernanceConfirm = errors.New("city.governance.confirm_error")
|
||||
)
|
||||
|
||||
type LocalityPolicy struct {
|
||||
Kind string `json:"kind"`
|
||||
UUID string `json:"uuid"`
|
||||
Policy string `json:"policy"`
|
||||
}
|
||||
type LocalityPolicy = schutzaria.LocalityPolicy
|
||||
|
||||
type LocalityOwner struct {
|
||||
MembershipUUID string `json:"membership_uuid"`
|
||||
@@ -72,24 +70,7 @@ func (s *Store) ensureCityGovernanceSchema() error {
|
||||
// EvaluateLocalityPublication combines ceilings with intent. A public Domain
|
||||
// merely allows publication: it never publishes its root or arbitrary Pages.
|
||||
func EvaluateLocalityPublication(chain []LocalityPolicy) (allows, published bool) {
|
||||
allows = true
|
||||
for i, p := range chain {
|
||||
if i == 0 && p.Policy == "inherit" {
|
||||
return false, false
|
||||
}
|
||||
switch p.Policy {
|
||||
case "authenticated_acl":
|
||||
allows = false
|
||||
case "public":
|
||||
if i > 0 {
|
||||
published = true
|
||||
}
|
||||
case "inherit":
|
||||
default:
|
||||
return false, false
|
||||
}
|
||||
}
|
||||
return allows, allows && published
|
||||
return schutzaria.EvaluateLocalityPublication(chain)
|
||||
}
|
||||
|
||||
// All authority, parentage and policy reads use this one snapshot. Mutation
|
||||
|
||||
@@ -2,205 +2,17 @@ package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/mestionora"
|
||||
)
|
||||
|
||||
type CitySearchResult = mestionora.CitySearchResult
|
||||
|
||||
const (
|
||||
CitySearchCandidateLimit = 1000 // Per locality kind, before authorization.
|
||||
CitySearchResultLimit = 30
|
||||
citySearchTimeout = 2 * time.Second
|
||||
CitySearchCandidateLimit = mestionora.CitySearchCandidateLimit
|
||||
CitySearchResultLimit = mestionora.CitySearchResultLimit
|
||||
)
|
||||
|
||||
// CitySearchResult describes a locality, not its contained Pages. Description
|
||||
// is a bounded plain-text excerpt around the match and is never trusted HTML.
|
||||
type CitySearchResult struct {
|
||||
UUID, Kind, Name, Title, Description, PathPrefix string
|
||||
QuartierTitle, StreetTitle, QuartierName string
|
||||
CreatedAtMS, UpdatedAtMS int64
|
||||
NameMatch, PathMatch, DescriptionMatch bool
|
||||
}
|
||||
|
||||
type citySearchCandidate struct {
|
||||
CitySearchResult
|
||||
quartierPath, streetPath string
|
||||
}
|
||||
|
||||
// SearchCityInDomain searches locality metadata only. It does not construct a
|
||||
// CityMap, expand descendants, read Page bodies, or count hidden localities.
|
||||
// Scope selects a locality's own boundary, not an ancestor of the scope. Every
|
||||
// boundary in its parent chain must be readable before the result is exposed.
|
||||
func (s *Store) SearchCityInDomain(parent context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []CitySearchResult, searchErr error) {
|
||||
if err := options.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
query = strings.TrimSpace(query)
|
||||
if len(query) > 512 || len(strings.Fields(query)) > 16 {
|
||||
return nil, errors.New("search query is too long")
|
||||
}
|
||||
// File filters are intentionally not applied to locality metadata: a Spot
|
||||
// containing a PDF is not itself a PDF and must not claim to match one.
|
||||
if allowed == nil || options.FileType != "" || options.Extension != "" || (query == "" && !options.HasFilters()) {
|
||||
return nil, nil
|
||||
}
|
||||
if limit <= 0 || limit > CitySearchResultLimit {
|
||||
limit = CitySearchResultLimit
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(parent, citySearchTimeout)
|
||||
defer cancel()
|
||||
defer func() {
|
||||
if searchErr != nil && ctx.Err() != nil {
|
||||
searchErr = ctx.Err()
|
||||
}
|
||||
}()
|
||||
tx, err := s.DB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
scope = normalizePath(scope)
|
||||
prefix, upper := strings.TrimRight(scope, "/")+"/", strings.TrimRight(scope, "/")+"0"
|
||||
args := []any{
|
||||
sql.Named("domain", domainUUID), sql.Named("scope", scope),
|
||||
sql.Named("prefix", prefix), sql.Named("upper", upper),
|
||||
sql.Named("query", query), sql.Named("limit", CitySearchCandidateLimit),
|
||||
}
|
||||
dateColumn := "n.updated_at_ms"
|
||||
if options.DateField == "created" {
|
||||
dateColumn = "n.created_at_ms"
|
||||
}
|
||||
dateWhere := ""
|
||||
if options.FromMS != nil {
|
||||
dateWhere += " AND " + dateColumn + ">=:from_ms"
|
||||
args = append(args, sql.Named("from_ms", *options.FromMS))
|
||||
}
|
||||
if options.BeforeMS != nil {
|
||||
dateWhere += " AND " + dateColumn + "<:before_ms"
|
||||
args = append(args, sql.Named("before_ms", *options.BeforeMS))
|
||||
}
|
||||
candidates := make([]citySearchCandidate, 0)
|
||||
for _, kind := range []string{"quartier", "street", "spot"} {
|
||||
rows, err := tx.QueryContext(ctx, citySearchSQL(kind, dateWhere), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var candidate citySearchCandidate
|
||||
candidate.Kind = kind
|
||||
if err := rows.Scan(&candidate.UUID, &candidate.Name, &candidate.Title, &candidate.Description,
|
||||
&candidate.PathPrefix, &candidate.CreatedAtMS, &candidate.UpdatedAtMS,
|
||||
&candidate.NameMatch, &candidate.PathMatch, &candidate.DescriptionMatch,
|
||||
&candidate.QuartierTitle, &candidate.StreetTitle, &candidate.quartierPath, &candidate.streetPath, &candidate.QuartierName); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
candidates = append(candidates, candidate)
|
||||
}
|
||||
err = rows.Err()
|
||||
closeErr := rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
}
|
||||
// Release the sole read connection before calling authorization, which may
|
||||
// itself need a database connection. HTTP normally passes its cached policy.
|
||||
if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) {
|
||||
return nil, err
|
||||
}
|
||||
sort.SliceStable(candidates, func(i, j int) bool {
|
||||
a, b := candidates[i], candidates[j]
|
||||
if a.Kind != b.Kind {
|
||||
return citySearchKindOrder(a.Kind) < citySearchKindOrder(b.Kind)
|
||||
}
|
||||
if a.NameMatch != b.NameMatch {
|
||||
return a.NameMatch
|
||||
}
|
||||
if a.PathMatch != b.PathMatch {
|
||||
return a.PathMatch
|
||||
}
|
||||
if a.PathPrefix != b.PathPrefix {
|
||||
return a.PathPrefix < b.PathPrefix
|
||||
}
|
||||
return a.UUID < b.UUID
|
||||
})
|
||||
readable := make(map[string]bool)
|
||||
canRead := func(path string) bool {
|
||||
if value, exists := readable[path]; exists {
|
||||
return value
|
||||
}
|
||||
value := allowed(path)
|
||||
readable[path] = value
|
||||
return value
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !canRead(candidate.PathPrefix) || (candidate.quartierPath != "" && !canRead(candidate.quartierPath)) || (candidate.streetPath != "" && !canRead(candidate.streetPath)) {
|
||||
continue
|
||||
}
|
||||
found = append(found, candidate.CitySearchResult)
|
||||
if len(found) == limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
func citySearchKindOrder(kind string) int {
|
||||
switch kind {
|
||||
case "quartier":
|
||||
return 0
|
||||
case "street":
|
||||
return 1
|
||||
default:
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
// All interpolated SQL fragments are package-owned constants. User input is
|
||||
// bound, so %, _, quotes and backslashes are literal substring characters.
|
||||
func citySearchSQL(kind, dateWhere string) string {
|
||||
from, description := "zones n", "n.description"
|
||||
parentColumns := "'','','','',''"
|
||||
where := "n.kind='quartier'"
|
||||
switch kind {
|
||||
case "street":
|
||||
from += ` JOIN zones q ON q.uuid=n.parent_uuid AND q.domain_uuid=n.domain_uuid AND q.kind='quartier' AND q.state!='archived'`
|
||||
parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),'',q.path_prefix,'',q.name"
|
||||
where = "n.kind='street' AND " + citySearchAbsolutePathSQL("q.path_prefix")
|
||||
case "spot":
|
||||
from = `spots n JOIN street_spots ss ON ss.spot_uuid=n.uuid
|
||||
JOIN zones st ON st.uuid=ss.street_uuid AND st.domain_uuid=n.domain_uuid AND st.kind='street' AND st.state!='archived'
|
||||
JOIN zones q ON q.uuid=st.parent_uuid AND q.domain_uuid=st.domain_uuid AND q.kind='quartier' AND q.state!='archived'`
|
||||
description = "n.summary"
|
||||
parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),substr(COALESCE(NULLIF(st.title,''),st.name),1,256),q.path_prefix,st.path_prefix,q.name"
|
||||
where = citySearchAbsolutePathSQL("q.path_prefix") + " AND " + citySearchAbsolutePathSQL("st.path_prefix")
|
||||
}
|
||||
nameMatch := `(:query!='' AND (instr(lower(n.name),lower(:query))>0 OR instr(lower(n.title),lower(:query))>0))`
|
||||
pathMatch := `(:query!='' AND instr(lower(n.path_prefix),lower(:query))>0)`
|
||||
descriptionMatch := `(:query!='' AND instr(lower(` + description + `),lower(:query))>0)`
|
||||
return `SELECT n.uuid,substr(n.name,1,256),substr(n.title,1,256),
|
||||
substr(` + description + `,max(1,instr(lower(` + description + `),lower(:query))-160),512),
|
||||
n.path_prefix,n.created_at_ms,n.updated_at_ms,` + nameMatch + `,` + pathMatch + `,` + descriptionMatch + `,` + parentColumns + `
|
||||
FROM ` + from + ` WHERE n.domain_uuid=:domain AND n.state!='archived' AND ` + where + `
|
||||
AND ` + citySearchAbsolutePathSQL("n.path_prefix") + `
|
||||
AND (n.path_prefix COLLATE BINARY=:scope OR (n.path_prefix COLLATE BINARY>=:prefix AND n.path_prefix COLLATE BINARY<:upper))
|
||||
AND (:query='' OR ` + nameMatch + ` OR ` + pathMatch + ` OR ` + descriptionMatch + `)` + dateWhere + `
|
||||
ORDER BY ` + nameMatch + ` DESC,` + pathMatch + ` DESC,n.path_prefix COLLATE BINARY,n.uuid LIMIT :limit`
|
||||
}
|
||||
|
||||
func citySearchAbsolutePathSQL(column string) string {
|
||||
// Keep authorization paths intact rather than truncating them. Malformed
|
||||
// legacy boundaries and impractically large metadata are not navigable hits.
|
||||
return `(substr(` + column + `,1,1)='/' AND length(` + column + `)<=4096 AND instr(` + column + `,char(0))=0)`
|
||||
func (s *Store) SearchCityInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) ([]CitySearchResult, error) {
|
||||
return mestionora.NewSQLite(s.DB).SearchCityInDomain(ctx, domainUUID, scope, query, limit, allowed, options)
|
||||
}
|
||||
@@ -33,8 +33,8 @@ func citySearchIDs(results []CitySearchResult) []string {
|
||||
|
||||
func TestCitySearchMetadataRankingAndMatchSources(t *testing.T) {
|
||||
st := cityReadFixture(t)
|
||||
// No Page table remains: this search must not expand a CityMap or bodies.
|
||||
cityTestExec(t, st, `DROP TABLE paths`)
|
||||
// The fixture has only Path metadata, no document/revision bodies. Search
|
||||
// may test boundary existence but must not expand a CityMap or read bodies.
|
||||
citySearchInsertZone(t, st, "q-description", "d", "quartier", "", "description", "Description", "needle in charter", "/a-description")
|
||||
citySearchInsertZone(t, st, "q-path", "d", "quartier", "", "path", "Path", "", "/b-needle")
|
||||
citySearchInsertZone(t, st, "q-title", "d", "quartier", "", "title", "NeEdLe quartier", "", "/c-title")
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
var documentBackfillMigrations = []db.SQLMigration{{Version: 1, Name: "document_backfill_progress", Statements: []string{
|
||||
`CREATE TABLE document_backfills (
|
||||
store_id TEXT PRIMARY KEY, last_uuid TEXT NOT NULL DEFAULT '',
|
||||
processed INTEGER NOT NULL DEFAULT 0, completed INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at_ms INTEGER NOT NULL)`,
|
||||
}}, {Version: 2, Name: "verify_backfill_contract", Assertions: []string{
|
||||
schemaColumns("document_backfills", "store_id,last_uuid,processed,completed,updated_at_ms"),
|
||||
schemaUnique("document_backfills", "store_id"),
|
||||
}}}
|
||||
|
||||
type DocumentBackfillProgress struct {
|
||||
StoreID string
|
||||
LastUUID string
|
||||
Processed int64
|
||||
Completed bool
|
||||
}
|
||||
|
||||
// BackfillDocuments performs one restartable, bounded batch. It is a maintenance
|
||||
// operation: legacy DB-only writers must be stopped. Normal application saves
|
||||
// persist canonical bytes before inserting revisions and need no backfill.
|
||||
// The UUID cursor is stable across SQLite VACUUM (rowid is not).
|
||||
func (s *Store) BackfillDocuments(ctx context.Context, batchSize int) (DocumentBackfillProgress, error) {
|
||||
var progress DocumentBackfillProgress
|
||||
if s.ReadOnly {
|
||||
return progress, ErrReadOnly
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
if s.DocumentRoot == "" {
|
||||
progress.Completed = true
|
||||
return progress, nil
|
||||
}
|
||||
if batchSize <= 0 || batchSize > 128 {
|
||||
batchSize = 128
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(s.DocumentRoot, "sha256"), 0755); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
root, err := os.OpenRoot(s.DocumentRoot)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
defer root.Close()
|
||||
unlock, err := lockDocumentBackfill(ctx, root)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
defer unlock()
|
||||
progress.StoreID, err = documentStoreIdentity(root)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
if err := db.ApplySQLiteMigrations(ctx, s.DB, "document-store", documentBackfillMigrations); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
if _, err := s.DB.ExecContext(ctx, `INSERT INTO document_backfills(store_id,updated_at_ms) VALUES(?,?) ON CONFLICT(store_id) DO NOTHING`, progress.StoreID, time.Now().UnixMilli()); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
if err := s.DB.QueryRowContext(ctx, `SELECT last_uuid,processed,completed FROM document_backfills WHERE store_id=?`, progress.StoreID).Scan(&progress.LastUUID, &progress.Processed, &progress.Completed); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
if progress.Completed {
|
||||
return progress, nil
|
||||
}
|
||||
rows, err := s.DB.QueryContext(ctx, `SELECT uuid,document_hash FROM revisions WHERE uuid>? AND document_hash IS NOT NULL AND document_hash!='' ORDER BY uuid LIMIT ?`, progress.LastUUID, batchSize)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
type ref struct{ uuid, hash string }
|
||||
var refs []ref
|
||||
for rows.Next() {
|
||||
var r ref
|
||||
if err := rows.Scan(&r.uuid, &r.hash); err != nil {
|
||||
rows.Close()
|
||||
return progress, err
|
||||
}
|
||||
refs = append(refs, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return progress, err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
backend, err := s.documentStorage()
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
for _, ref := range refs {
|
||||
// Existing canonical bytes outrank a stale/missing compatibility body.
|
||||
// Never overwrite a corrupt canonical file with a cache implicitly.
|
||||
err := backend.Verify(ctx, ref.hash)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
var body string
|
||||
if err = s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE uuid=? AND document_hash=?`, ref.uuid, ref.hash).Scan(&body); err == nil {
|
||||
err = backend.PutIfAbsent(ctx, ref.hash, int64(len(body)), strings.NewReader(body))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return progress, fmt.Errorf("Document backfill revision=%s: %w", ref.uuid, err)
|
||||
}
|
||||
}
|
||||
next := progress
|
||||
if len(refs) != 0 {
|
||||
next.LastUUID = refs[len(refs)-1].uuid
|
||||
}
|
||||
next.Processed += int64(len(refs))
|
||||
next.Completed = len(refs) < batchSize
|
||||
if _, err := s.DB.ExecContext(ctx, `UPDATE document_backfills SET last_uuid=?,processed=?,completed=?,updated_at_ms=? WHERE store_id=?`, next.LastUUID, next.Processed, next.Completed, time.Now().UnixMilli(), next.StoreID); err != nil {
|
||||
return progress, err
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func lockDocumentBackfill(ctx context.Context, root *os.Root) (func(), error) {
|
||||
file, err := root.OpenFile(".document-backfill.lock", os.O_CREATE|os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
file.Close()
|
||||
return nil, err
|
||||
}
|
||||
err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
if err == nil {
|
||||
return func() { _ = syscall.Flock(int(file.Fd()), syscall.LOCK_UN); file.Close() }, nil
|
||||
}
|
||||
if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) {
|
||||
file.Close()
|
||||
return nil, err
|
||||
}
|
||||
timer := time.NewTimer(20 * time.Millisecond)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
file.Close()
|
||||
return nil, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Identity follows the physical store, not its pathname. A replaced/empty root
|
||||
// gets a new identity and cannot inherit the DB's previous completion marker.
|
||||
func documentStoreIdentity(root *os.Root) (string, error) {
|
||||
file, err := root.Open(".document-store-id")
|
||||
if err == nil {
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Size() > 64 {
|
||||
return "", errors.New("invalid Document store identity file")
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(file, 128))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
id := strings.TrimSpace(string(body))
|
||||
if decoded, err := hex.DecodeString(id); err != nil || len(decoded) != 16 {
|
||||
return "", errors.New("invalid Document store identity")
|
||||
}
|
||||
// Also sync on resume: the previous process may have published the
|
||||
// identity then crashed before syncing its directory.
|
||||
return id, syncDocumentRoot(root)
|
||||
}
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return "", err
|
||||
}
|
||||
var nonce [16]byte
|
||||
if _, err := rand.Read(nonce[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
id := hex.EncodeToString(nonce[:])
|
||||
tmp := ".document-store-id-" + id
|
||||
file, err = root.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer root.Remove(tmp)
|
||||
_, err = file.WriteString(id + "\n")
|
||||
if err == nil {
|
||||
err = file.Sync()
|
||||
}
|
||||
closeErr := file.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := root.Link(tmp, ".document-store-id"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return id, syncDocumentRoot(root)
|
||||
}
|
||||
|
||||
func syncDocumentRoot(root *os.Root) error {
|
||||
dir, err := root.Open(".")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = dir.Sync()
|
||||
dir.Close()
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
func backfillFixture(t *testing.T, st *store.Store, id, body string) string {
|
||||
t.Helper()
|
||||
sum := sha256.Sum256([]byte(body))
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
_, err := st.DB.Exec(`INSERT INTO revisions(uuid,object_uuid,body,document_hash,lang,content_type,filename,message,created_at_ms) VALUES(?,'backfill-object',?,?,'und','text/plain','','',0)`, id, body, hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return hash
|
||||
}
|
||||
|
||||
func TestDocumentBackfillResumesAndDoesNotRescanCompletedStore(t *testing.T) {
|
||||
st := newTopologyStore(t)
|
||||
hashA := backfillFixture(t, st, "a", "first")
|
||||
hashB := backfillFixture(t, st, "b", "second")
|
||||
st.DocumentRoot = t.TempDir()
|
||||
first, err := st.BackfillDocuments(context.Background(), 1)
|
||||
if err != nil || first.LastUUID != "a" || first.Processed != 1 || first.Completed {
|
||||
t.Fatalf("first=%+v err=%v", first, err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`UPDATE revisions SET body='broken cache' WHERE uuid='a'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`VACUUM`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A new Store instance resumes the durable UUID cursor, not in-memory state.
|
||||
restarted := &store.Store{DB: st.DB, DocumentRoot: st.DocumentRoot}
|
||||
second, err := restarted.BackfillDocuments(context.Background(), 1)
|
||||
if err != nil || second.LastUUID != "b" || second.Processed != 2 {
|
||||
t.Fatalf("second=%+v err=%v", second, err)
|
||||
}
|
||||
final, err := restarted.BackfillDocuments(context.Background(), 1)
|
||||
if err != nil || !final.Completed {
|
||||
t.Fatalf("final=%+v err=%v", final, err)
|
||||
}
|
||||
for _, hash := range []string{hashA, hashB} {
|
||||
if _, err := os.Stat(filepath.Join(st.DocumentRoot, "sha256", hash[:2], hash)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Renaming the revisions table proves the completed path does not even
|
||||
// query it. This is synthetic data in a temporary DB, never production.
|
||||
if _, err := st.DB.Exec(`ALTER TABLE revisions RENAME TO hidden_revisions`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := restarted.BackfillDocuments(context.Background(), 128)
|
||||
if err != nil || again != final {
|
||||
t.Fatalf("completed progress=%+v err=%v", again, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentBackfillRootReplacementAndFailedBatch(t *testing.T) {
|
||||
st := newTopologyStore(t)
|
||||
hash := backfillFixture(t, st, "a", "original")
|
||||
st.DocumentRoot = filepath.Join(t.TempDir(), "documents")
|
||||
first, err := st.BackfillDocuments(context.Background(), 128)
|
||||
if err != nil || !first.Completed {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Keep exactly the same pathname, but replace its storage. Path-based
|
||||
// completion markers would incorrectly skip this empty replacement.
|
||||
if err := os.Rename(st.DocumentRoot, st.DocumentRoot+"-old"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`UPDATE revisions SET body='wrong' WHERE uuid='a'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed, err := st.BackfillDocuments(context.Background(), 128)
|
||||
if !errors.Is(err, store.ErrDocumentHashMismatch) || failed.Processed != 0 || failed.Completed || failed.StoreID == first.StoreID {
|
||||
t.Fatalf("failed=%+v err=%v", failed, err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`UPDATE revisions SET body='original' WHERE uuid='a'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retried, err := st.BackfillDocuments(context.Background(), 128)
|
||||
if err != nil || !retried.Completed || retried.Processed != 1 {
|
||||
t.Fatalf("retry=%+v err=%v", retried, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(st.DocumentRoot, "sha256", hash[:2], hash)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentBackfillPrefersCanonicalAndNeverRepairsCorruptionImplicitly(t *testing.T) {
|
||||
st := newTopologyStore(t)
|
||||
hash := backfillFixture(t, st, "a", "canonical")
|
||||
st.DocumentRoot = t.TempDir()
|
||||
dir := filepath.Join(st.DocumentRoot, "sha256", hash[:2])
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
filename := filepath.Join(dir, hash)
|
||||
if err := os.WriteFile(filename, []byte("canonical"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`UPDATE revisions SET body='' WHERE uuid='a'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
progress, err := st.BackfillDocuments(context.Background(), 128)
|
||||
if err != nil || !progress.Completed {
|
||||
t.Fatalf("canonical rejected: %v", err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`DELETE FROM document_backfills`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filename, []byte("broken"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.BackfillDocuments(context.Background(), 128); !errors.Is(err, store.ErrDocumentHashMismatch) {
|
||||
t.Fatalf("corrupt canonical: %v", err)
|
||||
}
|
||||
data, _ := os.ReadFile(filename)
|
||||
if string(data) != "broken" {
|
||||
t.Fatal("corruption overwritten")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentBackfillCancellationAndReadOnlyDoNotCreateFiles(t *testing.T) {
|
||||
st := newTopologyStore(t)
|
||||
st.DocumentRoot = filepath.Join(t.TempDir(), "new-root")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if err := st.EnsureDocumentStoreContext(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st.ReadOnly = true
|
||||
if err := st.EnsureDocumentStore(); !errors.Is(err, store.ErrReadOnly) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(st.DocumentRoot); !os.IsNotExist(err) {
|
||||
t.Fatal("cancelled/standby operation created root")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentBackfillSerializesAndLockWaitHonorsCancellation(t *testing.T) {
|
||||
st := newTopologyStore(t)
|
||||
backfillFixture(t, st, "a", "first")
|
||||
backfillFixture(t, st, "b", "second")
|
||||
st.DocumentRoot = t.TempDir()
|
||||
lock, err := os.OpenFile(filepath.Join(st.DocumentRoot, ".document-backfill.lock"), os.O_CREATE|os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer lock.Close()
|
||||
if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
if _, err := st.BackfillDocuments(ctx, 1); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("lock cancellation: %v", err)
|
||||
}
|
||||
if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_UN); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 2)
|
||||
for i := 0; i < 2; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
progress, err := st.BackfillDocuments(context.Background(), 1)
|
||||
if err != nil || progress.Completed {
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
progress, err := st.BackfillDocuments(context.Background(), 1)
|
||||
if err != nil || !progress.Completed || progress.Processed != 2 {
|
||||
t.Fatalf("double-counted migration: %+v %v", progress, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/geduldh"
|
||||
)
|
||||
|
||||
// MaterializeDocument supplies a verified node-local, read-only scratch copy
|
||||
// for tools needing a pathname. Callers must authorize the reference first and
|
||||
// close the lease after all subprocesses finish. DB-only legacy stores work too.
|
||||
// TMPDIR is the operator-controlled local scratch root, not DocumentRoot.
|
||||
func (s *Store) MaterializeDocument(ctx context.Context, hash string, maxBytes int64) (*geduldh.Lease, error) {
|
||||
s.documentMaterializerOnce.Do(func() {
|
||||
// These fixed safe defaults precede configurable deployment profiles.
|
||||
s.documentMaterializer, _ = geduldh.NewMaterializer(documentSource{s}, geduldh.MaterializeOptions{
|
||||
MaxBytes: 2 << 30, Concurrent: 2, Timeout: 2 * time.Minute,
|
||||
})
|
||||
})
|
||||
return s.documentMaterializer.Acquire(ctx, hash, maxBytes)
|
||||
}
|
||||
|
||||
type documentSource struct{ store *Store }
|
||||
|
||||
func (source documentSource) Stat(ctx context.Context, hash string) (geduldh.Info, error) {
|
||||
size, err := source.store.DocumentSizeContext(ctx, hash)
|
||||
return geduldh.Info{Hash: hash, Size: size}, err
|
||||
}
|
||||
|
||||
func (source documentSource) Open(ctx context.Context, hash string) (geduldh.Reader, error) {
|
||||
return source.store.OpenDocumentContext(ctx, hash)
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDocumentMaterializeContextAndDBByteSize(t *testing.T) {
|
||||
for _, disk := range []bool{false, true} {
|
||||
name := "database"
|
||||
if disk {
|
||||
name = "local"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
st := newTopologyStore(t)
|
||||
if disk {
|
||||
st.DocumentRoot = t.TempDir()
|
||||
if err := st.EnsureDocumentStore(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
object, err := st.CreatePageAtPathInDomain(mustDefaultDomainUUID(t, st), "/materialize.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("日本語\x00\xff\n")
|
||||
revision, err := st.SaveRevisionContent(object, body, "test", "und", "application/octet-stream", "materialize.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
size, err := st.DocumentSizeContext(context.Background(), revision.DocumentHash)
|
||||
if err != nil || size != int64(len(body)) {
|
||||
t.Fatalf("size=%d err=%v", size, err)
|
||||
}
|
||||
lease, err := st.MaterializeDocument(context.Background(), revision.DocumentHash, 1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(lease.Path)
|
||||
if err != nil || string(data) != string(body) {
|
||||
t.Fatalf("data=%q err=%v", data, err)
|
||||
}
|
||||
lease.Close()
|
||||
if _, err := os.Stat(lease.Path); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("cleanup: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
reader, err := st.OpenDocumentContext(ctx, revision.DocumentHash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
cancel()
|
||||
if _, err := reader.Read(make([]byte, 1)); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if _, err := reader.ReadAt(make([]byte, 1), 0); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("readat: %v", err)
|
||||
}
|
||||
if _, err := reader.Seek(0, io.SeekStart); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("seek: %v", err)
|
||||
}
|
||||
if _, err := st.ReadDocumentWindowContext(ctx, revision.DocumentHash, 0, 10); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("window: %v", err)
|
||||
}
|
||||
if _, err := st.DocumentSizeContext(ctx, revision.DocumentHash); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("stat: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,28 +2,50 @@ package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"pageswiki/internal/geduldh"
|
||||
)
|
||||
|
||||
type memoryDocumentReader struct {
|
||||
*bytes.Reader
|
||||
reader *bytes.Reader
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (memoryDocumentReader) Close() error { return nil }
|
||||
|
||||
func (r memoryDocumentReader) Read(p []byte) (int, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.reader.Read(p)
|
||||
}
|
||||
|
||||
func (r memoryDocumentReader) ReadAt(p []byte, offset int64) (int, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.reader.ReadAt(p, offset)
|
||||
}
|
||||
|
||||
func (r memoryDocumentReader) Seek(offset int64, whence int) (int64, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.reader.Seek(offset, whence)
|
||||
}
|
||||
|
||||
var (
|
||||
ErrInvalidDocumentHash = errors.New("invalid Document SHA-256")
|
||||
ErrDocumentHashMismatch = errors.New(
|
||||
"Document payload does not match SHA-256",
|
||||
)
|
||||
ErrInvalidDocumentHash = geduldh.ErrInvalidHash
|
||||
ErrDocumentHashMismatch = geduldh.ErrHashMismatch
|
||||
)
|
||||
|
||||
func MigrateLegacyDocumentRoot(
|
||||
@@ -131,18 +153,38 @@ func (s *Store) DocumentPath(hash string) (string, error) {
|
||||
// OpenDocument opens an immutable content-addressed Document. Callers must
|
||||
// perform authorization through a referencing object before invoking it.
|
||||
func (s *Store) OpenDocument(hash string) (io.ReadSeekCloser, error) {
|
||||
path, err := s.documentPath(hash)
|
||||
if err != nil {
|
||||
return s.OpenDocumentContext(context.Background(), hash)
|
||||
}
|
||||
|
||||
// OpenDocumentContext supports bounded ReaderAt access without hydrating a
|
||||
// revision body. A request/job context must be used by new storage consumers.
|
||||
func (s *Store) OpenDocumentContext(ctx context.Context, hash string) (geduldh.Reader, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if path != "" {
|
||||
return os.Open(path)
|
||||
if _, err := geduldh.Key(hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.DocumentRoot != "" {
|
||||
backend, err := s.documentStorage()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return backend.Open(ctx, hash)
|
||||
}
|
||||
var body string
|
||||
if err := s.DB.QueryRow(`SELECT COALESCE(body,'') FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, strings.ToLower(strings.TrimSpace(hash))).Scan(&body); err != nil {
|
||||
if err := s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, strings.ToLower(strings.TrimSpace(hash))).Scan(&body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return memoryDocumentReader{bytes.NewReader([]byte(body))}, nil
|
||||
return memoryDocumentReader{reader: bytes.NewReader([]byte(body)), ctx: ctx}, nil
|
||||
}
|
||||
|
||||
// documentStorage is the payload-only backend boundary. DocumentRoot still
|
||||
// anchors legacy calendars and instance paths; it must not become an S3 URL.
|
||||
// Remote selection is intentionally not exposed until canonical state, CLI,
|
||||
// deployment profiles and backend contract tests have been migrated.
|
||||
func (s *Store) documentStorage() (geduldh.Store, error) {
|
||||
return geduldh.NewLocal(s.DocumentRoot)
|
||||
}
|
||||
|
||||
type DocumentReference struct {
|
||||
@@ -205,191 +247,57 @@ func (s *Store) DocumentReferencesInDomain(domainUUID, hash string) ([]DocumentR
|
||||
return references, nil
|
||||
}
|
||||
|
||||
func (s *Store) persistDocument(
|
||||
expectedHash string,
|
||||
body string,
|
||||
) error {
|
||||
func (s *Store) persistDocument(expectedHash, body string) error {
|
||||
if s.ReadOnly {
|
||||
return ErrReadOnly
|
||||
}
|
||||
if s.DocumentRoot == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
actualHash :=
|
||||
revisionHash(
|
||||
body,
|
||||
)
|
||||
|
||||
if !strings.EqualFold(expectedHash, actualHash) {
|
||||
return fmt.Errorf(
|
||||
"%w: expected %s, got %s",
|
||||
ErrDocumentHashMismatch,
|
||||
expectedHash,
|
||||
actualHash,
|
||||
)
|
||||
}
|
||||
|
||||
path, err :=
|
||||
s.documentPath(
|
||||
actualHash,
|
||||
)
|
||||
|
||||
backend, err := s.documentStorage()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if existing, err := os.ReadFile(path); err == nil {
|
||||
if revisionHash(string(existing)) != actualHash {
|
||||
return fmt.Errorf(
|
||||
"%w: corrupt existing file %s",
|
||||
ErrDocumentHashMismatch,
|
||||
path,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
|
||||
dir :=
|
||||
filepath.Dir(
|
||||
path,
|
||||
)
|
||||
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
temp, err :=
|
||||
os.CreateTemp(
|
||||
dir,
|
||||
".document-*",
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tempPath :=
|
||||
temp.Name()
|
||||
|
||||
defer os.Remove(tempPath)
|
||||
|
||||
if err := temp.Chmod(0644); err != nil {
|
||||
temp.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := temp.WriteString(body); err != nil {
|
||||
temp.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
if err := temp.Sync(); err != nil {
|
||||
temp.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
if err := temp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.Rename(tempPath, path); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
return backend.PutIfAbsent(context.Background(), expectedHash, int64(len(body)), strings.NewReader(body))
|
||||
}
|
||||
|
||||
func (s *Store) readDocument(
|
||||
hash string,
|
||||
) (string, error) {
|
||||
func (s *Store) readDocument(hash string) (string, error) {
|
||||
if s.DocumentRoot == "" {
|
||||
return "", os.ErrNotExist
|
||||
}
|
||||
|
||||
path, err :=
|
||||
s.documentPath(
|
||||
hash,
|
||||
)
|
||||
|
||||
backend, err := s.documentStorage()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
payload, err :=
|
||||
os.ReadFile(
|
||||
path,
|
||||
)
|
||||
|
||||
reader, err := backend.Open(context.Background(), hash)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
body :=
|
||||
string(
|
||||
payload,
|
||||
)
|
||||
|
||||
if revisionHash(body) != strings.ToLower(hash) {
|
||||
return "", fmt.Errorf(
|
||||
"%w: corrupt file %s",
|
||||
ErrDocumentHashMismatch,
|
||||
path,
|
||||
)
|
||||
defer reader.Close()
|
||||
payload, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return body, nil
|
||||
if revisionHash(string(payload)) != strings.ToLower(strings.TrimSpace(hash)) {
|
||||
return "", ErrDocumentHashMismatch
|
||||
}
|
||||
return string(payload), nil
|
||||
}
|
||||
|
||||
func (s *Store) EnsureDocumentStore() error {
|
||||
if s.DocumentRoot == "" {
|
||||
return nil
|
||||
}
|
||||
return s.EnsureDocumentStoreContext(context.Background())
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(
|
||||
filepath.Join(
|
||||
s.DocumentRoot,
|
||||
"sha256",
|
||||
),
|
||||
0755,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rows, err := s.DB.Query(`
|
||||
SELECT
|
||||
document_hash,
|
||||
COALESCE(body, '')
|
||||
FROM revisions
|
||||
WHERE document_hash IS NOT NULL
|
||||
AND document_hash != ''
|
||||
`)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
var hash string
|
||||
var body string
|
||||
|
||||
if err := rows.Scan(&hash, &body); err != nil {
|
||||
rows.Close()
|
||||
func (s *Store) EnsureDocumentStoreContext(ctx context.Context) error {
|
||||
for {
|
||||
progress, err := s.BackfillDocuments(ctx, 128)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := s.persistDocument(hash, body); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
if progress.Completed {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
return rows.Close()
|
||||
}
|
||||
|
||||
func (s *Store) hydrateRevision(
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
const MaxDocumentWindowBytes int64 = 256 << 10
|
||||
@@ -17,6 +17,13 @@ type DocumentWindow struct {
|
||||
}
|
||||
|
||||
func (s *Store) ReadDocumentWindow(documentHash string, offset, limit int64) (DocumentWindow, error) {
|
||||
return s.ReadDocumentWindowContext(context.Background(), documentHash, offset, limit)
|
||||
}
|
||||
|
||||
func (s *Store) ReadDocumentWindowContext(ctx context.Context, documentHash string, offset, limit int64) (DocumentWindow, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return DocumentWindow{}, err
|
||||
}
|
||||
if offset < 0 {
|
||||
return DocumentWindow{}, errors.New("negative Document offset")
|
||||
}
|
||||
@@ -27,32 +34,32 @@ func (s *Store) ReadDocumentWindow(documentHash string, offset, limit int64) (Do
|
||||
limit = MaxDocumentWindowBytes
|
||||
}
|
||||
if s.DocumentRoot != "" {
|
||||
path, err := s.DocumentPath(documentHash)
|
||||
backend, err := s.documentStorage()
|
||||
if err != nil {
|
||||
return DocumentWindow{}, err
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
file, err := backend.Open(ctx, documentHash)
|
||||
if err != nil {
|
||||
return DocumentWindow{}, err
|
||||
}
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
size, err := file.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return DocumentWindow{}, err
|
||||
}
|
||||
if offset > info.Size() {
|
||||
offset = info.Size()
|
||||
if offset > size {
|
||||
offset = size
|
||||
}
|
||||
data := make([]byte, minInt64(limit, info.Size()-offset))
|
||||
data := make([]byte, minInt64(limit, size-offset))
|
||||
n, err := file.ReadAt(data, offset)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return DocumentWindow{}, err
|
||||
}
|
||||
data = data[:n]
|
||||
return DocumentWindow{Data: data, Offset: offset, NextOffset: offset + int64(n), TotalBytes: info.Size(), Truncated: offset+int64(n) < info.Size()}, nil
|
||||
return DocumentWindow{Data: data, Offset: offset, NextOffset: offset + int64(n), TotalBytes: size, Truncated: offset+int64(n) < size}, nil
|
||||
}
|
||||
var body string
|
||||
if err := s.DB.QueryRow(`SELECT COALESCE(body,'') FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, documentHash).Scan(&body); err != nil {
|
||||
if err := s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, documentHash).Scan(&body); err != nil {
|
||||
return DocumentWindow{}, err
|
||||
}
|
||||
total := int64(len(body))
|
||||
@@ -79,18 +86,25 @@ func (s *Store) GetRevisionMetadata(revisionUUID string) (*Revision, error) {
|
||||
}
|
||||
|
||||
func (s *Store) DocumentSize(documentHash string) (int64, error) {
|
||||
return s.DocumentSizeContext(context.Background(), documentHash)
|
||||
}
|
||||
|
||||
func (s *Store) DocumentSizeContext(ctx context.Context, documentHash string) (int64, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if s.DocumentRoot != "" {
|
||||
path, err := s.DocumentPath(documentHash)
|
||||
backend, err := s.documentStorage()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
info, err := backend.Stat(ctx, documentHash)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Size(), nil
|
||||
return info.Size, nil
|
||||
}
|
||||
var size int64
|
||||
err := s.DB.QueryRow(`SELECT length(COALESCE(body,'')) FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, documentHash).Scan(&size)
|
||||
err := s.DB.QueryRowContext(ctx, `SELECT length(CAST(COALESCE(body,'') AS BLOB)) FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, documentHash).Scan(&size)
|
||||
return size, err
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/dregarnuhr"
|
||||
)
|
||||
|
||||
func TestDregarnuhrPersistedLifecycle(t *testing.T) {
|
||||
for _, terminal := range []string{"succeeded", "failed", "skipped", "cancelled"} {
|
||||
t.Run(terminal, func(t *testing.T) {
|
||||
s := newSchemaTestStore(t)
|
||||
if err := s.EnsureEventJobsSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings, err := s.GetQueueSettings()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings.RetryBaseSeconds = 30
|
||||
if err := s.SetQueueSettings(settings); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
if _, err := s.EnqueueEventWithOptions("", "test.lifecycle", terminal, "{}", at, 7, 3); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
if _, err := s.ClaimDueEventAt(ctx, at.Add(-time.Millisecond)); !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatalf("early claim: %v", err)
|
||||
}
|
||||
for attempt := 1; attempt <= 3; attempt++ {
|
||||
job, err := s.ClaimDueEventAt(ctx, at)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job.Attempts != attempt || job.Status != "running" || job.StartedAtMS != at.UnixMilli() {
|
||||
t.Fatalf("claim: %+v", job)
|
||||
}
|
||||
finished := at.Add(time.Second)
|
||||
runErr := errors.New("synthetic failure")
|
||||
if attempt == 3 {
|
||||
switch terminal {
|
||||
case "succeeded":
|
||||
runErr = nil
|
||||
case "skipped":
|
||||
runErr = fmt.Errorf("inapplicable: %w", ErrEventSkipped)
|
||||
case "cancelled":
|
||||
if err := s.CancelEventJob(job.UUID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := s.FinishEventAt(ctx, job.UUID, runErr, finished); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
persisted, err := scanEventJob(s.DB.QueryRow(`SELECT `+eventJobColumns+` FROM event_jobs WHERE uuid=?`, job.UUID))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if persisted.UpdatedAt != finished.UnixMilli() {
|
||||
t.Fatalf("completion time: %+v", persisted)
|
||||
}
|
||||
if attempt < 3 {
|
||||
at = finished.Add(time.Duration(30*(1<<(attempt-1))) * time.Second)
|
||||
if persisted.Status != "queued" || persisted.FinishedAtMS != 0 || persisted.RunAtMS != at.UnixMilli() {
|
||||
t.Fatalf("retry: %+v", persisted)
|
||||
}
|
||||
if _, err := s.ClaimDueEventAt(ctx, at.Add(-time.Millisecond)); !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatalf("early retry: %v", err)
|
||||
}
|
||||
} else if persisted.Status != terminal || persisted.FinishedAtMS != finished.UnixMilli() {
|
||||
t.Fatalf("terminal: %+v", persisted)
|
||||
}
|
||||
}
|
||||
if _, err := s.ClaimDueEventAt(ctx, at.Add(48*time.Hour)); !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatalf("terminal claimed again: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if ErrEventSkipped != dregarnuhr.ErrSkipped {
|
||||
t.Fatal("legacy skip identity changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDregarnuhrCancellationErrorsPropagate(t *testing.T) {
|
||||
s := newSchemaTestStore(t)
|
||||
if err := s.EnsureEventJobsSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.EventCancellationState(context.Background(), "missing"); !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatalf("missing job: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := s.ClaimDueEventAt(ctx, time.Now()); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancelled claim: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"pageswiki/internal/db"
|
||||
"pageswiki/internal/store"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEditorRevisionCAS(t *testing.T) {
|
||||
file := t.TempDir() + "/editor.db"
|
||||
database, err := db.Open(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err = db.Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := t.TempDir() + "/documents"
|
||||
s := store.NewWithDocumentRoot(database, root)
|
||||
object, err := s.CreatePageAtPath("/editor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base, err := s.SaveRevisionContentIfHead(object, []byte("base"), "", "und", "text/html", "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = s.SaveRevisionContentIfHead(object, []byte("stale"), "", "und", "text/html", "", ""); !errors.Is(err, store.ErrRevisionConflict) {
|
||||
t.Fatalf("stale empty head: %v", err)
|
||||
}
|
||||
en, err := s.GetRevisionForExactLanguage(object, "en")
|
||||
if err != nil || en.UUID != "" || en.Body != "" {
|
||||
t.Fatalf("language fallback: %+v %v", en, err)
|
||||
}
|
||||
if _, err = s.SaveRevisionContentIfHead(object, []byte("english"), "", "en", "text/html", "", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Independent Store and connection: mutex alone cannot enforce this CAS.
|
||||
database2, err := db.Open(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database2.Close()
|
||||
s2 := store.NewWithDocumentRoot(database2, root)
|
||||
start := make(chan struct{})
|
||||
results := make(chan error, 2)
|
||||
var wg sync.WaitGroup
|
||||
for _, st := range []*store.Store{s, s2} {
|
||||
wg.Add(1)
|
||||
go func(st *store.Store) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
_, e := st.SaveRevisionContentIfHead(object, []byte("next"), "", "und", "text/html", "", base.UUID)
|
||||
results <- e
|
||||
}(st)
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(results)
|
||||
won, lost := 0, 0
|
||||
for err := range results {
|
||||
if err == nil {
|
||||
won++
|
||||
} else if errors.Is(err, store.ErrRevisionConflict) {
|
||||
lost++
|
||||
} else {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if won != 1 || lost != 1 {
|
||||
t.Fatalf("won=%d lost=%d", won, lost)
|
||||
}
|
||||
old, err := s.GetRevision(base.UUID)
|
||||
if err != nil || old.Body != "base" {
|
||||
t.Fatal("immutable base overwritten", err)
|
||||
}
|
||||
head, err := s.GetRevisionMetadataForExactLanguage(object, "und")
|
||||
if err != nil || head.UUID == base.UUID || head.Body != "" {
|
||||
t.Fatalf("head metadata: %+v %v", head, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
func TestClaimDueEventPriorityAndReturnedState(t *testing.T) {
|
||||
s := newSchemaTestStore(t)
|
||||
if err := s.EnsureEventJobsSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, item := range []struct {
|
||||
key string
|
||||
priority int
|
||||
due time.Time
|
||||
}{
|
||||
{"low", 1, time.Now().Add(-time.Hour)}, {"high", 9, time.Now().Add(-time.Minute)}, {"future", 99, time.Now().Add(time.Hour)},
|
||||
} {
|
||||
if _, err := s.EnqueueEventWithOptions("", "test", item.key, "opaque", item.due, item.priority, 3); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"high", "low"} {
|
||||
job, err := s.ClaimDueEvent()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job.Key != key || job.Status != "running" || job.Attempts != 1 || job.StartedAtMS == 0 || job.UpdatedAt != job.StartedAtMS {
|
||||
t.Fatalf("wrong claimed state: %+v", job)
|
||||
}
|
||||
}
|
||||
if _, err := s.ClaimDueEvent(); !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatalf("future/running claimed: %v", err)
|
||||
}
|
||||
s.ReadOnly = true
|
||||
if _, err := s.ClaimDueEvent(); !errors.Is(err, ErrReadOnly) {
|
||||
t.Fatalf("standby claim: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimDueEventConcurrentWriters(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "queue.db")
|
||||
database, err := db.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
s := &Store{DB: database}
|
||||
if err := s.EnsureEventJobsSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const total = 64
|
||||
for n := 0; n < total; n++ {
|
||||
if _, err := s.EnqueueEvent("test", fmt.Sprint(n), "opaque", time.Now().Add(-time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
jobs := make(chan string, total)
|
||||
errs := make(chan error, 9)
|
||||
// Independent DB handles exercise file-level contention, not an in-process
|
||||
// mutex or a single sql.DB connection serializing the test for us.
|
||||
for range 8 {
|
||||
other, err := db.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer other.Close()
|
||||
worker := &Store{DB: other}
|
||||
for {
|
||||
job, err := worker.ClaimDueEvent()
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
jobs <- job.UUID
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for n := 0; n < 100; n++ {
|
||||
if _, err := database.Exec(`UPDATE event_queue_settings SET max_depth=max_depth+1 WHERE id=1`); err != nil {
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
wg.Wait()
|
||||
close(jobs)
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Error(err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for id := range jobs {
|
||||
if seen[id] {
|
||||
t.Error("duplicate claim", id)
|
||||
}
|
||||
seen[id] = true
|
||||
}
|
||||
if len(seen) != total {
|
||||
t.Fatalf("claimed=%d want=%d", len(seen), total)
|
||||
}
|
||||
var once int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM event_jobs WHERE status='running' AND attempts=1 AND started_at_ms=updated_at_ms`).Scan(&once); err != nil || once != total {
|
||||
t.Fatalf("persisted claims=%d err=%v", once, err)
|
||||
}
|
||||
}
|
||||
+69
-181
@@ -9,26 +9,11 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"pageswiki/internal/dregarnuhr"
|
||||
)
|
||||
|
||||
type EventJob struct {
|
||||
UUID string `json:"uuid"`
|
||||
EventUUID string `json:"event_uuid,omitempty"`
|
||||
Type string `json:"event_type"`
|
||||
Key string `json:"event_key"`
|
||||
Payload string `json:"payload,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Priority int `json:"priority"`
|
||||
Attempts int `json:"attempts"`
|
||||
MaxAttempts int `json:"max_attempts"`
|
||||
RunAtMS int64 `json:"run_at_ms"`
|
||||
StartedAtMS int64 `json:"started_at_ms,omitempty"`
|
||||
FinishedAtMS int64 `json:"finished_at_ms,omitempty"`
|
||||
CancelRequested bool `json:"cancel_requested"`
|
||||
Error string `json:"error,omitempty"`
|
||||
CreatedAt int64 `json:"created_at_ms"`
|
||||
UpdatedAt int64 `json:"updated_at_ms"`
|
||||
}
|
||||
// EventJob retains the existing public JSON shape and source compatibility.
|
||||
type EventJob = dregarnuhr.Job
|
||||
|
||||
type EventDefinition struct {
|
||||
UUID string `json:"uuid"`
|
||||
@@ -74,13 +59,7 @@ type EventLogTrend struct {
|
||||
SampleMessage string `json:"sample_message,omitempty"`
|
||||
}
|
||||
|
||||
type QueueSettings struct {
|
||||
Paused bool `json:"paused"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
MaxDepth int `json:"max_depth"`
|
||||
RetryBaseSeconds int `json:"retry_base_seconds"`
|
||||
RetentionHours int `json:"retention_hours"`
|
||||
}
|
||||
type QueueSettings = dregarnuhr.Settings
|
||||
|
||||
type EventJobStats struct {
|
||||
Queued int `json:"queued"`
|
||||
@@ -91,97 +70,18 @@ type EventJobStats struct {
|
||||
Cancelled int `json:"cancelled"`
|
||||
}
|
||||
|
||||
var ErrEventSkipped = errors.New("event is not applicable")
|
||||
var ErrEventSkipped = dregarnuhr.ErrSkipped
|
||||
|
||||
func (s *Store) EnsureEventJobsSchema() error {
|
||||
if _, err := s.DB.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS event_jobs (
|
||||
uuid TEXT PRIMARY KEY,
|
||||
event_type TEXT NOT NULL,
|
||||
event_key TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
run_at_ms INTEGER NOT NULL,
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
created_at_ms INTEGER NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL
|
||||
)
|
||||
`); err != nil {
|
||||
if err := s.ensureSchemaComponent("event-jobs"); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, statement := range []string{
|
||||
`ALTER TABLE event_jobs ADD COLUMN event_uuid TEXT NOT NULL DEFAULT ''`,
|
||||
`ALTER TABLE event_jobs ADD COLUMN priority INTEGER NOT NULL DEFAULT 0`,
|
||||
`ALTER TABLE event_jobs ADD COLUMN max_attempts INTEGER NOT NULL DEFAULT 3`,
|
||||
`ALTER TABLE event_jobs ADD COLUMN started_at_ms INTEGER NOT NULL DEFAULT 0`,
|
||||
`ALTER TABLE event_jobs ADD COLUMN finished_at_ms INTEGER NOT NULL DEFAULT 0`,
|
||||
`ALTER TABLE event_jobs ADD COLUMN cancel_requested INTEGER NOT NULL DEFAULT 0`,
|
||||
} {
|
||||
_, _ = s.DB.Exec(statement)
|
||||
}
|
||||
statements := []string{
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_jobs_listing ON event_jobs (created_at_ms DESC, uuid DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_jobs_status_listing ON event_jobs (status, created_at_ms DESC, uuid DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_jobs_history ON event_jobs (created_at_ms DESC, uuid DESC) WHERE status IN ('succeeded','skipped','failed','cancelled')`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_jobs_due
|
||||
ON event_jobs (status, priority DESC, run_at_ms, created_at_ms)`,
|
||||
`CREATE TABLE IF NOT EXISTS event_definitions (
|
||||
uuid TEXT PRIMARY KEY, name TEXT NOT NULL, event_type TEXT NOT NULL,
|
||||
event_key TEXT NOT NULL, payload TEXT NOT NULL DEFAULT '',
|
||||
schedule_kind TEXT NOT NULL DEFAULT 'manual',
|
||||
interval_seconds INTEGER NOT NULL DEFAULT 0,
|
||||
next_run_at_ms INTEGER NOT NULL DEFAULT 0,
|
||||
enabled INTEGER NOT NULL DEFAULT 1, priority INTEGER NOT NULL DEFAULT 0,
|
||||
max_attempts INTEGER NOT NULL DEFAULT 3,
|
||||
created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_definitions_due
|
||||
ON event_definitions (enabled, next_run_at_ms)`,
|
||||
`CREATE TABLE IF NOT EXISTS event_logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, job_uuid TEXT NOT NULL,
|
||||
event_uuid TEXT NOT NULL DEFAULT '', level TEXT NOT NULL,
|
||||
message TEXT NOT NULL, created_at_ms INTEGER NOT NULL
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_logs_job
|
||||
ON event_logs (job_uuid, created_at_ms DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_logs_job_id ON event_logs (job_uuid, id DESC)`,
|
||||
`CREATE TABLE IF NOT EXISTS event_log_policies (
|
||||
event_type TEXT PRIMARY KEY,
|
||||
retention_hours INTEGER NOT NULL DEFAULT 168,
|
||||
aggregate_after_hours INTEGER NOT NULL DEFAULT 24,
|
||||
bucket_seconds INTEGER NOT NULL DEFAULT 60,
|
||||
max_rows INTEGER NOT NULL DEFAULT 5000
|
||||
)`,
|
||||
`INSERT OR IGNORE INTO event_log_policies(event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows)
|
||||
VALUES('*',168,24,60,5000)`,
|
||||
`CREATE TABLE IF NOT EXISTS event_log_buckets (
|
||||
event_type TEXT NOT NULL,
|
||||
bucket_start_ms INTEGER NOT NULL,
|
||||
level TEXT NOT NULL,
|
||||
count INTEGER NOT NULL DEFAULT 0,
|
||||
first_at_ms INTEGER NOT NULL,
|
||||
last_at_ms INTEGER NOT NULL,
|
||||
sample_message TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY(event_type,bucket_start_ms,level)
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_log_buckets_time ON event_log_buckets(event_type,bucket_start_ms DESC)`,
|
||||
`CREATE TABLE IF NOT EXISTS event_queue_settings (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1), paused INTEGER NOT NULL DEFAULT 0,
|
||||
concurrency INTEGER NOT NULL DEFAULT 1,
|
||||
max_depth INTEGER NOT NULL DEFAULT 10000,
|
||||
retry_base_seconds INTEGER NOT NULL DEFAULT 30,
|
||||
retention_hours INTEGER NOT NULL DEFAULT 168
|
||||
)`,
|
||||
`INSERT OR IGNORE INTO event_queue_settings
|
||||
(id, paused, concurrency, max_depth, retry_base_seconds, retention_hours)
|
||||
VALUES (1, 0, 1, 10000, 30, 168)`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := s.DB.Exec(statement); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.recoverInterruptedEventJobs()
|
||||
}
|
||||
|
||||
// Startup recovery mutates runtime state, not schema. Never call it from
|
||||
// status/preflight, read-only standby or a migration rehearsal.
|
||||
func (s *Store) recoverInterruptedEventJobs() error {
|
||||
now := time.Now().UnixMilli()
|
||||
if _, err := s.DB.Exec(`UPDATE event_jobs SET status='queued',
|
||||
error=CASE WHEN error='' THEN 'execution interrupted by process restart' ELSE error END,
|
||||
@@ -250,86 +150,67 @@ const eventJobColumns = `uuid, event_uuid, event_type, event_key, payload,
|
||||
finished_at_ms, cancel_requested, error, created_at_ms, updated_at_ms`
|
||||
|
||||
func (s *Store) ClaimDueEvent() (*EventJob, error) {
|
||||
tx, err := s.DB.Begin()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return s.ClaimDueEventAt(context.Background(), time.Now())
|
||||
}
|
||||
|
||||
// ClaimDueEventAt keeps selection and claim in one statement; injected time
|
||||
// allows scheduler tests without real sleeps or production data.
|
||||
func (s *Store) ClaimDueEventAt(ctx context.Context, at time.Time) (*EventJob, error) {
|
||||
if s.ReadOnly {
|
||||
return nil, ErrReadOnly
|
||||
}
|
||||
defer tx.Rollback()
|
||||
job, err := scanEventJob(tx.QueryRow(`
|
||||
SELECT `+eventJobColumns+` FROM event_jobs
|
||||
WHERE status = 'queued' AND run_at_ms <= ?
|
||||
ORDER BY priority DESC, run_at_ms, created_at_ms LIMIT 1`,
|
||||
time.Now().UnixMilli()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
result, err := tx.Exec(`
|
||||
now := at.UnixMilli()
|
||||
// A deferred SELECT transaction cannot always upgrade its WAL snapshot to a
|
||||
// writer (SQLITE_BUSY[_SNAPSHOT]), even with busy_timeout. Select and claim in
|
||||
// one write statement instead. SQLite serializes concurrent claimers; RETURNING
|
||||
// gives the committed attempt/status/timestamps, not the pre-claim snapshot.
|
||||
return scanEventJob(s.DB.QueryRowContext(ctx, `
|
||||
UPDATE event_jobs SET status='running', attempts=attempts+1,
|
||||
started_at_ms=?, updated_at_ms=? WHERE uuid=? AND status='queued'`,
|
||||
now, now, job.UUID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
count, err := result.RowsAffected()
|
||||
if err != nil || count != 1 {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
job.Status, job.Attempts, job.StartedAtMS = "running", job.Attempts+1, now
|
||||
return job, nil
|
||||
started_at_ms=?, updated_at_ms=?
|
||||
WHERE uuid=(SELECT uuid FROM event_jobs
|
||||
WHERE status='queued' AND run_at_ms<=?
|
||||
ORDER BY priority DESC,run_at_ms,created_at_ms,uuid LIMIT 1)
|
||||
AND status='queued'
|
||||
RETURNING `+eventJobColumns, now, now, now))
|
||||
}
|
||||
|
||||
func (s *Store) FinishEvent(uuid string, runErr error) error {
|
||||
job, err := scanEventJob(s.DB.QueryRow(`SELECT `+eventJobColumns+` FROM event_jobs WHERE uuid=?`, uuid))
|
||||
return s.FinishEventAt(context.Background(), uuid, runErr, time.Now())
|
||||
}
|
||||
|
||||
func (s *Store) FinishEventAt(ctx context.Context, uuid string, runErr error, at time.Time) error {
|
||||
job, err := scanEventJob(s.DB.QueryRowContext(ctx, `SELECT `+eventJobColumns+` FROM event_jobs WHERE uuid=?`, uuid))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status, message := "succeeded", ""
|
||||
if runErr != nil {
|
||||
message = runErr.Error()
|
||||
if errors.Is(runErr, ErrEventSkipped) {
|
||||
status = "skipped"
|
||||
} else if job.CancelRequested {
|
||||
status = "cancelled"
|
||||
} else if job.Attempts < job.MaxAttempts {
|
||||
settings, settingsErr := s.GetQueueSettings()
|
||||
if settingsErr != nil {
|
||||
return settingsErr
|
||||
}
|
||||
delay := time.Duration(settings.RetryBaseSeconds) * time.Second
|
||||
for i := 1; i < job.Attempts; i++ {
|
||||
delay *= 2
|
||||
if delay >= 24*time.Hour {
|
||||
delay = 24 * time.Hour
|
||||
break
|
||||
}
|
||||
}
|
||||
_, err = s.DB.Exec(`UPDATE event_jobs SET status='queued', error=?,
|
||||
run_at_ms=?, finished_at_ms=0, updated_at_ms=? WHERE uuid=?`,
|
||||
message, time.Now().Add(delay).UnixMilli(), time.Now().UnixMilli(), uuid)
|
||||
_ = s.AddEventLog(uuid, job.EventUUID, "warning", "execution failed; retry scheduled: "+message)
|
||||
base := 1
|
||||
if runErr != nil && !errors.Is(runErr, ErrEventSkipped) && !job.CancelRequested && job.Attempts < job.MaxAttempts {
|
||||
settings, err := s.GetQueueSettings()
|
||||
if err != nil {
|
||||
return err
|
||||
} else {
|
||||
status = "failed"
|
||||
}
|
||||
base = settings.RetryBaseSeconds
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
_, err = s.DB.Exec(`UPDATE event_jobs SET status=?, error=?,
|
||||
outcome := dregarnuhr.Complete(*job, runErr, base, at)
|
||||
if outcome.Status == "queued" {
|
||||
_, err = s.DB.ExecContext(ctx, `UPDATE event_jobs SET status='queued', error=?,
|
||||
run_at_ms=?, finished_at_ms=0, updated_at_ms=? WHERE uuid=?`,
|
||||
outcome.Message, outcome.RetryAt.UnixMilli(), at.UnixMilli(), uuid)
|
||||
if err == nil {
|
||||
_ = s.AddEventLog(uuid, job.EventUUID, outcome.Level, "execution failed; retry scheduled: "+outcome.Message)
|
||||
}
|
||||
return err
|
||||
}
|
||||
_, err = s.DB.ExecContext(ctx, `UPDATE event_jobs SET status=?, error=?,
|
||||
finished_at_ms=?, updated_at_ms=? WHERE uuid=?`,
|
||||
status, message, now, now, uuid)
|
||||
level := "info"
|
||||
if status == "failed" {
|
||||
level = "error"
|
||||
outcome.Status, outcome.Message, at.UnixMilli(), at.UnixMilli(), uuid)
|
||||
message := outcome.Status
|
||||
if outcome.Message != "" {
|
||||
message += ": " + outcome.Message
|
||||
}
|
||||
if err == nil {
|
||||
_ = s.AddEventLog(uuid, job.EventUUID, outcome.Level, message)
|
||||
}
|
||||
_ = s.AddEventLog(uuid, job.EventUUID, level, status+func() string {
|
||||
if message != "" {
|
||||
return ": " + message
|
||||
}
|
||||
return ""
|
||||
}())
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -442,9 +323,16 @@ func (s *Store) RetryEventJob(jobUUID string) error {
|
||||
}
|
||||
|
||||
func (s *Store) EventCancellationRequested(jobUUID string) bool {
|
||||
requested, err := s.EventCancellationState(context.Background(), jobUUID)
|
||||
return err == nil && requested
|
||||
}
|
||||
|
||||
// EventCancellationState lets the runner distinguish a read failure from a
|
||||
// confirmed absence of cancellation. The boolean compatibility API stays.
|
||||
func (s *Store) EventCancellationState(ctx context.Context, jobUUID string) (bool, error) {
|
||||
var requested int
|
||||
err := s.DB.QueryRow(`SELECT cancel_requested FROM event_jobs WHERE uuid=?`, jobUUID).Scan(&requested)
|
||||
return err == nil && requested != 0
|
||||
err := s.DB.QueryRowContext(ctx, `SELECT cancel_requested FROM event_jobs WHERE uuid=?`, jobUUID).Scan(&requested)
|
||||
return requested != 0, err
|
||||
}
|
||||
|
||||
func (s *Store) PurgeEventHistory(retentionHours int) error {
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
"pageswiki/internal/geduldh"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
func TestGeduldhPublicationPrecedesRevisionCommit(t *testing.T) {
|
||||
for _, mode := range []string{"missing-root", "corrupt-existing", "metadata-conflict"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
database, err := db.Open(filepath.Join(root, "pages.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err := db.Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
documents := filepath.Join(root, "documents")
|
||||
st := store.NewWithDocumentRoot(database, documents)
|
||||
page, err := st.CreatePageAtPath("/geduldh-contract")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base, err := st.SaveRevisionContent(page, []byte("base"), "base", "en", "text/plain", "base.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := []byte("new immutable payload")
|
||||
sum := sha256.Sum256(payload)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
filename := filepath.Join(documents, "sha256", hash[:2], hash)
|
||||
switch mode {
|
||||
case "missing-root":
|
||||
st.DocumentRoot = filepath.Join(root, "unavailable")
|
||||
case "corrupt-existing":
|
||||
if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filename, []byte("damaged"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
expected := base.UUID
|
||||
if mode == "metadata-conflict" {
|
||||
expected = "not-the-current-head"
|
||||
}
|
||||
_, err = st.SaveRevisionContentIfHead(page, payload, "must not commit", "en", "text/plain", "new.txt", expected)
|
||||
if err == nil {
|
||||
t.Fatal("unsafe revision committed")
|
||||
}
|
||||
if mode == "corrupt-existing" && (!errors.Is(err, geduldh.ErrHashMismatch) || !errors.Is(err, store.ErrDocumentHashMismatch)) {
|
||||
t.Fatalf("storage error identity changed: %v", err)
|
||||
}
|
||||
if mode == "metadata-conflict" && !errors.Is(err, store.ErrRevisionConflict) {
|
||||
t.Fatalf("conflict changed: %v", err)
|
||||
}
|
||||
var count int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM revisions WHERE object_uuid=?`, page).Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("failed save added revision: count=%d", count)
|
||||
}
|
||||
st.DocumentRoot = documents
|
||||
loaded, err := st.GetRevision(base.UUID)
|
||||
if err != nil || loaded.Body != "base" {
|
||||
t.Fatalf("previous revision lost: %v", err)
|
||||
}
|
||||
switch mode {
|
||||
case "corrupt-existing":
|
||||
data, err := os.ReadFile(filename)
|
||||
if err != nil || string(data) != "damaged" {
|
||||
t.Fatalf("corruption silently repaired: %v", err)
|
||||
}
|
||||
case "metadata-conflict":
|
||||
// A failed metadata transaction may leave a valid unreferenced blob;
|
||||
// the payload layer must not delete it without reference/retention proof.
|
||||
backend, err := geduldh.NewLocal(documents)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := backend.Verify(context.Background(), hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"pageswiki/internal/schutzaria"
|
||||
)
|
||||
|
||||
var groupNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{1,63}$`)
|
||||
@@ -25,19 +27,8 @@ type GroupPermission struct {
|
||||
UUID, GroupUUID, ScopePath, Permission string
|
||||
}
|
||||
|
||||
var groupPermissions = map[string]map[string]bool{
|
||||
"read": {"read": true},
|
||||
"read_write_once": {"read": true, "create": true, "upload": true},
|
||||
"read_write": {"read": true, "create": true, "write": true, "upload": true, "rename": true, "link": true},
|
||||
"upload_only": {"upload": true},
|
||||
}
|
||||
|
||||
func NormalizeGroupPermission(value string) (string, error) {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
if _, ok := groupPermissions[value]; !ok {
|
||||
return "", errors.New("invalid group permission")
|
||||
}
|
||||
return value, nil
|
||||
return schutzaria.NormalizeGroupPermission(value)
|
||||
}
|
||||
|
||||
func normalizeGroupName(name string) (string, error) {
|
||||
@@ -375,8 +366,7 @@ func groupPermissionAllowsFrom(q pathQueryer, domainUUID, principalUUID, capabil
|
||||
if err := rows.Scan(&scope, &permission); err != nil {
|
||||
return false, err
|
||||
}
|
||||
inside := targetPath == scope || scope == "/" || strings.HasPrefix(targetPath, strings.TrimRight(scope, "/")+"/")
|
||||
if inside && groupPermissions[permission][capability] {
|
||||
if schutzaria.GroupPermissionAllows(scope, permission, capability, targetPath) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -30,47 +29,7 @@ type ImportItem struct {
|
||||
}
|
||||
|
||||
func (s *Store) EnsureImportSchema() error {
|
||||
statements := []string{
|
||||
`CREATE TABLE IF NOT EXISTS import_sessions(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,actor_uuid TEXT NOT NULL DEFAULT '',archive_name TEXT NOT NULL,archive_path TEXT NOT NULL,target_prefix TEXT NOT NULL DEFAULT '/',status TEXT NOT NULL DEFAULT 'queued',total_files INTEGER NOT NULL DEFAULT 0,markdown_files INTEGER NOT NULL DEFAULT 0,error TEXT NOT NULL DEFAULT '',created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`,
|
||||
`CREATE TABLE IF NOT EXISTS import_items(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,source_path TEXT NOT NULL,target_path TEXT NOT NULL,size INTEGER NOT NULL DEFAULT 0,content_type TEXT NOT NULL DEFAULT '',title TEXT NOT NULL DEFAULT '',suggested_quartier TEXT NOT NULL DEFAULT '',suggested_street TEXT NOT NULL DEFAULT '',suggested_spot TEXT NOT NULL DEFAULT '',confidence REAL NOT NULL DEFAULT 0,selected INTEGER NOT NULL DEFAULT 1,PRIMARY KEY(session_uuid,sequence))`,
|
||||
`CREATE TABLE IF NOT EXISTS import_journal(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,path TEXT NOT NULL,object_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,sequence))`,
|
||||
`CREATE TABLE IF NOT EXISTS import_city_journal(session_uuid TEXT NOT NULL,kind TEXT NOT NULL,node_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,kind,node_uuid))`,
|
||||
`CREATE TABLE IF NOT EXISTS import_city_edges(session_uuid TEXT NOT NULL,spot_uuid TEXT NOT NULL,page_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,spot_uuid,page_uuid))`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_import_sessions_domain ON import_sessions(domain_uuid,created_at_ms DESC)`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := s.DB.Exec(statement); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_ = ensureImportColumn(s.DB, "import_journal", "completed", "INTEGER NOT NULL DEFAULT 0")
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureImportColumn(db *sql.DB, table, column, declaration string) error {
|
||||
rows, err := db.Query(`PRAGMA table_info(` + table + `)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found := false
|
||||
for rows.Next() {
|
||||
var cid, notnull, pk int
|
||||
var name, kind string
|
||||
var defaultValue any
|
||||
if err := rows.Scan(&cid, &name, &kind, ¬null, &defaultValue, &pk); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if name == column {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
if found {
|
||||
return nil
|
||||
}
|
||||
_, err = db.Exec(`ALTER TABLE ` + table + ` ADD COLUMN ` + column + ` ` + declaration)
|
||||
return err
|
||||
return s.ensureSchemaComponent("imports")
|
||||
}
|
||||
|
||||
func (s *Store) CreateImportSession(domainUUID, actor, archiveName, archivePath, targetPrefix string) (string, error) {
|
||||
|
||||
@@ -2,7 +2,6 @@ package store
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -246,10 +245,8 @@ func (s *Store) ListSelectedChildEntriesInDomain(domainUUID, parent string, page
|
||||
entry.Title = firstHTMLHeading(sourcePrefix)
|
||||
}
|
||||
if documentHash != "" && s.DocumentRoot != "" {
|
||||
if path, pathErr := s.documentPath(documentHash); pathErr == nil {
|
||||
if info, statErr := os.Stat(path); statErr == nil {
|
||||
entry.Size = int(info.Size())
|
||||
}
|
||||
if size, err := s.DocumentSize(documentHash); err == nil {
|
||||
entry.Size = int(size)
|
||||
}
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/schutzaria"
|
||||
)
|
||||
|
||||
var ErrLocalityAccessEnforcing = errors.New("city.access.enforcing_error")
|
||||
@@ -41,52 +43,14 @@ func (s *Store) ensureLocalityAccessSchema() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type localityAccessLayer struct{ read, write, grant bool }
|
||||
type localityAccessSnapshot struct {
|
||||
admin bool
|
||||
paths map[string][]localityAccessLayer
|
||||
}
|
||||
|
||||
func (p *localityAccessSnapshot) decide(path, capability string, basis bool) bool {
|
||||
if !strings.HasPrefix(path, "/") {
|
||||
return false
|
||||
}
|
||||
if p == nil || p.admin {
|
||||
return basis
|
||||
}
|
||||
grant := false
|
||||
for at := path; ; {
|
||||
for _, layer := range p.paths[at] {
|
||||
allowed := layer.write
|
||||
if capability == "read" {
|
||||
allowed = layer.read
|
||||
}
|
||||
if !allowed {
|
||||
return false
|
||||
}
|
||||
if layer.grant && (capability == "read" || capability == "write" || capability == "create" || capability == "upload") {
|
||||
grant = true
|
||||
}
|
||||
}
|
||||
if at == "/" {
|
||||
break
|
||||
}
|
||||
i := strings.LastIndex(at, "/")
|
||||
if i <= 0 {
|
||||
at = "/"
|
||||
} else {
|
||||
at = at[:i]
|
||||
}
|
||||
}
|
||||
return basis || grant
|
||||
}
|
||||
type localityAccessSnapshot = schutzaria.LocalityAccess
|
||||
|
||||
func (s *Store) CheckLocalityAccessCaps(domain, principal, capability, path string) error {
|
||||
p, err := loadLocalityAccess(s.DB, domain, principal)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !p.decide(path, capability, true) {
|
||||
if !p.Decide(path, capability, true) {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
return nil
|
||||
@@ -112,7 +76,7 @@ func (s *Store) CheckObjectLocalityAccessCaps(domain, principal, capability, obj
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if p.admin || len(p.paths) == 0 {
|
||||
if !p.RequiresPathChecks() {
|
||||
return nil
|
||||
}
|
||||
rows, err := s.DB.Query(`SELECT path FROM paths WHERE domain_uuid=? AND object_uuid=? LIMIT 10001`, domain, object)
|
||||
@@ -130,7 +94,7 @@ func (s *Store) CheckObjectLocalityAccessCaps(domain, principal, capability, obj
|
||||
if err = rows.Scan(&path); err != nil {
|
||||
return err
|
||||
}
|
||||
if !p.decide(path, capability, true) {
|
||||
if !p.Decide(path, capability, true) {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
}
|
||||
@@ -176,7 +140,8 @@ func installLocalityAccessGuards(ctx context.Context, tx *sql.Tx) error {
|
||||
}
|
||||
|
||||
func loadLocalityAccess(q pathQueryer, domain, principal string) (*localityAccessSnapshot, error) {
|
||||
p := &localityAccessSnapshot{paths: map[string][]localityAccessLayer{}}
|
||||
var layers []schutzaria.AccessLayer
|
||||
var admin bool
|
||||
rows, err := q.Query(`SELECT a.scope_path,a.grants_access,
|
||||
EXISTS(SELECT 1 FROM locality_access_entries e JOIN domain_memberships m ON m.domain_uuid=e.domain_uuid AND m.principal_uuid=? AND m.state='active' JOIN principals u ON u.uuid=m.principal_uuid AND u.state IN ('active','readonly') WHERE e.domain_uuid=a.domain_uuid AND e.node_kind=a.node_kind AND e.node_uuid=a.node_uuid AND ((e.subject_kind='user' AND e.subject_uuid=m.uuid) OR (e.subject_kind='group' AND EXISTS(SELECT 1 FROM groups g JOIN group_members gm ON gm.group_uuid=g.uuid AND gm.membership_uuid=m.uuid WHERE g.uuid=e.subject_uuid AND g.domain_uuid=m.domain_uuid AND g.state='active')))),
|
||||
EXISTS(SELECT 1 FROM locality_access_entries e JOIN domain_memberships m ON m.domain_uuid=e.domain_uuid AND m.principal_uuid=? AND m.state='active' JOIN principals u ON u.uuid=m.principal_uuid AND u.state='active' WHERE e.domain_uuid=a.domain_uuid AND e.node_kind=a.node_kind AND e.node_uuid=a.node_uuid AND e.permission='read_write' AND ((e.subject_kind='user' AND e.subject_uuid=m.uuid) OR (e.subject_kind='group' AND EXISTS(SELECT 1 FROM groups g JOIN group_members gm ON gm.group_uuid=g.uuid AND gm.membership_uuid=m.uuid WHERE g.uuid=e.subject_uuid AND g.domain_uuid=m.domain_uuid AND g.state='active')))),
|
||||
@@ -193,15 +158,17 @@ func loadLocalityAccess(q pathQueryer, domain, principal string) (*localityAcces
|
||||
return nil, ErrGovernanceInvalid
|
||||
}
|
||||
var path string
|
||||
var layer localityAccessLayer
|
||||
var admin bool
|
||||
if err = rows.Scan(&path, &layer.grant, &layer.read, &layer.write, &admin); err != nil {
|
||||
var layer schutzaria.AccessLayer
|
||||
if err = rows.Scan(&path, &layer.Grant, &layer.Read, &layer.Write, &admin); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.admin = admin
|
||||
p.paths[path] = append(p.paths[path], layer)
|
||||
layer.Path = path
|
||||
layers = append(layers, layer)
|
||||
}
|
||||
return p, rows.Err()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return schutzaria.NewLocalityAccess(admin, layers), nil
|
||||
}
|
||||
|
||||
func readLocalityAccess(ctx context.Context, tx *sql.Tx, domain, kind, id string, g CityGovernance) (LocalityAccess, error) {
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/schutzaria"
|
||||
)
|
||||
|
||||
var ErrPublicationImpact = errors.New("city.publication.impact_error")
|
||||
@@ -313,20 +315,7 @@ func (s *Store) LocalityPublication(ctx context.Context, domain, actor, kind, id
|
||||
// Explicitly exclude every application/user/group/editor namespace. Public
|
||||
// projections must never turn an arbitrary reserved handler into a document.
|
||||
func PublicSemanticPath(path string) bool {
|
||||
if !strings.HasPrefix(path, "/") || strings.ContainsAny(path, "\\\x00\r\n") || strings.Contains(path, "//") {
|
||||
return false
|
||||
}
|
||||
for _, segment := range strings.Split(path, "/") {
|
||||
if segment == "." || segment == ".." {
|
||||
return false
|
||||
}
|
||||
}
|
||||
first := strings.Split(strings.TrimPrefix(path, "/"), "/")[0]
|
||||
switch first {
|
||||
case "_", "-", ":", "o", "r", "v", "l", "e", "s", "u", "g", "static", "health", "new":
|
||||
return false
|
||||
}
|
||||
return true
|
||||
return schutzaria.PublicSemanticPath(path)
|
||||
}
|
||||
|
||||
// Lookup is metadata-only and Path-indexed. The manifest is fenced by current
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"pageswiki/internal/mestionora"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
func TestMestionoraStoreCompatibilityAndRequestIsolation(t *testing.T) {
|
||||
st := searchStoreFixture(t)
|
||||
domain := defaultDomainUUID(t, st)
|
||||
object := insertSearchFilterPage(t, st, domain, "/docs/needle", "text/html", "needle.html", "<h1>Needle Guide</h1><p>needle body</p>", 1, 2000)
|
||||
insertSearchFilterPage(t, st, domain, "/private/needle", "text/html", "secret.html", "<h1>Secret Needle</h1>", 1, 2000)
|
||||
if _, err := st.DB.Exec(`INSERT INTO paths(domain_uuid,path,object_uuid,created_at) VALUES(?,'/private/alias',?,1)`, domain, object); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine := mestionora.NewSQLite(st.DB)
|
||||
allow := func(p string) bool { return strings.HasPrefix(p, "/docs/") }
|
||||
ctx := context.Background()
|
||||
for _, options := range []store.SearchOptions{{}, {FileType: "html"}, {Extension: "html"}, {FromMS: searchFilterTime(1000), BeforeMS: searchFilterTime(3000)}} {
|
||||
legacy, err := st.SearchPathsInDomainWithOptions(ctx, domain, "/", "needle", 20, allow, options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := engine.SearchPathsInDomainWithOptions(ctx, domain, "/", "needle", 20, allow, options)
|
||||
if err != nil || !reflect.DeepEqual(legacy, current) || len(current) != 1 || current[0].Path != "/docs/needle" {
|
||||
t.Fatalf("incompatible/exposed results: %v %v", current, err)
|
||||
}
|
||||
}
|
||||
legacy, err := st.SearchPathSuggestionsInDomain(ctx, domain, "/", "needle", 8, allow)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := engine.SearchPathSuggestionsInDomain(ctx, domain, "/", "needle", 8, allow)
|
||||
if err != nil || !reflect.DeepEqual(legacy, current) || len(current) != 1 || current[0].Title != "Needle Guide" {
|
||||
t.Fatalf("suggestion compatibility: %v %v", current, err)
|
||||
}
|
||||
for _, request := range []struct {
|
||||
domain string
|
||||
allowed func(string) bool
|
||||
}{
|
||||
{domain, nil}, {domain, func(string) bool { return false }}, {"other-domain", allow},
|
||||
} {
|
||||
results, err := engine.SearchPathsInDomainFiltered(ctx, request.domain, "/", "needle", 20, request.allowed)
|
||||
if err != nil || len(results) != 0 {
|
||||
t.Fatalf("prior request leaked: %v %v", results, err)
|
||||
}
|
||||
}
|
||||
if store.ErrInvalidSearchOptions != mestionora.ErrInvalidSearchOptions || store.SearchCandidateLimit != mestionora.SearchCandidateLimit {
|
||||
t.Fatal("compatibility identity changed")
|
||||
}
|
||||
// Alias method sets preserve City display/navigation projections too.
|
||||
var city store.CitySearchResult = mestionora.CitySearchResult{Kind: "spot", Name: "home-undefined", Title: "Undefined", StreetTitle: "Home", PathPrefix: "/life/home/_undefined", NavigationPath: "/life/home"}
|
||||
if city.DisplayTitle() != "Home" || city.ContentPath() != "/life/home" {
|
||||
t.Fatal("City projection changed")
|
||||
}
|
||||
}
|
||||
@@ -8,9 +8,11 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/ordoschnellic"
|
||||
)
|
||||
|
||||
const NotificationPushEventType = "notification.push.deliver"
|
||||
const NotificationPushEventType = ordoschnellic.PushEventType
|
||||
|
||||
func (s *Store) ensureNotificationPushSchema() error {
|
||||
_, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS notification_push_keys(id INTEGER PRIMARY KEY CHECK(id=1),private_key TEXT NOT NULL,public_key TEXT NOT NULL);
|
||||
@@ -81,11 +83,7 @@ func (s *Store) DeleteNotificationPush(ctx context.Context, domain, owner, endpo
|
||||
return err
|
||||
}
|
||||
|
||||
type NotificationPushDelivery struct {
|
||||
ID, Notification, Domain, Owner, Subscription string
|
||||
Focused bool
|
||||
Attempt int
|
||||
}
|
||||
type NotificationPushDelivery = ordoschnellic.PushDelivery
|
||||
|
||||
func (s *Store) AcknowledgeNotificationPush(ctx context.Context, domain, owner, endpoint string, ids []string) error {
|
||||
if len(ids) == 0 {
|
||||
@@ -104,12 +102,20 @@ func (s *Store) AcknowledgeNotificationPush(ctx context.Context, domain, owner,
|
||||
return err
|
||||
}
|
||||
func (s *Store) DeferFocusedNotificationPush(ctx context.Context, d *NotificationPushDelivery) error {
|
||||
_, err := s.DB.ExecContext(ctx, `UPDATE notification_push_queue SET attempts=attempts-1,lease_until_ms=0,next_at_ms=? WHERE subscription_id=? AND notification_uuid=? AND attempts=?`, time.Now().Add(time.Minute).UnixMilli(), d.ID, d.Notification, d.Attempt)
|
||||
return s.DeferFocusedNotificationPushAt(ctx, d, time.Now())
|
||||
}
|
||||
|
||||
func (s *Store) DeferFocusedNotificationPushAt(ctx context.Context, d *NotificationPushDelivery, at time.Time) error {
|
||||
_, err := s.DB.ExecContext(ctx, `UPDATE notification_push_queue SET attempts=attempts-1,lease_until_ms=0,next_at_ms=? WHERE subscription_id=? AND notification_uuid=? AND attempts=?`, at.Add(ordoschnellic.PushFocusDelay).UnixMilli(), d.ID, d.Notification, d.Attempt)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) ClaimNotificationPush(ctx context.Context) (*NotificationPushDelivery, error) {
|
||||
now := time.Now().UnixMilli()
|
||||
return s.ClaimNotificationPushAt(ctx, time.Now())
|
||||
}
|
||||
|
||||
func (s *Store) ClaimNotificationPushAt(ctx context.Context, at time.Time) (*NotificationPushDelivery, error) {
|
||||
now := at.UnixMilli()
|
||||
tx, err := s.DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -117,7 +123,7 @@ func (s *Store) ClaimNotificationPush(ctx context.Context) (*NotificationPushDel
|
||||
defer tx.Rollback()
|
||||
// Atomic claim across scheduler workers/processes. Outbound I/O is never inside this transaction.
|
||||
var d NotificationPushDelivery
|
||||
err = tx.QueryRowContext(ctx, `UPDATE notification_push_queue SET lease_until_ms=?,attempts=attempts+1 WHERE (subscription_id,notification_uuid)=(SELECT subscription_id,notification_uuid FROM notification_push_queue WHERE next_at_ms<=? AND lease_until_ms<=? ORDER BY next_at_ms LIMIT 1) RETURNING subscription_id,notification_uuid,attempts`, now+60000, now, now).Scan(&d.ID, &d.Notification, &d.Attempt)
|
||||
err = tx.QueryRowContext(ctx, `UPDATE notification_push_queue SET lease_until_ms=?,attempts=attempts+1 WHERE (subscription_id,notification_uuid)=(SELECT subscription_id,notification_uuid FROM notification_push_queue WHERE next_at_ms<=? AND lease_until_ms<=? ORDER BY next_at_ms LIMIT 1) RETURNING subscription_id,notification_uuid,attempts`, at.Add(ordoschnellic.PushLeaseDuration).UnixMilli(), now, now).Scan(&d.ID, &d.Notification, &d.Attempt)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -141,15 +147,32 @@ func (s *Store) ClaimNotificationPush(ctx context.Context) (*NotificationPushDel
|
||||
}
|
||||
|
||||
func (s *Store) FinishNotificationPush(ctx context.Context, d *NotificationPushDelivery, retry, expired bool) error {
|
||||
return s.FinishNotificationPushAt(ctx, d, retry, expired, time.Now())
|
||||
}
|
||||
|
||||
func (s *Store) FinishNotificationPushAt(ctx context.Context, d *NotificationPushDelivery, retry, expired bool, at time.Time) error {
|
||||
if expired {
|
||||
if _, err := s.DB.ExecContext(ctx, `DELETE FROM notification_push_subscriptions WHERE id=?`, d.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if retry && d.Attempt < 5 {
|
||||
_, err := s.DB.ExecContext(ctx, `UPDATE notification_push_queue SET next_at_ms=?,lease_until_ms=0 WHERE subscription_id=? AND notification_uuid=? AND attempts=?`, time.Now().Add(time.Duration(1<<d.Attempt)*time.Minute).UnixMilli(), d.ID, d.Notification, d.Attempt)
|
||||
if next, again := ordoschnellic.RetryAt(d.Attempt, at); retry && again {
|
||||
_, err := s.DB.ExecContext(ctx, `UPDATE notification_push_queue SET next_at_ms=?,lease_until_ms=0 WHERE subscription_id=? AND notification_uuid=? AND attempts=?`, next.UnixMilli(), d.ID, d.Notification, d.Attempt)
|
||||
return err
|
||||
}
|
||||
_, err := s.DB.ExecContext(ctx, `DELETE FROM notification_push_queue WHERE subscription_id=? AND notification_uuid=? AND attempts=?`, d.ID, d.Notification, d.Attempt)
|
||||
return err
|
||||
}
|
||||
|
||||
// CleanupNotificationPush only removes bounded stale delivery state, not Inbox.
|
||||
func (s *Store) CleanupNotificationPush(ctx context.Context, at time.Time) error {
|
||||
if s.ReadOnly {
|
||||
return ErrReadOnly
|
||||
}
|
||||
_, err := s.DB.ExecContext(ctx, `DELETE FROM notification_push_subscriptions WHERE id IN (SELECT id FROM notification_push_subscriptions WHERE expires_at_ms<? LIMIT 100)`, at.UnixMilli())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.DB.ExecContext(ctx, `DELETE FROM notification_push_queue WHERE rowid IN (SELECT rowid FROM notification_push_queue WHERE created_at_ms<? LIMIT 500)`, at.Add(-ordoschnellic.PushRetention).UnixMilli())
|
||||
return err
|
||||
}
|
||||
@@ -8,23 +8,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"pageswiki/internal/ordoschnellic"
|
||||
)
|
||||
|
||||
type Notification struct {
|
||||
UUID string `json:"uuid"`
|
||||
DomainUUID string `json:"domain_uuid"`
|
||||
RecipientUUID string `json:"recipient_uuid"`
|
||||
EventUUID string `json:"event_uuid,omitempty"`
|
||||
Kind string `json:"kind"`
|
||||
Status string `json:"status"`
|
||||
RequestUUID string `json:"request_uuid,omitempty"`
|
||||
Href string `json:"href,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
CreatedAtMS int64 `json:"created_at_ms"`
|
||||
ReadAtMS int64 `json:"read_at_ms,omitempty"`
|
||||
}
|
||||
type Notification = ordoschnellic.Notification
|
||||
|
||||
var ErrNotificationInput = errors.New("invalid notification")
|
||||
var ErrNotificationInput = ordoschnellic.ErrNotificationInput
|
||||
|
||||
func (s *Store) EnsureNotificationSchema() error {
|
||||
for _, statement := range []string{
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"pageswiki/internal/ordoschnellic"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
func TestOrdoschnellicStoreCompatibility(t *testing.T) {
|
||||
var notification store.Notification = ordoschnellic.Notification{UUID: "notification", DomainUUID: "domain", RecipientUUID: "owner", Kind: "task.due", Status: "unread", CreatedAtMS: 1234}
|
||||
var delivery store.NotificationPushDelivery = ordoschnellic.PushDelivery{Notification: notification.UUID, Attempt: 1}
|
||||
if delivery.Notification != notification.UUID || store.ErrNotificationInput != ordoschnellic.ErrNotificationInput || store.NotificationPushEventType != ordoschnellic.PushEventType {
|
||||
t.Fatal("legacy identities changed")
|
||||
}
|
||||
data, err := json.Marshal(notification)
|
||||
if err != nil || string(data) != `{"uuid":"notification","domain_uuid":"domain","recipient_uuid":"owner","kind":"task.due","status":"unread","created_at_ms":1234}` {
|
||||
t.Fatalf("notification wire contract changed: %s %v", data, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Completion needs names, never Revision bodies, titles, thumbnails or counts.
|
||||
// Cap the candidate budget before the caller applies its request's ACL filter.
|
||||
func (s *Store) ListPathCompletionNames(ctx context.Context, domain, parent, prefix string) ([]BrowserEntry, bool, error) {
|
||||
base := strings.TrimRight(normalizeBrowserPath(parent), "/") + "/"
|
||||
rows, err := s.DB.QueryContext(ctx, `WITH relative AS (
|
||||
SELECT substr(path,length(?)+1) AS rest FROM paths
|
||||
WHERE domain_uuid=? AND path>=? AND path<? AND path!=?
|
||||
), children AS (
|
||||
SELECT CASE WHEN instr(rest,'/')=0 THEN rest ELSE substr(rest,1,instr(rest,'/')-1) END AS name,
|
||||
MAX(instr(rest,'/')>0) AS is_dir FROM relative WHERE rest!='' GROUP BY 1
|
||||
) SELECT name,is_dir FROM children
|
||||
WHERE substr(lower(name),1,length(?))=lower(?) ORDER BY name LIMIT 1001`, base, domain, base, strings.TrimSuffix(base, "/")+"0", parent, prefix, prefix)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
entries := make([]BrowserEntry, 0)
|
||||
for rows.Next() {
|
||||
var entry BrowserEntry
|
||||
if err := rows.Scan(&entry.Name, &entry.IsDir); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
entry.Path = base + entry.Name
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
truncated := len(entries) > 1000
|
||||
if truncated {
|
||||
entries = entries[:1000]
|
||||
}
|
||||
return entries, truncated, nil
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPathCompletionNamesNoRevisionsAndBounded(t *testing.T) {
|
||||
s := newSchemaTestStore(t)
|
||||
// Deliberately omit revisions and documents: completion must never hydrate them.
|
||||
_, err := s.DB.Exec(`CREATE TABLE paths(domain_uuid TEXT,path TEXT,object_uuid TEXT);
|
||||
INSERT INTO paths VALUES('d','/docs/日本語 #?/child','o'),('d','/docs/a%b','o'),('d','/docs/a_b','o'),('other','/docs/secret','o'),('d','/docs-other/no','o');`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, prefix := range []string{"日本", "a%", "a_"} {
|
||||
entries, more, err := s.ListPathCompletionNames(context.Background(), "d", "/docs", prefix)
|
||||
if err != nil || more || len(entries) != 1 {
|
||||
t.Fatalf("%q: %+v more=%v err=%v", prefix, entries, more, err)
|
||||
}
|
||||
if prefix == "日本" && (!entries[0].IsDir || entries[0].Path != "/docs/日本語 #?") {
|
||||
t.Fatal(entries)
|
||||
}
|
||||
}
|
||||
_, err = s.DB.Exec(`WITH RECURSIVE n(v) AS (VALUES(1) UNION ALL SELECT v+1 FROM n WHERE v<1200) INSERT INTO paths SELECT 'd','/many/'||printf('%04d',v),'o' FROM n`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries, more, err := s.ListPathCompletionNames(context.Background(), "d", "/many", "")
|
||||
if err != nil || !more || len(entries) != 1000 {
|
||||
t.Fatalf("budget: %d %v %v", len(entries), more, err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, _, err = s.ListPathCompletionNames(ctx, "d", "/many", ""); err == nil {
|
||||
t.Fatal("ignored cancellation")
|
||||
}
|
||||
}
|
||||
@@ -252,7 +252,7 @@ func (s *Store) changePeriodical(ctx context.Context, domain, actor, kind string
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !caps.decide(e.Path, capability, basis == nil) {
|
||||
if !caps.Decide(e.Path, capability, basis == nil) {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
var profileContactMigrations = []db.SQLMigration{{Version: 1, Name: "private_profile_phone", Statements: []string{
|
||||
`CREATE TABLE profile_contacts (
|
||||
domain_uuid TEXT NOT NULL, principal_uuid TEXT NOT NULL,
|
||||
phone_display TEXT NOT NULL DEFAULT '', phone_number TEXT NOT NULL DEFAULT '',
|
||||
visibility TEXT NOT NULL DEFAULT 'private' CHECK(visibility IN ('private','domain','public')),
|
||||
version INTEGER NOT NULL DEFAULT 1, updated_at_ms INTEGER NOT NULL,
|
||||
PRIMARY KEY(domain_uuid,principal_uuid))`,
|
||||
}, Assertions: []string{
|
||||
schemaColumns("profile_contacts", "domain_uuid,principal_uuid,phone_display,phone_number,visibility,version,updated_at_ms"),
|
||||
schemaUnique("profile_contacts", "domain_uuid,principal_uuid"),
|
||||
}}}
|
||||
|
||||
// Kept outside Principal: account lists, identity federation and auth responses
|
||||
// must not accidentally export personal contact details.
|
||||
type ProfileContact struct {
|
||||
PhoneDisplay string
|
||||
PhoneNumber string
|
||||
Visibility string
|
||||
Version int64
|
||||
}
|
||||
|
||||
var ErrContactPhone = errors.New("contact.invalid_phone")
|
||||
|
||||
// Initial scope: ordinary local/international numbers only. No service codes,
|
||||
// extensions, pauses, URI parameters, or control characters are dialled.
|
||||
func NormalizeContactPhone(value string) (string, string, error) {
|
||||
if len(value) > 64 {
|
||||
return "", "", ErrContactPhone
|
||||
}
|
||||
for _, c := range value {
|
||||
if !(c >= '0' && c <= '9') && c != '+' && c != ' ' && c != '-' && c != '(' && c != ')' {
|
||||
return "", "", ErrContactPhone
|
||||
}
|
||||
}
|
||||
display := strings.TrimSpace(value)
|
||||
if display == "" {
|
||||
return "", "", nil
|
||||
}
|
||||
var digits strings.Builder
|
||||
for i, c := range display {
|
||||
if c >= '0' && c <= '9' || c == '+' && i == 0 {
|
||||
digits.WriteRune(c)
|
||||
} else if c == '+' {
|
||||
return "", "", ErrContactPhone
|
||||
}
|
||||
}
|
||||
number := digits.String()
|
||||
n := len(strings.TrimPrefix(number, "+"))
|
||||
if n < 3 || n > 15 {
|
||||
return "", "", ErrContactPhone
|
||||
}
|
||||
return display, number, nil
|
||||
}
|
||||
|
||||
func loadProfileContact(ctx context.Context, q interface {
|
||||
QueryRowContext(context.Context, string, ...any) *sql.Row
|
||||
}, domain, owner string) (ProfileContact, error) {
|
||||
c := ProfileContact{Visibility: "private"}
|
||||
err := q.QueryRowContext(ctx, `SELECT phone_display,phone_number,visibility,version FROM profile_contacts WHERE domain_uuid=? AND principal_uuid=?`, domain, owner).Scan(&c.PhoneDisplay, &c.PhoneNumber, &c.Visibility, &c.Version)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
err = nil
|
||||
}
|
||||
return c, err
|
||||
}
|
||||
|
||||
// ProfileContactForViewer returns no private data, including visibility/version,
|
||||
// to unauthorized readers. Administrator status is not a contact consent grant.
|
||||
func (s *Store) ProfileContactForViewer(ctx context.Context, domain, owner, viewer string) (ProfileContact, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
tx, err := s.DB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
|
||||
if err != nil {
|
||||
return ProfileContact{}, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var username string
|
||||
err = tx.QueryRowContext(ctx, `SELECT i.subject FROM principals p JOIN domain_memberships m ON m.principal_uuid=p.uuid AND m.domain_uuid=? AND m.state='active' JOIN principal_identities i ON i.principal_uuid=p.uuid AND i.provider_type='password' WHERE p.uuid=? AND p.state IN ('active','readonly')`, domain, owner).Scan(&username)
|
||||
if err != nil {
|
||||
return ProfileContact{}, err
|
||||
}
|
||||
if viewer != "" {
|
||||
var member bool
|
||||
if err = tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM domain_memberships m JOIN principals p ON p.uuid=m.principal_uuid WHERE m.domain_uuid=? AND m.principal_uuid=? AND m.state='active' AND p.state IN ('active','readonly'))`, domain, viewer).Scan(&member); err != nil {
|
||||
return ProfileContact{}, err
|
||||
}
|
||||
if !member {
|
||||
return ProfileContact{}, nil
|
||||
}
|
||||
}
|
||||
c, err := loadProfileContact(ctx, tx, domain, owner)
|
||||
if err != nil {
|
||||
return ProfileContact{}, err
|
||||
}
|
||||
if viewer != owner {
|
||||
policy, err := loadUserSpacePolicy(ctx, tx, domain, owner, username)
|
||||
if err != nil {
|
||||
return ProfileContact{}, err
|
||||
}
|
||||
if viewer != "" {
|
||||
if userSpaceVisibilityRank(c.Visibility) < 1 || userSpaceVisibilityRank(policy.ProfileVisibility) < 1 {
|
||||
return ProfileContact{}, nil
|
||||
}
|
||||
} else {
|
||||
allowed, err := s.domainAllowsUserSpacePublic(ctx, tx, domain)
|
||||
if err != nil {
|
||||
return ProfileContact{}, err
|
||||
}
|
||||
if !allowed || c.Visibility != "public" || policy.ProfileVisibility != "public" {
|
||||
return ProfileContact{}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return ProfileContact{}, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// The actor is the owner; callers cannot pass a separate target principal.
|
||||
func (s *Store) SaveProfileContact(ctx context.Context, domain, actor string, proposed ProfileContact, confirmPublic bool) error {
|
||||
if s.ReadOnly {
|
||||
return ErrReadOnly
|
||||
}
|
||||
display, number, err := NormalizeContactPhone(proposed.PhoneDisplay)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if userSpaceVisibilityRank(proposed.Visibility) < 0 || proposed.Version < 0 {
|
||||
return ErrGovernanceInvalid
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
tx, err := s.DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var username string
|
||||
if err = tx.QueryRowContext(ctx, `SELECT i.subject FROM principals p JOIN domain_memberships m ON m.principal_uuid=p.uuid AND m.domain_uuid=? AND m.state='active' JOIN principal_identities i ON i.principal_uuid=p.uuid AND i.provider_type='password' WHERE p.uuid=? AND p.state='active'`, domain, actor).Scan(&username); err != nil {
|
||||
return err
|
||||
}
|
||||
current, err := loadProfileContact(ctx, tx, domain, actor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current.Version != proposed.Version {
|
||||
return ErrGovernanceConflict
|
||||
}
|
||||
if number == "" {
|
||||
proposed.Visibility = "private"
|
||||
}
|
||||
if proposed.Visibility == "public" {
|
||||
policy, err := loadUserSpacePolicy(ctx, tx, domain, actor, username)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
allowed, err := s.domainAllowsUserSpacePublic(ctx, tx, domain)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !allowed || policy.ProfileVisibility != "public" {
|
||||
return ErrGovernanceCeiling
|
||||
}
|
||||
if (current.Visibility != "public" || current.PhoneNumber != number) && !confirmPublic {
|
||||
return ErrGovernanceInvalid
|
||||
}
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO profile_contacts(domain_uuid,principal_uuid,phone_display,phone_number,visibility,version,updated_at_ms) VALUES(?,?,?,?,?,1,?) ON CONFLICT(domain_uuid,principal_uuid) DO UPDATE SET phone_display=excluded.phone_display,phone_number=excluded.phone_number,visibility=excluded.visibility,version=profile_contacts.version+1,updated_at_ms=excluded.updated_at_ms`, domain, actor, display, number, proposed.Visibility, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package store
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNormalizeContactPhone(t *testing.T) {
|
||||
for _, tc := range []struct{ input, want string }{
|
||||
{"", ""}, {" 03-1234-5678 ", "0312345678"}, {"+81 (90) 1234-5678", "+819012345678"},
|
||||
} {
|
||||
_, got, err := NormalizeContactPhone(tc.input)
|
||||
if err != nil || got != tc.want {
|
||||
t.Fatalf("%q: %q %v", tc.input, got, err)
|
||||
}
|
||||
}
|
||||
for _, input := range []string{"*21*123#", "tel:1234", "javascript:alert(1)", "123;ext=4", "123,456", "123\r\n456", "\t123", "123", "12+34", "+", "12", "1234567890123456", "(+)123", "123\x1b"} {
|
||||
if _, _, err := NormalizeContactPhone(input); err == nil {
|
||||
t.Errorf("accepted %q", input)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,20 +27,7 @@ type ProjectionMetadata struct {
|
||||
}
|
||||
|
||||
func (s *Store) EnsureProjectionMetadataSchema() error {
|
||||
for _, statement := range []string{
|
||||
`CREATE TABLE IF NOT EXISTS projection_metadata (
|
||||
document_hash TEXT NOT NULL,kind TEXT NOT NULL,version INTEGER NOT NULL,
|
||||
payload TEXT NOT NULL DEFAULT '{}',error TEXT NOT NULL DEFAULT '',
|
||||
updated_at_ms INTEGER NOT NULL,retry_at_ms INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY(document_hash,kind,version))`,
|
||||
`CREATE INDEX IF NOT EXISTS projection_metadata_eviction ON projection_metadata(updated_at_ms DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_jobs_identity_status ON event_jobs(event_type,event_key,status)`,
|
||||
} {
|
||||
if _, err := s.DB.Exec(statement); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return s.ensureSchemaComponent("projection-metadata")
|
||||
}
|
||||
|
||||
func (s *Store) GetProjectionMetadata(hash, kind string, version int) (*ProjectionMetadata, error) {
|
||||
|
||||
@@ -11,9 +11,10 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"pageswiki/internal/schutzaria"
|
||||
)
|
||||
|
||||
var ErrApprovalTransition = errors.New("invalid approval request transition")
|
||||
var ErrApprovalTransition = schutzaria.ErrApprovalTransition
|
||||
|
||||
type ApprovalRequest struct {
|
||||
UUID string `json:"uuid"`
|
||||
@@ -127,7 +128,7 @@ func (s *Store) DecideApprovalRequestScoped(ctx context.Context, domainUUID, req
|
||||
}
|
||||
|
||||
func (s *Store) decideApprovalRequest(ctx context.Context, domainUUID, requestUUID, reviewerUUID, decision, justification string, generation int64) error {
|
||||
if requestUUID == "" || reviewerUUID == "" || generation < 1 || (decision != "approved" && decision != "rejected" && decision != "changes_requested") {
|
||||
if requestUUID == "" || reviewerUUID == "" || generation < 1 || !schutzaria.ValidApprovalDecision(decision) {
|
||||
return ErrApprovalTransition
|
||||
}
|
||||
tx, err := s.DB.BeginTx(ctx, nil)
|
||||
@@ -146,17 +147,14 @@ func (s *Store) decideApprovalRequest(ctx context.Context, domainUUID, requestUU
|
||||
if err := tx.QueryRowContext(ctx, query, args...).Scan(&state, ¤tGeneration); err != nil {
|
||||
return err
|
||||
}
|
||||
if state != "pending" || currentGeneration != generation {
|
||||
return ErrApprovalTransition
|
||||
newState, err := schutzaria.ApprovalNextState(state, currentGeneration, generation, decision)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO publication_decisions(uuid,request_uuid,reviewer_uuid,generation,decision,justification,created_at_ms) VALUES(?,?,?,?,?,?,?)`, uuid.NewString(), requestUUID, reviewerUUID, generation, decision, justification, now); err != nil {
|
||||
return err
|
||||
}
|
||||
newState := decision
|
||||
if decision == "changes_requested" {
|
||||
newState = "rejected"
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE publication_requests SET state=?,updated_at_ms=? WHERE uuid=? AND state='pending' AND generation=?`, newState, now, requestUUID, generation); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -3,38 +3,12 @@ package store
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"pageswiki/internal/schutzaria"
|
||||
)
|
||||
|
||||
// ReadPathPolicy is an immutable, request-scoped snapshot of positive Path
|
||||
// read grants. Never share it across requests or use it to authorize writes.
|
||||
type ReadPathPolicy struct {
|
||||
access *localityAccessSnapshot
|
||||
all bool
|
||||
exact map[string]bool
|
||||
subtrees map[string]bool
|
||||
}
|
||||
|
||||
func (p *ReadPathPolicy) Allows(path string) bool {
|
||||
if p == nil {
|
||||
return false
|
||||
}
|
||||
return p.access.decide(path, "read", p.allowsBase(path))
|
||||
}
|
||||
|
||||
func (p *ReadPathPolicy) allowsBase(path string) bool {
|
||||
if p == nil || !strings.HasPrefix(path, "/") {
|
||||
return false
|
||||
}
|
||||
if p.all || p.exact[path] {
|
||||
return true
|
||||
}
|
||||
for at := strings.LastIndex(path, "/"); at > 0; at = strings.LastIndex(path[:at], "/") {
|
||||
if p.subtrees[path[:at]] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
// ReadPathPolicy is a request-scoped, read-only policy snapshot.
|
||||
type ReadPathPolicy = schutzaria.ReadPathPolicy
|
||||
|
||||
// LoadReadPathPolicy uses one bounded query, independent of the number of
|
||||
// directory entries. It deliberately excludes Object grants: an Object grant
|
||||
@@ -44,9 +18,10 @@ func (s *Store) LoadReadPathPolicy(domain, principal string) (*ReadPathPolicy, e
|
||||
}
|
||||
|
||||
func loadReadPathPolicy(q pathQueryer, domain, principal string) (*ReadPathPolicy, error) {
|
||||
p := &ReadPathPolicy{exact: map[string]bool{}, subtrees: map[string]bool{}}
|
||||
var all bool
|
||||
var exact, subtrees []string
|
||||
if domain == "" || principal == "" {
|
||||
return p, ErrAccessDenied
|
||||
return &ReadPathPolicy{}, ErrAccessDenied
|
||||
}
|
||||
rows, err := q.Query(`
|
||||
SELECT 'role',g.target_type,g.target_id FROM principals p
|
||||
@@ -79,15 +54,15 @@ func loadReadPathPolicy(q pathQueryer, domain, principal string) (*ReadPathPolic
|
||||
}
|
||||
if source == "role" {
|
||||
if kind == "domain" && target == domain || kind == "path" && target == "*" {
|
||||
p.all = true
|
||||
all = true
|
||||
} else if kind == "path" {
|
||||
p.exact[target] = true
|
||||
exact = append(exact, target)
|
||||
}
|
||||
} else if target == "/" {
|
||||
p.all = true
|
||||
all = true
|
||||
} else if strings.HasPrefix(target, "/") {
|
||||
p.exact[target] = true
|
||||
p.subtrees[strings.TrimRight(target, "/")] = true
|
||||
exact = append(exact, target)
|
||||
subtrees = append(subtrees, strings.TrimRight(target, "/"))
|
||||
}
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
@@ -96,6 +71,9 @@ func loadReadPathPolicy(q pathQueryer, domain, principal string) (*ReadPathPolic
|
||||
if err = rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.access, err = loadLocalityAccess(q, domain, principal)
|
||||
return p, err
|
||||
access, err := loadLocalityAccess(q, domain, principal)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return schutzaria.NewReadPathPolicy(all, exact, subtrees, access), nil
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var ErrReadOnly = errors.New("DR standby is read-only")
|
||||
|
||||
// No migrations, schema repair, key generation or file creation on standby.
|
||||
func NewReadOnlyWithDocumentRoot(database *sql.DB, root string) (*Store, error) {
|
||||
var readOnly int
|
||||
if err := database.QueryRow("PRAGMA query_only").Scan(&readOnly); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if readOnly != 1 {
|
||||
return nil, errors.New("standby requires a read-only database connection")
|
||||
}
|
||||
st := &Store{DB: database, DocumentRoot: root, childCache: map[string][]string{}, ReadOnly: true}
|
||||
if encoded := strings.TrimSpace(os.Getenv("PAGESWIKI_AUTH_KEY")); encoded != "" {
|
||||
sum := sha256.Sum256([]byte(encoded))
|
||||
st.authKey = sum[:]
|
||||
} else {
|
||||
key, err := os.ReadFile(filepath.Join(filepath.Dir(root), "auth.key"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(key) != 32 {
|
||||
return nil, errors.New("standby requires the original 32-byte auth.key")
|
||||
}
|
||||
st.authKey = key
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
@@ -1,7 +1,5 @@
|
||||
package store
|
||||
|
||||
import "os"
|
||||
|
||||
type RecentObject struct {
|
||||
UUID string
|
||||
Path string
|
||||
@@ -117,9 +115,9 @@ func (s *Store) ListRecentObjectsInDomain(
|
||||
if out[i].documentHash == "" {
|
||||
continue
|
||||
}
|
||||
if path, err := s.documentPath(out[i].documentHash); err == nil && path != "" {
|
||||
if info, err := os.Stat(path); err == nil {
|
||||
out[i].Size = int(info.Size())
|
||||
if s.DocumentRoot != "" {
|
||||
if size, err := s.DocumentSize(out[i].documentHash); err == nil {
|
||||
out[i].Size = int(size)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,17 +4,15 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/seheweit"
|
||||
)
|
||||
|
||||
const ResourceEventID = "builtin.resource.sample"
|
||||
const ResourceEventType = "resource.sample"
|
||||
const ResourceEventID = seheweit.ResourceEventID
|
||||
const ResourceEventType = seheweit.ResourceEventType
|
||||
|
||||
func (s *Store) EnsureResourceHistorySchema() error {
|
||||
_, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS resource_samples (
|
||||
id INTEGER PRIMARY KEY, job_uuid TEXT NOT NULL UNIQUE,
|
||||
timestamp_ms INTEGER NOT NULL, payload TEXT NOT NULL
|
||||
); CREATE INDEX IF NOT EXISTS idx_resource_samples_time ON resource_samples(timestamp_ms)`)
|
||||
return err
|
||||
return s.ensureSchemaComponent("resource-history")
|
||||
}
|
||||
|
||||
func (s *Store) EnsureBuiltinResourceEvent() error {
|
||||
@@ -26,37 +24,34 @@ func (s *Store) EnsureBuiltinResourceEvent() error {
|
||||
}
|
||||
|
||||
func (s *Store) SaveResourceSample(job string, at int64, payload []byte) error {
|
||||
tx, err := s.DB.Begin()
|
||||
return s.SaveResourceSampleContext(context.Background(), job, at, payload)
|
||||
}
|
||||
|
||||
func (s *Store) SaveResourceSampleContext(ctx context.Context, job string, at int64, payload []byte) error {
|
||||
if s.ReadOnly {
|
||||
return ErrReadOnly
|
||||
}
|
||||
tx, err := s.DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err = tx.Exec(`INSERT OR IGNORE INTO resource_samples(job_uuid,timestamp_ms,payload) VALUES(?,?,?)`, job, at, string(payload)); err != nil {
|
||||
if _, err = tx.ExecContext(ctx, `INSERT OR IGNORE INTO resource_samples(job_uuid,timestamp_ms,payload) VALUES(?,?,?)`, job, at, string(payload)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(`DELETE FROM resource_samples WHERE timestamp_ms < ?`, at-int64(7*24*time.Hour/time.Millisecond)); err != nil {
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM resource_samples WHERE timestamp_ms < ?`, seheweit.RetentionCutoff(at)); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// Return the last sample in each time bucket, not averages disguised as peaks.
|
||||
// Return the last inserted sample in each time bucket, not averages or peaks.
|
||||
// At most 361 points and an indexed seven-day scan; no request-side collection.
|
||||
func (s *Store) ResourceHistory(ctx context.Context, hours int, now time.Time) ([]json.RawMessage, error) {
|
||||
if hours < 1 {
|
||||
hours = 1
|
||||
}
|
||||
if hours > 168 {
|
||||
hours = 168
|
||||
}
|
||||
span := int64(hours) * 3600000
|
||||
bucket := span / 360
|
||||
if bucket < 60000 {
|
||||
bucket = 60000
|
||||
}
|
||||
window := seheweit.Window(hours, now)
|
||||
rows, err := s.DB.QueryContext(ctx, `SELECT payload FROM resource_samples WHERE id IN
|
||||
(SELECT MAX(id) FROM resource_samples WHERE timestamp_ms>=? AND timestamp_ms<=? GROUP BY timestamp_ms / ?)
|
||||
ORDER BY timestamp_ms LIMIT 361`, now.UnixMilli()-span, now.UnixMilli(), bucket)
|
||||
ORDER BY timestamp_ms LIMIT ?`, window.FromMS, window.ToMS, window.BucketMS, seheweit.MaxHistoryPoints)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -107,6 +107,19 @@ func (s *Store) saveRevisionContentUnlocked(
|
||||
contentType string,
|
||||
filename string,
|
||||
) (*Revision, error) {
|
||||
return s.saveRevisionContentExpected(objectUUID, payload, message, rawLanguage, contentType, filename, nil)
|
||||
}
|
||||
|
||||
// SaveRevisionContentIfHead compares the exact language head and appends within
|
||||
// one SQLite write transaction, including writers outside this Store process.
|
||||
// An empty expected UUID asserts that this language has no revision yet.
|
||||
func (s *Store) SaveRevisionContentIfHead(objectUUID string, payload []byte, message, lang, contentType, filename, expected string) (*Revision, error) {
|
||||
s.revisionWriteMu.Lock()
|
||||
defer s.revisionWriteMu.Unlock()
|
||||
return s.saveRevisionContentExpected(objectUUID, payload, message, lang, contentType, filename, &expected)
|
||||
}
|
||||
|
||||
func (s *Store) saveRevisionContentExpected(objectUUID string, payload []byte, message, rawLanguage, contentType, filename string, expected *string) (*Revision, error) {
|
||||
lang, err :=
|
||||
normalizeRevisionLanguage(
|
||||
rawLanguage,
|
||||
@@ -165,6 +178,29 @@ func (s *Store) saveRevisionContentUnlocked(
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if expected != nil {
|
||||
// Acquire a write lock before reading the head. Avoid a read-to-write
|
||||
// upgrade race with CLI/another process writing this same database.
|
||||
if _, err = tx.Exec(`UPDATE objects SET uuid=uuid WHERE uuid=?`, objectUUID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var current string
|
||||
var currentMS int64
|
||||
err = tx.QueryRow(`SELECT uuid,COALESCE(created_at_ms,created_at*1000) FROM revisions WHERE object_uuid=? AND COALESCE(NULLIF(lang,''),'und')=? ORDER BY COALESCE(created_at_ms,created_at*1000) DESC,rowid DESC LIMIT 1`, objectUUID, lang).Scan(¤t, ¤tMS)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, err
|
||||
}
|
||||
if current != *expected {
|
||||
return nil, ErrRevisionConflict
|
||||
}
|
||||
// Hashing/persisting may wait behind another process. The revision we
|
||||
// accepted must sort after its base even if the wall clock moved back.
|
||||
revision.CreatedAtMS = time.Now().UnixMilli()
|
||||
if revision.CreatedAtMS <= currentMS {
|
||||
revision.CreatedAtMS = currentMS + 1
|
||||
}
|
||||
revision.CreatedAt = revision.CreatedAtMS / 1000
|
||||
}
|
||||
if err = insertRevision(tx, revision, embeddedHashes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -178,6 +214,36 @@ func (s *Store) saveRevisionContentUnlocked(
|
||||
return revision, nil
|
||||
}
|
||||
|
||||
// GetRevisionForExactLanguage deliberately has no viewer-language fallback.
|
||||
func (s *Store) GetRevisionForExactLanguage(objectUUID, rawLanguage string) (*Revision, error) {
|
||||
return s.revisionForExactLanguage(objectUUID, rawLanguage, false)
|
||||
}
|
||||
|
||||
// GetRevisionMetadataForExactLanguage does not read the document file. Polling
|
||||
// an editor head must not hydrate a potentially large body on every request.
|
||||
func (s *Store) GetRevisionMetadataForExactLanguage(objectUUID, rawLanguage string) (*Revision, error) {
|
||||
return s.revisionForExactLanguage(objectUUID, rawLanguage, true)
|
||||
}
|
||||
|
||||
func (s *Store) revisionForExactLanguage(objectUUID, rawLanguage string, metadata bool) (*Revision, error) {
|
||||
lang, err := normalizeRevisionLanguage(rawLanguage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var id string
|
||||
err = s.DB.QueryRow(`SELECT uuid FROM revisions WHERE object_uuid=? AND COALESCE(NULLIF(lang,''),'und')=? ORDER BY COALESCE(created_at_ms,created_at*1000) DESC,rowid DESC LIMIT 1`, objectUUID, lang).Scan(&id)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return &Revision{ObjectUUID: objectUUID, Language: lang, ContentType: "text/html; charset=utf-8"}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if metadata {
|
||||
return s.GetRevisionMetadata(id)
|
||||
}
|
||||
return s.GetRevision(id)
|
||||
}
|
||||
|
||||
func insertRevision(tx *sql.Tx, revision *Revision, embeddedHashes []string) error {
|
||||
result, err := tx.Exec(`
|
||||
INSERT INTO revisions (
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
type SchemaInspection struct {
|
||||
Coverage string `json:"coverage"`
|
||||
Components []db.MigrationInspection `json:"components"`
|
||||
LegacyAreas []string `json:"legacy_areas"`
|
||||
}
|
||||
|
||||
// InspectSchema deliberately does not construct a Store: constructors can
|
||||
// migrate, backfill, generate keys, seed defaults and requeue interrupted jobs.
|
||||
func InspectSchema(ctx context.Context, database *sql.DB) (SchemaInspection, error) {
|
||||
plans := make(map[string][]db.SQLMigration, len(managedStoreSchemas)+1)
|
||||
for name, steps := range managedStoreSchemas {
|
||||
plans[name] = steps
|
||||
}
|
||||
plans["document-store"] = documentBackfillMigrations
|
||||
components, err := db.InspectSQLiteMigrations(ctx, database, plans)
|
||||
return SchemaInspection{Coverage: "managed-components-only", Components: components, LegacyAreas: []string{
|
||||
"core-bootstrap", "campuses/domains/federation", "auth/ACL/user-space", "outbox/notifications/publication-approval",
|
||||
"City", "tasks/calendars", "periodicals", "search/object-listing/assets/GC",
|
||||
}}, err
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
func TestSchemaInspectionDoesNotRecoverJobsOrBackfill(t *testing.T) {
|
||||
st := newSchemaTestStore(t)
|
||||
for name := range managedStoreSchemas {
|
||||
if err := st.ensureSchemaComponent(name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := db.ApplySQLiteMigrations(context.Background(), st.DB, "document-store", documentBackfillMigrations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`INSERT INTO event_jobs(uuid,event_type,event_key,payload,status,run_at_ms,created_at_ms,updated_at_ms) VALUES('running','test','key','','running',1,1,1); INSERT INTO document_backfills VALUES('store','cursor',7,0,1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report, err := InspectSchema(context.Background(), st.DB)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, component := range report.Components {
|
||||
if component.State != "current" {
|
||||
t.Fatalf("component=%+v", component)
|
||||
}
|
||||
}
|
||||
var status string
|
||||
if err := st.DB.QueryRow(`SELECT status FROM event_jobs WHERE uuid='running'`).Scan(&status); err != nil || status != "running" {
|
||||
t.Fatalf("job modified: %q %v", status, err)
|
||||
}
|
||||
var processed, complete int
|
||||
if err := st.DB.QueryRow(`SELECT processed,completed FROM document_backfills WHERE store_id='store'`).Scan(&processed, &complete); err != nil || processed != 7 || complete != 0 {
|
||||
t.Fatalf("backfill modified: %d %d %v", processed, complete, err)
|
||||
}
|
||||
if err := st.EnsureEventJobsSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.DB.QueryRow(`SELECT status FROM event_jobs WHERE uuid='running'`).Scan(&status); err != nil || status != "queued" {
|
||||
t.Fatalf("startup recovery broken: %q %v", status, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventJobMigrationAdoptsLegacyAndPreservesSettings(t *testing.T) {
|
||||
st := newSchemaTestStore(t)
|
||||
if _, err := st.DB.Exec(eventJobMigrations[0].Statements[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`INSERT INTO event_jobs(uuid,event_type,event_key,payload,status,run_at_ms,created_at_ms,updated_at_ms) VALUES('old','test','key','keep','running',1,1,1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.ensureSchemaComponent("event-jobs"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var payload, status string
|
||||
var attempts int
|
||||
if err := st.DB.QueryRow(`SELECT payload,status,max_attempts FROM event_jobs WHERE uuid='old'`).Scan(&payload, &status, &attempts); err != nil || payload != "keep" || status != "running" || attempts != 3 {
|
||||
t.Fatalf("legacy job=%q %q %d %v", payload, status, attempts, err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`UPDATE event_queue_settings SET paused=1,concurrency=3; UPDATE event_log_policies SET max_rows=42 WHERE event_type='*'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.ensureSchemaComponent("event-jobs"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var paused, workers, rows int
|
||||
if err := st.DB.QueryRow(`SELECT paused,concurrency FROM event_queue_settings WHERE id=1`).Scan(&paused, &workers); err != nil || paused != 1 || workers != 3 {
|
||||
t.Fatalf("settings overwritten %d %d %v", paused, workers, err)
|
||||
}
|
||||
if err := st.DB.QueryRow(`SELECT max_rows FROM event_log_policies WHERE event_type='*'`).Scan(&rows); err != nil || rows != 42 {
|
||||
t.Fatalf("policy overwritten %d %v", rows, err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`DROP INDEX idx_event_jobs_due`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.ensureSchemaComponent("event-jobs"); err == nil {
|
||||
t.Fatal("missing queue index accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package store
|
||||
|
||||
import "pageswiki/internal/db"
|
||||
|
||||
// Version 1 repairs only the known legacy columns; version 2 can then safely
|
||||
// create indexes using them. Recovery of running jobs is intentionally separate.
|
||||
var eventJobMigrations = []db.SQLMigration{
|
||||
{Version: 1, Name: "adopt_event_job_columns", Statements: []string{`
|
||||
CREATE TABLE IF NOT EXISTS event_jobs (
|
||||
uuid TEXT PRIMARY KEY,
|
||||
event_type TEXT NOT NULL,
|
||||
event_key TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
run_at_ms INTEGER NOT NULL,
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
created_at_ms INTEGER NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL
|
||||
)
|
||||
`}, AddColumns: []db.SQLiteColumn{
|
||||
{Table: "event_jobs", Name: "event_uuid", Declaration: "TEXT NOT NULL DEFAULT ''"},
|
||||
{Table: "event_jobs", Name: "priority", Declaration: "INTEGER NOT NULL DEFAULT 0"},
|
||||
{Table: "event_jobs", Name: "max_attempts", Declaration: "INTEGER NOT NULL DEFAULT 3"},
|
||||
{Table: "event_jobs", Name: "started_at_ms", Declaration: "INTEGER NOT NULL DEFAULT 0"},
|
||||
{Table: "event_jobs", Name: "finished_at_ms", Declaration: "INTEGER NOT NULL DEFAULT 0"},
|
||||
{Table: "event_jobs", Name: "cancel_requested", Declaration: "INTEGER NOT NULL DEFAULT 0"},
|
||||
}},
|
||||
{Version: 2, Name: "adopt_event_queue_and_logs", Statements: []string{
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_jobs_identity_status ON event_jobs(event_type,event_key,status)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_jobs_listing ON event_jobs (created_at_ms DESC, uuid DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_jobs_status_listing ON event_jobs (status, created_at_ms DESC, uuid DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_jobs_history ON event_jobs (created_at_ms DESC, uuid DESC) WHERE status IN ('succeeded','skipped','failed','cancelled')`,
|
||||
// Legacy installations already have this name without priority. IF NOT
|
||||
// EXISTS would retain that definition and fail the contract below. This
|
||||
// non-unique scheduling index is rebuilt once, in the adoption transaction;
|
||||
// neither jobs nor their status/attempts are changed.
|
||||
`DROP INDEX IF EXISTS idx_event_jobs_due`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_jobs_due
|
||||
ON event_jobs (status, priority DESC, run_at_ms, created_at_ms)`,
|
||||
`CREATE TABLE IF NOT EXISTS event_definitions (
|
||||
uuid TEXT PRIMARY KEY, name TEXT NOT NULL, event_type TEXT NOT NULL,
|
||||
event_key TEXT NOT NULL, payload TEXT NOT NULL DEFAULT '',
|
||||
schedule_kind TEXT NOT NULL DEFAULT 'manual',
|
||||
interval_seconds INTEGER NOT NULL DEFAULT 0,
|
||||
next_run_at_ms INTEGER NOT NULL DEFAULT 0,
|
||||
enabled INTEGER NOT NULL DEFAULT 1, priority INTEGER NOT NULL DEFAULT 0,
|
||||
max_attempts INTEGER NOT NULL DEFAULT 3,
|
||||
created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_definitions_due
|
||||
ON event_definitions (enabled, next_run_at_ms)`,
|
||||
`CREATE TABLE IF NOT EXISTS event_logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, job_uuid TEXT NOT NULL,
|
||||
event_uuid TEXT NOT NULL DEFAULT '', level TEXT NOT NULL,
|
||||
message TEXT NOT NULL, created_at_ms INTEGER NOT NULL
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_logs_job
|
||||
ON event_logs (job_uuid, created_at_ms DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_logs_job_id ON event_logs (job_uuid, id DESC)`,
|
||||
`CREATE TABLE IF NOT EXISTS event_log_policies (
|
||||
event_type TEXT PRIMARY KEY,
|
||||
retention_hours INTEGER NOT NULL DEFAULT 168,
|
||||
aggregate_after_hours INTEGER NOT NULL DEFAULT 24,
|
||||
bucket_seconds INTEGER NOT NULL DEFAULT 60,
|
||||
max_rows INTEGER NOT NULL DEFAULT 5000
|
||||
)`,
|
||||
`INSERT OR IGNORE INTO event_log_policies(event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows)
|
||||
VALUES('*',168,24,60,5000)`,
|
||||
`CREATE TABLE IF NOT EXISTS event_log_buckets (
|
||||
event_type TEXT NOT NULL,
|
||||
bucket_start_ms INTEGER NOT NULL,
|
||||
level TEXT NOT NULL,
|
||||
count INTEGER NOT NULL DEFAULT 0,
|
||||
first_at_ms INTEGER NOT NULL,
|
||||
last_at_ms INTEGER NOT NULL,
|
||||
sample_message TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY(event_type,bucket_start_ms,level)
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_event_log_buckets_time ON event_log_buckets(event_type,bucket_start_ms DESC)`,
|
||||
`CREATE TABLE IF NOT EXISTS event_queue_settings (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1), paused INTEGER NOT NULL DEFAULT 0,
|
||||
concurrency INTEGER NOT NULL DEFAULT 1,
|
||||
max_depth INTEGER NOT NULL DEFAULT 10000,
|
||||
retry_base_seconds INTEGER NOT NULL DEFAULT 30,
|
||||
retention_hours INTEGER NOT NULL DEFAULT 168
|
||||
)`,
|
||||
`INSERT OR IGNORE INTO event_queue_settings
|
||||
(id, paused, concurrency, max_depth, retry_base_seconds, retention_hours)
|
||||
VALUES (1, 0, 1, 10000, 30, 168)`,
|
||||
}, Assertions: []string{
|
||||
schemaColumns("event_jobs", "uuid,event_type,event_key,payload,status,attempts,run_at_ms,error,created_at_ms,updated_at_ms,event_uuid,priority,max_attempts,started_at_ms,finished_at_ms,cancel_requested"),
|
||||
schemaColumns("event_definitions", "uuid,name,event_type,event_key,payload,schedule_kind,interval_seconds,next_run_at_ms,enabled,priority,max_attempts,created_at_ms,updated_at_ms"),
|
||||
schemaColumns("event_logs", "id,job_uuid,event_uuid,level,message,created_at_ms"),
|
||||
schemaColumns("event_log_policies", "event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows"),
|
||||
schemaColumns("event_log_buckets", "event_type,bucket_start_ms,level,count,first_at_ms,last_at_ms,sample_message"),
|
||||
schemaColumns("event_queue_settings", "id,paused,concurrency,max_depth,retry_base_seconds,retention_hours"),
|
||||
schemaUnique("event_jobs", "uuid"), schemaUnique("event_definitions", "uuid"),
|
||||
schemaUnique("event_log_policies", "event_type"), schemaUnique("event_log_buckets", "event_type,bucket_start_ms,level"),
|
||||
schemaIndex("event_jobs", "idx_event_jobs_identity_status", "event_type,event_key,status"),
|
||||
schemaIndex("event_jobs", "idx_event_jobs_due", "status,priority,run_at_ms,created_at_ms"),
|
||||
schemaIndex("event_jobs", "idx_event_jobs_listing", "created_at_ms,uuid"),
|
||||
schemaIndex("event_jobs", "idx_event_jobs_status_listing", "status,created_at_ms,uuid"),
|
||||
schemaIndex("event_jobs", "idx_event_jobs_history", "created_at_ms,uuid"),
|
||||
schemaIndexPredicate("idx_event_jobs_history", "status IN ('succeeded','skipped','failed','cancelled')"),
|
||||
schemaIndex("event_definitions", "idx_event_definitions_due", "enabled,next_run_at_ms"),
|
||||
schemaIndex("event_logs", "idx_event_logs_job", "job_uuid,created_at_ms"),
|
||||
schemaIndex("event_logs", "idx_event_logs_job_id", "job_uuid,id"),
|
||||
schemaIndex("event_log_buckets", "idx_event_log_buckets_time", "event_type,bucket_start_ms"),
|
||||
schemaAddedColumn("event_jobs", "priority", "INTEGER", "0"),
|
||||
schemaAddedColumn("event_jobs", "max_attempts", "INTEGER", "3"),
|
||||
schemaAddedColumn("event_jobs", "event_uuid", "TEXT", "''"),
|
||||
schemaAddedColumn("event_jobs", "started_at_ms", "INTEGER", "0"),
|
||||
schemaAddedColumn("event_jobs", "finished_at_ms", "INTEGER", "0"),
|
||||
schemaAddedColumn("event_jobs", "cancel_requested", "INTEGER", "0"),
|
||||
`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_table_info('event_logs') WHERE name='id' AND type='INTEGER' AND pk=1)`,
|
||||
`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_table_info('event_queue_settings') WHERE name='id' AND type='INTEGER' AND pk=1)`,
|
||||
`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM event_queue_settings WHERE id=1)`,
|
||||
}},
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
// These definitions adopt known legacy tables without replacing their rows.
|
||||
// Once released, add a new version; never edit an applied definition.
|
||||
// Auth, City, tasks and calendars remain in their existing migration paths.
|
||||
var managedStoreSchemas = map[string][]db.SQLMigration{
|
||||
"profile-contacts": profileContactMigrations,
|
||||
"event-jobs": eventJobMigrations,
|
||||
"projection-metadata": {{Version: 1, Name: "adopt_projection_metadata", Statements: []string{
|
||||
`CREATE TABLE IF NOT EXISTS projection_metadata (
|
||||
document_hash TEXT NOT NULL,kind TEXT NOT NULL,version INTEGER NOT NULL,
|
||||
payload TEXT NOT NULL DEFAULT '{}',error TEXT NOT NULL DEFAULT '',
|
||||
updated_at_ms INTEGER NOT NULL,retry_at_ms INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY(document_hash,kind,version))`,
|
||||
`CREATE INDEX IF NOT EXISTS projection_metadata_eviction ON projection_metadata(updated_at_ms DESC)`,
|
||||
}, Assertions: []string{
|
||||
schemaColumns("projection_metadata", "document_hash,kind,version,payload,error,updated_at_ms,retry_at_ms"),
|
||||
schemaUnique("projection_metadata", "document_hash,kind,version"),
|
||||
schemaIndex("projection_metadata", "projection_metadata_eviction", "updated_at_ms"),
|
||||
}}},
|
||||
"resource-history": {{Version: 1, Name: "adopt_resource_history", Statements: []string{
|
||||
`CREATE TABLE IF NOT EXISTS resource_samples (
|
||||
id INTEGER PRIMARY KEY, job_uuid TEXT NOT NULL UNIQUE,
|
||||
timestamp_ms INTEGER NOT NULL, payload TEXT NOT NULL)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_resource_samples_time ON resource_samples(timestamp_ms)`,
|
||||
}, Assertions: []string{
|
||||
schemaColumns("resource_samples", "id,job_uuid,timestamp_ms,payload"),
|
||||
schemaUnique("resource_samples", "job_uuid"),
|
||||
`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_table_info('resource_samples') WHERE name='id' AND type='INTEGER' AND pk=1)`,
|
||||
schemaIndex("resource_samples", "idx_resource_samples_time", "timestamp_ms"),
|
||||
}}},
|
||||
"broadcasts": {{Version: 1, Name: "adopt_broadcasts", Statements: []string{
|
||||
`CREATE TABLE IF NOT EXISTS broadcasts (uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,title TEXT NOT NULL,body TEXT NOT NULL,severity TEXT NOT NULL DEFAULT 'info' CHECK(severity IN ('info','success','warning','critical')),starts_at_ms INTEGER NOT NULL DEFAULT 0,ends_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_broadcast_active ON broadcasts(domain_uuid,starts_at_ms,ends_at_ms,updated_at_ms DESC)`,
|
||||
}, Assertions: []string{
|
||||
schemaColumns("broadcasts", "uuid,domain_uuid,title,body,severity,starts_at_ms,ends_at_ms,created_at_ms,updated_at_ms"),
|
||||
schemaUnique("broadcasts", "uuid"),
|
||||
schemaIndex("broadcasts", "idx_broadcast_active", "domain_uuid,starts_at_ms,ends_at_ms,updated_at_ms"),
|
||||
}}},
|
||||
"browser-navigation": {{Version: 1, Name: "adopt_browser_navigation", Statements: []string{
|
||||
`CREATE TABLE IF NOT EXISTS browser_navigation_links (
|
||||
uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,principal_uuid TEXT NOT NULL,
|
||||
label TEXT NOT NULL,path TEXT NOT NULL,is_directory INTEGER NOT NULL DEFAULT 0,
|
||||
position INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL,
|
||||
UNIQUE(domain_uuid, principal_uuid, path))`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_browser_navigation_owner ON browser_navigation_links(domain_uuid, principal_uuid, position, created_at_ms)`,
|
||||
}, AddColumns: []db.SQLiteColumn{{Table: "browser_navigation_links", Name: "is_directory", Declaration: "INTEGER NOT NULL DEFAULT 0"}}, Assertions: []string{
|
||||
schemaColumns("browser_navigation_links", "uuid,domain_uuid,principal_uuid,label,path,is_directory,position,created_at_ms,updated_at_ms"),
|
||||
schemaUnique("browser_navigation_links", "uuid"),
|
||||
schemaUnique("browser_navigation_links", "domain_uuid,principal_uuid,path"),
|
||||
schemaAddedColumn("browser_navigation_links", "is_directory", "INTEGER", "0"),
|
||||
schemaIndex("browser_navigation_links", "idx_browser_navigation_owner", "domain_uuid,principal_uuid,position,created_at_ms"),
|
||||
}}},
|
||||
"adapter-sessions": {{Version: 1, Name: "adopt_adapter_sessions", Statements: []string{
|
||||
`CREATE TABLE IF NOT EXISTS adapter_sessions (
|
||||
ticket_hash TEXT PRIMARY KEY,session_hash TEXT NOT NULL DEFAULT '',
|
||||
domain_uuid TEXT NOT NULL,plugin_id TEXT NOT NULL,object_uuid TEXT NOT NULL,
|
||||
principal_uuid TEXT NOT NULL DEFAULT '',base_revision_uuid TEXT NOT NULL,
|
||||
ticket_expires_at_ms INTEGER NOT NULL,session_expires_at_ms INTEGER NOT NULL DEFAULT 0,
|
||||
exchanged_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL)`,
|
||||
`CREATE UNIQUE INDEX IF NOT EXISTS idx_adapter_session_hash ON adapter_sessions(session_hash) WHERE session_hash != ''`,
|
||||
}, AddColumns: []db.SQLiteColumn{
|
||||
{Table: "adapter_sessions", Name: "root_path", Declaration: "TEXT NOT NULL DEFAULT ''"},
|
||||
{Table: "adapter_sessions", Name: "principal_uuid", Declaration: "TEXT NOT NULL DEFAULT ''"},
|
||||
}, Assertions: []string{
|
||||
schemaColumns("adapter_sessions", "ticket_hash,session_hash,domain_uuid,plugin_id,object_uuid,principal_uuid,base_revision_uuid,ticket_expires_at_ms,session_expires_at_ms,exchanged_at_ms,created_at_ms,root_path"),
|
||||
schemaUnique("adapter_sessions", "ticket_hash"),
|
||||
schemaAddedColumn("adapter_sessions", "root_path", "TEXT", "''"),
|
||||
schemaAddedColumn("adapter_sessions", "principal_uuid", "TEXT", "''"),
|
||||
schemaIndex("adapter_sessions", "idx_adapter_session_hash", "session_hash"),
|
||||
`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_index_list('adapter_sessions') WHERE name='idx_adapter_session_hash' AND "unique"=1 AND partial=1)`,
|
||||
schemaIndexPredicate("idx_adapter_session_hash", "session_hash != ''"),
|
||||
}}},
|
||||
"imports": {{Version: 1, Name: "adopt_imports", Statements: []string{
|
||||
`CREATE TABLE IF NOT EXISTS import_sessions(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,actor_uuid TEXT NOT NULL DEFAULT '',archive_name TEXT NOT NULL,archive_path TEXT NOT NULL,target_prefix TEXT NOT NULL DEFAULT '/',status TEXT NOT NULL DEFAULT 'queued',total_files INTEGER NOT NULL DEFAULT 0,markdown_files INTEGER NOT NULL DEFAULT 0,error TEXT NOT NULL DEFAULT '',created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`,
|
||||
`CREATE TABLE IF NOT EXISTS import_items(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,source_path TEXT NOT NULL,target_path TEXT NOT NULL,size INTEGER NOT NULL DEFAULT 0,content_type TEXT NOT NULL DEFAULT '',title TEXT NOT NULL DEFAULT '',suggested_quartier TEXT NOT NULL DEFAULT '',suggested_street TEXT NOT NULL DEFAULT '',suggested_spot TEXT NOT NULL DEFAULT '',confidence REAL NOT NULL DEFAULT 0,selected INTEGER NOT NULL DEFAULT 1,PRIMARY KEY(session_uuid,sequence))`,
|
||||
`CREATE TABLE IF NOT EXISTS import_journal(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,path TEXT NOT NULL,object_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,sequence))`,
|
||||
`CREATE TABLE IF NOT EXISTS import_city_journal(session_uuid TEXT NOT NULL,kind TEXT NOT NULL,node_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,kind,node_uuid))`,
|
||||
`CREATE TABLE IF NOT EXISTS import_city_edges(session_uuid TEXT NOT NULL,spot_uuid TEXT NOT NULL,page_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,spot_uuid,page_uuid))`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_import_sessions_domain ON import_sessions(domain_uuid,created_at_ms DESC)`,
|
||||
}, AddColumns: []db.SQLiteColumn{{Table: "import_journal", Name: "completed", Declaration: "INTEGER NOT NULL DEFAULT 0"}}, Assertions: []string{
|
||||
schemaColumns("import_sessions", "uuid,domain_uuid,actor_uuid,archive_name,archive_path,target_prefix,status,total_files,markdown_files,error,created_at_ms,updated_at_ms"),
|
||||
schemaColumns("import_items", "session_uuid,sequence,source_path,target_path,size,content_type,title,suggested_quartier,suggested_street,suggested_spot,confidence,selected"),
|
||||
schemaColumns("import_journal", "session_uuid,sequence,path,object_uuid,created_at_ms,completed"),
|
||||
schemaColumns("import_city_journal", "session_uuid,kind,node_uuid,created_at_ms"),
|
||||
schemaColumns("import_city_edges", "session_uuid,spot_uuid,page_uuid,created_at_ms"),
|
||||
schemaUnique("import_sessions", "uuid"),
|
||||
schemaUnique("import_items", "session_uuid,sequence"),
|
||||
schemaUnique("import_journal", "session_uuid,sequence"),
|
||||
schemaUnique("import_city_journal", "session_uuid,kind,node_uuid"),
|
||||
schemaUnique("import_city_edges", "session_uuid,spot_uuid,page_uuid"),
|
||||
schemaAddedColumn("import_journal", "completed", "INTEGER", "0"),
|
||||
schemaIndex("import_sessions", "idx_import_sessions_domain", "domain_uuid,created_at_ms"),
|
||||
}}},
|
||||
}
|
||||
|
||||
func (s *Store) ensureSchemaComponent(component string) error {
|
||||
if s.ReadOnly {
|
||||
return ErrReadOnly
|
||||
}
|
||||
steps, ok := managedStoreSchemas[component]
|
||||
if !ok {
|
||||
return fmt.Errorf("unknown schema component %q", component)
|
||||
}
|
||||
return db.ApplySQLiteMigrations(context.Background(), s.DB, component, steps)
|
||||
}
|
||||
|
||||
// These helpers consume only static definitions above, never request input.
|
||||
func schemaColumns(table, columns string) string {
|
||||
return fmt.Sprintf("SELECT %s FROM %q LIMIT 0", columns, table)
|
||||
}
|
||||
|
||||
func schemaUnique(table, columns string) string {
|
||||
return fmt.Sprintf(`SELECT 1 WHERE NOT EXISTS (
|
||||
SELECT 1 FROM pragma_index_list('%s') AS i WHERE i."unique"=1 AND i.partial=0
|
||||
AND (SELECT group_concat(name,',') FROM (SELECT name FROM pragma_index_info(i.name) ORDER BY seqno))='%s')`, table, columns)
|
||||
}
|
||||
|
||||
func schemaIndex(table, name, columns string) string {
|
||||
return fmt.Sprintf(`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_index_list('%s') WHERE name='%s')
|
||||
OR (SELECT group_concat(name,',') FROM (SELECT name FROM pragma_index_info('%s') ORDER BY seqno)) IS NOT '%s'`, table, name, name, strings.TrimSpace(columns))
|
||||
}
|
||||
|
||||
func schemaAddedColumn(table, column, kind, defaultSQL string) string {
|
||||
return fmt.Sprintf(`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_table_info('%s') WHERE name='%s' AND upper(type)='%s' AND "notnull"=1 AND dflt_value='%s')`, table, column, kind, strings.ReplaceAll(defaultSQL, "'", "''"))
|
||||
}
|
||||
|
||||
func schemaIndexPredicate(name, predicate string) string {
|
||||
normalized := strings.ToLower(strings.Join(strings.Fields(predicate), ""))
|
||||
return fmt.Sprintf(`SELECT 1 WHERE NOT EXISTS (
|
||||
SELECT 1 FROM (SELECT lower(replace(replace(replace(replace(sql,' ',''),char(9),''),char(10),''),char(13),'')) AS normalized FROM sqlite_master WHERE type='index' AND name='%s')
|
||||
WHERE substr(normalized,instr(normalized,'where')+5)='%s')`, name, strings.ReplaceAll(normalized, "'", "''"))
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
func newSchemaTestStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "schema.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
return &Store{DB: database}
|
||||
}
|
||||
|
||||
func TestManagedSchemasFreshRepeatAndReadOnly(t *testing.T) {
|
||||
for component := range managedStoreSchemas {
|
||||
t.Run(component, func(t *testing.T) {
|
||||
st := newSchemaTestStore(t)
|
||||
if err := st.ensureSchemaComponent(component); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.ensureSchemaComponent(component); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var count int
|
||||
if err := st.DB.QueryRow(`SELECT count(*) FROM schema_migration_steps WHERE component=?`, component).Scan(&count); err != nil || count != len(managedStoreSchemas[component]) {
|
||||
t.Fatalf("ledger=%d err=%v", count, err)
|
||||
}
|
||||
st.ReadOnly = true
|
||||
if err := st.ensureSchemaComponent(component); !errors.Is(err, ErrReadOnly) {
|
||||
t.Fatalf("standby migration: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedSchemasAdoptLegacyRowsWithoutRecreatingTables(t *testing.T) {
|
||||
for _, tc := range []struct{ component, ddl, insert, query, want string }{
|
||||
{"browser-navigation", `CREATE TABLE browser_navigation_links(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,principal_uuid TEXT NOT NULL,label TEXT NOT NULL,path TEXT NOT NULL,position INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL,UNIQUE(domain_uuid,principal_uuid,path))`,
|
||||
`INSERT INTO browser_navigation_links VALUES('u','d','p','keep','/page',3,10,20)`, `SELECT label||':'||position||':'||is_directory FROM browser_navigation_links WHERE uuid='u'`, "keep:3:0"},
|
||||
{"adapter-sessions", `CREATE TABLE adapter_sessions(ticket_hash TEXT PRIMARY KEY,session_hash TEXT NOT NULL DEFAULT '',domain_uuid TEXT NOT NULL,plugin_id TEXT NOT NULL,object_uuid TEXT NOT NULL,base_revision_uuid TEXT NOT NULL,ticket_expires_at_ms INTEGER NOT NULL,session_expires_at_ms INTEGER NOT NULL DEFAULT 0,exchanged_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL)`,
|
||||
`INSERT INTO adapter_sessions VALUES('ticket','session','d','code','o','r',100,200,50,10)`, `SELECT session_hash||':'||root_path||':'||principal_uuid FROM adapter_sessions WHERE ticket_hash='ticket'`, "session::"},
|
||||
{"imports", `CREATE TABLE import_journal(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,path TEXT NOT NULL,object_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,sequence))`,
|
||||
`INSERT INTO import_journal VALUES('s',2,'/keep','o',20)`, `SELECT path||':'||completed FROM import_journal WHERE session_uuid='s' AND sequence=2`, "/keep:0"},
|
||||
{"resource-history", `CREATE TABLE resource_samples(id INTEGER PRIMARY KEY,job_uuid TEXT NOT NULL UNIQUE,timestamp_ms INTEGER NOT NULL,payload TEXT NOT NULL)`,
|
||||
`INSERT INTO resource_samples VALUES(9,'j',100,'keep')`, `SELECT id||':'||payload FROM resource_samples WHERE job_uuid='j'`, "9:keep"},
|
||||
{"projection-metadata", `CREATE TABLE projection_metadata(document_hash TEXT NOT NULL,kind TEXT NOT NULL,version INTEGER NOT NULL,payload TEXT NOT NULL DEFAULT '{}',error TEXT NOT NULL DEFAULT '',updated_at_ms INTEGER NOT NULL,retry_at_ms INTEGER NOT NULL DEFAULT 0,PRIMARY KEY(document_hash,kind,version))`,
|
||||
`INSERT INTO projection_metadata VALUES('h','pdf',1,'keep','',10,0)`, `SELECT payload FROM projection_metadata WHERE document_hash='h'`, "keep"},
|
||||
{"broadcasts", `CREATE TABLE broadcasts(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,title TEXT NOT NULL,body TEXT NOT NULL,severity TEXT NOT NULL DEFAULT 'info' CHECK(severity IN ('info','success','warning','critical')),starts_at_ms INTEGER NOT NULL DEFAULT 0,ends_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`,
|
||||
`INSERT INTO broadcasts VALUES('u','d','title','keep','info',0,0,10,20)`, `SELECT body FROM broadcasts WHERE uuid='u'`, "keep"},
|
||||
} {
|
||||
t.Run(tc.component, func(t *testing.T) {
|
||||
st := newSchemaTestStore(t)
|
||||
for _, statement := range []string{tc.ddl, tc.insert} {
|
||||
if _, err := st.DB.Exec(statement); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := st.ensureSchemaComponent(tc.component); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var value string
|
||||
if err := st.DB.QueryRow(tc.query).Scan(&value); err != nil || value != tc.want {
|
||||
t.Fatalf("row=%q want=%q err=%v", value, tc.want, err)
|
||||
}
|
||||
if err := st.ensureSchemaComponent(tc.component); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedSchemasRejectBrokenAdoptionAndLaterDrift(t *testing.T) {
|
||||
st := newSchemaTestStore(t)
|
||||
// Matching columns without the conflict-target uniqueness is not sufficient.
|
||||
if _, err := st.DB.Exec(`CREATE TABLE resource_samples(id INTEGER PRIMARY KEY,job_uuid TEXT NOT NULL,timestamp_ms INTEGER NOT NULL,payload TEXT NOT NULL); INSERT INTO resource_samples VALUES(1,'j',10,'keep')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.EnsureResourceHistorySchema(); err == nil {
|
||||
t.Fatal("missing unique constraint accepted")
|
||||
}
|
||||
var payload string
|
||||
if err := st.DB.QueryRow(`SELECT payload FROM resource_samples WHERE id=1`).Scan(&payload); err != nil || payload != "keep" {
|
||||
t.Fatal("failed adoption modified data", err)
|
||||
}
|
||||
var count int
|
||||
if err := st.DB.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='idx_resource_samples_time'`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatal("failed adoption left partial DDL", err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`CREATE UNIQUE INDEX restored_resource_unique ON resource_samples(job_uuid)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.EnsureResourceHistorySchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`DROP INDEX idx_resource_samples_time`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.EnsureResourceHistorySchema(); err == nil {
|
||||
t.Fatal("ledger masked missing index")
|
||||
}
|
||||
if err := st.EnsureAdapterSessionsSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.DB.Exec(`DROP INDEX idx_adapter_session_hash; CREATE UNIQUE INDEX idx_adapter_session_hash ON adapter_sessions(session_hash) WHERE session_hash = ''`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.EnsureAdapterSessionsSchema(); err == nil {
|
||||
t.Fatal("incorrect partial index predicate accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventJobsAdoptLegacyDueIndex(t *testing.T) {
|
||||
st := newSchemaTestStore(t)
|
||||
// The deployed pre-ledger scheduler used the same index name without priority.
|
||||
for _, statement := range []string{
|
||||
eventJobMigrations[0].Statements[0],
|
||||
`CREATE INDEX idx_event_jobs_due ON event_jobs(status,run_at_ms,created_at_ms)`,
|
||||
`INSERT INTO event_jobs(uuid,event_type,event_key,payload,status,attempts,run_at_ms,created_at_ms,updated_at_ms) VALUES('keep','test','key','unchanged','running',2,10,20,30)`,
|
||||
} {
|
||||
if _, err := st.DB.Exec(statement); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for range 2 {
|
||||
if err := st.ensureSchemaComponent("event-jobs"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var columns, row string
|
||||
if err := st.DB.QueryRow(`SELECT group_concat(name,',') FROM (SELECT name FROM pragma_index_info('idx_event_jobs_due') ORDER BY seqno)`).Scan(&columns); err != nil || columns != "status,priority,run_at_ms,created_at_ms" {
|
||||
t.Fatalf("index=%q err=%v", columns, err)
|
||||
}
|
||||
if err := st.DB.QueryRow(`SELECT payload||':'||status||':'||attempts||':'||run_at_ms||':'||created_at_ms||':'||updated_at_ms FROM event_jobs WHERE uuid='keep'`).Scan(&row); err != nil || row != "unchanged:running:2:10:20:30" {
|
||||
t.Fatalf("adoption changed job: %q err=%v", row, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"pageswiki/internal/schutzaria"
|
||||
)
|
||||
|
||||
func schutzariaStoreFixture(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
t.Cleanup(func() { db.Close() })
|
||||
s := &Store{DB: db}
|
||||
if err = s.EnsureAuthSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, stmt := range []string{
|
||||
`INSERT INTO principals(uuid,state,created_at_ms,updated_at_ms) VALUES('p','active',1,1)`,
|
||||
`INSERT INTO domain_memberships(uuid,domain_uuid,principal_uuid,state,created_at_ms) VALUES('m','d','p','active',1)`,
|
||||
`INSERT INTO auth_roles(uuid,domain_uuid,name,created_at_ms) VALUES('r','d','reader',1)`,
|
||||
`INSERT INTO auth_role_bindings(membership_uuid,role_uuid,created_at_ms) VALUES('m','r',1)`,
|
||||
} {
|
||||
if _, err = db.Exec(stmt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestSchutzariaReadLoaderBudgetsAndErrors(t *testing.T) {
|
||||
for _, kind := range []string{"grant", "locality", "db-error"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
s := schutzariaStoreFixture(t)
|
||||
if _, err := s.DB.Exec(`INSERT INTO auth_capability_grants(uuid,role_uuid,capability,target_type,target_id,created_at_ms) VALUES('all','r','read','domain','d',1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
switch kind {
|
||||
case "grant":
|
||||
_, err := s.DB.Exec(`WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<4096) INSERT INTO auth_capability_grants(uuid,role_uuid,capability,target_type,target_id,created_at_ms) SELECT 'grant-'||x,'r','read','path','/page/'||x,1 FROM n`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "locality":
|
||||
_, err := s.DB.Exec(`WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<4097) INSERT INTO locality_access_policies(domain_uuid,node_kind,node_uuid,scope_path,grants_access,actor_uuid,authority_rank) SELECT 'd','spot','spot-'||x,'/scope/'||x,0,'p',3 FROM n`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "db-error":
|
||||
if _, err := s.DB.Exec(`DROP TABLE locality_access_policies`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
policy, err := s.LoadReadPathPolicy("d", "p")
|
||||
if err == nil || policy != nil || policy.Allows("/would-be-allowed") {
|
||||
t.Fatal("incomplete ACL snapshot escaped", policy, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchutzariaStoreCompatibility(t *testing.T) {
|
||||
var policy *ReadPathPolicy = schutzaria.NewReadPathPolicy(false, []string{"/exact"}, nil, nil)
|
||||
if !policy.Allows("/exact") || policy.Allows("/exact/child") {
|
||||
t.Fatal("alias mismatch")
|
||||
}
|
||||
var locality LocalityPolicy = schutzaria.LocalityPolicy{Kind: "spot", UUID: "id", Policy: "public"}
|
||||
data, err := json.Marshal(locality)
|
||||
if err != nil || string(data) != `{"kind":"spot","uuid":"id","policy":"public"}` {
|
||||
t.Fatal(string(data), err)
|
||||
}
|
||||
var user UserSpacePolicy = schutzaria.UserSpacePolicy{Username: "alice", RootVisibility: "private", DescendantVisibility: "domain"}
|
||||
for _, p := range []string{"/u/alice", "/u/alice/page", "/u/bob"} {
|
||||
if EffectiveUserSpaceVisibility(user, p) != schutzaria.EffectiveUserSpaceVisibility(user, p) {
|
||||
t.Fatal("user policy wrapper mismatch")
|
||||
}
|
||||
}
|
||||
if ErrApprovalTransition != schutzaria.ErrApprovalTransition {
|
||||
t.Fatal("error identity changed")
|
||||
}
|
||||
for _, p := range []string{"/public", "/u/private", "/v/tree"} {
|
||||
if PublicSemanticPath(p) != schutzaria.PublicSemanticPath(p) {
|
||||
t.Fatal("publication wrapper mismatch")
|
||||
}
|
||||
}
|
||||
}
|
||||
+16
-360
@@ -2,378 +2,34 @@ package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/mestionora"
|
||||
)
|
||||
|
||||
const SearchCandidateLimit = 1000
|
||||
const SearchExcerptDocumentBytes = 64 << 10
|
||||
const SearchExcerptTotalBytes = 1 << 20
|
||||
const searchExcerptTimeout = 400 * time.Millisecond
|
||||
type SearchResult = mestionora.SearchResult
|
||||
type PathSuggestion = mestionora.PathSuggestion
|
||||
|
||||
const searchResultColumns = `p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.revision_uuid,''),COALESCE(r.created_at_ms/1000,0)`
|
||||
const searchPathBoundary = `(p.path=? OR (p.path>=? AND p.path<?))`
|
||||
const searchPathCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,0
|
||||
FROM paths p LEFT JOIN latest_revision_index r ON r.object_uuid=p.object_uuid
|
||||
WHERE p.domain_uuid=? AND ` + searchPathBoundary
|
||||
const searchPathQueryPredicate = ` AND (instr(lower(p.path),lower(?))>0 OR instr(lower(COALESCE(r.filename,'')),lower(?))>0)`
|
||||
const searchPathCandidatesSQL = searchPathCandidatesBaseSQL + searchPathQueryPredicate + ` ORDER BY p.path LIMIT ?`
|
||||
const (
|
||||
SearchCandidateLimit = mestionora.SearchCandidateLimit
|
||||
SearchExcerptDocumentBytes = mestionora.SearchExcerptDocumentBytes
|
||||
SearchExcerptTotalBytes = mestionora.SearchExcerptTotalBytes
|
||||
)
|
||||
|
||||
// CROSS JOIN is intentional: revision_uuid is an UNINDEXED FTS payload column.
|
||||
// A Path-first plan re-runs the complete MATCH scan for every Path. Start with
|
||||
// MATCH exactly once, then seek current Revision metadata and Domain bindings
|
||||
// through their B-tree indexes. Keep scope filtering before the candidate cap.
|
||||
const searchFTSCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,revision_search.rowid
|
||||
FROM revision_search
|
||||
CROSS JOIN latest_revision_index r ON r.revision_uuid=revision_search.revision_uuid
|
||||
CROSS JOIN paths p ON p.object_uuid=r.object_uuid
|
||||
WHERE revision_search MATCH ? AND p.domain_uuid=? AND ` + searchPathBoundary
|
||||
const searchFTSCandidatesSQL = searchFTSCandidatesBaseSQL + ` LIMIT ?`
|
||||
|
||||
func searchBodyExcerptsSQL(count int) string {
|
||||
// UUID is not indexed by FTS5. Restrict the MATCH/snippet work using the
|
||||
// integer rowids collected from the same read snapshot, not payload UUIDs.
|
||||
return `SELECT revision_uuid,substr(snippet(revision_search,3,'','',' … ',36),1,4096)
|
||||
FROM revision_search WHERE revision_search MATCH ? AND rowid IN (` + strings.TrimRight(strings.Repeat("?,", count), ",") + `)`
|
||||
}
|
||||
|
||||
// Excerpts are optional enrichment. Even small documents can have expensive
|
||||
// prefix hit-position scans, so their time budget is separate from finding
|
||||
// results. A spent excerpt budget omits snippets, not already-authorized hits.
|
||||
func loadSearchBodyExcerpts(parent context.Context, tx *sql.Tx, expression string, ids []any, budget time.Duration) (excerpts map[string]string, err error) {
|
||||
ctx, cancel := context.WithTimeout(parent, budget)
|
||||
defer cancel()
|
||||
defer func() {
|
||||
if err != nil && ctx.Err() != nil {
|
||||
if parent.Err() != nil {
|
||||
err = parent.Err()
|
||||
} else {
|
||||
excerpts, err = nil, nil
|
||||
}
|
||||
}
|
||||
}()
|
||||
args := append([]any{"body : (" + expression + ")"}, ids...)
|
||||
rows, err := tx.QueryContext(ctx, searchBodyExcerptsSQL(len(ids)), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
excerpts = make(map[string]string)
|
||||
for rows.Next() {
|
||||
var id, excerpt string
|
||||
if err := rows.Scan(&id, &excerpt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
excerpts[id] = excerpt
|
||||
}
|
||||
return excerpts, rows.Err()
|
||||
}
|
||||
|
||||
type SearchResult struct {
|
||||
Path string
|
||||
ContentType string
|
||||
Filename string
|
||||
DocumentHash string
|
||||
RevisionUUID string
|
||||
Size int64 // Kept for API compatibility; search does not stat Document files.
|
||||
UpdatedAt int64
|
||||
PathMatch bool
|
||||
FilenameMatch bool
|
||||
BodyMatch bool
|
||||
Excerpt string // Untrusted source text, never HTML-safe markup.
|
||||
ExcerptOmitted bool // A body match remains visible when snippet work exceeds its budget.
|
||||
}
|
||||
|
||||
type PathSuggestion struct {
|
||||
SearchResult
|
||||
Title string
|
||||
}
|
||||
|
||||
// SearchPathsInDomain preserves the store API for trusted callers. HTTP callers
|
||||
// must use SearchPathsInDomainFiltered with a request-scoped authorization filter.
|
||||
// SearchPathsInDomain is the legacy trusted-caller entry point. HTTP callers
|
||||
// must pass their request policy to the filtered APIs; there is no implicit
|
||||
// allow-all entry point in Mestionora itself.
|
||||
func (s *Store) SearchPathsInDomain(domainUUID, scope, query string, limit int) ([]SearchResult, error) {
|
||||
return s.SearchPathsInDomainFiltered(context.Background(), domainUUID, scope, query, limit, func(string) bool { return true })
|
||||
}
|
||||
|
||||
// SearchPathsInDomainFiltered bounds SQL candidates before materialization and
|
||||
// authorizes each Path before loading an excerpt. It never computes a global hit
|
||||
// count or stats Document files. A narrow scope improves both recall and cost.
|
||||
func (s *Store) SearchPathsInDomainFiltered(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]SearchResult, error) {
|
||||
return s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{})
|
||||
return mestionora.NewSQLite(s.DB).SearchPathsInDomainFiltered(ctx, domainUUID, scope, query, limit, allowed)
|
||||
}
|
||||
|
||||
// SearchPathsInDomainWithOptions also permits filter-only searches. File/date
|
||||
// predicates apply before SQL candidate caps, while ACL checks stay ahead of all
|
||||
// snippet enrichment. Zero options preserve the original indexed query plans.
|
||||
func (s *Store) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, searchErr error) {
|
||||
return s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options, true)
|
||||
func (s *Store) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) ([]SearchResult, error) {
|
||||
return mestionora.NewSQLite(s.DB).SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options)
|
||||
}
|
||||
|
||||
// SearchPathSuggestionsInDomain shares the current-Revision FTS and ACL path
|
||||
// with full search, but deliberately skips snippet generation. Typeahead calls
|
||||
// are frequent and only need a bounded display label, never a body excerpt.
|
||||
func (s *Store) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) {
|
||||
results, err := s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}, false)
|
||||
if err != nil || len(results) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
ids := make([]any, 0, len(results))
|
||||
seen := make(map[string]bool, len(results))
|
||||
for _, result := range results {
|
||||
if result.RevisionUUID != "" && strings.HasPrefix(strings.ToLower(result.ContentType), "text/html") && !seen[result.RevisionUUID] {
|
||||
ids = append(ids, result.RevisionUUID)
|
||||
seen[result.RevisionUUID] = true
|
||||
}
|
||||
}
|
||||
headings := make(map[string]string, len(ids))
|
||||
if len(ids) > 0 {
|
||||
rows, queryErr := s.DB.QueryContext(ctx, `SELECT uuid,substr(CAST(body AS TEXT),1,65536) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
|
||||
if queryErr != nil {
|
||||
return nil, queryErr
|
||||
}
|
||||
for rows.Next() {
|
||||
var revision, source string
|
||||
if queryErr = rows.Scan(&revision, &source); queryErr != nil {
|
||||
rows.Close()
|
||||
return nil, queryErr
|
||||
}
|
||||
headings[revision] = firstHTMLHeading(source)
|
||||
}
|
||||
queryErr = rows.Err()
|
||||
rows.Close()
|
||||
if queryErr != nil {
|
||||
return nil, queryErr
|
||||
}
|
||||
}
|
||||
out := make([]PathSuggestion, 0, len(results))
|
||||
for _, result := range results {
|
||||
out = append(out, PathSuggestion{SearchResult: result, Title: headings[result.RevisionUUID]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Store) searchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions, loadExcerpts bool) (found []SearchResult, searchErr error) {
|
||||
if err := options.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scope, query = normalizePath(scope), strings.TrimSpace(query)
|
||||
if query == "" && !options.HasFilters() || allowed == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if len(query) > 512 || len(strings.Fields(query)) > 16 {
|
||||
return nil, errors.New("search query is too long")
|
||||
}
|
||||
if limit <= 0 || limit > 200 {
|
||||
limit = 100
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
defer func() {
|
||||
// SQLite can report an interrupted statement instead of wrapping the
|
||||
// context error. Preserve cancellation/deadline semantics for HTTP.
|
||||
if searchErr != nil && ctx.Err() != nil {
|
||||
searchErr = ctx.Err()
|
||||
}
|
||||
}()
|
||||
tx, err := s.DB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
prefix := strings.TrimRight(scope, "/") + "/"
|
||||
upper := strings.TrimRight(scope, "/") + "0"
|
||||
results := make(map[string]SearchResult)
|
||||
ftsRows := make(map[string]int64)
|
||||
collect := func(statement string, body, filename bool, args ...any) error {
|
||||
rows, err := tx.QueryContext(ctx, statement, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var result SearchResult
|
||||
var rowID int64
|
||||
if err := rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt, &rowID); err != nil {
|
||||
return err
|
||||
}
|
||||
if !allowed(result.Path) {
|
||||
continue
|
||||
}
|
||||
if rowID != 0 {
|
||||
ftsRows[result.RevisionUUID] = rowID
|
||||
}
|
||||
result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query))
|
||||
result.FilenameMatch = filename || results[result.Path].FilenameMatch || query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query))
|
||||
result.BodyMatch = body || results[result.Path].BodyMatch
|
||||
results[result.Path] = result
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
filterSQL, filterArgs := searchOptionsPredicate(options)
|
||||
pathSQL := searchPathCandidatesBaseSQL
|
||||
pathArgs := []any{domainUUID, scope, prefix, upper}
|
||||
if query != "" {
|
||||
pathSQL += searchPathQueryPredicate
|
||||
pathArgs = append(pathArgs, query, query)
|
||||
}
|
||||
pathSQL += filterSQL + ` ORDER BY p.path LIMIT ?`
|
||||
pathArgs = append(pathArgs, filterArgs...)
|
||||
pathArgs = append(pathArgs, SearchCandidateLimit)
|
||||
if err := collect(pathSQL, false, false, pathArgs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expression := ftsPrefixExpression(query)
|
||||
if expression != "" {
|
||||
// No global ORDER BY or unbounded IN materialization: stop the FTS scan
|
||||
// after at most this many current, in-Domain, in-scope Path candidates.
|
||||
for _, field := range []string{"body", "filename"} {
|
||||
args := []any{field + " : (" + expression + ")", domainUUID, scope, prefix, upper}
|
||||
args = append(args, filterArgs...)
|
||||
args = append(args, SearchCandidateLimit)
|
||||
if err := collect(searchFTSCandidatesBaseSQL+filterSQL+` LIMIT ?`, field == "body", field == "filename", args...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]SearchResult, 0, len(results))
|
||||
for _, result := range results {
|
||||
out = append(out, result)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
// Direct name/Path matches precede body-only hits; all ties are stable.
|
||||
iName, jName := out[i].PathMatch || out[i].FilenameMatch, out[j].PathMatch || out[j].FilenameMatch
|
||||
if iName != jName {
|
||||
return iName
|
||||
}
|
||||
if len(out[i].Path) != len(out[j].Path) {
|
||||
return len(out[i].Path) < len(out[j].Path)
|
||||
}
|
||||
return out[i].Path < out[j].Path
|
||||
})
|
||||
if len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
if !loadExcerpts {
|
||||
return out, nil
|
||||
}
|
||||
// Only authorized, displayed results receive text excerpts. Batch the FTS
|
||||
// lookup so a broad query is not re-scanned once for every visible result.
|
||||
ids := make([]any, 0, len(out))
|
||||
seen := make(map[string]bool)
|
||||
for _, result := range out {
|
||||
if result.BodyMatch && !seen[result.RevisionUUID] {
|
||||
ids = append(ids, ftsRows[result.RevisionUUID])
|
||||
seen[result.RevisionUUID] = true
|
||||
}
|
||||
}
|
||||
if len(ids) != 0 {
|
||||
// snippet's output token/character limit does not bound its input work:
|
||||
// FTS5 may tokenize the entire matched document. Read byte lengths only
|
||||
// for authorized displayed rows before spending the snippet budget.
|
||||
rows, err := tx.QueryContext(ctx, `SELECT rowid,octet_length(body)
|
||||
FROM revision_search WHERE rowid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sizes := make(map[int64]int64)
|
||||
for rows.Next() {
|
||||
var rowID, size int64
|
||||
if err := rows.Scan(&rowID, &size); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
sizes[rowID] = size
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids = nil
|
||||
selected := make(map[string]bool)
|
||||
remaining := int64(SearchExcerptTotalBytes)
|
||||
for i := range out {
|
||||
if !out[i].BodyMatch {
|
||||
continue
|
||||
}
|
||||
revision := out[i].RevisionUUID
|
||||
include, decided := selected[revision]
|
||||
if !decided {
|
||||
rowID := ftsRows[revision]
|
||||
size, exists := sizes[rowID]
|
||||
include = exists && size >= 0 && size <= SearchExcerptDocumentBytes && size <= remaining
|
||||
if include {
|
||||
ids = append(ids, rowID)
|
||||
remaining -= size
|
||||
}
|
||||
selected[revision] = include
|
||||
}
|
||||
out[i].ExcerptOmitted = !include
|
||||
}
|
||||
}
|
||||
if len(ids) != 0 {
|
||||
excerpts, err := loadSearchBodyExcerpts(ctx, tx, expression, ids, searchExcerptTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range out {
|
||||
if out[i].BodyMatch {
|
||||
var present bool
|
||||
out[i].Excerpt, present = excerpts[out[i].RevisionUUID]
|
||||
out[i].ExcerptOmitted = !present
|
||||
}
|
||||
}
|
||||
}
|
||||
// A filename/Path-only hit still benefits from a short Page summary. Read
|
||||
// only a bounded prefix from displayed textual revisions, in one batch.
|
||||
ids, seen = nil, make(map[string]bool)
|
||||
for _, result := range out {
|
||||
media := strings.ToLower(result.ContentType)
|
||||
textual := strings.HasPrefix(media, "text/") || strings.HasPrefix(media, "application/json") || strings.HasPrefix(media, "application/xml") || strings.HasPrefix(media, "application/javascript") || strings.HasPrefix(media, "application/yaml") || strings.HasPrefix(media, "application/x-yaml")
|
||||
if !result.BodyMatch && textual && result.RevisionUUID != "" && !seen[result.RevisionUUID] {
|
||||
ids = append(ids, result.RevisionUUID)
|
||||
seen[result.RevisionUUID] = true
|
||||
}
|
||||
}
|
||||
if len(ids) != 0 {
|
||||
rows, err := tx.QueryContext(ctx, `SELECT uuid,substr(body,1,1024) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
excerpts := make(map[string]string)
|
||||
for rows.Next() {
|
||||
var id, excerpt string
|
||||
if err := rows.Scan(&id, &excerpt); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
excerpts[id] = excerpt
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range out {
|
||||
if !out[i].BodyMatch {
|
||||
out[i].Excerpt = excerpts[out[i].RevisionUUID]
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func ftsPrefixExpression(query string) string {
|
||||
words := strings.Fields(query)
|
||||
terms := make([]string, 0, len(words))
|
||||
for _, word := range words {
|
||||
word = strings.ReplaceAll(word, `"`, `""`)
|
||||
if word != "" {
|
||||
terms = append(terms, `"`+word+`"*`)
|
||||
}
|
||||
}
|
||||
return strings.Join(terms, " AND ")
|
||||
return mestionora.NewSQLite(s.DB).SearchPathSuggestionsInDomain(ctx, domainUUID, scope, query, limit, allowed)
|
||||
}
|
||||
@@ -1,135 +1,7 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
import "pageswiki/internal/mestionora"
|
||||
|
||||
var ErrInvalidSearchOptions = errors.New("invalid search options")
|
||||
type SearchOptions = mestionora.SearchOptions
|
||||
|
||||
// SearchOptions applies to current Page metadata, not historical Revisions.
|
||||
// Dates are Unix milliseconds: FromMS is inclusive and BeforeMS is exclusive.
|
||||
// The created field refers to Object creation, not creation of a Path alias.
|
||||
// FileType and Extension are normalized lower-case values; Extension has no dot
|
||||
// prefix and may contain a compound suffix such as tar.gz.
|
||||
type SearchOptions struct {
|
||||
FileType string
|
||||
Extension string
|
||||
DateField string
|
||||
FromMS *int64
|
||||
BeforeMS *int64
|
||||
}
|
||||
|
||||
func (o SearchOptions) HasFilters() bool {
|
||||
return o.FileType != "" || o.Extension != "" || o.FromMS != nil || o.BeforeMS != nil
|
||||
}
|
||||
|
||||
func (o SearchOptions) Validate() error {
|
||||
switch o.FileType {
|
||||
case "", "html", "text", "image", "audio", "video", "pdf", "archive", "csv", "json", "code", "office", "other":
|
||||
default:
|
||||
return fmt.Errorf("%w: unknown file type", ErrInvalidSearchOptions)
|
||||
}
|
||||
switch o.DateField {
|
||||
case "", "updated", "created":
|
||||
default:
|
||||
return fmt.Errorf("%w: unknown date field", ErrInvalidSearchOptions)
|
||||
}
|
||||
if len(o.Extension) > 32 {
|
||||
return fmt.Errorf("%w: extension is too long", ErrInvalidSearchOptions)
|
||||
}
|
||||
for i, c := range o.Extension {
|
||||
alnum := c >= 'a' && c <= 'z' || c >= '0' && c <= '9'
|
||||
if !alnum && (i == 0 || !strings.ContainsRune("._+-", c)) {
|
||||
return fmt.Errorf("%w: invalid extension", ErrInvalidSearchOptions)
|
||||
}
|
||||
}
|
||||
if o.FromMS != nil && o.BeforeMS != nil && *o.FromMS >= *o.BeforeMS {
|
||||
return fmt.Errorf("%w: empty or reversed date range", ErrInvalidSearchOptions)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const searchMIMESQL = `lower(trim(substr(COALESCE(r.content_type,''),1,instr(COALESCE(r.content_type,'')||';',';')-1)))`
|
||||
|
||||
// The original filename wins over the Path suffix when present. Names remain
|
||||
// data: even extension patterns are bound arguments and do not become SQL.
|
||||
const searchFilenameSQL = `lower(CASE WHEN COALESCE(r.filename,'')<>'' THEN r.filename ELSE p.path END)`
|
||||
|
||||
func searchSuffixesSQL(suffixes ...string) string {
|
||||
clauses := make([]string, 0, len(suffixes))
|
||||
for _, suffix := range suffixes {
|
||||
// Only package-owned ASCII constants reach this helper. GLOB has no
|
||||
// escaping ambiguity for underscores, unlike LIKE.
|
||||
clauses = append(clauses, searchFilenameSQL+` GLOB '*.`+suffix+`'`)
|
||||
}
|
||||
return "(" + strings.Join(clauses, " OR ") + ")"
|
||||
}
|
||||
|
||||
// Specific MIME types are authoritative. Generic text/binary types may use a
|
||||
// known suffix (e.g. a Python source uploaded as text/plain); unrecognized,
|
||||
// specific MIME types stay Other instead of pretending to be a different type.
|
||||
// This uses metadata only and never hydrates a Document to sniff its bytes.
|
||||
var searchFileTypeSQL = `CASE
|
||||
WHEN ` + searchMIMESQL + ` IN ('text/html','application/xhtml+xml') THEN 'html'
|
||||
WHEN ` + searchMIMESQL + `='application/pdf' THEN 'pdf'
|
||||
WHEN ` + searchMIMESQL + ` LIKE 'image/%' THEN 'image'
|
||||
WHEN ` + searchMIMESQL + ` LIKE 'audio/%' THEN 'audio'
|
||||
WHEN ` + searchMIMESQL + ` LIKE 'video/%' THEN 'video'
|
||||
WHEN ` + searchMIMESQL + ` IN ('text/csv','application/csv','text/tab-separated-values') THEN 'csv'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/json','text/json') OR ` + searchMIMESQL + ` GLOB '*+json' THEN 'json'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/zip','application/x-zip-compressed','application/x-7z-compressed','application/vnd.rar','application/x-rar','application/x-rar-compressed','application/x-tar','application/gzip','application/x-gzip','application/x-xz','application/x-bzip2','application/zstd','application/x-zstd','application/x-compress','application/x-cpio') THEN 'archive'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/msword','application/vnd.ms-excel','application/vnd.ms-powerpoint','application/rtf','text/rtf') OR ` + searchMIMESQL + ` GLOB 'application/vnd.openxmlformats-officedocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.oasis.opendocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.ms-*.macroenabled.*' THEN 'office'
|
||||
WHEN ` + searchMIMESQL + ` IN ('text/markdown','text/x-markdown') THEN 'text'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/javascript','application/x-javascript','text/javascript','application/typescript','text/typescript','application/xml','text/xml','application/yaml','application/x-yaml','text/yaml','application/toml','application/x-sh','application/x-shellscript','application/x-python','application/x-python-code','application/x-httpd-php','application/sql','text/css') OR ` + searchMIMESQL + ` GLOB 'text/x-*' OR ` + searchMIMESQL + ` GLOB '*+xml' THEN 'code'
|
||||
WHEN ` + searchMIMESQL + ` NOT IN ('','text/plain','application/octet-stream','binary/octet-stream','application/x-empty') THEN CASE WHEN ` + searchMIMESQL + ` LIKE 'text/%' THEN 'text' ELSE 'other' END
|
||||
WHEN ` + searchSuffixesSQL("html", "htm", "xhtml") + ` THEN 'html'
|
||||
WHEN ` + searchSuffixesSQL("pdf") + ` THEN 'pdf'
|
||||
WHEN ` + searchSuffixesSQL("png", "jpg", "jpeg", "gif", "webp", "avif", "bmp", "svg", "tif", "tiff", "ico", "heic", "heif") + ` THEN 'image'
|
||||
WHEN ` + searchSuffixesSQL("mp3", "flac", "wav", "ogg", "oga", "opus", "m4a", "aac", "aiff", "alac", "mid", "midi") + ` THEN 'audio'
|
||||
WHEN ` + searchSuffixesSQL("mp4", "webm", "mkv", "mov", "avi", "m4v", "ogv", "mpeg", "mpg") + ` THEN 'video'
|
||||
WHEN ` + searchSuffixesSQL("zip", "7z", "rar", "tar", "gz", "tgz", "xz", "txz", "bz2", "tbz", "tbz2", "zst", "tzst", "cbz", "cbr", "cpio") + ` THEN 'archive'
|
||||
WHEN ` + searchSuffixesSQL("csv", "tsv") + ` THEN 'csv'
|
||||
WHEN ` + searchSuffixesSQL("json", "jsonl", "ndjson", "excalidraw", "ipynb") + ` THEN 'json'
|
||||
WHEN ` + searchSuffixesSQL("doc", "docx", "docm", "xls", "xlsx", "xlsm", "ppt", "pptx", "pptm", "odt", "ods", "odp", "rtf") + ` THEN 'office'
|
||||
WHEN ` + searchSuffixesSQL("py", "pyw", "sh", "bash", "zsh", "fish", "ps1", "bat", "cmd", "js", "mjs", "cjs", "jsx", "ts", "tsx", "go", "rs", "c", "h", "cpp", "hpp", "cc", "cs", "java", "kt", "kts", "swift", "rb", "php", "pl", "pm", "lua", "r", "sql", "css", "scss", "sass", "less", "xml", "xsl", "yaml", "yml", "toml", "ini", "conf", "vue", "svelte", "dart", "ex", "exs", "erl", "hs", "scala", "clj", "lisp", "el", "vim", "wasm") + ` THEN 'code'
|
||||
WHEN ` + searchMIMESQL + `='text/plain' OR ` + searchSuffixesSQL("txt", "text", "log", "md", "markdown", "rst", "adoc", "org") + ` THEN 'text'
|
||||
ELSE 'other' END`
|
||||
|
||||
// The returned predicate is inserted before each candidate stream's LIMIT,
|
||||
// including FTS-first streams. Filtering after their cap silently loses hits.
|
||||
func searchOptionsPredicate(options SearchOptions) (string, []any) {
|
||||
var predicate strings.Builder
|
||||
var args []any
|
||||
if options.FromMS != nil || options.BeforeMS != nil {
|
||||
dateSQL := "r.created_at_ms"
|
||||
if options.DateField == "created" {
|
||||
// Objects.uuid is a primary-key lookup. Avoid joining Objects in the
|
||||
// normal search plan when Object creation is not being filtered.
|
||||
predicate.WriteString(" AND EXISTS (SELECT 1 FROM objects o WHERE o.uuid=p.object_uuid")
|
||||
dateSQL = "o.created_at*1000"
|
||||
}
|
||||
if options.FromMS != nil {
|
||||
predicate.WriteString(" AND " + dateSQL + ">=?")
|
||||
args = append(args, *options.FromMS)
|
||||
}
|
||||
if options.BeforeMS != nil {
|
||||
predicate.WriteString(" AND " + dateSQL + "<?")
|
||||
args = append(args, *options.BeforeMS)
|
||||
}
|
||||
if options.DateField == "created" {
|
||||
predicate.WriteString(")")
|
||||
}
|
||||
}
|
||||
// Evaluate inexpensive date/suffix exclusions before the metadata category.
|
||||
if options.Extension != "" {
|
||||
predicate.WriteString(" AND " + searchFilenameSQL + " GLOB ?")
|
||||
args = append(args, "*."+options.Extension)
|
||||
}
|
||||
if options.FileType != "" {
|
||||
predicate.WriteString(" AND (" + searchFileTypeSQL + ")=?")
|
||||
args = append(args, options.FileType)
|
||||
}
|
||||
return predicate.String(), args
|
||||
}
|
||||
var ErrInvalidSearchOptions = mestionora.ErrInvalidSearchOptions
|
||||
@@ -1,211 +1,7 @@
|
||||
package store
|
||||
|
||||
import "database/sql"
|
||||
import "pageswiki/internal/mestionora"
|
||||
|
||||
// EnsureSearchIndexSchema installs an FTS5 index over immutable Revisions.
|
||||
// Paths stay outside the index because links can be renamed independently; the
|
||||
// search query joins matching latest Revisions to the current domain Paths.
|
||||
func (s *Store) EnsureSearchIndexSchema() error {
|
||||
var revisionsTable string
|
||||
if err := s.DB.QueryRow(`
|
||||
SELECT name FROM sqlite_master
|
||||
WHERE type = 'table' AND name = 'revisions'
|
||||
`).Scan(&revisionsTable); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if _, err := s.DB.Exec(`
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS revision_search USING fts5(
|
||||
revision_uuid UNINDEXED,
|
||||
object_uuid UNINDEXED,
|
||||
filename,
|
||||
body,
|
||||
tokenize = 'unicode61'
|
||||
)
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.DB.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS revision_search_state (
|
||||
revision_uuid TEXT PRIMARY KEY
|
||||
)
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.DB.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS latest_revision_index (
|
||||
object_uuid TEXT PRIMARY KEY,
|
||||
revision_uuid TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
filename TEXT NOT NULL,
|
||||
document_hash TEXT NOT NULL,
|
||||
created_at_ms INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS search_schema_state (
|
||||
name TEXT PRIMARY KEY
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_latest_revision_search_uuid ON latest_revision_index(revision_uuid)
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
var latestBackfilled int
|
||||
if err := s.DB.QueryRow(`
|
||||
SELECT EXISTS(
|
||||
SELECT 1 FROM search_schema_state
|
||||
WHERE name = 'latest_revision_index_v1'
|
||||
)
|
||||
`).Scan(&latestBackfilled); err != nil {
|
||||
return err
|
||||
}
|
||||
if latestBackfilled == 0 {
|
||||
if _, err := s.DB.Exec(`
|
||||
INSERT OR REPLACE INTO latest_revision_index (
|
||||
object_uuid, revision_uuid, content_type, filename,
|
||||
document_hash, created_at_ms
|
||||
)
|
||||
SELECT
|
||||
object_uuid, uuid, COALESCE(content_type, ''),
|
||||
COALESCE(filename, ''), COALESCE(document_hash, ''),
|
||||
COALESCE(created_at_ms, created_at * 1000, 0)
|
||||
FROM (
|
||||
SELECT
|
||||
uuid, object_uuid, content_type, filename,
|
||||
document_hash, created_at_ms, created_at,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY object_uuid
|
||||
ORDER BY COALESCE(created_at_ms, created_at * 1000) DESC,
|
||||
rowid DESC
|
||||
) AS rn
|
||||
FROM revisions
|
||||
)
|
||||
WHERE rn = 1;
|
||||
INSERT OR IGNORE INTO search_schema_state (name)
|
||||
VALUES ('latest_revision_index_v1');
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Populate the cheap membership table once for installations that already
|
||||
// created revision_search before revision_search_state was introduced.
|
||||
var stateExists int
|
||||
if err := s.DB.QueryRow(`
|
||||
SELECT EXISTS(SELECT 1 FROM revision_search_state LIMIT 1)
|
||||
`).Scan(&stateExists); err != nil {
|
||||
return err
|
||||
}
|
||||
if stateExists == 0 {
|
||||
if _, err := s.DB.Exec(`
|
||||
INSERT OR IGNORE INTO revision_search_state (revision_uuid)
|
||||
SELECT revision_uuid FROM revision_search
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Trigger definitions evolve with the projection schema; recreate them so
|
||||
// existing installations receive the current maintenance logic.
|
||||
if _, err := s.DB.Exec(`
|
||||
DROP TRIGGER IF EXISTS revision_search_insert;
|
||||
DROP TRIGGER IF EXISTS revision_search_delete;
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.DB.Exec(`
|
||||
CREATE TRIGGER IF NOT EXISTS revision_search_insert
|
||||
AFTER INSERT ON revisions BEGIN
|
||||
INSERT INTO revision_search (
|
||||
revision_uuid, object_uuid, filename, body
|
||||
) VALUES (
|
||||
new.uuid,
|
||||
new.object_uuid,
|
||||
COALESCE(new.filename, ''),
|
||||
CASE
|
||||
WHEN lower(COALESCE(new.content_type, '')) LIKE 'text/%'
|
||||
OR lower(COALESCE(new.content_type, '')) LIKE 'application/json%'
|
||||
OR lower(COALESCE(new.content_type, '')) LIKE 'application/xml%'
|
||||
OR lower(COALESCE(new.content_type, '')) LIKE 'application/javascript%'
|
||||
OR lower(COALESCE(new.content_type, '')) LIKE 'application/x-yaml%'
|
||||
OR lower(COALESCE(new.content_type, '')) LIKE 'application/yaml%'
|
||||
THEN COALESCE(new.body, '')
|
||||
ELSE ''
|
||||
END
|
||||
);
|
||||
INSERT OR IGNORE INTO revision_search_state (revision_uuid)
|
||||
VALUES (new.uuid);
|
||||
INSERT INTO latest_revision_index (
|
||||
object_uuid, revision_uuid, content_type, filename,
|
||||
document_hash, created_at_ms
|
||||
) VALUES (
|
||||
new.object_uuid, new.uuid, COALESCE(new.content_type, ''),
|
||||
COALESCE(new.filename, ''), COALESCE(new.document_hash, ''),
|
||||
COALESCE(new.created_at_ms, new.created_at * 1000, 0)
|
||||
)
|
||||
ON CONFLICT(object_uuid) DO UPDATE SET
|
||||
revision_uuid = excluded.revision_uuid,
|
||||
content_type = excluded.content_type,
|
||||
filename = excluded.filename,
|
||||
document_hash = excluded.document_hash,
|
||||
created_at_ms = excluded.created_at_ms
|
||||
WHERE excluded.created_at_ms >= latest_revision_index.created_at_ms;
|
||||
END
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.DB.Exec(`
|
||||
CREATE TRIGGER IF NOT EXISTS revision_search_delete
|
||||
AFTER DELETE ON revisions BEGIN
|
||||
DELETE FROM revision_search WHERE revision_uuid = old.uuid;
|
||||
DELETE FROM revision_search_state WHERE revision_uuid = old.uuid;
|
||||
DELETE FROM latest_revision_index
|
||||
WHERE object_uuid = old.object_uuid
|
||||
AND revision_uuid = old.uuid;
|
||||
INSERT OR REPLACE INTO latest_revision_index (
|
||||
object_uuid, revision_uuid, content_type, filename,
|
||||
document_hash, created_at_ms
|
||||
)
|
||||
SELECT
|
||||
object_uuid, uuid, COALESCE(content_type, ''),
|
||||
COALESCE(filename, ''), COALESCE(document_hash, ''),
|
||||
COALESCE(created_at_ms, created_at * 1000, 0)
|
||||
FROM revisions
|
||||
WHERE object_uuid = old.object_uuid
|
||||
ORDER BY COALESCE(created_at_ms, created_at * 1000) DESC, rowid DESC
|
||||
LIMIT 1;
|
||||
END
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := s.DB.Exec(`
|
||||
INSERT INTO revision_search (
|
||||
revision_uuid, object_uuid, filename, body
|
||||
)
|
||||
SELECT
|
||||
r.uuid,
|
||||
r.object_uuid,
|
||||
COALESCE(r.filename, ''),
|
||||
CASE
|
||||
WHEN lower(COALESCE(r.content_type, '')) LIKE 'text/%'
|
||||
OR lower(COALESCE(r.content_type, '')) LIKE 'application/json%'
|
||||
OR lower(COALESCE(r.content_type, '')) LIKE 'application/xml%'
|
||||
OR lower(COALESCE(r.content_type, '')) LIKE 'application/javascript%'
|
||||
OR lower(COALESCE(r.content_type, '')) LIKE 'application/x-yaml%'
|
||||
OR lower(COALESCE(r.content_type, '')) LIKE 'application/yaml%'
|
||||
THEN COALESCE(r.body, '')
|
||||
ELSE ''
|
||||
END
|
||||
FROM revisions r
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM revision_search_state state
|
||||
WHERE state.revision_uuid = r.uuid
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.DB.Exec(`
|
||||
INSERT OR IGNORE INTO revision_search_state (revision_uuid)
|
||||
SELECT uuid FROM revisions
|
||||
`)
|
||||
return err
|
||||
return mestionora.NewSQLite(s.DB).EnsureSearchIndexSchema()
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
func searchQueryPlan(t *testing.T, database *sql.DB, query string, args ...any) []string {
|
||||
t.Helper()
|
||||
rows, err := database.Query("EXPLAIN QUERY PLAN "+query, args...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var plan []string
|
||||
for rows.Next() {
|
||||
var id, parent, unused int
|
||||
var detail string
|
||||
if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan = append(plan, detail)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
func TestSearchExcerptDeadlineIsOptionalButParentCancellationIsNot(t *testing.T) {
|
||||
database, err := db.Open(t.TempDir() + "/excerpt-deadline.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err := db.Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
New(database)
|
||||
tx, err := database.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
excerpts, err := loadSearchBodyExcerpts(context.Background(), tx, `"needle"*`, []any{1}, -time.Second)
|
||||
if err != nil || len(excerpts) != 0 {
|
||||
t.Fatalf("optional excerpt deadline discarded search: excerpts=%v err=%v", excerpts, err)
|
||||
}
|
||||
var one int
|
||||
if err := tx.QueryRow(`SELECT 1`).Scan(&one); err != nil || one != 1 {
|
||||
t.Fatalf("excerpt timeout left search transaction unusable: %d %v", one, err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := loadSearchBodyExcerpts(ctx, tx, `"needle"*`, []any{1}, time.Second); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("parent cancellation was swallowed: %v", err)
|
||||
}
|
||||
if _, err := loadSearchBodyExcerpts(context.Background(), tx, `NEAR(`, []any{1}, time.Second); err == nil {
|
||||
t.Fatal("non-timeout SQL failure was swallowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchFTSPlanScansMatchesOnce(t *testing.T) {
|
||||
database, err := db.Open(t.TempDir() + "/search-plan.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err := db.Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
New(database)
|
||||
for _, field := range []string{"body", "filename"} {
|
||||
plan := searchQueryPlan(t, database, searchFTSCandidatesSQL,
|
||||
field+` : ("needle"*)`, "domain", "/", "/", "0", SearchCandidateLimit)
|
||||
if len(plan) != 3 || !strings.Contains(plan[0], "SCAN revision_search VIRTUAL TABLE") ||
|
||||
!strings.Contains(plan[1], "revision_uuid=?") || !strings.Contains(plan[2], "object_uuid=? AND domain_uuid=?") {
|
||||
t.Fatalf("%s search must scan FTS once, then use indexed Revision/Path seeks:\n%s", field, strings.Join(plan, "\n"))
|
||||
}
|
||||
}
|
||||
plan := searchQueryPlan(t, database, searchBodyExcerptsSQL(2), `body : ("needle"*)`, 1, 2)
|
||||
if len(plan) != 1 || !strings.Contains(plan[0], "VIRTUAL TABLE INDEX 0:=M4") {
|
||||
t.Fatalf("snippets must use both rowid and MATCH constraints:\n%s", strings.Join(plan, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchFilteredFTSPlanKeepsMatchFirstAndIndexedSeeks(t *testing.T) {
|
||||
database, err := db.Open(t.TempDir() + "/search-filter-plan.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err := db.Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
New(database)
|
||||
from, before := int64(1000), int64(2000)
|
||||
for _, options := range []SearchOptions{
|
||||
{FileType: "code"}, {Extension: "py"}, {FromMS: &from, BeforeMS: &before},
|
||||
{DateField: "created", FromMS: &from, BeforeMS: &before, FileType: "code", Extension: "py"},
|
||||
} {
|
||||
predicate, filterArgs := searchOptionsPredicate(options)
|
||||
args := []any{`body : ("needle"*)`, "domain", "/", "/", "0"}
|
||||
args = append(args, filterArgs...)
|
||||
args = append(args, SearchCandidateLimit)
|
||||
plan := searchQueryPlan(t, database, searchFTSCandidatesBaseSQL+predicate+` LIMIT ?`, args...)
|
||||
if len(plan) < 3 || !strings.Contains(plan[0], "SCAN revision_search VIRTUAL TABLE") ||
|
||||
!strings.Contains(plan[1], "revision_uuid=?") || !strings.Contains(plan[2], "object_uuid=? AND domain_uuid=?") {
|
||||
t.Fatalf("filtered FTS must scan MATCH once, then indexed Revision/Path seeks: %+v\n%s", options, strings.Join(plan, "\n"))
|
||||
}
|
||||
if options.DateField == "created" {
|
||||
if len(plan) != 5 || !strings.Contains(plan[4], "SEARCH o USING INDEX") || !strings.Contains(plan[4], "uuid=?") {
|
||||
t.Fatalf("Object date needs one indexed PK lookup, not a scan: %s", strings.Join(plan, "\n"))
|
||||
}
|
||||
}
|
||||
args = []any{"domain", "/", "/", "0"}
|
||||
args = append(args, filterArgs...)
|
||||
args = append(args, SearchCandidateLimit)
|
||||
plan = searchQueryPlan(t, database, searchPathCandidatesBaseSQL+predicate+` ORDER BY p.path LIMIT ?`, args...)
|
||||
joinedPlan := strings.Join(plan, "\n")
|
||||
if len(plan) < 2 || !strings.Contains(plan[0], "SEARCH p USING INDEX") || !strings.Contains(plan[0], "domain_uuid=?") ||
|
||||
!strings.Contains(joinedPlan, "SEARCH r USING INDEX") || !strings.Contains(joinedPlan, "object_uuid=?") || strings.Contains(joinedPlan, "TEMP B-TREE") {
|
||||
t.Fatalf("filter-only search must keep indexed Domain order and bounded candidate memory: %s", strings.Join(plan, "\n"))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/seheweit"
|
||||
)
|
||||
|
||||
func resourceTestStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
s := newSchemaTestStore(t)
|
||||
if err := s.EnsureResourceHistorySchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestSeheweitHistoryBucketCompatibility(t *testing.T) {
|
||||
s := resourceTestStore(t)
|
||||
now := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC)
|
||||
// Boundary-inclusive query. Later insertion, not later timestamp, wins a
|
||||
// bucket, matching the old MAX(id) query even with out-of-order samples.
|
||||
for i, tc := range []struct {
|
||||
offset time.Duration
|
||||
payload string
|
||||
}{
|
||||
{-time.Hour - time.Millisecond, `{"label":"too-old"}`},
|
||||
{-time.Hour, `{"label":"start"}`},
|
||||
{-10 * time.Second, `{"label":"later-time"}`},
|
||||
{-20 * time.Second, `{"label":"last-insert"}`},
|
||||
{0, `{"label":"end"}`},
|
||||
{time.Millisecond, `{"label":"future"}`},
|
||||
} {
|
||||
if err := s.SaveResourceSample(fmt.Sprint(i), now.Add(tc.offset).UnixMilli(), []byte(tc.payload)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := s.ResourceHistory(t.Context(), 1, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{`{"label":"start"}`, `{"label":"last-insert"}`, `{"label":"end"}`}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("samples %s", got)
|
||||
}
|
||||
for i := range want {
|
||||
if string(got[i]) != want[i] {
|
||||
t.Fatalf("sample %d: %s", i, got[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeheweitHistoryRetentionAndIdempotence(t *testing.T) {
|
||||
s := resourceTestStore(t)
|
||||
at := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC).UnixMilli()
|
||||
cutoff := seheweit.RetentionCutoff(at)
|
||||
for _, tc := range []struct {
|
||||
job string
|
||||
at int64
|
||||
}{{"expired", cutoff - 1}, {"boundary", cutoff}, {"current", at}} {
|
||||
if err := s.SaveResourceSample(tc.job, tc.at, []byte(`{"original":true}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := s.SaveResourceSample("current", at+1, []byte(`{"replacement":true}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var payload string
|
||||
if err := s.DB.QueryRow(`SELECT payload FROM resource_samples WHERE job_uuid='current'`).Scan(&payload); err != nil || payload != `{"original":true}` {
|
||||
t.Fatalf("retry replaced sample: %s %v", payload, err)
|
||||
}
|
||||
// The retry uses its collection time for pruning, so the old cutoff is now
|
||||
// expired too. A distinct test covers exact retention-boundary preservation.
|
||||
var count int
|
||||
if err := s.DB.QueryRow(`SELECT COUNT(*) FROM resource_samples`).Scan(&count); err != nil || count != 1 {
|
||||
t.Fatalf("retention count=%d %v", count, err)
|
||||
}
|
||||
s2 := resourceTestStore(t)
|
||||
if err := s2.SaveResourceSample("boundary", cutoff, []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s2.SaveResourceSample("current", at, []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s2.DB.QueryRow(`SELECT COUNT(*) FROM resource_samples`).Scan(&count); err != nil || count != 2 {
|
||||
t.Fatalf("inclusive cutoff count=%d %v", count, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeheweitHistoryCancelledReadOnlyAndRollback(t *testing.T) {
|
||||
s := resourceTestStore(t)
|
||||
at := time.Now().UnixMilli()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
if err := s.SaveResourceSampleContext(ctx, "cancelled", at, []byte(`{}`)); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancel: %v", err)
|
||||
}
|
||||
if _, err := s.ResourceHistory(ctx, 24, time.Now()); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("read cancel: %v", err)
|
||||
}
|
||||
s.ReadOnly = true
|
||||
if err := s.SaveResourceSample("readonly", at, []byte(`{}`)); !errors.Is(err, ErrReadOnly) {
|
||||
t.Fatalf("readonly: %v", err)
|
||||
}
|
||||
s.ReadOnly = false
|
||||
if err := s.SaveResourceSample("old", seheweit.RetentionCutoff(at)-1, []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Failure during expiration must roll back the insertion as well.
|
||||
if _, err := s.DB.Exec(`CREATE TRIGGER prevent_resource_expire BEFORE DELETE ON resource_samples BEGIN SELECT RAISE(ABORT,'synthetic failure'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.SaveResourceSample("rollback", at, []byte(`{}`)); err == nil {
|
||||
t.Fatal("expiration failure ignored")
|
||||
}
|
||||
var count int
|
||||
if err := s.DB.QueryRow(`SELECT COUNT(*) FROM resource_samples WHERE job_uuid<>'old'`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatalf("partial transaction or cancelled write count=%d %v", count, err)
|
||||
}
|
||||
}
|
||||
+18
-6
@@ -8,15 +8,24 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/geduldh"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
DB *sql.DB
|
||||
DocumentRoot string
|
||||
pathCacheMu sync.RWMutex
|
||||
revisionWriteMu sync.Mutex
|
||||
childCache map[string][]string
|
||||
authKey []byte
|
||||
// Standby is immutable for the lifetime of this process. Role changes require restart.
|
||||
ReadOnly bool
|
||||
StandbyReadUntil time.Time
|
||||
BackgroundJobsDisabled bool
|
||||
DB *sql.DB
|
||||
DocumentRoot string
|
||||
documentMaterializerOnce sync.Once
|
||||
documentMaterializer *geduldh.Materializer
|
||||
pathCacheMu sync.RWMutex
|
||||
revisionWriteMu sync.Mutex
|
||||
childCache map[string][]string
|
||||
authKey []byte
|
||||
}
|
||||
|
||||
func New(
|
||||
@@ -108,6 +117,9 @@ func NewWithDocumentRoot(
|
||||
if err := st.EnsureUserSpaceSchema(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := st.ensureSchemaComponent("profile-contacts"); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := st.EnsureBuiltinAdminGroups(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
@@ -8,26 +8,14 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/schutzaria"
|
||||
)
|
||||
|
||||
const MaxUserSpaceOverrides = 256
|
||||
|
||||
type UserSpacePathPolicy struct {
|
||||
Path string `json:"path"`
|
||||
Visibility string `json:"visibility"`
|
||||
}
|
||||
|
||||
type UserSpacePolicy struct {
|
||||
PrincipalUUID string `json:"principal_uuid"`
|
||||
Username string `json:"username"`
|
||||
ProfileVisibility string `json:"profile_visibility"`
|
||||
AvatarVisibility string `json:"avatar_visibility"`
|
||||
RootVisibility string `json:"root_visibility"`
|
||||
DescendantVisibility string `json:"descendant_visibility"`
|
||||
Version int64 `json:"version"`
|
||||
DomainAllowsPublic bool `json:"domain_allows_public"`
|
||||
Overrides []UserSpacePathPolicy `json:"overrides"`
|
||||
}
|
||||
type UserSpacePathPolicy = schutzaria.UserSpacePathPolicy
|
||||
type UserSpacePolicy = schutzaria.UserSpacePolicy
|
||||
|
||||
type UserSpacePage struct {
|
||||
Path string
|
||||
@@ -76,18 +64,7 @@ func defaultUserSpacePolicy(principal, username string) UserSpacePolicy {
|
||||
return UserSpacePolicy{PrincipalUUID: principal, Username: username, ProfileVisibility: "domain", AvatarVisibility: "domain", RootVisibility: "domain", DescendantVisibility: "domain", Overrides: []UserSpacePathPolicy{}}
|
||||
}
|
||||
|
||||
func userSpaceVisibilityRank(value string) int {
|
||||
switch value {
|
||||
case "private":
|
||||
return 0
|
||||
case "domain":
|
||||
return 1
|
||||
case "public":
|
||||
return 2
|
||||
default:
|
||||
return -1
|
||||
}
|
||||
}
|
||||
func userSpaceVisibilityRank(value string) int { return schutzaria.UserSpaceVisibilityRank(value) }
|
||||
|
||||
func userSpaceRoot(username string) string { return "/u/" + username }
|
||||
|
||||
@@ -98,18 +75,7 @@ func escapeUserSpaceLike(value string) string {
|
||||
}
|
||||
|
||||
func userSpaceDocumentPath(username, value string, descendant bool) bool {
|
||||
root := userSpaceRoot(username)
|
||||
if value != path.Clean(value) || strings.ContainsAny(value, "\\\x00\r\n") {
|
||||
return false
|
||||
}
|
||||
if descendant {
|
||||
if !strings.HasPrefix(value, root+"/") {
|
||||
return false
|
||||
}
|
||||
rest := strings.TrimPrefix(value, root+"/")
|
||||
return rest != "" && !strings.HasPrefix(rest, "_/") && rest != "_"
|
||||
}
|
||||
return value == root
|
||||
return schutzaria.UserSpaceDocumentPath(username, value, descendant)
|
||||
}
|
||||
|
||||
func (s *Store) domainAllowsUserSpacePublic(ctx context.Context, q interface {
|
||||
@@ -170,22 +136,7 @@ func (s *Store) UserSpacePolicy(ctx context.Context, domain, principal string) (
|
||||
}
|
||||
|
||||
func userSpaceEffectiveVisibility(p UserSpacePolicy, target string) string {
|
||||
root := userSpaceRoot(p.Username)
|
||||
if target == root {
|
||||
return p.RootVisibility
|
||||
}
|
||||
if !userSpaceDocumentPath(p.Username, target, true) {
|
||||
return "private"
|
||||
}
|
||||
visibility := p.DescendantVisibility
|
||||
for _, item := range p.Overrides {
|
||||
if target == item.Path || strings.HasPrefix(target, strings.TrimRight(item.Path, "/")+"/") {
|
||||
if userSpaceVisibilityRank(item.Visibility) < userSpaceVisibilityRank(visibility) {
|
||||
visibility = item.Visibility
|
||||
}
|
||||
}
|
||||
}
|
||||
return visibility
|
||||
return schutzaria.EffectiveUserSpaceVisibility(p, target)
|
||||
}
|
||||
|
||||
func EffectiveUserSpaceVisibility(p UserSpacePolicy, target string) string {
|
||||
|
||||
Reference in new issue
Block a user