Refactor architecture into eight components and consolidate reliability work
Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts. Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation. Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
This commit is contained in:
1 parent
9dd425f765
commit
d5d21b7841
302 files changed
+20037
-3492
No files matched your search
@@ -0,0 +1,15 @@
|
||||
package compiler
|
||||
|
||||
import "pageswiki/internal/grammaratur"
|
||||
|
||||
// HTMLToMarkdownAuthoring delegates loss-preserving conversion to Grammaratur.
|
||||
// Deprecated: use grammaratur.HTMLToMarkdownAuthoring in new application code.
|
||||
func HTMLToMarkdownAuthoring(source string) (string, error) {
|
||||
return grammaratur.HTMLToMarkdownAuthoring(source)
|
||||
}
|
||||
|
||||
// MarkdownAuthoringHTML preserves the existing authoring entry point.
|
||||
// Deprecated: use grammaratur.MarkdownAuthoringHTML in new application code.
|
||||
func MarkdownAuthoringHTML(source string) string {
|
||||
return grammaratur.MarkdownAuthoringHTML(source)
|
||||
}
|
||||
@@ -1,24 +1,8 @@
|
||||
package compiler
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
)
|
||||
import "pageswiki/internal/kunstzeal"
|
||||
|
||||
func DiagramHash(
|
||||
diagramType string,
|
||||
source string,
|
||||
format string,
|
||||
) string {
|
||||
|
||||
sum :=
|
||||
sha256.Sum256(
|
||||
[]byte(
|
||||
diagramType + "\x00" + format + "\x00" + source,
|
||||
),
|
||||
)
|
||||
|
||||
return hex.EncodeToString(
|
||||
sum[:],
|
||||
)
|
||||
// DiagramHash preserves the legacy key format through the rendering component.
|
||||
func DiagramHash(diagramType, source, format string) string {
|
||||
return kunstzeal.DiagramHash(diagramType, source, format)
|
||||
}
|
||||
@@ -1,14 +1,14 @@
|
||||
package compiler
|
||||
|
||||
import (
|
||||
htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
)
|
||||
import "pageswiki/internal/grammaratur"
|
||||
|
||||
// HTMLToMarkdown preserves the legacy conversion contract (without protection
|
||||
// fences). Deprecated: use grammaratur.HTMLToMarkdown in new application code.
|
||||
func HTMLToMarkdown(
|
||||
html string,
|
||||
) (string, error) {
|
||||
|
||||
return htmltomarkdown.ConvertString(
|
||||
return grammaratur.HTMLToMarkdown(
|
||||
html,
|
||||
)
|
||||
}
|
||||
+17
-1
@@ -1,9 +1,12 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
@@ -24,7 +27,20 @@ func Open(path string) (*sql.DB, error) {
|
||||
},
|
||||
}.Encode(),
|
||||
}).String()
|
||||
return sql.Open("sqlite", dsn)
|
||||
database, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// sql.Open is lazy. Establish WAL once before exposing the pool, otherwise
|
||||
// simultaneous first borrowers race the initial journal_mode transition in
|
||||
// their connection pragmas (SQLITE_BUSY before BEGIN IMMEDIATE is reached).
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := database.PingContext(ctx); err != nil {
|
||||
database.Close()
|
||||
return nil, fmt.Errorf("initialize SQLite connection: %w", err)
|
||||
}
|
||||
return database, nil
|
||||
}
|
||||
|
||||
func Init(db *sql.DB) error {
|
||||
|
||||
+87
-90
@@ -4,6 +4,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func Migrate(
|
||||
@@ -96,54 +97,21 @@ func Migrate(
|
||||
}
|
||||
}
|
||||
|
||||
alterStmts := []string{
|
||||
|
||||
`
|
||||
ALTER TABLE paths
|
||||
ADD COLUMN created_at INTEGER
|
||||
`,
|
||||
|
||||
`
|
||||
ALTER TABLE objects
|
||||
ADD COLUMN created_at INTEGER
|
||||
`,
|
||||
|
||||
`
|
||||
ALTER TABLE revisions
|
||||
ADD COLUMN created_at INTEGER
|
||||
`,
|
||||
|
||||
`
|
||||
ALTER TABLE revisions
|
||||
ADD COLUMN document_hash TEXT
|
||||
`,
|
||||
|
||||
`
|
||||
ALTER TABLE revisions
|
||||
ADD COLUMN lang TEXT NOT NULL DEFAULT 'und'
|
||||
`,
|
||||
|
||||
`
|
||||
ALTER TABLE revisions
|
||||
ADD COLUMN created_at_ms INTEGER
|
||||
`,
|
||||
`
|
||||
ALTER TABLE revisions
|
||||
ADD COLUMN content_type TEXT NOT NULL DEFAULT 'text/html; charset=utf-8'
|
||||
`,
|
||||
`
|
||||
ALTER TABLE revisions
|
||||
ADD COLUMN filename TEXT NOT NULL DEFAULT ''
|
||||
`,
|
||||
|
||||
`
|
||||
ALTER TABLE tags
|
||||
ADD COLUMN created_at INTEGER
|
||||
`,
|
||||
columns := []struct{ table, name, declaration string }{
|
||||
{"paths", "created_at", "INTEGER"},
|
||||
{"objects", "created_at", "INTEGER"},
|
||||
{"revisions", "created_at", "INTEGER"},
|
||||
{"revisions", "document_hash", "TEXT"},
|
||||
{"revisions", "lang", "TEXT NOT NULL DEFAULT 'und'"},
|
||||
{"revisions", "created_at_ms", "INTEGER"},
|
||||
{"revisions", "content_type", "TEXT NOT NULL DEFAULT 'text/html; charset=utf-8'"},
|
||||
{"revisions", "filename", "TEXT NOT NULL DEFAULT ''"},
|
||||
{"tags", "created_at", "INTEGER"},
|
||||
}
|
||||
|
||||
for _, stmt := range alterStmts {
|
||||
_, _ = db.Exec(stmt)
|
||||
for _, column := range columns {
|
||||
if err := ensureSQLiteColumn(db, column.table, column.name, column.declaration); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := migrateRevisionMetadata(db); err != nil {
|
||||
@@ -185,54 +153,47 @@ func migrateRevisionMetadata(db *sql.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
rows, err := db.Query(`
|
||||
SELECT
|
||||
uuid,
|
||||
COALESCE(body, '')
|
||||
FROM revisions
|
||||
WHERE document_hash IS NULL
|
||||
OR document_hash = ''
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
type legacyRevision struct {
|
||||
uuid string
|
||||
body string
|
||||
}
|
||||
|
||||
var revisions []legacyRevision
|
||||
|
||||
for rows.Next() {
|
||||
var revision legacyRevision
|
||||
|
||||
if err := rows.Scan(&revision.uuid, &revision.body); err != nil {
|
||||
// Fetch only identifiers in batches, then hydrate one legacy body at a
|
||||
// time. The old implementation accumulated all payloads in memory.
|
||||
lastUUID := ""
|
||||
for {
|
||||
rows, err := db.Query(`SELECT uuid FROM revisions WHERE uuid>? AND (document_hash IS NULL OR document_hash='') ORDER BY uuid LIMIT 128`, lastUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
revisions = append(revisions, revision)
|
||||
}
|
||||
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, revision := range revisions {
|
||||
sum := sha256.Sum256([]byte(revision.body))
|
||||
documentHash := hex.EncodeToString(sum[:])
|
||||
|
||||
if _, err := db.Exec(`
|
||||
UPDATE revisions
|
||||
SET document_hash = ?
|
||||
WHERE uuid = ?
|
||||
`, documentHash, revision.uuid); err != nil {
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, id := range ids {
|
||||
var body string
|
||||
if err := db.QueryRow(`SELECT COALESCE(body,'') FROM revisions WHERE uuid=?`, id).Scan(&body); err != nil {
|
||||
return err
|
||||
}
|
||||
sum := sha256.Sum256([]byte(body))
|
||||
if _, err := db.Exec(`UPDATE revisions SET document_hash=? WHERE uuid=? AND (document_hash IS NULL OR document_hash='')`, hex.EncodeToString(sum[:]), id); err != nil {
|
||||
return err
|
||||
}
|
||||
lastUUID = id
|
||||
}
|
||||
if len(ids) < 128 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if _, err = db.Exec(`
|
||||
if _, err := db.Exec(`
|
||||
UPDATE revisions
|
||||
SET
|
||||
lang = COALESCE(NULLIF(lang, ''), 'und'),
|
||||
@@ -247,7 +208,7 @@ func migrateRevisionMetadata(db *sql.DB) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = db.Exec(`
|
||||
if _, err := db.Exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_revisions_object_created
|
||||
ON revisions (
|
||||
object_uuid,
|
||||
@@ -257,9 +218,45 @@ func migrateRevisionMetadata(db *sql.DB) error {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
_, err := db.Exec(`
|
||||
INSERT OR IGNORE INTO schema_migrations (name, applied_at)
|
||||
VALUES ('revision_metadata_v1', strftime('%s','now'))
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
func sqliteColumnExists(database *sql.DB, table, column string) (bool, error) {
|
||||
// All identifiers originate in the fixed legacy bootstrap above.
|
||||
rows, err := database.Query(fmt.Sprintf("PRAGMA table_info(%q)", table))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
found := false
|
||||
for rows.Next() {
|
||||
var cid, notNull, primaryKey int
|
||||
var name, kind string
|
||||
var defaultValue any
|
||||
if err := rows.Scan(&cid, &name, &kind, ¬Null, &defaultValue, &primaryKey); err != nil {
|
||||
return false, err
|
||||
}
|
||||
found = found || name == column
|
||||
}
|
||||
return found, rows.Err()
|
||||
}
|
||||
|
||||
func ensureSQLiteColumn(database *sql.DB, table, column, declaration string) error {
|
||||
exists, err := sqliteColumnExists(database, table, column)
|
||||
if err != nil || exists {
|
||||
return err
|
||||
}
|
||||
_, err = database.Exec(fmt.Sprintf("ALTER TABLE %q ADD COLUMN %q %s", table, column, declaration))
|
||||
if err != nil {
|
||||
// Only tolerate another bootstrap adding the exact column concurrently.
|
||||
if exists, checkErr := sqliteColumnExists(database, table, column); checkErr == nil && exists {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("add column %s.%s: %w", table, column, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
type MigrationPlanStep struct {
|
||||
Version int `json:"version"`
|
||||
Name string `json:"name"`
|
||||
Statements int `json:"statements"`
|
||||
AddColumns []SQLiteColumn `json:"add_columns,omitempty"`
|
||||
}
|
||||
|
||||
type MigrationInspection struct {
|
||||
Component string `json:"component"`
|
||||
State string `json:"state"` // current, pending, blocked, unknown
|
||||
AppliedVersion int `json:"applied_version"`
|
||||
LatestVersion int `json:"latest_version"`
|
||||
Pending []MigrationPlanStep `json:"pending,omitempty"`
|
||||
ContractChecked bool `json:"contract_checked"`
|
||||
Issues []string `json:"issues,omitempty"`
|
||||
}
|
||||
|
||||
// InspectSQLiteMigrations takes a read transaction and never applies DDL/DML.
|
||||
// Pending definitions are a plan, not a successful rehearsal: data-dependent
|
||||
// DDL and assertions on not-yet-created objects require a disposable DB copy.
|
||||
func InspectSQLiteMigrations(ctx context.Context, database *sql.DB, plans map[string][]SQLMigration) ([]MigrationInspection, error) {
|
||||
tx, err := database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var ledger bool
|
||||
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_migration_steps')`).Scan(&ledger); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type record struct {
|
||||
version int
|
||||
name, checksum string
|
||||
}
|
||||
history := map[string][]record{}
|
||||
if ledger {
|
||||
rows, err := tx.QueryContext(ctx, `SELECT component,version,name,checksum FROM schema_migration_steps ORDER BY component,version`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var component string
|
||||
var r record
|
||||
if err := rows.Scan(&component, &r.version, &r.name, &r.checksum); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
history[component] = append(history[component], r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
names := make([]string, 0, len(plans)+len(history))
|
||||
for name := range plans {
|
||||
names = append(names, name)
|
||||
}
|
||||
for name := range history {
|
||||
if _, ok := plans[name]; !ok {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
result := make([]MigrationInspection, 0, len(names))
|
||||
for _, name := range names {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
steps, known := plans[name]
|
||||
item := MigrationInspection{Component: name, State: "current", LatestVersion: len(steps)}
|
||||
for i, step := range steps {
|
||||
if step.Version != i+1 || step.Name == "" || (len(step.Statements) == 0 && len(step.AddColumns) == 0 && len(step.Assertions) == 0) {
|
||||
return nil, fmt.Errorf("invalid migration definition %s/%d", name, step.Version)
|
||||
}
|
||||
}
|
||||
if !known {
|
||||
item.State = "unknown"
|
||||
item.Issues = []string{"component is not registered in this binary"}
|
||||
}
|
||||
for i, record := range history[name] {
|
||||
item.AppliedVersion = record.version
|
||||
if known && (record.version != i+1 || record.version > len(steps) || record.name != steps[record.version-1].Name || record.checksum != migrationChecksum(steps[record.version-1])) {
|
||||
item.State = "blocked"
|
||||
item.Issues = append(item.Issues, fmt.Sprintf("migration history mismatch at version %d", record.version))
|
||||
}
|
||||
}
|
||||
if item.State == "current" {
|
||||
for _, step := range steps[len(history[name]):] {
|
||||
item.Pending = append(item.Pending, MigrationPlanStep{Version: step.Version, Name: step.Name, Statements: len(step.Statements), AddColumns: step.AddColumns})
|
||||
}
|
||||
if len(item.Pending) > 0 {
|
||||
item.State = "pending"
|
||||
}
|
||||
if len(steps) > 0 && len(steps[len(steps)-1].Assertions) > 0 {
|
||||
item.ContractChecked = true
|
||||
for i, assertion := range steps[len(steps)-1].Assertions {
|
||||
rows, err := tx.QueryContext(ctx, assertion)
|
||||
if err != nil {
|
||||
item.Issues = append(item.Issues, fmt.Sprintf("assertion %d: %v", i+1, err))
|
||||
continue
|
||||
}
|
||||
violated := rows.Next()
|
||||
err = rows.Err()
|
||||
closeErr := rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
if violated {
|
||||
item.Issues = append(item.Issues, fmt.Sprintf("assertion %d failed", i+1))
|
||||
}
|
||||
}
|
||||
if len(item.Issues) > 0 && item.State == "current" {
|
||||
item.State = "blocked"
|
||||
}
|
||||
}
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, tx.Commit()
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInspectMigrationsReadOnlyLifecycle(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "inspect.db")
|
||||
writer, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer writer.Close()
|
||||
if err := writer.Ping(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reader, err := OpenReadOnly(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample(id INTEGER PRIMARY KEY, value TEXT)`, `INSERT INTO sample VALUES(1,'keep')`}, Assertions: []string{`SELECT id,value FROM sample LIMIT 0`}}}
|
||||
plans := map[string][]SQLMigration{"sample": steps}
|
||||
inspect := func(state string, issues bool) {
|
||||
t.Helper()
|
||||
got, err := InspectSQLiteMigrations(context.Background(), reader, plans)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].State != state || (len(got[0].Issues) > 0) != issues || !got[0].ContractChecked {
|
||||
t.Fatalf("report: %+v", got)
|
||||
}
|
||||
}
|
||||
inspect("pending", true)
|
||||
var count int
|
||||
if err := writer.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name IN ('sample','schema_migration_steps')`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatalf("inspection created tables: %d %v", count, err)
|
||||
}
|
||||
if _, err := reader.Exec(`CREATE TABLE forbidden(id INTEGER)`); err == nil {
|
||||
t.Fatal("connection is writable")
|
||||
}
|
||||
// Keep the writer open: the reader must see committed WAL contents too.
|
||||
if err := ApplySQLiteMigrations(context.Background(), writer, "sample", steps); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspect("current", false)
|
||||
if _, err := writer.Exec(`ALTER TABLE sample RENAME COLUMN value TO lost`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspect("blocked", true)
|
||||
var value string
|
||||
if err := writer.QueryRow(`SELECT lost FROM sample WHERE id=1`).Scan(&value); err != nil || value != "keep" {
|
||||
t.Fatalf("inspection changed data: %q %v", value, err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := InspectSQLiteMigrations(ctx, reader, plans); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancel: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectMigrationsRejectHistoryMismatch(t *testing.T) {
|
||||
for _, tc := range []struct{ name, mutation string }{
|
||||
{"checksum", `UPDATE schema_migration_steps SET checksum='changed'`},
|
||||
{"name", `UPDATE schema_migration_steps SET name='changed'`},
|
||||
{"zero", `UPDATE schema_migration_steps SET version=0`},
|
||||
{"negative", `UPDATE schema_migration_steps SET version=-1`},
|
||||
{"newer", `UPDATE schema_migration_steps SET version=2`},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
database, err := Open(filepath.Join(t.TempDir(), "inspect.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample(id INTEGER)`}}}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "sample", steps); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := database.Exec(tc.mutation); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := InspectSQLiteMigrations(context.Background(), database, map[string][]SQLMigration{"sample": steps})
|
||||
if err != nil || len(got) != 1 || got[0].State != "blocked" {
|
||||
t.Fatalf("report=%+v err=%v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectMigrationsUnknownAndPendingPlan(t *testing.T) {
|
||||
database, err := Open(filepath.Join(t.TempDir(), "inspect.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample(id INTEGER)`}}}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "sample", steps); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := InspectSQLiteMigrations(context.Background(), database, nil)
|
||||
if err != nil || len(got) != 1 || got[0].State != "unknown" {
|
||||
t.Fatalf("unknown=%+v %v", got, err)
|
||||
}
|
||||
steps = append(steps, SQLMigration{Version: 2, Name: "add_value", AddColumns: []SQLiteColumn{{Table: "sample", Name: "value", Declaration: "TEXT NOT NULL DEFAULT ''"}}, Assertions: []string{`SELECT value FROM sample LIMIT 0`}})
|
||||
got, err = InspectSQLiteMigrations(context.Background(), database, map[string][]SQLMigration{"sample": steps})
|
||||
if err != nil || len(got) != 1 {
|
||||
t.Fatalf("pending=%+v %v", got, err)
|
||||
}
|
||||
item := got[0]
|
||||
if item.State != "pending" || item.AppliedVersion != 1 || item.LatestVersion != 2 || len(item.Pending) != 1 || len(item.Pending[0].AddColumns) != 1 || len(item.Issues) == 0 {
|
||||
t.Fatalf("pending=%+v", item)
|
||||
}
|
||||
var count int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM pragma_table_info('sample') WHERE name='value'`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatalf("pending DDL applied: %d %v", count, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SQLMigration is append-only once released. Edit a new version, never an
|
||||
// applied statement. This runner is SQLite-specific; other DBs need their own
|
||||
// locking/DDL implementations rather than a driver-name substitution.
|
||||
type SQLMigration struct {
|
||||
Version int
|
||||
Name string
|
||||
Statements []string
|
||||
// Omit empty new fields to preserve checksums of previously released steps.
|
||||
AddColumns []SQLiteColumn `json:",omitempty"`
|
||||
// Each assertion must execute successfully and return no rows. The latest
|
||||
// version defines the whole current contract; older assertions are superseded
|
||||
// to permit deliberate column removal in a later, immutable migration.
|
||||
Assertions []string `json:",omitempty"`
|
||||
}
|
||||
|
||||
type SQLiteColumn struct {
|
||||
Table string
|
||||
Name string
|
||||
Declaration string
|
||||
}
|
||||
|
||||
func migrationChecksum(step SQLMigration) string {
|
||||
encoded, _ := json.Marshal(step)
|
||||
sum := sha256.Sum256(encoded)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// ApplySQLiteMigrations serializes processes on a dedicated connection with
|
||||
// BEGIN IMMEDIATE. DDL, data statements and ledger updates commit together.
|
||||
// The legacy schema_migrations table remains untouched for old installations.
|
||||
func ApplySQLiteMigrations(ctx context.Context, database *sql.DB, component string, steps []SQLMigration) error {
|
||||
if component == "" {
|
||||
return fmt.Errorf("migration component is required")
|
||||
}
|
||||
for i, step := range steps {
|
||||
if step.Version != i+1 || step.Name == "" || (len(step.Statements) == 0 && len(step.AddColumns) == 0 && len(step.Assertions) == 0) {
|
||||
return fmt.Errorf("invalid migration sequence for %s", component)
|
||||
}
|
||||
}
|
||||
conn, err := database.Conn(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("migration %s acquire connection: %w", component, err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil {
|
||||
return fmt.Errorf("migration %s begin immediate: %w", component, err)
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if committed {
|
||||
return
|
||||
}
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if _, err := conn.ExecContext(cleanup, "ROLLBACK"); err != nil {
|
||||
// Never return a connection with uncertain transaction state to
|
||||
// the pool. A failed/ambiguous COMMIT also takes this path.
|
||||
_ = conn.Raw(func(any) error { return driver.ErrBadConn })
|
||||
}
|
||||
}()
|
||||
if _, err := conn.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_migration_steps (
|
||||
component TEXT NOT NULL, version INTEGER NOT NULL, name TEXT NOT NULL,
|
||||
checksum TEXT NOT NULL, applied_at_ms INTEGER NOT NULL,
|
||||
PRIMARY KEY(component, version))`); err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := conn.QueryContext(ctx, `SELECT version,name,checksum FROM schema_migration_steps WHERE component=? ORDER BY version`, component)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
applied := 0
|
||||
for rows.Next() {
|
||||
var version int
|
||||
var name, checksum string
|
||||
if err := rows.Scan(&version, &name, &checksum); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if version != applied+1 || version > len(steps) || steps[version-1].Name != name || migrationChecksum(steps[version-1]) != checksum {
|
||||
rows.Close()
|
||||
return fmt.Errorf("migration history mismatch: component=%s version=%d (changed, missing or newer migration)", component, version)
|
||||
}
|
||||
applied++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, step := range steps[applied:] {
|
||||
for i, statement := range step.Statements {
|
||||
if _, err := conn.ExecContext(ctx, statement); err != nil {
|
||||
return fmt.Errorf("migration %s/%d %s statement %d: %w", component, step.Version, step.Name, i+1, err)
|
||||
}
|
||||
}
|
||||
for _, column := range step.AddColumns {
|
||||
if err := addSQLiteColumn(ctx, conn, column); err != nil {
|
||||
return fmt.Errorf("migration %s/%d column %s.%s: %w", component, step.Version, column.Table, column.Name, err)
|
||||
}
|
||||
}
|
||||
if _, err := conn.ExecContext(ctx, `INSERT INTO schema_migration_steps(component,version,name,checksum,applied_at_ms) VALUES(?,?,?,?,?)`, component, step.Version, step.Name, migrationChecksum(step), time.Now().UnixMilli()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(steps) > 0 {
|
||||
step := steps[len(steps)-1]
|
||||
for i, assertion := range step.Assertions {
|
||||
rows, err := conn.QueryContext(ctx, assertion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("schema contract %s/%d assertion %d: %w", component, step.Version, i+1, err)
|
||||
}
|
||||
violated := rows.Next()
|
||||
err = rows.Err()
|
||||
closeErr := rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if violated {
|
||||
return fmt.Errorf("schema contract %s/%d assertion %d failed", component, step.Version, i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
_, err = conn.ExecContext(ctx, "COMMIT")
|
||||
committed = err == nil
|
||||
return err
|
||||
}
|
||||
|
||||
func addSQLiteColumn(ctx context.Context, conn *sql.Conn, column SQLiteColumn) error {
|
||||
// Identifiers are trusted migration definitions, but validate before SQL
|
||||
// interpolation so accidental punctuation cannot change the statement.
|
||||
for _, identifier := range []string{column.Table, column.Name} {
|
||||
if identifier == "" || strings.IndexFunc(identifier, func(r rune) bool {
|
||||
return !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_')
|
||||
}) >= 0 {
|
||||
return fmt.Errorf("invalid schema identifier")
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(column.Declaration) == "" {
|
||||
return fmt.Errorf("column declaration is required")
|
||||
}
|
||||
rows, err := conn.QueryContext(ctx, `SELECT name FROM pragma_table_info(?)`, column.Table)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found, tableExists := false, false
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
found, tableExists = found || name == column.Name, true
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !tableExists {
|
||||
return fmt.Errorf("table %s is missing", column.Table)
|
||||
}
|
||||
if found {
|
||||
return nil
|
||||
}
|
||||
_, err = conn.ExecContext(ctx, fmt.Sprintf("ALTER TABLE %q ADD COLUMN %q %s", column.Table, column.Name, column.Declaration))
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSQLMigrationLedgerConcurrencyAndChecksum(t *testing.T) {
|
||||
database, err := Open(filepath.Join(t.TempDir(), "migration.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample (id INTEGER PRIMARY KEY)`, `INSERT INTO sample VALUES (1)`}}}
|
||||
var wg sync.WaitGroup
|
||||
errors := make(chan error, 6)
|
||||
for i := 0; i < 6; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
errors <- ApplySQLiteMigrations(context.Background(), database, "test", steps)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errors)
|
||||
for err := range errors {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var count int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM sample`).Scan(&count); err != nil || count != 1 {
|
||||
t.Fatalf("count=%d err=%v", count, err)
|
||||
}
|
||||
changed := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample (id TEXT)`}}}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", changed); err == nil || !strings.Contains(err.Error(), "history mismatch") {
|
||||
t.Fatalf("changed migration accepted: %v", err)
|
||||
}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", nil); err == nil {
|
||||
t.Fatal("downgrade accepted")
|
||||
}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "other", []SQLMigration{{Version: 2, Name: "gap", Statements: []string{`SELECT 1`}}}); err == nil {
|
||||
t.Fatal("version gap accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLMigrationFailureRollsBackDDLAndLedger(t *testing.T) {
|
||||
database, err := Open(filepath.Join(t.TempDir(), "migration.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
steps := []SQLMigration{{Version: 1, Name: "broken", Statements: []string{`CREATE TABLE partial (id INTEGER)`, `NOT VALID SQL`}}}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err == nil {
|
||||
t.Fatal("DDL failure ignored")
|
||||
}
|
||||
var count int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name IN ('partial','schema_migration_steps')`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatalf("partial migration remains count=%d err=%v", count, err)
|
||||
}
|
||||
steps[0].Statements = steps[0].Statements[:1]
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil {
|
||||
t.Fatal("retry after rollback", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if err := ApplySQLiteMigrations(ctx, database, "test", steps); err == nil {
|
||||
t.Fatal("cancel ignored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyRevisionHashMigrationSpansBoundedBatches(t *testing.T) {
|
||||
database, err := Open(filepath.Join(t.TempDir(), "legacy.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if _, err := database.Exec(`CREATE TABLE revisions (uuid TEXT PRIMARY KEY,object_uuid TEXT,body TEXT,message TEXT)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 260; i++ {
|
||||
if _, err := database.Exec(`INSERT INTO revisions(uuid,body) VALUES(?,?)`, fmt.Sprintf("revision-%04d", i), "body"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var count int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM revisions WHERE length(document_hash)=64 AND lang='und' AND created_at_ms=0`).Scan(&count); err != nil || count != 260 {
|
||||
t.Fatalf("migrated=%d err=%v", count, err)
|
||||
}
|
||||
if err := ensureSQLiteColumn(database, "does_not_exist", "created_at", "INTEGER"); err == nil {
|
||||
t.Fatal("ALTER failure swallowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationChecksumPreservesLegacyFormat(t *testing.T) {
|
||||
step := SQLMigration{Version: 1, Name: "legacy", Statements: []string{"SELECT 1"}}
|
||||
sum := sha256.Sum256([]byte(`{"Version":1,"Name":"legacy","Statements":["SELECT 1"]}`))
|
||||
if got := migrationChecksum(step); got != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("legacy checksum changed: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationConditionalColumnsAndAssertionsAreAtomic(t *testing.T) {
|
||||
database, err := Open(filepath.Join(t.TempDir(), "schema.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if _, err := database.Exec(`CREATE TABLE sample(id INTEGER PRIMARY KEY); INSERT INTO sample VALUES(1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := []SQLMigration{{Version: 1, Name: "adopt", AddColumns: []SQLiteColumn{{Table: "sample", Name: "value", Declaration: "TEXT NOT NULL DEFAULT ''"}}, Assertions: []string{`SELECT missing FROM sample LIMIT 0`}}}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err == nil {
|
||||
t.Fatal("invalid schema accepted")
|
||||
}
|
||||
var count int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM pragma_table_info('sample') WHERE name='value'`).Scan(&count); err != nil || count != 0 {
|
||||
t.Fatalf("ALTER was not rolled back: %d %v", count, err)
|
||||
}
|
||||
steps[0].Assertions = []string{`SELECT id,value FROM sample LIMIT 0`}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil {
|
||||
t.Fatal("repeated apply", err)
|
||||
}
|
||||
var value string
|
||||
if err := database.QueryRow(`SELECT value FROM sample WHERE id=1`).Scan(&value); err != nil || value != "" {
|
||||
t.Fatalf("legacy row lost: %q %v", value, err)
|
||||
}
|
||||
steps = append(steps, SQLMigration{Version: 2, Name: "remove_value", Statements: []string{`ALTER TABLE sample DROP COLUMN value`}, Assertions: []string{`SELECT id FROM sample LIMIT 0`}})
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil {
|
||||
t.Fatal("new contract must supersede old contract", err)
|
||||
}
|
||||
if _, err := database.Exec(`DROP TABLE sample`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err == nil {
|
||||
t.Fatal("recorded history masked missing table")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// OpenReadOnly deliberately does not migrate, set journal_mode, or create a DB.
|
||||
func OpenReadOnly(path string) (*sql.DB, error) {
|
||||
path, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: url.Values{"mode": {"ro"}, "_pragma": {"query_only(1)", "busy_timeout(5000)"}}.Encode()}).String()
|
||||
database, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = database.Ping(); err != nil {
|
||||
database.Close()
|
||||
return nil, err
|
||||
}
|
||||
return database, nil
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package documentstore_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/documentstore"
|
||||
"pageswiki/internal/geduldh"
|
||||
)
|
||||
|
||||
// These assignments intentionally fail compilation if aliases become wrappers
|
||||
// with different type identities or constructor signatures.
|
||||
var (
|
||||
_ func(string) (*geduldh.Local, error) = documentstore.NewLocal
|
||||
_ func(documentstore.Source, documentstore.MaterializeOptions) (*documentstore.Materializer, error) = geduldh.NewMaterializer
|
||||
_ documentstore.Store = (*geduldh.Local)(nil)
|
||||
_ geduldh.Store = (*documentstore.Local)(nil)
|
||||
)
|
||||
|
||||
func TestGeduldhLegacyLayoutAndSharedIdentity(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
body := []byte("existing Document 日本語\x00\xff")
|
||||
sum := sha256.Sum256(body)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
// Seed the pre-extraction layout independently of either Key implementation.
|
||||
filename := filepath.Join(root, "sha256", hash[:2], hash)
|
||||
if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filename, body, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := os.Stat(filename)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old, err := documentstore.NewLocal(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := geduldh.NewLocal(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, backend := range []geduldh.Store{old, current} {
|
||||
// The old normalization rules and payload bytes stay unchanged.
|
||||
inputHash := " " + strings.ToUpper(hash) + " "
|
||||
info, err := backend.Stat(ctx, inputHash)
|
||||
if err != nil || info.Hash != hash || info.Size != int64(len(body)) {
|
||||
t.Fatalf("stat=%+v err=%v", info, err)
|
||||
}
|
||||
reader, err := backend.Open(ctx, inputHash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
func() {
|
||||
defer reader.Close()
|
||||
if _, err := reader.Seek(2, io.SeekStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buf := make([]byte, 4)
|
||||
if _, err := reader.ReadAt(buf, 5); err != nil || !bytes.Equal(buf, body[5:9]) {
|
||||
t.Fatalf("ReadAt=%q err=%v", buf, err)
|
||||
}
|
||||
// ReaderAt must not move the sequential stream cursor (ZIP and Range).
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil || !bytes.Equal(data, body[2:]) {
|
||||
t.Fatalf("stream=%q err=%v", data, err)
|
||||
}
|
||||
}()
|
||||
if err := backend.PutIfAbsent(ctx, hash, int64(len(body)), bytes.NewReader(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := backend.Verify(ctx, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
after, err := os.Stat(filename)
|
||||
if err != nil || !os.SameFile(before, after) {
|
||||
t.Fatalf("canonical inode replaced: %v", err)
|
||||
}
|
||||
oldKey, err := documentstore.Key(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newKey, err := geduldh.Key(hash)
|
||||
if err != nil || oldKey != newKey || oldKey != "sha256/"+hash[:2]+"/"+hash {
|
||||
t.Fatalf("key changed: %s %s %v", oldKey, newKey, err)
|
||||
}
|
||||
if documentstore.ErrInvalidHash != geduldh.ErrInvalidHash || documentstore.ErrHashMismatch != geduldh.ErrHashMismatch || documentstore.ErrSizeMismatch != geduldh.ErrSizeMismatch || documentstore.ErrMaterializeLimit != geduldh.ErrMaterializeLimit {
|
||||
t.Fatal("legacy error identity changed")
|
||||
}
|
||||
if err := os.WriteFile(filename, []byte("corrupt"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, backend := range []geduldh.Store{old, current} {
|
||||
if err := backend.PutIfAbsent(ctx, hash, int64(len(body)), bytes.NewReader(body)); !errors.Is(err, documentstore.ErrHashMismatch) {
|
||||
t.Fatalf("corruption repaired or error changed: %v", err)
|
||||
}
|
||||
}
|
||||
corrupt, err := os.ReadFile(filename)
|
||||
if err != nil || string(corrupt) != "corrupt" {
|
||||
t.Fatalf("corrupt canonical overwritten: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeduldhCompatibilityMaterializer(t *testing.T) {
|
||||
root, scratch := t.TempDir(), t.TempDir()
|
||||
backend, err := geduldh.NewLocal(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("shared scratch bytes")
|
||||
sum := sha256.Sum256(body)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
ctx := context.Background()
|
||||
if err := backend.PutIfAbsent(ctx, hash, int64(len(body)), bytes.NewReader(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var materializer *geduldh.Materializer
|
||||
materializer, err = documentstore.NewMaterializer(backend, documentstore.MaterializeOptions{TempDir: scratch, MaxBytes: 1024, Concurrent: 1, Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var first *documentstore.Lease
|
||||
first, err = materializer.Acquire(ctx, hash, 1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer first.Close()
|
||||
second, err := materializer.Acquire(ctx, hash, 1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer second.Close()
|
||||
if first.Path != second.Path {
|
||||
t.Fatal("legacy/current leases no longer share the same copy")
|
||||
}
|
||||
first.Close()
|
||||
data, err := os.ReadFile(second.Path)
|
||||
if err != nil || !bytes.Equal(data, body) {
|
||||
t.Fatalf("active lease lost: %v", err)
|
||||
}
|
||||
second.Close()
|
||||
if _, err := os.Stat(second.Path); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("scratch not reclaimed: %v", err)
|
||||
}
|
||||
if err := backend.Verify(ctx, hash); err != nil {
|
||||
t.Fatal("lease cleanup affected canonical content", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeduldhConstructorsDoNotProvisionRoots(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "missing")
|
||||
for _, create := range []func(string) (*geduldh.Local, error){documentstore.NewLocal, geduldh.NewLocal} {
|
||||
if _, err := create(root); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(root); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("constructor wrote to storage root: %v", err)
|
||||
}
|
||||
if _, err := create(" "); err == nil {
|
||||
t.Fatal("empty root accepted")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
// Package documentstore preserves the original Document Store entry points.
|
||||
// Payload storage and scratch materialization are implemented by Geduldh.
|
||||
// New consumers should use pageswiki/internal/geduldh directly.
|
||||
package documentstore
|
||||
|
||||
import "pageswiki/internal/geduldh"
|
||||
|
||||
// Aliases preserve interface/type identity, method sets and error matching.
|
||||
// This package must not acquire its own persistence or validation logic.
|
||||
type (
|
||||
Info = geduldh.Info
|
||||
Reader = geduldh.Reader
|
||||
Store = geduldh.Store
|
||||
Local = geduldh.Local
|
||||
Source = geduldh.Source
|
||||
MaterializeOptions = geduldh.MaterializeOptions
|
||||
Materializer = geduldh.Materializer
|
||||
Lease = geduldh.Lease
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidHash = geduldh.ErrInvalidHash
|
||||
ErrHashMismatch = geduldh.ErrHashMismatch
|
||||
ErrSizeMismatch = geduldh.ErrSizeMismatch
|
||||
ErrMaterializeLimit = geduldh.ErrMaterializeLimit
|
||||
)
|
||||
|
||||
func Key(hash string) (string, error) { return geduldh.Key(hash) }
|
||||
|
||||
func NewLocal(root string) (*Local, error) { return geduldh.NewLocal(root) }
|
||||
|
||||
func NewMaterializer(source Source, options MaterializeOptions) (*Materializer, error) {
|
||||
return geduldh.NewMaterializer(source, options)
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
package dr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
func fixture(t *testing.T) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
database, err := db.Open(filepath.Join(root, "pages.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = database.Exec("CREATE TABLE revisions(document_hash TEXT); INSERT INTO revisions(rowid,document_hash) VALUES(42,'')"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = database.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = Init(root); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.WriteFile(filepath.Join(root, "auth.key"), make([]byte, 32), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func TestSnapshotRestorePromote(t *testing.T) {
|
||||
root := fixture(t)
|
||||
ctx := context.Background()
|
||||
dest := filepath.Join(t.TempDir(), "snapshot")
|
||||
body := []byte("synthetic DR test document")
|
||||
sum := sha256.Sum256(body)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
path := filepath.Join(root, "documents", "sha256", hash[:2], hash)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, body, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
database, err := db.Open(filepath.Join(root, "pages.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Hold a connection open to keep committed WAL pages present. No writer is
|
||||
// active during snapshot; this covers recovery after an uncheckpointed stop.
|
||||
if _, err = database.Exec("PRAGMA wal_autocheckpoint=0; UPDATE revisions SET document_hash=? WHERE rowid=42", hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err = os.MkdirAll(filepath.Join(root, "previews"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.WriteFile(filepath.Join(root, "previews", "ignored"), []byte("cache"), 0600)
|
||||
m, digest, err := Snapshot(ctx, root, dest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = Verify(ctx, dest, digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = Verify(ctx, dest, digest); err != nil {
|
||||
t.Fatalf("verification must not mutate snapshot: %v", err)
|
||||
}
|
||||
restored := filepath.Join(t.TempDir(), "standby")
|
||||
node, err := Restore(ctx, dest, digest, restored)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if node.Role != "standby" || node.NodeID == node.PrimaryNodeID || node.SnapshotID != m.ID || !node.Readable(time.Now()) {
|
||||
t.Fatalf("bad standby: %+v", node)
|
||||
}
|
||||
if _, err = os.Stat(filepath.Join(restored, "previews")); !os.IsNotExist(err) {
|
||||
t.Fatal("copied cache")
|
||||
}
|
||||
ro, err := db.OpenReadOnly(filepath.Join(restored, "pages.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ro.Close()
|
||||
var got string
|
||||
if err = ro.QueryRow("SELECT document_hash FROM revisions WHERE rowid=42").Scan(&got); err != nil || got != hash {
|
||||
t.Fatalf("WAL or rowid lost: %s %v", got, err)
|
||||
}
|
||||
if _, err = ro.Exec("DELETE FROM revisions"); err == nil {
|
||||
t.Fatal("standby accepted write")
|
||||
}
|
||||
shared, err := Lock(restored, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = Promote(restored, node.PrimaryNodeID, "test fence", true); err == nil {
|
||||
t.Fatal("promoted running standby")
|
||||
}
|
||||
shared.Close()
|
||||
if _, err = Promote(restored, "wrong", "test fence", true); err == nil {
|
||||
t.Fatal("accepted wrong primary")
|
||||
}
|
||||
if _, err = Promote(restored, node.PrimaryNodeID, "test fence", false); err == nil {
|
||||
t.Fatal("accepted implicit data loss")
|
||||
}
|
||||
promoted, err := Promote(restored, node.PrimaryNodeID, "test fence", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if promoted.Role != "primary" || promoted.Epoch != node.Epoch+1 || len(promoted.History) != 1 {
|
||||
t.Fatalf("bad promotion: %+v", promoted)
|
||||
}
|
||||
if promoted.WorkersEnabled {
|
||||
t.Fatal("promotion replayed jobs without review")
|
||||
}
|
||||
if _, err = ResumeWorkers(restored, false); err == nil {
|
||||
t.Fatal("resumed jobs without review")
|
||||
}
|
||||
resumed, err := ResumeWorkers(restored, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !resumed.WorkersEnabled || resumed.WorkersResumedAt.IsZero() {
|
||||
t.Fatal("worker resumption not audited")
|
||||
}
|
||||
if _, err = Restore(ctx, dest, digest, restored); err == nil {
|
||||
t.Fatal("overwrote existing restore")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotCalendarReferencesAndRestoreSafety(t *testing.T) {
|
||||
root := fixture(t)
|
||||
dir := filepath.Join(root, "documents", "calendars", "synthetic-domain", "synthetic-calendar")
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("synthetic opaque monthly generation; no external calendar data")
|
||||
sum := sha256.Sum256(body)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
metadata, _ := json.Marshal(map[string]any{"months": map[string]string{"2026-09": hash}})
|
||||
if err := os.WriteFile(filepath.Join(dir, "calendar.json"), metadata, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dest := filepath.Join(t.TempDir(), "snapshot")
|
||||
if _, _, err := Snapshot(context.Background(), root, dest); err == nil {
|
||||
t.Fatal("accepted missing calendar month")
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "2026-09."+hash+".ics"), body, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, digest, err := Snapshot(context.Background(), root, dest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = Verify(context.Background(), dest, digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err = Snapshot(context.Background(), root, filepath.Join(root, "inside")); err == nil {
|
||||
t.Fatal("snapshot inside source accepted")
|
||||
}
|
||||
if _, err = Restore(context.Background(), dest, digest, root); err == nil {
|
||||
t.Fatal("restore overwrote existing data")
|
||||
}
|
||||
if _, err = Restore(context.Background(), dest, digest, filepath.Join(dest, "nested")); err == nil {
|
||||
t.Fatal("restore corrupted source snapshot")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err = Verify(ctx, dest, digest); err == nil {
|
||||
t.Fatal("verification ignored cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotRefusesActiveMissingAndSymlink(t *testing.T) {
|
||||
root := fixture(t)
|
||||
dest := filepath.Join(t.TempDir(), "snapshot")
|
||||
ctx := context.Background()
|
||||
shared, err := Lock(root, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err = Snapshot(ctx, root, dest); err == nil {
|
||||
t.Fatal("snapshotted running instance")
|
||||
}
|
||||
shared.Close()
|
||||
database, err := db.Open(filepath.Join(root, "pages.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = database.Exec("UPDATE revisions SET document_hash=?", string(make([]byte, 64)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
database.Close()
|
||||
if _, _, err = Snapshot(ctx, root, dest); err == nil {
|
||||
t.Fatal("accepted missing canonical document")
|
||||
}
|
||||
if _, err = os.Stat(dest); !os.IsNotExist(err) {
|
||||
t.Fatal("published failed snapshot")
|
||||
}
|
||||
if err = os.Symlink(t.TempDir(), filepath.Join(root, "themes")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err = Snapshot(ctx, root, dest); err == nil {
|
||||
t.Fatal("followed source symlink")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotTamperAndPathValidation(t *testing.T) {
|
||||
root := fixture(t)
|
||||
dest := filepath.Join(t.TempDir(), "snapshot")
|
||||
ctx := context.Background()
|
||||
_, digest, err := Snapshot(ctx, root, dest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = Verify(ctx, dest, ""); err == nil {
|
||||
t.Fatal("accepted unauthenticated manifest")
|
||||
}
|
||||
manifestPath := filepath.Join(dest, "manifest.json")
|
||||
encoded, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m Manifest
|
||||
if err = json.Unmarshal(encoded, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Files[0].Path = "../outside"
|
||||
altered, _ := json.Marshal(m)
|
||||
sum := sha256.Sum256(altered)
|
||||
os.WriteFile(manifestPath, altered, 0600)
|
||||
if _, err = Verify(ctx, dest, digest); err == nil {
|
||||
t.Fatal("accepted changed manifest")
|
||||
}
|
||||
if _, err = Verify(ctx, dest, hex.EncodeToString(sum[:])); err == nil {
|
||||
t.Fatal("accepted traversal even with matching manifest digest")
|
||||
}
|
||||
os.WriteFile(manifestPath, encoded, 0600)
|
||||
os.WriteFile(filepath.Join(dest, "data", "auth.key"), []byte("tampered"), 0600)
|
||||
if _, err = Verify(ctx, dest, digest); err == nil {
|
||||
t.Fatal("accepted payload tamper")
|
||||
}
|
||||
for _, path := range []string{"/etc/passwd", "documents/../auth.key", "documents\\foo", "dr-node.json", "pages.db-wal", "cache/foo"} {
|
||||
if canonical(path) {
|
||||
t.Errorf("accepted %q", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigFailsClosedAndExpiry(t *testing.T) {
|
||||
root := fixture(t)
|
||||
c, err := Load(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.Role = "standby"
|
||||
if err = Save(root, *c); err == nil {
|
||||
t.Fatal("accepted inconsistent role")
|
||||
}
|
||||
os.WriteFile(filepath.Join(root, ConfigFile), []byte(`{"role":"broken"}`), 0600)
|
||||
if _, err = Load(root); err == nil {
|
||||
t.Fatal("invalid config accepted")
|
||||
}
|
||||
c.Role = "standby"
|
||||
c.CapturedAt = time.Now().Add(-2 * time.Hour)
|
||||
c.MaxReadAgeSeconds = 3600
|
||||
if c.Readable(time.Now()) {
|
||||
t.Fatal("stale snapshot is readable")
|
||||
}
|
||||
c.CapturedAt = time.Now().Add(time.Hour)
|
||||
if c.Readable(time.Now()) {
|
||||
t.Fatal("future snapshot is readable")
|
||||
}
|
||||
if err = RequireRole(nil, "standby"); err == nil {
|
||||
t.Fatal("empty standby volume would start writable")
|
||||
}
|
||||
if err = RequireRole(c, "primary"); err == nil {
|
||||
t.Fatal("mismatched deployment role accepted")
|
||||
}
|
||||
if err = RequireRole(c, "standby"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = RequireRole(nil, ""); err != nil {
|
||||
t.Fatal("legacy startup broken")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package dr
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
"pageswiki/internal/geduldh"
|
||||
)
|
||||
|
||||
func TestGeduldhSnapshotRestoreCompatibility(t *testing.T) {
|
||||
root := fixture(t)
|
||||
documents := filepath.Join(root, "documents")
|
||||
if err := os.MkdirAll(documents, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend, err := geduldh.NewLocal(documents)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
body := []byte("Geduldh DR compatibility 日本語\x00\xff")
|
||||
sum := sha256.Sum256(body)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
if err := backend.PutIfAbsent(ctx, hash, int64(len(body)), bytes.NewReader(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
database, err := db.Open(filepath.Join(root, "pages.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if _, err := database.Exec(`UPDATE revisions SET document_hash=? WHERE rowid=42`, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshot := filepath.Join(t.TempDir(), "snapshot")
|
||||
_, digest, err := Snapshot(ctx, root, snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Verify(ctx, snapshot, digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restored := filepath.Join(t.TempDir(), "standby")
|
||||
if _, err := Restore(ctx, snapshot, digest, restored); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recovered, err := geduldh.NewLocal(filepath.Join(restored, "documents"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recovered.Verify(ctx, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reader, err := recovered.Open(ctx, hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil || !bytes.Equal(data, body) {
|
||||
t.Fatalf("restored payload changed: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
// Package dr implements offline, operator-fenced recovery, not leader election.
|
||||
package dr
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const ConfigFile = "dr-node.json"
|
||||
const LockFile = ".pageswiki-instance.lock"
|
||||
|
||||
// RequireRole is an optional deployment guard against mounting an empty/wrong
|
||||
// data volume on a DR host and accidentally starting a new writable instance.
|
||||
func RequireRole(c *Config, expected string) error {
|
||||
if expected == "" {
|
||||
return nil
|
||||
}
|
||||
if expected != "primary" && expected != "standby" {
|
||||
return errors.New("PAGESWIKI_DR_EXPECT_ROLE must be primary or standby")
|
||||
}
|
||||
if c == nil || c.Role != expected {
|
||||
return errors.New("DR role does not match PAGESWIKI_DR_EXPECT_ROLE; refusing startup")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Promotion struct {
|
||||
At time.Time `json:"at"`
|
||||
PreviousNode string `json:"previous_node"`
|
||||
FenceReference string `json:"fence_reference"`
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
}
|
||||
|
||||
// Config is node-local and MUST NOT be overwritten by a primary's snapshot.
|
||||
// Epoch is an audit generation only; it is NOT a distributed fencing token.
|
||||
type Config struct {
|
||||
Version int `json:"version"`
|
||||
ClusterID string `json:"cluster_id"`
|
||||
NodeID string `json:"node_id"`
|
||||
PrimaryNodeID string `json:"primary_node_id"`
|
||||
Role string `json:"role"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
SnapshotID string `json:"snapshot_id,omitempty"`
|
||||
CapturedAt time.Time `json:"captured_at,omitempty"`
|
||||
MaxReadAgeSeconds int64 `json:"max_read_age_seconds"`
|
||||
WorkersEnabled bool `json:"workers_enabled"`
|
||||
WorkersResumedAt time.Time `json:"workers_resumed_at,omitempty"`
|
||||
History []Promotion `json:"promotion_history,omitempty"`
|
||||
}
|
||||
|
||||
func (c Config) Validate() error {
|
||||
for _, id := range []string{c.ClusterID, c.NodeID, c.PrimaryNodeID} {
|
||||
u, err := uuid.Parse(id)
|
||||
if err != nil || u == uuid.Nil || u.String() != id {
|
||||
return errors.New("DR identity must be a canonical nonzero UUID")
|
||||
}
|
||||
}
|
||||
if c.Version != 1 || c.Epoch == 0 || c.MaxReadAgeSeconds < 1 || c.MaxReadAgeSeconds > 86400 {
|
||||
return errors.New("invalid DR version, epoch or max_read_age_seconds (1..86400)")
|
||||
}
|
||||
if c.Role != "primary" && c.Role != "standby" {
|
||||
return errors.New("DR role must be primary or standby")
|
||||
}
|
||||
if (c.Role == "primary") != (c.PrimaryNodeID == c.NodeID) {
|
||||
return errors.New("DR role and primary identity disagree")
|
||||
}
|
||||
if c.Role == "standby" && (c.SnapshotID == "" || c.CapturedAt.IsZero()) {
|
||||
return errors.New("standby requires a verified snapshot")
|
||||
}
|
||||
if c.Role == "standby" && c.WorkersEnabled {
|
||||
return errors.New("standby cannot enable background workers")
|
||||
}
|
||||
if c.SnapshotID != "" {
|
||||
u, err := uuid.Parse(c.SnapshotID)
|
||||
if err != nil || u == uuid.Nil || u.String() != c.SnapshotID {
|
||||
return errors.New("invalid snapshot identity")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c Config) Readable(now time.Time) bool {
|
||||
return c.Role == "standby" && !c.CapturedAt.After(now.Add(time.Minute)) && now.Sub(c.CapturedAt) <= time.Duration(c.MaxReadAgeSeconds)*time.Second
|
||||
}
|
||||
|
||||
func (c Config) ReadUntil() time.Time {
|
||||
if c.CapturedAt.After(time.Now().Add(time.Minute)) {
|
||||
return time.Time{}
|
||||
}
|
||||
return c.CapturedAt.Add(time.Duration(c.MaxReadAgeSeconds) * time.Second)
|
||||
}
|
||||
|
||||
func Load(dataDir string) (*Config, error) {
|
||||
f, err := os.Open(filepath.Join(dataDir, ConfigFile))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Size() > 1<<20 {
|
||||
return nil, errors.New("invalid or oversized DR configuration")
|
||||
}
|
||||
var c Config
|
||||
d := json.NewDecoder(io.LimitReader(f, 1<<20))
|
||||
d.DisallowUnknownFields()
|
||||
if err = d.Decode(&c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = d.Decode(new(any)); err != io.EOF {
|
||||
return nil, errors.New("trailing DR configuration data")
|
||||
}
|
||||
return &c, c.Validate()
|
||||
}
|
||||
|
||||
func Save(dataDir string, c Config) error {
|
||||
if err := c.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.MarshalIndent(c, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.CreateTemp(dataDir, ".dr-node-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
if _, err = f.Write(append(b, '\n')); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err = f.Sync(); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err = f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = os.Rename(f.Name(), filepath.Join(dataDir, ConfigFile)); err != nil {
|
||||
return err
|
||||
}
|
||||
return syncDir(dataDir)
|
||||
}
|
||||
|
||||
// Lock is shared for a running server/CLI, exclusive for offline maintenance.
|
||||
// Advisory locking cannot fence older binaries or an independent SQLite client.
|
||||
func Lock(dataDir string, exclusive bool) (*os.File, error) {
|
||||
// Shared locks also work with a read-only standby mount.
|
||||
path := filepath.Join(dataDir, LockFile)
|
||||
f, err := os.Open(path)
|
||||
if exclusive || errors.Is(err, os.ErrNotExist) {
|
||||
if f != nil {
|
||||
f.Close()
|
||||
}
|
||||
f, err = os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mode := syscall.LOCK_SH
|
||||
if exclusive {
|
||||
mode = syscall.LOCK_EX
|
||||
}
|
||||
if err = syscall.Flock(int(f.Fd()), mode|syscall.LOCK_NB); err != nil {
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("DR data directory is in use; stop all writers/readers before maintenance: %w", err)
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func Init(dataDir string) (*Config, error) {
|
||||
l, err := Lock(dataDir, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer l.Close()
|
||||
if c, err := Load(dataDir); err != nil || c != nil {
|
||||
return nil, errors.New("DR configuration exists or cannot be read; refusing initialization")
|
||||
}
|
||||
id := uuid.NewString()
|
||||
c := Config{Version: 1, ClusterID: uuid.NewString(), NodeID: id, PrimaryNodeID: id, Role: "primary", Epoch: 1, MaxReadAgeSeconds: 3600, WorkersEnabled: true}
|
||||
return &c, Save(dataDir, c)
|
||||
}
|
||||
|
||||
// Promote records an operator's fencing attestation. It does not contact,
|
||||
// power off, or independently verify the former primary. No automatic promotion.
|
||||
func Promote(dataDir, fencedNode, reference string, acceptLoss bool) (*Config, error) {
|
||||
l, err := Lock(dataDir, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer l.Close()
|
||||
c, err := Load(dataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c == nil || c.Role != "standby" {
|
||||
return nil, errors.New("only an offline standby can be promoted")
|
||||
}
|
||||
if err = checkAuthKey(dataDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fencedNode != c.PrimaryNodeID || !acceptLoss || len(strings.TrimSpace(reference)) < 4 || len(reference) > 1024 {
|
||||
return nil, errors.New("promotion requires exact former primary ID, fencing reference and explicit acceptance of snapshot data loss")
|
||||
}
|
||||
if c.Epoch == ^uint64(0) {
|
||||
return nil, errors.New("DR epoch exhausted")
|
||||
}
|
||||
c.Epoch++
|
||||
c.History = append(c.History, Promotion{At: time.Now().UTC(), PreviousNode: fencedNode, FenceReference: strings.TrimSpace(reference), SnapshotID: c.SnapshotID, Epoch: c.Epoch})
|
||||
c.Role, c.PrimaryNodeID = "primary", c.NodeID
|
||||
// Replaying pre-snapshot jobs can duplicate notifications or external writes.
|
||||
// Resume them only after an operator reviews the restored queue.
|
||||
c.WorkersEnabled = false
|
||||
return c, Save(dataDir, *c)
|
||||
}
|
||||
|
||||
func checkAuthKey(dataDir string) error {
|
||||
if strings.TrimSpace(os.Getenv("PAGESWIKI_AUTH_KEY")) != "" {
|
||||
return nil
|
||||
}
|
||||
key, err := os.ReadFile(filepath.Join(dataDir, "auth.key"))
|
||||
if err != nil || len(key) != 32 {
|
||||
return errors.New("original auth.key or PAGESWIKI_AUTH_KEY is required; refusing implicit key regeneration")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ResumeWorkers(dataDir string, reviewed bool) (*Config, error) {
|
||||
if !reviewed {
|
||||
return nil, errors.New("review restored jobs and federation state before resuming workers")
|
||||
}
|
||||
l, err := Lock(dataDir, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer l.Close()
|
||||
c, err := Load(dataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c == nil || c.Role != "primary" {
|
||||
return nil, errors.New("only an offline primary can resume workers")
|
||||
}
|
||||
c.WorkersEnabled = true
|
||||
c.WorkersResumedAt = time.Now().UTC()
|
||||
return c, Save(dataDir, *c)
|
||||
}
|
||||
|
||||
func syncDir(path string) error {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
return f.Sync()
|
||||
}
|
||||
@@ -0,0 +1,579 @@
|
||||
package dr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/sys/unix"
|
||||
"modernc.org/sqlite"
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
type File struct {
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
type Manifest struct {
|
||||
Version int `json:"version"`
|
||||
ID string `json:"id"`
|
||||
ClusterID string `json:"cluster_id"`
|
||||
PrimaryNodeID string `json:"primary_node_id"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
CapturedAt time.Time `json:"captured_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
Scope string `json:"scope"`
|
||||
Files []File `json:"files"`
|
||||
}
|
||||
|
||||
// Include canonical stores, auth/signing keys and managed-campus enrollment.
|
||||
// Caches, temporary uploads, editor workspaces and node-local DR state are NOT
|
||||
// recovery authorities. Extra mounted storage/environment secrets are external.
|
||||
func canonical(path string) bool {
|
||||
if !filepath.IsLocal(path) || filepath.ToSlash(path) != path || strings.Contains(path, "\\") || path != filepath.Clean(path) {
|
||||
return false
|
||||
}
|
||||
root := strings.Split(path, "/")[0]
|
||||
switch root {
|
||||
case "documents", "themes", "federation":
|
||||
return true
|
||||
case "pages.db", "auth.key", "instance-ed25519.key":
|
||||
return path == root
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func fileHash(path string) (File, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return File{}, err
|
||||
}
|
||||
defer f.Close()
|
||||
i, err := f.Stat()
|
||||
if err != nil || !i.Mode().IsRegular() {
|
||||
return File{}, errors.New("snapshot entry is not a regular file")
|
||||
}
|
||||
h := sha256.New()
|
||||
n, err := io.CopyBuffer(h, f, make([]byte, 256<<10))
|
||||
return File{Size: n, SHA256: hex.EncodeToString(h.Sum(nil))}, err
|
||||
}
|
||||
|
||||
func copyFile(source, target string) error {
|
||||
i, err := os.Lstat(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !i.Mode().IsRegular() {
|
||||
return fmt.Errorf("refusing nonregular source: %s", source)
|
||||
}
|
||||
in, err := os.Open(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
opened, err := in.Stat()
|
||||
if err != nil || !os.SameFile(i, opened) {
|
||||
return errors.New("source changed during snapshot")
|
||||
}
|
||||
if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.CopyBuffer(out, in, make([]byte, 256<<10))
|
||||
if err == nil {
|
||||
err = out.Sync()
|
||||
}
|
||||
closeErr := out.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Backup API preserves rowids (unlike VACUUM INTO) and includes committed WAL.
|
||||
func backupDB(ctx context.Context, source, target string) error {
|
||||
database, err := db.OpenReadOnly(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer database.Close()
|
||||
conn, err := database.Conn(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
err = conn.Raw(func(raw any) error {
|
||||
b, err := raw.(interface {
|
||||
NewBackup(string) (*sqlite.Backup, error)
|
||||
}).NewBackup((&url.URL{Scheme: "file", Path: target}).String())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
if err = ctx.Err(); err != nil {
|
||||
b.Finish()
|
||||
return err
|
||||
}
|
||||
more, e := b.Step(256)
|
||||
if e != nil {
|
||||
b.Finish()
|
||||
return e
|
||||
}
|
||||
if !more {
|
||||
return b.Finish()
|
||||
}
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Do not leave a WAL-mode snapshot that creates sidecars during verification.
|
||||
dst, err := sql.Open("sqlite", (&url.URL{Scheme: "file", Path: target, RawQuery: "mode=rw"}).String())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = dst.ExecContext(ctx, "PRAGMA journal_mode=DELETE")
|
||||
closeErr := dst.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
// Finish leaves a standalone DB; fsync before publishing the generation.
|
||||
f, err := os.OpenFile(target, os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
if err = f.Chmod(0600); err != nil {
|
||||
return err
|
||||
}
|
||||
return f.Sync()
|
||||
}
|
||||
|
||||
// Snapshot requires every writer stopped, including older binaries and CLIs.
|
||||
// The lock enforces this for upgraded PagesWiki processes, not arbitrary tools.
|
||||
func Snapshot(ctx context.Context, dataDir, target string) (*Manifest, string, error) {
|
||||
source, err := filepath.EvalSymlinks(dataDir)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
source, err = filepath.Abs(source)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
target, err = filepath.Abs(target)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
parent, err := filepath.EvalSymlinks(filepath.Dir(target))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
target = filepath.Join(parent, filepath.Base(target))
|
||||
if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) {
|
||||
return nil, "", errors.New("snapshot destination must be outside the data directory")
|
||||
}
|
||||
l, err := Lock(source, true)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer l.Close()
|
||||
c, err := Load(source)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if c == nil || c.Role != "primary" {
|
||||
return nil, "", errors.New("initialize primary DR configuration before snapshot")
|
||||
}
|
||||
if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, "", errors.New("snapshot destination must not exist")
|
||||
}
|
||||
stage, err := os.MkdirTemp(parent, ".dr-snapshot-")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer os.RemoveAll(stage)
|
||||
m := &Manifest{Version: 1, ID: uuid.NewString(), ClusterID: c.ClusterID, PrimaryNodeID: c.NodeID, Epoch: c.Epoch, CapturedAt: time.Now().UTC(), Scope: "instance-offline-v1"}
|
||||
root := filepath.Join(stage, "data")
|
||||
if err = os.Mkdir(root, 0700); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
info, err := os.Lstat(filepath.Join(source, "pages.db"))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil, "", errors.New("snapshot database must be a regular file, not a symlink")
|
||||
}
|
||||
if err = checkAuthKey(source); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err = backupDB(ctx, filepath.Join(source, "pages.db"), filepath.Join(root, "pages.db")); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
for _, name := range []string{"documents", "themes", "federation", "auth.key", "instance-ed25519.key"} {
|
||||
base := filepath.Join(source, name)
|
||||
if _, e := os.Lstat(base); errors.Is(e, os.ErrNotExist) {
|
||||
continue
|
||||
} else if e != nil {
|
||||
return nil, "", e
|
||||
}
|
||||
err = filepath.WalkDir(base, func(path string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.Type()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("snapshot refuses symlink: %s", path)
|
||||
}
|
||||
rel, _ := filepath.Rel(source, path)
|
||||
if entry.IsDir() {
|
||||
return os.MkdirAll(filepath.Join(root, rel), 0700)
|
||||
}
|
||||
return copyFile(path, filepath.Join(root, rel))
|
||||
})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, e error) error {
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return syncDir(path)
|
||||
}
|
||||
if len(m.Files) >= 100000 {
|
||||
return errors.New("snapshot exceeds 100000 files; partitioned snapshots require a later format")
|
||||
}
|
||||
f, e := fileHash(path)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
f.Path, _ = filepath.Rel(root, path)
|
||||
m.Files = append(m.Files, f)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
m.CompletedAt = time.Now().UTC()
|
||||
encoded, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err = os.WriteFile(filepath.Join(stage, "manifest.json"), encoded, 0600); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
manifestFile, err := os.OpenFile(filepath.Join(stage, "manifest.json"), os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
err = manifestFile.Sync()
|
||||
manifestFile.Close()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
hash := sha256.Sum256(encoded)
|
||||
if _, err = Verify(ctx, stage, hex.EncodeToString(hash[:])); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err = syncDir(stage); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err = publish(stage, target); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return m, hex.EncodeToString(hash[:]), syncDir(parent)
|
||||
}
|
||||
|
||||
func Verify(ctx context.Context, root, expectedHash string) (*Manifest, error) {
|
||||
if len(expectedHash) != 64 {
|
||||
return nil, errors.New("expected manifest SHA-256 is required from a trusted channel")
|
||||
}
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Lstat(filepath.Join(root, "manifest.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Size() > 64<<20 {
|
||||
return nil, errors.New("invalid or oversized snapshot manifest")
|
||||
}
|
||||
encoded, err := os.ReadFile(filepath.Join(root, "manifest.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sum := sha256.Sum256(encoded)
|
||||
if hex.EncodeToString(sum[:]) != expectedHash {
|
||||
return nil, errors.New("manifest SHA-256 mismatch")
|
||||
}
|
||||
var m Manifest
|
||||
if err = json.Unmarshal(encoded, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if m.Version != 1 || m.Scope != "instance-offline-v1" || m.Epoch == 0 || len(m.Files) == 0 || len(m.Files) > 100000 || m.CapturedAt.IsZero() || m.CompletedAt.Before(m.CapturedAt) || m.CompletedAt.After(time.Now().Add(time.Minute)) {
|
||||
return nil, errors.New("invalid snapshot manifest")
|
||||
}
|
||||
for _, id := range []string{m.ID, m.ClusterID, m.PrimaryNodeID} {
|
||||
u, e := uuid.Parse(id)
|
||||
if e != nil || u == uuid.Nil || u.String() != id {
|
||||
return nil, errors.New("invalid snapshot identity")
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
hashes := map[string]string{}
|
||||
for _, item := range m.Files {
|
||||
if err = ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !canonical(item.Path) || seen[item.Path] {
|
||||
return nil, errors.New("invalid/duplicate snapshot path")
|
||||
}
|
||||
seen[item.Path] = true
|
||||
hashes[item.Path] = item.SHA256
|
||||
path := filepath.Join(root, "data", item.Path)
|
||||
// Refuse symlinks in every component, including parent directories.
|
||||
for p := path; p != filepath.Dir(root) && p != "."; p = filepath.Dir(p) {
|
||||
info, e := os.Lstat(p)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return nil, errors.New("snapshot contains symlink")
|
||||
}
|
||||
if p == root || filepath.Dir(p) == p {
|
||||
break
|
||||
}
|
||||
}
|
||||
actual, e := fileHash(path)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if actual.Size != item.Size || actual.SHA256 != item.SHA256 {
|
||||
return nil, fmt.Errorf("snapshot payload mismatch: %s", item.Path)
|
||||
}
|
||||
if strings.HasPrefix(item.Path, "documents/sha256/") && filepath.Base(item.Path) != actual.SHA256 {
|
||||
return nil, errors.New("CAS filename does not match payload hash")
|
||||
}
|
||||
}
|
||||
if !seen["pages.db"] {
|
||||
return nil, errors.New("snapshot database missing")
|
||||
}
|
||||
if err = verifyCalendarFiles(root, hashes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = filepath.WalkDir(filepath.Join(root, "data"), func(path string, entry fs.DirEntry, e error) error {
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(filepath.Join(root, "data"), path)
|
||||
if !seen[rel] {
|
||||
return errors.New("unlisted snapshot payload")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
database, err := db.OpenReadOnly(filepath.Join(root, "data", "pages.db"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer database.Close()
|
||||
var integrity string
|
||||
if err = database.QueryRowContext(ctx, "PRAGMA quick_check").Scan(&integrity); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if integrity != "ok" {
|
||||
return nil, errors.New("snapshot database integrity check failed")
|
||||
}
|
||||
if err = verifyReferences(ctx, database, seen); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
// Restore never merges into or overwrites a running/existing data directory.
|
||||
// It publishes a new generation with a fresh physical node ID, always standby.
|
||||
func Restore(ctx context.Context, snapshot, expectedHash, target string) (*Config, error) {
|
||||
m, err := Verify(ctx, snapshot, expectedHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target, err = filepath.Abs(target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parent, err := filepath.EvalSymlinks(filepath.Dir(target))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target = filepath.Join(parent, filepath.Base(target))
|
||||
source, err := filepath.EvalSymlinks(snapshot)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
source, err = filepath.Abs(source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) {
|
||||
return nil, errors.New("restore destination must be outside the snapshot")
|
||||
}
|
||||
if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, errors.New("restore target must not exist; use a new generation")
|
||||
}
|
||||
stage, err := os.MkdirTemp(filepath.Dir(target), ".dr-restore-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(stage)
|
||||
for _, item := range m.Files {
|
||||
if err = ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = copyFile(filepath.Join(snapshot, "data", item.Path), filepath.Join(stage, item.Path)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
actual, e := fileHash(filepath.Join(stage, item.Path))
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if actual.Size != item.Size || actual.SHA256 != item.SHA256 {
|
||||
return nil, errors.New("snapshot changed during restore")
|
||||
}
|
||||
}
|
||||
c := &Config{Version: 1, ClusterID: m.ClusterID, NodeID: uuid.NewString(), PrimaryNodeID: m.PrimaryNodeID, Role: "standby", Epoch: m.Epoch, SnapshotID: m.ID, CapturedAt: m.CapturedAt, MaxReadAgeSeconds: 3600}
|
||||
if err = Save(stage, *c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = os.MkdirAll(filepath.Join(stage, "documents", "sha256"), 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Create the lock before a read-only data mount is applied.
|
||||
l, err := Lock(stage, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l.Close()
|
||||
err = filepath.WalkDir(stage, func(path string, entry fs.DirEntry, e error) error {
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return syncDir(path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = publish(stage, target); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c, syncDir(filepath.Dir(target))
|
||||
}
|
||||
|
||||
// Linux deployment: atomically publish without replacing even an empty directory.
|
||||
func publish(stage, target string) error {
|
||||
return unix.Renameat2(unix.AT_FDCWD, stage, unix.AT_FDCWD, target, unix.RENAME_NOREPLACE)
|
||||
}
|
||||
|
||||
func verifyReferences(ctx context.Context, database *sql.DB, seen map[string]bool) error {
|
||||
// Only authoritative references, not derived projection/preview caches. Some
|
||||
// installations predate optional modules, hence the explicit table check.
|
||||
for _, ref := range [][2]string{{"revisions", "document_hash"}, {"assets", "head_document_hash"}, {"asset_head_history", "document_hash"}, {"organizer_items", "document_hash"}, {"organizer_revisions", "document_hash"}} {
|
||||
var exists bool
|
||||
if err := database.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?)", ref[0]).Scan(&exists); err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
rows, err := database.QueryContext(ctx, "SELECT DISTINCT "+ref[1]+" FROM "+ref[0]+" WHERE "+ref[1]+" IS NOT NULL AND "+ref[1]+"<>''")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var hash string
|
||||
if err = rows.Scan(&hash); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if len(hash) != 64 || !seen["documents/sha256/"+hash[:2]+"/"+hash] {
|
||||
rows.Close()
|
||||
return fmt.Errorf("snapshot lacks canonical payload referenced by %s", ref[0])
|
||||
}
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verify file-based calendar authority without parsing event bodies or making
|
||||
// external calendar requests. Derived calendar indexes may be repaired only
|
||||
// after promotion, never by a standby read request.
|
||||
func verifyCalendarFiles(root string, hashes map[string]string) error {
|
||||
for path := range hashes {
|
||||
if !strings.HasPrefix(path, "documents/calendars/") || filepath.Base(path) != "calendar.json" {
|
||||
continue
|
||||
}
|
||||
filename := filepath.Join(root, "data", path)
|
||||
info, err := os.Stat(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Size() > 16<<20 {
|
||||
return errors.New("calendar metadata exceeds snapshot verification limit")
|
||||
}
|
||||
encoded, err := os.ReadFile(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var metadata struct {
|
||||
Months map[string]string `json:"months"`
|
||||
}
|
||||
if err = json.Unmarshal(encoded, &metadata); err != nil {
|
||||
return errors.New("invalid calendar metadata in snapshot")
|
||||
}
|
||||
for month, hash := range metadata.Months {
|
||||
if _, err = time.Parse("2006-01", month); err != nil {
|
||||
return errors.New("invalid calendar month in snapshot")
|
||||
}
|
||||
if len(hash) != 64 || hashes[filepath.ToSlash(filepath.Join(filepath.Dir(path), month+"."+hash+".ics"))] != hash {
|
||||
return errors.New("snapshot lacks a valid calendar monthly generation")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package dregarnuhr
|
||||
|
||||
import (
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestComponentBoundary(t *testing.T) {
|
||||
if Component != "dregarnuhr" {
|
||||
t.Fatal("component identity changed")
|
||||
}
|
||||
files, err := filepath.Glob("*.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, file := range files {
|
||||
if strings.HasSuffix(file, "_test.go") {
|
||||
continue
|
||||
}
|
||||
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, spec := range ast.Imports {
|
||||
path, err := strconv.Unquote(spec.Path.Value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"pageswiki/internal", "database", "net", "os"} {
|
||||
if path == forbidden || strings.HasPrefix(path, forbidden+"/") {
|
||||
t.Errorf("%s imports %s: scheduler core must not own persistence or business I/O", file, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package dregarnuhr
|
||||
|
||||
import "time"
|
||||
|
||||
type Timer interface {
|
||||
C() <-chan time.Time
|
||||
Stop() bool
|
||||
}
|
||||
type Clock interface {
|
||||
Now() time.Time
|
||||
NewTimer(time.Duration) Timer
|
||||
}
|
||||
type SystemClock struct{}
|
||||
|
||||
func (SystemClock) Now() time.Time { return time.Now() }
|
||||
func (SystemClock) NewTimer(d time.Duration) Timer { return systemTimer{time.NewTimer(d)} }
|
||||
|
||||
type systemTimer struct{ *time.Timer }
|
||||
|
||||
func (t systemTimer) C() <-chan time.Time { return t.Timer.C }
|
||||
@@ -0,0 +1,65 @@
|
||||
package dregarnuhr
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeClock struct {
|
||||
mu sync.Mutex
|
||||
now time.Time
|
||||
timers map[*fakeTimer]time.Time
|
||||
created chan struct{}
|
||||
}
|
||||
type fakeTimer struct {
|
||||
clock *fakeClock
|
||||
ch chan time.Time
|
||||
}
|
||||
|
||||
func newFakeClock() *fakeClock {
|
||||
return &fakeClock{now: time.Unix(100000, 0), timers: map[*fakeTimer]time.Time{}, created: make(chan struct{}, 100)}
|
||||
}
|
||||
func (c *fakeClock) Now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.now }
|
||||
func (c *fakeClock) NewTimer(delay time.Duration) Timer {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
t := &fakeTimer{clock: c, ch: make(chan time.Time, 1)}
|
||||
if delay <= 0 {
|
||||
t.ch <- c.now
|
||||
} else {
|
||||
c.timers[t] = c.now.Add(delay)
|
||||
}
|
||||
c.created <- struct{}{}
|
||||
return t
|
||||
}
|
||||
func (t *fakeTimer) C() <-chan time.Time { return t.ch }
|
||||
func (t *fakeTimer) Stop() bool {
|
||||
t.clock.mu.Lock()
|
||||
defer t.clock.mu.Unlock()
|
||||
_, ok := t.clock.timers[t]
|
||||
delete(t.clock.timers, t)
|
||||
return ok
|
||||
}
|
||||
func (c *fakeClock) advance(d time.Duration) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.now = c.now.Add(d)
|
||||
for timer, at := range c.timers {
|
||||
if !at.After(c.now) {
|
||||
timer.ch <- c.now
|
||||
delete(c.timers, timer)
|
||||
}
|
||||
}
|
||||
}
|
||||
func receive[T any](t *testing.T, ch <-chan T) T {
|
||||
t.Helper()
|
||||
select {
|
||||
case v := <-ch:
|
||||
return v
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for test worker")
|
||||
var zero T
|
||||
return zero
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package dregarnuhr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Trigger struct {
|
||||
Name string
|
||||
After, Every time.Duration
|
||||
Run func(context.Context, time.Time) error
|
||||
}
|
||||
type Loop struct {
|
||||
Clock Clock
|
||||
Every time.Duration
|
||||
Wake <-chan struct{}
|
||||
OnWake func(context.Context, time.Time) error
|
||||
Triggers []Trigger
|
||||
Step func(context.Context, time.Time) error
|
||||
Report func(string, error)
|
||||
}
|
||||
|
||||
// Run coalesces overdue ticks (no catch-up storm), stops its timer on
|
||||
// cancellation, and invokes caller-supplied hooks. It knows no event types.
|
||||
func (l Loop) Run(ctx context.Context) error {
|
||||
if l.Every <= 0 || l.Step == nil {
|
||||
return errors.New("loop requires positive cadence and step")
|
||||
}
|
||||
clock := l.Clock
|
||||
if clock == nil {
|
||||
clock = SystemClock{}
|
||||
}
|
||||
now := clock.Now()
|
||||
next := now.Add(l.Every)
|
||||
due := make([]time.Time, len(l.Triggers))
|
||||
seen := map[string]bool{}
|
||||
for i, t := range l.Triggers {
|
||||
if t.Name == "" || seen[t.Name] || t.Run == nil || t.After < 0 || t.Every < 0 || (t.After == 0 && t.Every == 0) {
|
||||
return fmt.Errorf("invalid trigger %q", t.Name)
|
||||
}
|
||||
seen[t.Name] = true
|
||||
delay := t.After
|
||||
if delay == 0 {
|
||||
delay = t.Every
|
||||
}
|
||||
due[i] = now.Add(delay)
|
||||
}
|
||||
wake := l.Wake
|
||||
call := func(name string, fn func(context.Context, time.Time) error, now time.Time) {
|
||||
if fn != nil {
|
||||
if err := fn(ctx, now); err != nil && l.Report != nil {
|
||||
l.Report(name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
at := next
|
||||
for _, d := range due {
|
||||
if !d.IsZero() && d.Before(at) {
|
||||
at = d
|
||||
}
|
||||
}
|
||||
timer := clock.NewTimer(max(time.Duration(0), at.Sub(clock.Now())))
|
||||
woken := false
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return ctx.Err()
|
||||
case _, ok := <-wake:
|
||||
if !ok {
|
||||
wake = nil
|
||||
timer.Stop()
|
||||
continue
|
||||
}
|
||||
woken = true
|
||||
case <-timer.C():
|
||||
}
|
||||
timer.Stop()
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
now = clock.Now()
|
||||
if woken {
|
||||
call("wake", l.OnWake, now)
|
||||
}
|
||||
for i, t := range l.Triggers {
|
||||
if !due[i].IsZero() && !due[i].After(now) {
|
||||
call(t.Name, t.Run, now)
|
||||
if t.Every > 0 {
|
||||
due[i] = now.Add(t.Every)
|
||||
} else {
|
||||
due[i] = time.Time{}
|
||||
}
|
||||
}
|
||||
}
|
||||
call("step", l.Step, now)
|
||||
if !next.After(now) {
|
||||
next = now.Add(l.Every)
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package dregarnuhr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoopFakeClockWakeAndCoalescing(t *testing.T) {
|
||||
clock := newFakeClock()
|
||||
wake := make(chan struct{}, 1)
|
||||
steps := make(chan time.Time, 20)
|
||||
reports := make(chan string, 10)
|
||||
var once, periodic, awakened atomic.Int64
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
loop := Loop{Clock: clock, Every: time.Second, Wake: wake,
|
||||
OnWake: func(context.Context, time.Time) error { awakened.Add(1); return nil },
|
||||
Triggers: []Trigger{
|
||||
{Name: "once", After: 5 * time.Second, Run: func(context.Context, time.Time) error { once.Add(1); return nil }},
|
||||
{Name: "periodic", Every: 10 * time.Second, Run: func(context.Context, time.Time) error { periodic.Add(1); return errors.New("synthetic") }},
|
||||
},
|
||||
Step: func(_ context.Context, at time.Time) error { steps <- at; return nil },
|
||||
Report: func(stage string, _ error) { reports <- stage },
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- loop.Run(ctx) }()
|
||||
receive(t, clock.created)
|
||||
wake <- struct{}{}
|
||||
receive(t, steps)
|
||||
receive(t, clock.created)
|
||||
if awakened.Load() != 1 || once.Load() != 0 {
|
||||
t.Fatal("wake changed timer deadlines")
|
||||
}
|
||||
clock.advance(5 * time.Second)
|
||||
receive(t, steps)
|
||||
receive(t, clock.created)
|
||||
if once.Load() != 1 || periodic.Load() != 0 {
|
||||
t.Fatal("initial cadence")
|
||||
}
|
||||
clock.advance(time.Hour)
|
||||
receive(t, steps)
|
||||
receive(t, clock.created)
|
||||
if once.Load() != 1 || periodic.Load() != 1 {
|
||||
t.Fatal("catch-up storm", once.Load(), periodic.Load())
|
||||
}
|
||||
if receive(t, reports) != "periodic" {
|
||||
t.Fatal("hook failure missing")
|
||||
}
|
||||
close(wake)
|
||||
receive(t, clock.created) // closed wake channel must not spin
|
||||
clock.advance(10 * time.Second)
|
||||
receive(t, steps)
|
||||
receive(t, clock.created)
|
||||
if periodic.Load() != 2 {
|
||||
t.Fatal("periodic stopped after error")
|
||||
}
|
||||
cancel()
|
||||
if !errors.Is(receive(t, done), context.Canceled) {
|
||||
t.Fatal("loop did not stop")
|
||||
}
|
||||
clock.mu.Lock()
|
||||
defer clock.mu.Unlock()
|
||||
if len(clock.timers) != 0 {
|
||||
t.Fatal("timer leaked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoopRejectsInvalidHooks(t *testing.T) {
|
||||
for _, l := range []Loop{{}, {Every: time.Second, Step: func(context.Context, time.Time) error { return nil }, Triggers: []Trigger{{Name: "invalid"}}}} {
|
||||
if l.Run(context.Background()) == nil {
|
||||
t.Fatal("invalid loop accepted")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package dregarnuhr
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CanStart retains a single background worker under foreground load, avoiding
|
||||
// starvation by long-lived requests while protecting foreground capacity.
|
||||
func CanStart(s Settings, active int64, foreground bool) bool {
|
||||
return !s.Paused && active < int64(s.Concurrency) && (!foreground || active == 0)
|
||||
}
|
||||
|
||||
// RetryDelay doubles per completed attempt and saturates without overflowing.
|
||||
func RetryDelay(baseSeconds, attempt int) time.Duration {
|
||||
if baseSeconds < 1 {
|
||||
baseSeconds = 1
|
||||
}
|
||||
if baseSeconds >= 86400 {
|
||||
return 24 * time.Hour
|
||||
}
|
||||
delay := time.Duration(baseSeconds) * time.Second
|
||||
for i := 1; i < attempt; i++ {
|
||||
if delay >= 12*time.Hour {
|
||||
return 24 * time.Hour
|
||||
}
|
||||
delay *= 2
|
||||
}
|
||||
return delay
|
||||
}
|
||||
|
||||
type Completion struct {
|
||||
Status, Message, Level string
|
||||
RetryAt time.Time
|
||||
}
|
||||
|
||||
// Complete is independent of DB access. The repository persists this decision.
|
||||
func Complete(job Job, runErr error, baseSeconds int, now time.Time) Completion {
|
||||
out := Completion{Status: "succeeded", Level: "info"}
|
||||
if runErr == nil {
|
||||
return out
|
||||
}
|
||||
out.Message = runErr.Error()
|
||||
switch {
|
||||
case errors.Is(runErr, ErrSkipped):
|
||||
out.Status = "skipped"
|
||||
case job.CancelRequested:
|
||||
out.Status = "cancelled"
|
||||
case job.Attempts < job.MaxAttempts:
|
||||
out.Status = "queued"
|
||||
out.Level = "warning"
|
||||
out.RetryAt = now.Add(RetryDelay(baseSeconds, job.Attempts))
|
||||
default:
|
||||
out.Status = "failed"
|
||||
out.Level = "error"
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package dregarnuhr
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRetryDelay(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
base, attempt int
|
||||
want time.Duration
|
||||
}{
|
||||
{30, 1, 30 * time.Second}, {30, 2, time.Minute}, {30, 3, 2 * time.Minute},
|
||||
{30, 1000000, 24 * time.Hour}, {86401, 1, 24 * time.Hour}, {math.MaxInt, math.MaxInt, 24 * time.Hour}, {0, 1, time.Second},
|
||||
} {
|
||||
if got := RetryDelay(tc.base, tc.attempt); got != tc.want {
|
||||
t.Errorf("%+v got=%s", tc, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompletionPolicy(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
job Job
|
||||
err error
|
||||
status, level string
|
||||
}{
|
||||
{"success", Job{Attempts: 1, MaxAttempts: 3}, nil, "succeeded", "info"},
|
||||
{"retry", Job{Attempts: 2, MaxAttempts: 3}, errors.New("failed"), "queued", "warning"},
|
||||
{"exhausted", Job{Attempts: 3, MaxAttempts: 3}, errors.New("failed"), "failed", "error"},
|
||||
{"cancel", Job{Attempts: 1, MaxAttempts: 3, CancelRequested: true}, errors.New("cancelled"), "cancelled", "info"},
|
||||
{"skip-before-cancel", Job{Attempts: 1, MaxAttempts: 3, CancelRequested: true}, fmt.Errorf("wrapped: %w", ErrSkipped), "skipped", "info"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := Complete(tc.job, tc.err, 30, now)
|
||||
if got.Status != tc.status || got.Level != tc.level {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if tc.status == "queued" && !got.RetryAt.Equal(now.Add(time.Minute)) {
|
||||
t.Fatal(got)
|
||||
}
|
||||
if tc.status != "queued" && !got.RetryAt.IsZero() {
|
||||
t.Fatal("terminal job retried")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdmissionPolicy(t *testing.T) {
|
||||
s := Settings{Concurrency: 4}
|
||||
if !CanStart(s, 0, true) || CanStart(s, 1, true) || !CanStart(s, 3, false) || CanStart(s, 4, false) {
|
||||
t.Fatal("admission policy changed")
|
||||
}
|
||||
s.Paused = true
|
||||
if CanStart(s, 0, false) {
|
||||
t.Fatal("paused worker admitted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package dregarnuhr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Handler func(context.Context, *Job) error
|
||||
|
||||
// Registry holds existing event identifiers, not new provider or consumer IDs.
|
||||
type Registry struct {
|
||||
mu sync.RWMutex
|
||||
handlers map[string]Handler
|
||||
}
|
||||
|
||||
func (r *Registry) Register(kind string, handler Handler) error {
|
||||
if strings.TrimSpace(kind) == "" || handler == nil {
|
||||
return errors.New("event type and handler are required")
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.handlers == nil {
|
||||
r.handlers = make(map[string]Handler)
|
||||
}
|
||||
if _, ok := r.handlers[kind]; ok {
|
||||
return fmt.Errorf("duplicate event type %q", kind)
|
||||
}
|
||||
r.handlers[kind] = handler
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Registry) Run(ctx context.Context, job *Job) error {
|
||||
if job == nil {
|
||||
return errors.New("job is required")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
r.mu.RLock()
|
||||
handler := r.handlers[job.Type]
|
||||
r.mu.RUnlock()
|
||||
if handler == nil {
|
||||
return fmt.Errorf("unknown event type %q", job.Type)
|
||||
}
|
||||
return handler(ctx, job)
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package dregarnuhr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Queue owns atomic priority-ordered claims and persisted completion. ErrNoJob
|
||||
// means no eligible job. The initial adapter is single-writer-process, not HA.
|
||||
type Queue interface {
|
||||
Settings(context.Context) (Settings, error)
|
||||
Claim(context.Context, time.Time) (*Job, error)
|
||||
Cancellation(context.Context, string) (bool, error)
|
||||
Started(context.Context, *Job) error
|
||||
Finish(context.Context, *Job, error, time.Time) error
|
||||
}
|
||||
|
||||
type RunnerOptions struct {
|
||||
Clock Clock
|
||||
// Active is an optional compatibility counter for an existing monitor.
|
||||
Active *atomic.Int64
|
||||
Foreground func() bool
|
||||
Wake func()
|
||||
Report func(stage string, job *Job, err error)
|
||||
}
|
||||
|
||||
type Runner struct {
|
||||
queue Queue
|
||||
registry *Registry
|
||||
opts RunnerOptions
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func NewRunner(queue Queue, registry *Registry, opts RunnerOptions) *Runner {
|
||||
if opts.Clock == nil {
|
||||
opts.Clock = SystemClock{}
|
||||
}
|
||||
if opts.Active == nil {
|
||||
opts.Active = new(atomic.Int64)
|
||||
}
|
||||
return &Runner{queue: queue, registry: registry, opts: opts}
|
||||
}
|
||||
|
||||
func (r *Runner) report(stage string, job *Job, err error) {
|
||||
if err != nil && r.opts.Report != nil {
|
||||
r.opts.Report(stage, job, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) Running() int64 { return r.opts.Active.Load() }
|
||||
|
||||
// Dispatch serializes admission plus claim across concurrent callers. Each
|
||||
// invocation is bounded even if workers finish immediately and refill slots.
|
||||
func (r *Runner) Dispatch(ctx context.Context) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for n := 0; n < 32 && ctx.Err() == nil; n++ {
|
||||
settings, err := r.queue.Settings(ctx)
|
||||
if err != nil {
|
||||
r.report("settings", nil, err)
|
||||
return
|
||||
}
|
||||
foreground := r.opts.Foreground != nil && r.opts.Foreground()
|
||||
if !CanStart(settings, r.Running(), foreground) {
|
||||
return
|
||||
}
|
||||
job, err := r.queue.Claim(ctx, r.opts.Clock.Now())
|
||||
if errors.Is(err, ErrNoJob) || errors.Is(err, context.Canceled) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
r.report("claim", nil, err)
|
||||
return
|
||||
}
|
||||
if job == nil {
|
||||
r.report("claim", nil, errors.New("queue returned nil job"))
|
||||
return
|
||||
}
|
||||
r.opts.Active.Add(1)
|
||||
r.report("start-log", job, r.queue.Started(ctx, job))
|
||||
go r.execute(ctx, job)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) execute(ctx context.Context, job *Job) {
|
||||
defer func() {
|
||||
r.opts.Active.Add(-1)
|
||||
if r.opts.Wake != nil {
|
||||
r.opts.Wake()
|
||||
}
|
||||
}()
|
||||
runErr := r.invoke(ctx, job)
|
||||
// Completion must still be attempted when the worker context is cancelled.
|
||||
finishCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
r.report("finish", job, r.queue.Finish(finishCtx, job, runErr, r.opts.Clock.Now()))
|
||||
r.report("execute", job, runErr)
|
||||
}
|
||||
|
||||
func (r *Runner) invoke(ctx context.Context, job *Job) (err error) {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
err = errors.New("event handler panicked")
|
||||
}
|
||||
}()
|
||||
cancelled, err := r.queue.Cancellation(ctx, job.UUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cancelled {
|
||||
return errors.New("cancelled before execution")
|
||||
}
|
||||
if err = r.registry.Run(ctx, job); err != nil {
|
||||
return err
|
||||
}
|
||||
cancelled, err = r.queue.Cancellation(ctx, job.UUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cancelled {
|
||||
return errors.New("cancelled during execution")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package dregarnuhr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type finishedJob struct {
|
||||
id string
|
||||
err error
|
||||
at time.Time
|
||||
}
|
||||
type fakeQueue struct {
|
||||
mu sync.Mutex
|
||||
settings Settings
|
||||
jobs []Job
|
||||
claimed int
|
||||
cancelled bool
|
||||
cancelErr error
|
||||
finished chan finishedJob
|
||||
claimAt time.Time
|
||||
}
|
||||
|
||||
func newFakeQueue(concurrency int) *fakeQueue {
|
||||
q := &fakeQueue{settings: Settings{Concurrency: concurrency}, finished: make(chan finishedJob, 100)}
|
||||
for n := 0; n < 64; n++ {
|
||||
q.jobs = append(q.jobs, Job{UUID: fmt.Sprint(n), Type: "synthetic"})
|
||||
}
|
||||
return q
|
||||
}
|
||||
func (q *fakeQueue) Settings(context.Context) (Settings, error) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
return q.settings, nil
|
||||
}
|
||||
func (q *fakeQueue) Claim(_ context.Context, at time.Time) (*Job, error) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
if len(q.jobs) == 0 {
|
||||
return nil, ErrNoJob
|
||||
}
|
||||
j := q.jobs[0]
|
||||
q.jobs = q.jobs[1:]
|
||||
q.claimed++
|
||||
q.claimAt = at
|
||||
return &j, nil
|
||||
}
|
||||
func (q *fakeQueue) Cancellation(context.Context, string) (bool, error) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
return q.cancelled, q.cancelErr
|
||||
}
|
||||
func (q *fakeQueue) Started(context.Context, *Job) error { return nil }
|
||||
func (q *fakeQueue) Finish(ctx context.Context, j *Job, err error, at time.Time) error {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
q.finished <- finishedJob{j.UUID, err, at}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestRunnerConcurrentAdmission(t *testing.T) {
|
||||
for _, foreground := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(foreground), func(t *testing.T) {
|
||||
q := newFakeQueue(4)
|
||||
reg := &Registry{}
|
||||
block := make(chan struct{})
|
||||
started := make(chan struct{}, 4)
|
||||
wakes := make(chan struct{}, 4)
|
||||
clock := newFakeClock()
|
||||
defer close(block)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
if err := reg.Register("synthetic", func(ctx context.Context, _ *Job) error {
|
||||
started <- struct{}{}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-block:
|
||||
return nil
|
||||
}
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := NewRunner(q, reg, RunnerOptions{Clock: clock, Foreground: func() bool { return foreground }, Wake: func() { wakes <- struct{}{} }})
|
||||
var wg sync.WaitGroup
|
||||
for n := 0; n < 16; n++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); r.Dispatch(ctx) }()
|
||||
}
|
||||
wg.Wait()
|
||||
want := 4
|
||||
if foreground {
|
||||
want = 1
|
||||
}
|
||||
for n := 0; n < want; n++ {
|
||||
receive(t, started)
|
||||
}
|
||||
if r.Running() != int64(want) || q.claimed != want || !q.claimAt.Equal(clock.Now()) {
|
||||
t.Fatalf("active=%d claimed=%d want=%d", r.Running(), q.claimed, want)
|
||||
}
|
||||
cancel()
|
||||
for n := 0; n < want; n++ {
|
||||
receive(t, q.finished)
|
||||
receive(t, wakes)
|
||||
}
|
||||
if r.Running() != 0 {
|
||||
t.Fatal("worker slots leaked")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerOutcomesAndCancellation(t *testing.T) {
|
||||
for _, mode := range []string{"success", "cancel-before", "cancel-after", "cancellation-read-error", "panic", "unknown", "failure"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
q := newFakeQueue(1)
|
||||
q.jobs = q.jobs[:1]
|
||||
q.cancelled = mode == "cancel-before"
|
||||
if mode == "cancellation-read-error" {
|
||||
q.cancelErr = errors.New("database unavailable")
|
||||
}
|
||||
reg := &Registry{}
|
||||
called := false
|
||||
if mode != "unknown" {
|
||||
if err := reg.Register("synthetic", func(context.Context, *Job) error {
|
||||
called = true
|
||||
if mode == "panic" {
|
||||
panic("sensitive panic payload")
|
||||
}
|
||||
if mode == "failure" {
|
||||
return errors.New("handler failure")
|
||||
}
|
||||
if mode == "cancel-after" {
|
||||
q.mu.Lock()
|
||||
q.cancelled = true
|
||||
q.mu.Unlock()
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
wakes := make(chan struct{}, 1)
|
||||
clock := newFakeClock()
|
||||
r := NewRunner(q, reg, RunnerOptions{Clock: clock, Wake: func() { wakes <- struct{}{} }})
|
||||
r.Dispatch(context.Background())
|
||||
out := receive(t, q.finished)
|
||||
receive(t, wakes)
|
||||
if (out.err == nil) != (mode == "success") {
|
||||
t.Fatal(mode, out.err)
|
||||
}
|
||||
if (mode == "cancel-before" || mode == "cancellation-read-error") && called {
|
||||
t.Fatal("handler ran without cancellation check")
|
||||
}
|
||||
if mode == "panic" && out.err.Error() != "event handler panicked" {
|
||||
t.Fatal("panic payload leaked")
|
||||
}
|
||||
if r.Running() != 0 || !out.at.Equal(clock.Now()) {
|
||||
t.Fatal("completion/clock failure")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryAndPausedRunner(t *testing.T) {
|
||||
r := &Registry{}
|
||||
h := func(context.Context, *Job) error { return nil }
|
||||
if r.Register("", h) == nil || r.Register("nil", nil) == nil {
|
||||
t.Fatal("invalid registration")
|
||||
}
|
||||
if r.Register("synthetic", h) != nil || r.Register("synthetic", h) == nil {
|
||||
t.Fatal("duplicate replaced")
|
||||
}
|
||||
q := newFakeQueue(3)
|
||||
q.settings.Paused = true
|
||||
NewRunner(q, r, RunnerOptions{}).Dispatch(context.Background())
|
||||
if q.claimed != 0 {
|
||||
t.Fatal("paused claim")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
// Package dregarnuhr owns in-process scheduling, worker admission and retry
|
||||
// policy. Queue provides persistence; Registry provides business operations.
|
||||
// Atomic claims are not cross-instance expiring leases.
|
||||
package dregarnuhr
|
||||
|
||||
import "errors"
|
||||
|
||||
const Component = "dregarnuhr"
|
||||
|
||||
var (
|
||||
ErrNoJob = errors.New("no due job")
|
||||
ErrSkipped = errors.New("event is not applicable")
|
||||
)
|
||||
|
||||
type Job struct {
|
||||
UUID string `json:"uuid"`
|
||||
EventUUID string `json:"event_uuid,omitempty"`
|
||||
Type string `json:"event_type"`
|
||||
Key string `json:"event_key"`
|
||||
Payload string `json:"payload,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Priority int `json:"priority"`
|
||||
Attempts int `json:"attempts"`
|
||||
MaxAttempts int `json:"max_attempts"`
|
||||
RunAtMS int64 `json:"run_at_ms"`
|
||||
StartedAtMS int64 `json:"started_at_ms,omitempty"`
|
||||
FinishedAtMS int64 `json:"finished_at_ms,omitempty"`
|
||||
CancelRequested bool `json:"cancel_requested"`
|
||||
Error string `json:"error,omitempty"`
|
||||
CreatedAt int64 `json:"created_at_ms"`
|
||||
UpdatedAt int64 `json:"updated_at_ms"`
|
||||
}
|
||||
|
||||
type Settings struct {
|
||||
Paused bool `json:"paused"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
MaxDepth int `json:"max_depth"`
|
||||
RetryBaseSeconds int `json:"retry_base_seconds"`
|
||||
RetentionHours int `json:"retention_hours"`
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package geduldh
|
||||
|
||||
import (
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestComponentBoundary(t *testing.T) {
|
||||
if Component != "geduldh" {
|
||||
t.Fatal("component identity changed")
|
||||
}
|
||||
files, err := filepath.Glob("*.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, file := range files {
|
||||
if strings.HasSuffix(file, "_test.go") {
|
||||
continue
|
||||
}
|
||||
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, spec := range ast.Imports {
|
||||
path, err := strconv.Unquote(spec.Path.Value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"pageswiki/internal", "database", "net/http"} {
|
||||
if path == forbidden || strings.HasPrefix(path, forbidden+"/") {
|
||||
t.Errorf("%s imports %s: payload storage must not own metadata, HTTP, authorization or the compatibility facade", file, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Ordinary payload access cannot delete objects or make retention decisions.
|
||||
contract := reflect.TypeFor[Store]()
|
||||
var methods []string
|
||||
for i := 0; i < contract.NumMethod(); i++ {
|
||||
methods = append(methods, contract.Method(i).Name)
|
||||
}
|
||||
if want := []string{"Open", "PutIfAbsent", "Stat", "Verify"}; !reflect.DeepEqual(methods, want) {
|
||||
t.Fatalf("Document Store contract changed: %v", methods)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package geduldh
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Local requires hard-link no-replace publication and directory fsync support.
|
||||
// A mounted NFS/SMB/SSHFS share is not certified just because it has a path.
|
||||
// Failed persistence primitives return errors; there is no unsafe rename fallback.
|
||||
type Local struct{ root string }
|
||||
|
||||
var _ Store = (*Local)(nil)
|
||||
|
||||
// NewLocal has no I/O side effects. The root must be provisioned by the caller.
|
||||
func NewLocal(root string) (*Local, error) {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return nil, errors.New("Document root is required")
|
||||
}
|
||||
absolute, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Local{root: absolute}, nil
|
||||
}
|
||||
|
||||
func (s *Local) Open(ctx context.Context, hash string) (Reader, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := Key(hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root, err := os.OpenRoot(s.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer root.Close()
|
||||
file, err := root.Open(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
file.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, errors.New("Document is not a regular file")
|
||||
}
|
||||
return &localReader{file: file, ctx: ctx}, nil
|
||||
}
|
||||
|
||||
func (s *Local) Stat(ctx context.Context, hash string) (Info, error) {
|
||||
reader, err := s.Open(ctx, hash)
|
||||
if err != nil {
|
||||
return Info{}, err
|
||||
}
|
||||
defer reader.Close()
|
||||
info, err := reader.(*localReader).file.Stat()
|
||||
if err != nil {
|
||||
return Info{}, err
|
||||
}
|
||||
return Info{Hash: strings.ToLower(strings.TrimSpace(hash)), Size: info.Size()}, nil
|
||||
}
|
||||
|
||||
func verify(ctx context.Context, hash string, reader io.Reader) error {
|
||||
sum := sha256.New()
|
||||
if _, err := io.CopyBuffer(sum, contextReader{ctx, reader}, make([]byte, 64<<10)); err != nil {
|
||||
return err
|
||||
}
|
||||
if hex.EncodeToString(sum.Sum(nil)) != strings.ToLower(strings.TrimSpace(hash)) {
|
||||
return ErrHashMismatch
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func (s *Local) Verify(ctx context.Context, hash string) error {
|
||||
reader, err := s.Open(ctx, hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer reader.Close()
|
||||
return verify(ctx, hash, reader)
|
||||
}
|
||||
|
||||
func (s *Local) PutIfAbsent(ctx context.Context, hash string, size int64, input io.Reader) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := Key(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if size < 0 || size == math.MaxInt64 {
|
||||
return ErrSizeMismatch
|
||||
}
|
||||
if input == nil {
|
||||
return errors.New("Document reader is required")
|
||||
}
|
||||
root, err := os.OpenRoot(s.root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer root.Close()
|
||||
// The common retry/backfill path must not rewrite or stage existing blobs.
|
||||
// Still validate the supplied bytes, then verify the stored content.
|
||||
if existing, err := root.Open(key); err == nil {
|
||||
defer existing.Close()
|
||||
info, err := existing.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return errors.New("Document is not a regular file")
|
||||
}
|
||||
if err := copyValidated(ctx, hash, size, input, io.Discard); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verify(ctx, hash, existing); err != nil {
|
||||
return err
|
||||
}
|
||||
return syncPublishedDirectories(root, path.Dir(key))
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
dir := path.Dir(key)
|
||||
if err := root.MkdirAll(dir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
var nonce [16]byte
|
||||
if _, err := rand.Read(nonce[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := dir + "/.document-" + hex.EncodeToString(nonce[:])
|
||||
file, err := root.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer root.Remove(tmp)
|
||||
defer file.Close()
|
||||
if err := copyValidated(ctx, hash, size, input, file); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := file.Chmod(0644); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Link is atomic and fails if the destination exists. Rename would silently
|
||||
// replace an existing payload between an existence check and publication.
|
||||
if err := root.Link(tmp, key); err != nil {
|
||||
if !errors.Is(err, os.ErrExist) {
|
||||
return err
|
||||
}
|
||||
existing, err := root.Open(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer existing.Close()
|
||||
info, err := existing.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return errors.New("Document is not a regular file")
|
||||
}
|
||||
if err := verify(ctx, hash, existing); err != nil {
|
||||
return fmt.Errorf("existing Document: %w", err)
|
||||
}
|
||||
}
|
||||
// Persist the new entry and newly created shard directories before allowing
|
||||
// a metadata transaction to reference the payload. Errors are not swallowed.
|
||||
if err := syncPublishedDirectories(root, dir); err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func syncPublishedDirectories(root *os.Root, shard string) error {
|
||||
for _, name := range []string{shard, "sha256", "."} {
|
||||
directory, err := root.Open(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = directory.Sync()
|
||||
closeErr := directory.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyValidated(ctx context.Context, hash string, size int64, input io.Reader, output io.Writer) error {
|
||||
sum := sha256.New()
|
||||
n, err := io.CopyBuffer(io.MultiWriter(output, sum), contextReader{ctx, io.LimitReader(input, size+1)}, make([]byte, 64<<10))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return ErrSizeMismatch
|
||||
}
|
||||
if hex.EncodeToString(sum.Sum(nil)) != strings.ToLower(strings.TrimSpace(hash)) {
|
||||
return ErrHashMismatch
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
type contextReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func (r contextReader) Read(p []byte) (int, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.reader.Read(p)
|
||||
}
|
||||
|
||||
type localReader struct {
|
||||
file *os.File
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (r *localReader) Close() error { return r.file.Close() }
|
||||
|
||||
func (r *localReader) Read(p []byte) (int, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.file.Read(p)
|
||||
}
|
||||
func (r *localReader) ReadAt(p []byte, offset int64) (int, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.file.ReadAt(p, offset)
|
||||
}
|
||||
func (r *localReader) Seek(offset int64, whence int) (int64, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.file.Seek(offset, whence)
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package geduldh
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func digest(data []byte) string { sum := sha256.Sum256(data); return hex.EncodeToString(sum[:]) }
|
||||
func local(t *testing.T) (*Local, string) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
s, err := NewLocal(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s, root
|
||||
}
|
||||
|
||||
func TestLocalRoundTripAndRange(t *testing.T) {
|
||||
for _, payload := range [][]byte{nil, []byte("text 日本語"), {0, 0xff, 1, 0, 3}} {
|
||||
s, _ := local(t)
|
||||
ctx := context.Background()
|
||||
hash := digest(payload)
|
||||
if err := s.PutIfAbsent(ctx, hash, int64(len(payload)), bytes.NewReader(payload)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := s.Stat(ctx, hash)
|
||||
if err != nil || info.Hash != hash || info.Size != int64(len(payload)) {
|
||||
t.Fatalf("info=%+v err=%v", info, err)
|
||||
}
|
||||
reader, err := s.Open(ctx, hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil || !bytes.Equal(data, payload) {
|
||||
t.Fatalf("data=%v err=%v", data, err)
|
||||
}
|
||||
if size, err := reader.Seek(0, io.SeekEnd); err != nil || size != int64(len(payload)) {
|
||||
t.Fatalf("size=%d err=%v", size, err)
|
||||
}
|
||||
if len(payload) > 2 {
|
||||
buf := make([]byte, 2)
|
||||
if _, err := reader.ReadAt(buf, 1); err != nil || !bytes.Equal(buf, payload[1:3]) {
|
||||
t.Fatalf("range=%v err=%v", buf, err)
|
||||
}
|
||||
}
|
||||
reader.Close()
|
||||
if err := s.Verify(ctx, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalRejectsIncompleteOrCorruptInput(t *testing.T) {
|
||||
payload := []byte("good")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
data []byte
|
||||
size int64
|
||||
want error
|
||||
}{
|
||||
{"hash", []byte("evil"), 4, ErrHashMismatch},
|
||||
{"short", []byte("goo"), 4, ErrSizeMismatch},
|
||||
{"long", []byte("good!"), 4, ErrSizeMismatch},
|
||||
{"negative", payload, -1, ErrSizeMismatch},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s, root := local(t)
|
||||
err := s.PutIfAbsent(context.Background(), digest(payload), tc.size, bytes.NewReader(tc.data))
|
||||
if !errors.Is(err, tc.want) {
|
||||
t.Fatalf("got %v want %v", err, tc.want)
|
||||
}
|
||||
if _, err := s.Stat(context.Background(), digest(payload)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("published bad content: %v", err)
|
||||
}
|
||||
assertNoTemps(t, root)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalExistingAndConcurrentWritesNeverReplace(t *testing.T) {
|
||||
s, root := local(t)
|
||||
payload := []byte("immutable payload")
|
||||
hash := digest(payload)
|
||||
ctx := context.Background()
|
||||
var wg sync.WaitGroup
|
||||
errorsCh := make(chan error, 12)
|
||||
for range 12 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
errorsCh <- s.PutIfAbsent(ctx, hash, int64(len(payload)), bytes.NewReader(payload))
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errorsCh)
|
||||
for err := range errorsCh {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
key, _ := Key(hash)
|
||||
filename := filepath.Join(root, filepath.FromSlash(key))
|
||||
before, _ := os.Stat(filename)
|
||||
if err := s.PutIfAbsent(ctx, hash, int64(len(payload)), bytes.NewReader(payload)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, _ := os.Stat(filename)
|
||||
if !os.SameFile(before, after) {
|
||||
t.Fatal("existing inode replaced")
|
||||
}
|
||||
if err := os.WriteFile(filename, []byte("corrupt"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Verify(ctx, hash); !errors.Is(err, ErrHashMismatch) {
|
||||
t.Fatalf("verify=%v", err)
|
||||
}
|
||||
if err := s.PutIfAbsent(ctx, hash, int64(len(payload)), bytes.NewReader(payload)); !errors.Is(err, ErrHashMismatch) {
|
||||
t.Fatalf("put=%v", err)
|
||||
}
|
||||
content, _ := os.ReadFile(filename)
|
||||
if string(content) != "corrupt" {
|
||||
t.Fatal("corruption silently overwritten")
|
||||
}
|
||||
assertNoTemps(t, root)
|
||||
}
|
||||
|
||||
func TestLocalCancellationAndReaderFailure(t *testing.T) {
|
||||
s, root := local(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
hash := digest([]byte("data"))
|
||||
if err := s.PutIfAbsent(ctx, hash, 4, strings.NewReader("data")); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Open(ctx, hash); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failure := errors.New("source failed")
|
||||
if err := s.PutIfAbsent(context.Background(), hash, 4, failingReader{failure}); !errors.Is(err, failure) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Stat(context.Background(), hash); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.PutIfAbsent(context.Background(), hash, 4, strings.NewReader("data")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
r, err := s.Open(ctx, hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
cancel()
|
||||
if _, err := r.Read(make([]byte, 1)); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.ReadAt(make([]byte, 1), 0); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertNoTemps(t, root)
|
||||
}
|
||||
|
||||
type failingReader struct{ err error }
|
||||
|
||||
func (r failingReader) Read([]byte) (int, error) { return 0, r.err }
|
||||
|
||||
func TestLocalStreamsWithBoundedReadsAndMidStreamCancellation(t *testing.T) {
|
||||
s, root := local(t)
|
||||
payload := bytes.Repeat([]byte("large document\x00"), 100000)
|
||||
hash := digest(payload)
|
||||
input := &boundedReader{reader: bytes.NewReader(payload)}
|
||||
if err := s.PutIfAbsent(context.Background(), hash, int64(len(payload)), input); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if input.reads < 2 {
|
||||
t.Fatal("expected multiple bounded reads")
|
||||
}
|
||||
// Cancellation while staging must never publish the final content address.
|
||||
s2, root2 := local(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
input = &boundedReader{reader: bytes.NewReader(payload), cancel: cancel}
|
||||
if err := s2.PutIfAbsent(ctx, hash, int64(len(payload)), input); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("put=%v", err)
|
||||
}
|
||||
if _, err := s2.Stat(context.Background(), hash); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("published canceled upload: %v", err)
|
||||
}
|
||||
assertNoTemps(t, root)
|
||||
assertNoTemps(t, root2)
|
||||
}
|
||||
|
||||
type boundedReader struct {
|
||||
reader io.Reader
|
||||
reads int
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (r *boundedReader) Read(p []byte) (int, error) {
|
||||
if len(p) > 64<<10 {
|
||||
return 0, errors.New("unbounded input read")
|
||||
}
|
||||
r.reads++
|
||||
n, err := r.reader.Read(p)
|
||||
if r.cancel != nil {
|
||||
r.cancel()
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func TestLocalKeysAndSymlinkEscape(t *testing.T) {
|
||||
for _, hash := range []string{"", "../../etc/passwd", strings.Repeat("g", 64), strings.Repeat("a", 63)} {
|
||||
if _, err := Key(hash); !errors.Is(err, ErrInvalidHash) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
hash := digest([]byte("data"))
|
||||
key, err := Key(" " + strings.ToUpper(hash) + " ")
|
||||
if err != nil || key != "sha256/"+hash[:2]+"/"+hash {
|
||||
t.Fatalf("key=%q err=%v", key, err)
|
||||
}
|
||||
s, root := local(t)
|
||||
outside := t.TempDir()
|
||||
if err := os.Symlink(outside, filepath.Join(root, "sha256")); err != nil {
|
||||
t.Skip(err)
|
||||
}
|
||||
if err := s.PutIfAbsent(context.Background(), hash, 4, strings.NewReader("data")); err == nil {
|
||||
t.Fatal("escaped storage root")
|
||||
}
|
||||
if _, err := s.Open(context.Background(), hash); err == nil {
|
||||
t.Fatal("opened escaped path")
|
||||
}
|
||||
entries, err := os.ReadDir(outside)
|
||||
if err != nil || len(entries) != 0 {
|
||||
t.Fatalf("outside modified: %v %v", entries, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoTemps(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.HasPrefix(entry.Name(), ".document-") {
|
||||
t.Errorf("temporary file left: %s", path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
package geduldh
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrMaterializeLimit = errors.New("Document materialization capacity exceeded")
|
||||
|
||||
// Source deliberately excludes mutation. Authorization precedes Acquire.
|
||||
type Source interface {
|
||||
Stat(context.Context, string) (Info, error)
|
||||
Open(context.Context, string) (Reader, error)
|
||||
}
|
||||
|
||||
type MaterializeOptions struct {
|
||||
// TempDir must be node-local scratch, never canonical storage. Empty uses TMPDIR.
|
||||
TempDir string
|
||||
MaxBytes int64
|
||||
Concurrent int
|
||||
// MaxEntries bounds queued/leased objects too, including empty payloads.
|
||||
// Zero selects 64.
|
||||
MaxEntries int
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Materializer shares in-flight/leased copies only. The last lease removes the
|
||||
// copy; it is not a persistent cache and needs no idle cache eviction goroutine.
|
||||
// Limits include both copies being downloaded and copies in use by tools.
|
||||
type Materializer struct {
|
||||
source Source
|
||||
options MaterializeOptions
|
||||
slots chan struct{}
|
||||
mu sync.Mutex
|
||||
entries map[string]*materializedEntry
|
||||
reserved int64
|
||||
active int
|
||||
}
|
||||
|
||||
type materializedEntry struct {
|
||||
hash string
|
||||
size int64
|
||||
refs int
|
||||
ready chan struct{}
|
||||
cancel context.CancelFunc
|
||||
finished bool
|
||||
dir, path string
|
||||
err error
|
||||
}
|
||||
|
||||
// Lease pins an immutable scratch copy until Close. Tools must finish before
|
||||
// Close; never give its path to a detached process. Close is idempotent.
|
||||
type Lease struct {
|
||||
Path string
|
||||
Size int64
|
||||
once sync.Once
|
||||
release func()
|
||||
}
|
||||
|
||||
func (l *Lease) Close() { l.once.Do(l.release) }
|
||||
|
||||
func NewMaterializer(source Source, options MaterializeOptions) (*Materializer, error) {
|
||||
if source == nil || options.MaxBytes <= 0 || options.Concurrent <= 0 || options.Timeout <= 0 {
|
||||
return nil, errors.New("invalid Document materializer configuration")
|
||||
}
|
||||
if options.MaxEntries == 0 {
|
||||
options.MaxEntries = 64
|
||||
}
|
||||
if options.MaxEntries < 0 {
|
||||
return nil, errors.New("invalid materializer entry limit")
|
||||
}
|
||||
return &Materializer{source: source, options: options, slots: make(chan struct{}, options.Concurrent), entries: make(map[string]*materializedEntry)}, nil
|
||||
}
|
||||
|
||||
func (m *Materializer) Acquire(ctx context.Context, hash string, maxBytes int64) (*Lease, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := Key(hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hash = strings.ToLower(strings.TrimSpace(hash))
|
||||
info, err := m.source.Stat(ctx, hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.Size < 0 || maxBytes < 0 || info.Size > maxBytes || info.Size > m.options.MaxBytes {
|
||||
return nil, ErrMaterializeLimit
|
||||
}
|
||||
m.mu.Lock()
|
||||
e := m.entries[hash]
|
||||
if e != nil && e.size != info.Size {
|
||||
m.mu.Unlock()
|
||||
return nil, ErrSizeMismatch
|
||||
}
|
||||
if e == nil {
|
||||
if info.Size > m.options.MaxBytes-m.reserved || m.active >= m.options.MaxEntries {
|
||||
m.mu.Unlock()
|
||||
return nil, ErrMaterializeLimit
|
||||
}
|
||||
// One caller leaving must not cancel another caller's transfer. The
|
||||
// shared deadline is propagated to backend operations. Backends must
|
||||
// honor cancellation; it cannot interrupt an uninterruptible OS syscall.
|
||||
loadCtx, cancel := context.WithTimeout(context.Background(), m.options.Timeout)
|
||||
e = &materializedEntry{hash: hash, size: info.Size, ready: make(chan struct{}), cancel: cancel}
|
||||
m.entries[hash] = e
|
||||
m.reserved += info.Size
|
||||
m.active++
|
||||
go m.load(loadCtx, e)
|
||||
}
|
||||
e.refs++
|
||||
m.mu.Unlock()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
m.release(e)
|
||||
return nil, ctx.Err()
|
||||
case <-e.ready:
|
||||
if err := ctx.Err(); err != nil {
|
||||
m.release(e)
|
||||
return nil, err
|
||||
}
|
||||
if e.err != nil {
|
||||
m.release(e)
|
||||
return nil, e.err
|
||||
}
|
||||
return &Lease{Path: e.path, Size: e.size, release: func() { m.release(e) }}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Materializer) load(ctx context.Context, e *materializedEntry) {
|
||||
defer e.cancel()
|
||||
var dir, filename string
|
||||
var err error
|
||||
select {
|
||||
case m.slots <- struct{}{}:
|
||||
dir, filename, err = m.copy(ctx, e.hash, e.size)
|
||||
<-m.slots
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
}
|
||||
m.mu.Lock()
|
||||
e.dir, e.path, e.err, e.finished = dir, filename, err, true
|
||||
close(e.ready)
|
||||
if e.refs == 0 {
|
||||
m.remove(e)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *Materializer) release(e *materializedEntry) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
e.refs--
|
||||
if e.refs == 0 {
|
||||
e.cancel()
|
||||
// A later caller must not join an already-cancelled download.
|
||||
if m.entries[e.hash] == e {
|
||||
delete(m.entries, e.hash)
|
||||
}
|
||||
if e.finished {
|
||||
m.remove(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Materializer) remove(e *materializedEntry) {
|
||||
if m.entries[e.hash] == e {
|
||||
delete(m.entries, e.hash)
|
||||
}
|
||||
// Failed removal keeps the byte reservation: do not pretend disk space
|
||||
// was reclaimed. Recovery of crash leftovers is an operator/scratch task.
|
||||
if e.dir != "" {
|
||||
if err := os.RemoveAll(e.dir); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
m.reserved -= e.size
|
||||
m.active--
|
||||
}
|
||||
|
||||
func (m *Materializer) copy(ctx context.Context, hash string, size int64) (dir, filename string, err error) {
|
||||
reader, err := m.source.Open(ctx, hash)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer reader.Close()
|
||||
dir, err = os.MkdirTemp(m.options.TempDir, "pageswiki-document-*")
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
// Failed partial copies are retained in dir until load performs cleanup.
|
||||
filename = filepath.Join(dir, hash)
|
||||
file, err := os.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return dir, "", err
|
||||
}
|
||||
err = copyValidated(ctx, hash, size, reader, file)
|
||||
if err == nil {
|
||||
err = file.Chmod(0400)
|
||||
}
|
||||
closeErr := file.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
return dir, "", fmt.Errorf("materialize Document: %w", err)
|
||||
}
|
||||
// Scratch is reproducible, not durable canonical storage: no fsync here.
|
||||
return dir, filename, ctx.Err()
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
package geduldh
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type materializeSource struct {
|
||||
body []byte
|
||||
size int64
|
||||
gate chan struct{}
|
||||
started chan struct{}
|
||||
opens atomic.Int64
|
||||
}
|
||||
|
||||
func (s *materializeSource) Stat(ctx context.Context, hash string) (Info, error) {
|
||||
return Info{Hash: hash, Size: s.size}, ctx.Err()
|
||||
}
|
||||
func (s *materializeSource) Open(ctx context.Context, hash string) (Reader, error) {
|
||||
s.opens.Add(1)
|
||||
if s.started != nil {
|
||||
s.started <- struct{}{}
|
||||
}
|
||||
if s.gate != nil {
|
||||
select {
|
||||
case <-s.gate:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
return materializeReader{bytes.NewReader(s.body)}, ctx.Err()
|
||||
}
|
||||
|
||||
type materializeReader struct{ *bytes.Reader }
|
||||
|
||||
func (materializeReader) Close() error { return nil }
|
||||
|
||||
func newTestMaterializer(t *testing.T, source Source, quota int64) (*Materializer, string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
m, err := NewMaterializer(source, MaterializeOptions{TempDir: dir, MaxBytes: quota, Concurrent: 1, Timeout: 5 * time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, dir
|
||||
}
|
||||
|
||||
func waitMaterializer(t *testing.T, m *Materializer, predicate func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
m.mu.Lock()
|
||||
ok := predicate()
|
||||
m.mu.Unlock()
|
||||
if ok {
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
t.Fatal("materializer condition did not become true")
|
||||
}
|
||||
|
||||
func TestMaterializerSharesPinnedCopiesAndCleansLastLease(t *testing.T) {
|
||||
body := []byte("materialized 日本語")
|
||||
s := &materializeSource{body: body, size: int64(len(body))}
|
||||
m, root := newTestMaterializer(t, s, 1024)
|
||||
a, err := m.Acquire(context.Background(), digest(body), 1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := m.Acquire(context.Background(), digest(body), 1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.Path != b.Path || s.opens.Load() != 1 {
|
||||
t.Fatal("copy was not shared")
|
||||
}
|
||||
data, err := os.ReadFile(a.Path)
|
||||
if err != nil || !bytes.Equal(data, body) {
|
||||
t.Fatalf("body=%q err=%v", data, err)
|
||||
}
|
||||
info, _ := os.Stat(a.Path)
|
||||
if info.Mode().Perm() != 0400 {
|
||||
t.Fatalf("permissions=%v", info.Mode())
|
||||
}
|
||||
dirInfo, _ := os.Stat(filepath.Dir(a.Path))
|
||||
if dirInfo.Mode().Perm() != 0700 {
|
||||
t.Fatalf("directory permissions=%v", dirInfo.Mode())
|
||||
}
|
||||
a.Close()
|
||||
a.Close()
|
||||
if _, err := os.Stat(b.Path); err != nil {
|
||||
t.Fatal("active lease lost its file", err)
|
||||
}
|
||||
b.Close()
|
||||
items, _ := os.ReadDir(root)
|
||||
if len(items) != 0 || m.reserved != 0 || m.active != 0 {
|
||||
t.Fatal("scratch was not reclaimed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializerCoalescesDownloadsAndCancelsOnlyLastWaiter(t *testing.T) {
|
||||
body := []byte("shared")
|
||||
s := &materializeSource{body: body, size: int64(len(body)), gate: make(chan struct{}), started: make(chan struct{}, 2)}
|
||||
m, _ := newTestMaterializer(t, s, 1024)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
errA := make(chan error, 1)
|
||||
go func() {
|
||||
lease, err := m.Acquire(ctx, digest(body), 1024)
|
||||
if lease != nil {
|
||||
lease.Close()
|
||||
}
|
||||
errA <- err
|
||||
}()
|
||||
<-s.started
|
||||
leaseB := make(chan *Lease, 1)
|
||||
errB := make(chan error, 1)
|
||||
go func() {
|
||||
lease, err := m.Acquire(context.Background(), digest(body), 1024)
|
||||
leaseB <- lease
|
||||
errB <- err
|
||||
}()
|
||||
waitMaterializer(t, m, func() bool { return m.entries[digest(body)].refs == 2 })
|
||||
cancel()
|
||||
if err := <-errA; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("first waiter: %v", err)
|
||||
}
|
||||
close(s.gate)
|
||||
b := <-leaseB
|
||||
if err := <-errB; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.opens.Load() != 1 {
|
||||
t.Fatal("concurrent transfer not coalesced")
|
||||
}
|
||||
b.Close()
|
||||
|
||||
s.gate = make(chan struct{})
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
go func() {
|
||||
lease, err := m.Acquire(ctx, digest(body), 1024)
|
||||
if lease != nil {
|
||||
lease.Close()
|
||||
}
|
||||
errA <- err
|
||||
}()
|
||||
<-s.started
|
||||
cancel()
|
||||
if err := <-errA; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("last waiter: %v", err)
|
||||
}
|
||||
waitMaterializer(t, m, func() bool { return m.reserved == 0 && m.active == 0 })
|
||||
}
|
||||
|
||||
func TestMaterializerRejectsCorruptionSizeAndCapacity(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body []byte
|
||||
size int64
|
||||
hash string
|
||||
want error
|
||||
}{
|
||||
{"hash", []byte("bad"), 3, digest([]byte("yes")), ErrHashMismatch},
|
||||
{"short", []byte("x"), 2, digest([]byte("x")), ErrSizeMismatch},
|
||||
{"long", []byte("xyz"), 2, digest([]byte("xyz")), ErrSizeMismatch},
|
||||
{"oversize", []byte("xyz"), 3, digest([]byte("xyz")), ErrMaterializeLimit},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := &materializeSource{body: tc.body, size: tc.size}
|
||||
m, root := newTestMaterializer(t, s, 10)
|
||||
limit := int64(10)
|
||||
if tc.name == "oversize" {
|
||||
limit = 2
|
||||
}
|
||||
lease, err := m.Acquire(context.Background(), tc.hash, limit)
|
||||
if lease != nil || !errors.Is(err, tc.want) {
|
||||
t.Fatalf("lease=%v err=%v", lease, err)
|
||||
}
|
||||
items, _ := os.ReadDir(root)
|
||||
if len(items) != 0 || m.reserved != 0 {
|
||||
t.Fatal("failed input leaked scratch")
|
||||
}
|
||||
})
|
||||
}
|
||||
s := &materializeSource{body: []byte("x"), size: 1}
|
||||
m, _ := newTestMaterializer(t, s, 1)
|
||||
lease, err := m.Acquire(context.Background(), digest(s.body), 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer lease.Close()
|
||||
if _, err := m.Acquire(context.Background(), digest([]byte("y")), 1); !errors.Is(err, ErrMaterializeLimit) {
|
||||
t.Fatalf("quota: %v", err)
|
||||
}
|
||||
if s.opens.Load() != 1 {
|
||||
t.Fatal("over-quota transfer started")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializerTimeoutAndEntryBound(t *testing.T) {
|
||||
s := &materializeSource{body: []byte("x"), size: 1, gate: make(chan struct{}), started: make(chan struct{}, 3)}
|
||||
m, err := NewMaterializer(s, MaterializeOptions{TempDir: t.TempDir(), MaxBytes: 100, Concurrent: 1, MaxEntries: 1, Timeout: 50 * time.Millisecond})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := m.Acquire(context.Background(), digest(s.body), 10); done <- err }()
|
||||
<-s.started
|
||||
if _, err := m.Acquire(context.Background(), digest([]byte("y")), 10); !errors.Is(err, ErrMaterializeLimit) {
|
||||
t.Fatalf("entry bound: %v", err)
|
||||
}
|
||||
if err := <-done; !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("timeout: %v", err)
|
||||
}
|
||||
waitMaterializer(t, m, func() bool { return m.active == 0 })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := m.Acquire(ctx, digest(s.body), 10); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializerConcurrencyBoundAndCanonicalIsolation(t *testing.T) {
|
||||
s := &materializeSource{body: []byte("x"), size: 1, gate: make(chan struct{}), started: make(chan struct{}, 3)}
|
||||
m, _ := newTestMaterializer(t, s, 100)
|
||||
done := make(chan error, 2)
|
||||
for _, data := range []string{"x", "y"} {
|
||||
go func(data string) {
|
||||
lease, err := m.Acquire(context.Background(), digest([]byte(data)), 100)
|
||||
if lease != nil {
|
||||
lease.Close()
|
||||
}
|
||||
done <- err
|
||||
}(data)
|
||||
}
|
||||
<-s.started
|
||||
waitMaterializer(t, m, func() bool { return m.active == 2 })
|
||||
if s.opens.Load() != 1 {
|
||||
t.Fatal("concurrency limit exceeded")
|
||||
}
|
||||
close(s.gate)
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := <-done; err != nil && !errors.Is(err, ErrHashMismatch) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
waitMaterializer(t, m, func() bool { return m.active == 0 })
|
||||
|
||||
canonical, root := local(t)
|
||||
body := []byte("canonical")
|
||||
hash := digest(body)
|
||||
if err := canonical.PutIfAbsent(context.Background(), hash, int64(len(body)), bytes.NewReader(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, _ = newTestMaterializer(t, canonical, 100)
|
||||
lease, err := m.Acquire(context.Background(), hash, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer lease.Close()
|
||||
key, _ := Key(hash)
|
||||
sourceInfo, _ := os.Stat(filepath.Join(root, filepath.FromSlash(key)))
|
||||
tempInfo, _ := os.Stat(lease.Path)
|
||||
if os.SameFile(sourceInfo, tempInfo) {
|
||||
t.Fatal("canonical inode exposed to external tools")
|
||||
}
|
||||
if err := os.Chmod(lease.Path, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(lease.Path, []byte("modified!"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := canonical.Verify(context.Background(), hash); err != nil {
|
||||
t.Fatal("scratch mutation affected canonical data", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
// Package geduldh stores immutable, content-addressed Document payloads.
|
||||
// Authorization and reference/retention decisions belong to the application,
|
||||
// not to a storage backend. A hash or storage key is never an access token.
|
||||
package geduldh
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const Component = "geduldh"
|
||||
|
||||
var (
|
||||
ErrInvalidHash = errors.New("invalid Document SHA-256")
|
||||
ErrHashMismatch = errors.New("Document payload does not match SHA-256")
|
||||
ErrSizeMismatch = errors.New("Document payload does not match declared size")
|
||||
)
|
||||
|
||||
type Info struct {
|
||||
Hash string
|
||||
Size int64
|
||||
}
|
||||
|
||||
// Reader supports bounded and seekable access, including HTTP Range and ZIP.
|
||||
// A remote backend must not download the entire object merely to open it.
|
||||
type Reader interface {
|
||||
io.ReadSeekCloser
|
||||
io.ReaderAt
|
||||
}
|
||||
|
||||
// Store is the immutable payload boundary, not a filesystem abstraction.
|
||||
// PutIfAbsent must verify the size and SHA-256 before publishing a complete
|
||||
// object, never replace an existing object, and reject corrupt existing bytes.
|
||||
// A successful write must precede the metadata transaction which references it.
|
||||
// Delete is deliberately absent: GC requires reference and retention protection.
|
||||
type Store interface {
|
||||
Stat(context.Context, string) (Info, error)
|
||||
Open(context.Context, string) (Reader, error)
|
||||
PutIfAbsent(context.Context, string, int64, io.Reader) error
|
||||
Verify(context.Context, string) error
|
||||
}
|
||||
|
||||
// Key preserves the existing on-disk layout and is also suitable for S3 keys.
|
||||
// Backend placement never changes Page/Revision identity or public URLs.
|
||||
func Key(hash string) (string, error) {
|
||||
hash = strings.ToLower(strings.TrimSpace(hash))
|
||||
if len(hash) != sha256.Size*2 {
|
||||
return "", ErrInvalidHash
|
||||
}
|
||||
if _, err := hex.DecodeString(hash); err != nil {
|
||||
return "", ErrInvalidHash
|
||||
}
|
||||
return "sha256/" + hash[:2] + "/" + hash, nil
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package grammaratur
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// HTMLToMarkdownAuthoring keeps source HTML when conversion loses structure,
|
||||
// attributes, comments or text. This is not a sanitization boundary.
|
||||
func HTMLToMarkdownAuthoring(source string) (string, error) {
|
||||
converted, err := HTMLToMarkdown(source)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if htmlStructure(source) != htmlStructure(MarkdownToHTML(converted)) {
|
||||
return rawHTMLBlock(source), nil
|
||||
}
|
||||
return converted, nil
|
||||
}
|
||||
|
||||
// {=html} is an explicit authoring fence, not an ordinary code sample. The
|
||||
// closing delimiter is longer than any run in the source, so arbitrary HTML
|
||||
// (including Markdown examples) cannot accidentally terminate it.
|
||||
func rawHTMLBlock(source string) string {
|
||||
longest, run := 2, 0
|
||||
for _, c := range source {
|
||||
if c == '`' {
|
||||
run++
|
||||
if run > longest {
|
||||
longest = run
|
||||
}
|
||||
} else {
|
||||
run = 0
|
||||
}
|
||||
}
|
||||
fence := strings.Repeat("`", longest+1)
|
||||
return fence + "{=html}\n" + source + "\n" + fence
|
||||
}
|
||||
|
||||
// MarkdownAuthoringHTML preserves protected HTML byte-for-byte while rendering
|
||||
// ordinary Markdown. Callers must still apply projection sanitization for UI.
|
||||
func MarkdownAuthoringHTML(source string) string {
|
||||
lines := strings.SplitAfter(source, "\n")
|
||||
var markdown strings.Builder
|
||||
type rawBlock struct{ token, html string }
|
||||
var blocks []rawBlock
|
||||
var ordinary byte
|
||||
ordinaryLen := 0
|
||||
for i := 0; i < len(lines); i++ {
|
||||
line := strings.TrimSuffix(strings.TrimSuffix(lines[i], "\n"), "\r")
|
||||
char, n := byte(0), 0
|
||||
if len(line) > 0 && (line[0] == '`' || line[0] == '~') {
|
||||
char = line[0]
|
||||
for n < len(line) && line[n] == char {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if ordinary != 0 {
|
||||
markdown.WriteString(lines[i])
|
||||
if char == ordinary && n >= ordinaryLen && strings.TrimSpace(line[n:]) == "" {
|
||||
ordinary = 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
if char == '`' && n >= 3 && strings.TrimSpace(line[n:]) == "{=html}" {
|
||||
end := i + 1
|
||||
for ; end < len(lines); end++ {
|
||||
close := strings.TrimSpace(lines[end])
|
||||
if len(close) >= n && strings.Trim(close, "`") == "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if end < len(lines) {
|
||||
raw := strings.TrimSuffix(strings.Join(lines[i+1:end], ""), "\n")
|
||||
token := "PAGESWIKIRAW" + strings.ReplaceAll(uuid.NewString(), "-", "")
|
||||
blocks = append(blocks, rawBlock{token, raw})
|
||||
markdown.WriteString("\n\n" + token + "\n\n")
|
||||
i = end
|
||||
continue
|
||||
}
|
||||
}
|
||||
if n >= 3 {
|
||||
ordinary, ordinaryLen = char, n
|
||||
}
|
||||
markdown.WriteString(lines[i])
|
||||
}
|
||||
if len(blocks) == 1 && strings.TrimSpace(markdown.String()) == blocks[0].token {
|
||||
return blocks[0].html
|
||||
}
|
||||
out := MarkdownToHTML(markdown.String())
|
||||
for _, block := range blocks {
|
||||
out = strings.Replace(out, "<p>"+block.token+"</p>", block.html, 1)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func htmlStructure(source string) string {
|
||||
n, err := html.Parse(strings.NewReader(source))
|
||||
if err != nil {
|
||||
return source
|
||||
}
|
||||
var out bytes.Buffer
|
||||
var visit func(*html.Node, bool)
|
||||
visit = func(n *html.Node, literal bool) {
|
||||
literal = literal || n.Data == "pre" || n.Data == "code" || n.Data == "script" || n.Data == "style"
|
||||
data := n.Data
|
||||
if n.Type == html.TextNode && !literal {
|
||||
if strings.TrimSpace(data) == "" && (n.Parent == nil || n.Parent.Data == "body" || n.Parent.Data == "html") {
|
||||
return
|
||||
}
|
||||
// Inline whitespace can separate words (including around an inline
|
||||
// element). Never normalize it away when deciding conversion safety.
|
||||
}
|
||||
attrs := append([]html.Attribute(nil), n.Attr...)
|
||||
sort.Slice(attrs, func(i, j int) bool { return attrs[i].Namespace+":"+attrs[i].Key < attrs[j].Namespace+":"+attrs[j].Key })
|
||||
b, _ := json.Marshal([]any{n.Type, n.Namespace, data, attrs})
|
||||
out.Write(b)
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
visit(c, literal)
|
||||
}
|
||||
out.WriteByte(')')
|
||||
}
|
||||
visit(n, false)
|
||||
return out.String()
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package grammaratur
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAuthoringRoundTrip(t *testing.T) {
|
||||
for _, source := range []string{
|
||||
"<h1>Heading</h1><p>Hello <strong>world</strong>.</p>",
|
||||
"<p>{{children:/guide}}</p>",
|
||||
"<figure data-extra=\"kept\"><img src=\"/photo\" alt=\"Photo\"><figcaption>A caption</figcaption></figure>",
|
||||
"<!-- keep comment --><p id=\"anchor\" class=\"custom\">text</p>",
|
||||
"<table><tr><td rowspan=\"2\">a</td><td>b</td></tr><tr><td>c</td></tr></table>",
|
||||
"<p>a <em>b</em> c</p>",
|
||||
"<pre><code class=\"language-python\"> print('x')\n</code></pre>",
|
||||
"<custom-element data-x=\"1\">opaque</custom-element>",
|
||||
"<div class=\"custom\">\n<p>one</p>\n\n<p>two</p>\n</div>",
|
||||
"<script>const x = '<b>untouched</b>';</script>",
|
||||
} {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
md, err := HTMLToMarkdownAuthoring(source)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if htmlStructure(MarkdownAuthoringHTML(md)) != htmlStructure(source) {
|
||||
t.Fatalf("lossy conversion: %q", md)
|
||||
}
|
||||
})
|
||||
}
|
||||
md, err := HTMLToMarkdownAuthoring("<h1>Hello</h1>")
|
||||
if err != nil || md != "# Hello" {
|
||||
t.Fatalf("simple Markdown conversion: %q %v", md, err)
|
||||
}
|
||||
for _, source := range []string{`<p data-custom="yes">Hello</p>`, `<!-- saved --><h1>Title</h1>`, `<p>a <em>b</em> c</p>`} {
|
||||
md, err := HTMLToMarkdownAuthoring(source)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if htmlStructure(MarkdownAuthoringHTML(md)) != htmlStructure(source) {
|
||||
t.Fatal("whitespace/attributes lost")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthoringProtectedHTML(t *testing.T) {
|
||||
for _, raw := range []string{"<figure data-x='a'>a</figure>", "<pre>```\n{=html}\n````</pre>\n", "<script>const raw = `x`</script>", "<p>line\r\n</p>"} {
|
||||
md := rawHTMLBlock(raw)
|
||||
if got := MarkdownAuthoringHTML(md); got != raw {
|
||||
t.Fatalf("raw bytes changed: %q want %q", got, raw)
|
||||
}
|
||||
mixed := MarkdownAuthoringHTML("# Added\n\n" + md + "\n\nTail")
|
||||
if !strings.Contains(mixed, raw) || !strings.Contains(mixed, "<h1>Added</h1>") || !strings.Contains(mixed, "<p>Tail</p>") {
|
||||
t.Fatalf("mixed blocks: %q", mixed)
|
||||
}
|
||||
}
|
||||
code := "````text\n```{=html}\n<script>unsafe()</script>\n```\n````"
|
||||
if strings.Contains(MarkdownAuthoringHTML(code), "<script>") {
|
||||
t.Fatal("HTML escaped out of ordinary code sample")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthoringWhitespaceComparisonIsConservative(t *testing.T) {
|
||||
if htmlStructure("<p>a <em>b</em> c</p>") == htmlStructure("<p>a<em>b</em>c</p>") {
|
||||
t.Fatal("word boundaries ignored")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package grammaratur
|
||||
|
||||
import (
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestComponentBoundary(t *testing.T) {
|
||||
if Component != "grammaratur" {
|
||||
t.Fatal("component identity changed")
|
||||
}
|
||||
files, err := filepath.Glob("*.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, file := range files {
|
||||
if strings.HasSuffix(file, "_test.go") {
|
||||
continue
|
||||
}
|
||||
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, spec := range ast.Imports {
|
||||
path, err := strconv.Unquote(spec.Path.Value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"pageswiki/internal/", "database/", "net", "os"} {
|
||||
if path == forbidden || strings.HasPrefix(path, strings.TrimSuffix(forbidden, "/")+"/") {
|
||||
t.Errorf("%s imports %s: Grammaratur must not own I/O, persistence or application policy", file, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package grammaratur_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
"github.com/gomarkdown/markdown"
|
||||
"pageswiki/internal/compiler"
|
||||
"pageswiki/internal/grammaratur"
|
||||
"pageswiki/internal/render"
|
||||
)
|
||||
|
||||
func TestLegacyMarkdownCompatibility(t *testing.T) {
|
||||
for _, source := range []string{
|
||||
"", "# Heading\n\n本文", "{{children:/guide}}", "- one\n- two\n",
|
||||
"```python\nprint('hello')\n```", "[link](/path)\n\n",
|
||||
"<p data-custom='keep'>raw HTML</p>",
|
||||
"```{=html}\n<figure data-x='1'>source</figure>\n```",
|
||||
} {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
want := string(markdown.ToHTML([]byte(source), nil, nil))
|
||||
if got := grammaratur.MarkdownToHTML(source); got != want {
|
||||
t.Fatalf("dialect changed: %q want %q", got, want)
|
||||
}
|
||||
if got := render.Markdown(source); got != want {
|
||||
t.Fatalf("render entry changed: %q want %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyHTMLAndAuthoringCompatibility(t *testing.T) {
|
||||
for _, source := range []string{"", "<h1>Heading</h1>", "<p>{{children}}</p>", "<!--retain--><figure data-custom='keep'><figcaption>Caption</figcaption></figure>", "<script>never execute</script>"} {
|
||||
want, err := htmltomarkdown.ConvertString(source)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, convert := range []func(string) (string, error){grammaratur.HTMLToMarkdown, compiler.HTMLToMarkdown} {
|
||||
got, err := convert(source)
|
||||
if err != nil || got != want {
|
||||
t.Fatalf("ordinary conversion changed: %q %v", got, err)
|
||||
}
|
||||
}
|
||||
protected, err := grammaratur.HTMLToMarkdownAuthoring(source)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compat, err := compiler.HTMLToMarkdownAuthoring(source)
|
||||
if err != nil || protected != compat {
|
||||
t.Fatal("authoring facade changed", err)
|
||||
}
|
||||
if compiler.MarkdownAuthoringHTML(compat) != grammaratur.MarkdownAuthoringHTML(protected) {
|
||||
t.Fatal("protected render facade changed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthoringIsExplicitAndNotASanitizer(t *testing.T) {
|
||||
const source = "```{=html}\n<script>untrusted()</script>\n```"
|
||||
if strings.Contains(grammaratur.MarkdownToHTML(source), "<script>") {
|
||||
t.Fatal("ordinary conversion opted into authoring")
|
||||
}
|
||||
if got := grammaratur.MarkdownAuthoringHTML(source); got != "<script>untrusted()</script>" {
|
||||
t.Fatalf("authoring removed source: %q", got)
|
||||
}
|
||||
// Authorization, macro expansion and sanitization stay in the projection
|
||||
// layer. Grammaratur returns source, not HTML trusted for a browser sink.
|
||||
if got := grammaratur.MarkdownToHTML("{{children}}"); !strings.Contains(got, "{{children}}") {
|
||||
t.Fatal("macro unexpectedly expanded")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
// Package grammaratur owns document-format conversion and authoring fidelity.
|
||||
//
|
||||
// Ordinary conversion and protected authoring conversion are distinct contracts:
|
||||
// callers must opt into {=html} fences. Neither path authorizes content, expands
|
||||
// macros, sanitizes a display projection, persists revisions, or fetches URLs.
|
||||
// Those responsibilities remain with the caller. In particular, preserving
|
||||
// source HTML never grants permission to execute its scripts or event handlers.
|
||||
//
|
||||
// Grammaratur is an in-process component, not a service or a new public API.
|
||||
// The name denotes the words/format-conversion responsibility in PagesWiki's
|
||||
// architecture; see docs/design/ja/component-names.md.
|
||||
package grammaratur
|
||||
|
||||
// Component is the stable component label for diagnostics and instrumentation.
|
||||
// It is not a replacement for API routes, provider IDs or persisted settings.
|
||||
const Component = "grammaratur"
|
||||
@@ -0,0 +1,18 @@
|
||||
package grammaratur
|
||||
|
||||
import (
|
||||
htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
"github.com/gomarkdown/markdown"
|
||||
)
|
||||
|
||||
// HTMLToMarkdown is the existing ordinary conversion, used by integrations
|
||||
// which do not understand protected authoring blocks. It can be lossy.
|
||||
func HTMLToMarkdown(source string) (string, error) {
|
||||
return htmltomarkdown.ConvertString(source)
|
||||
}
|
||||
|
||||
// MarkdownToHTML preserves the existing Markdown dialect and renderer options.
|
||||
// It neither interprets authoring fences nor sanitizes the returned HTML.
|
||||
func MarkdownToHTML(source string) string {
|
||||
return string(markdown.ToHTML([]byte(source), nil, nil))
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
// Package kunstzeal selects projections and executes bounded rendering providers.
|
||||
// Authentication, source provenance and canonical storage remain caller-owned.
|
||||
package kunstzeal
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Delivery string
|
||||
|
||||
const (
|
||||
DeliveryHTML Delivery = "html"
|
||||
DeliveryText Delivery = "text"
|
||||
DeliveryRawMedia Delivery = "raw-media"
|
||||
DeliveryRawFrame Delivery = "raw-frame"
|
||||
DeliveryDownload Delivery = "download"
|
||||
)
|
||||
|
||||
type Descriptor struct {
|
||||
MediaType string
|
||||
Filename string
|
||||
SizeBytes int64
|
||||
}
|
||||
|
||||
type Adapter struct {
|
||||
ID string
|
||||
Kind string
|
||||
Delivery Delivery
|
||||
NeedsText bool
|
||||
WindowableText bool
|
||||
Match func(Descriptor) bool
|
||||
}
|
||||
|
||||
const InlineTextWindowThreshold = int64(2 << 20)
|
||||
|
||||
var projectionAdapters = []Adapter{
|
||||
{
|
||||
ID: "html",
|
||||
Kind: "html",
|
||||
Delivery: DeliveryHTML,
|
||||
Match: mediaTypes("text/html"),
|
||||
},
|
||||
{
|
||||
ID: "excalidraw",
|
||||
Kind: "excalidraw",
|
||||
Delivery: DeliveryText,
|
||||
NeedsText: true,
|
||||
WindowableText: true,
|
||||
Match: extensions(".excalidraw"),
|
||||
},
|
||||
{
|
||||
ID: "csv",
|
||||
Kind: "csv",
|
||||
Delivery: DeliveryText,
|
||||
NeedsText: true,
|
||||
WindowableText: true,
|
||||
Match: matchAny(
|
||||
mediaTypes("text/csv", "application/csv"),
|
||||
extensions(".csv"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "archive",
|
||||
Kind: "archive",
|
||||
Delivery: DeliveryDownload,
|
||||
Match: matchAny(
|
||||
mediaTypes(
|
||||
"application/zip", "application/x-zip-compressed",
|
||||
"application/x-rar-compressed", "application/vnd.rar",
|
||||
"application/x-7z-compressed", "application/x-tar",
|
||||
"application/gzip", "application/x-xz",
|
||||
),
|
||||
extensions(".zip", ".rar", ".7z", ".tar", ".gz", ".tgz", ".xz", ".txz"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "debian-package",
|
||||
Kind: "metadata",
|
||||
Delivery: DeliveryDownload,
|
||||
Match: matchAny(
|
||||
mediaTypes(
|
||||
"application/vnd.debian.binary-package",
|
||||
"application/x-debian-package",
|
||||
),
|
||||
extensions(".deb"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "eml",
|
||||
Kind: "email",
|
||||
Delivery: DeliveryText,
|
||||
Match: matchAny(
|
||||
mediaTypes("message/rfc822"),
|
||||
extensions(".eml"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "pe",
|
||||
Kind: "metadata",
|
||||
Delivery: DeliveryDownload,
|
||||
Match: extensions(".exe", ".dll", ".sys"),
|
||||
},
|
||||
{
|
||||
ID: "json",
|
||||
Kind: "json",
|
||||
Delivery: DeliveryText,
|
||||
WindowableText: true,
|
||||
Match: matchAny(mediaTypes("application/json", "application/ld+json"), extensions(".json", ".jsonl")),
|
||||
},
|
||||
{
|
||||
ID: "text",
|
||||
Kind: "code",
|
||||
Delivery: DeliveryText,
|
||||
NeedsText: true,
|
||||
WindowableText: true,
|
||||
Match: textMediaType,
|
||||
},
|
||||
{
|
||||
ID: "image",
|
||||
Kind: "image",
|
||||
Delivery: DeliveryRawMedia,
|
||||
Match: matchAny(
|
||||
mediaTypePrefix("image/"),
|
||||
extensions(".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif", ".svg"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "audio",
|
||||
Kind: "audio",
|
||||
Delivery: DeliveryRawMedia,
|
||||
Match: matchAny(
|
||||
mediaTypePrefix("audio/"),
|
||||
extensions(".mp3", ".flac", ".wav", ".ogg", ".m4a", ".aac", ".opus"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "video",
|
||||
Kind: "video",
|
||||
Delivery: DeliveryRawMedia,
|
||||
Match: matchAny(
|
||||
mediaTypePrefix("video/"),
|
||||
extensions(".mp4", ".webm", ".mkv", ".mov", ".m4v", ".ogv"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "pdf",
|
||||
Kind: "pdf",
|
||||
Delivery: DeliveryRawFrame,
|
||||
Match: matchAny(
|
||||
mediaTypes("application/pdf"),
|
||||
extensions(".pdf"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "download",
|
||||
Kind: "download",
|
||||
Delivery: DeliveryDownload,
|
||||
Match: func(Descriptor) bool { return true },
|
||||
},
|
||||
}
|
||||
|
||||
func SelectAdapter(descriptor Descriptor) Adapter {
|
||||
for _, adapter := range projectionAdapters {
|
||||
if adapter.Match(descriptor) {
|
||||
return adapter
|
||||
}
|
||||
}
|
||||
return projectionAdapters[len(projectionAdapters)-1]
|
||||
}
|
||||
|
||||
func mediaTypes(types ...string) func(Descriptor) bool {
|
||||
return func(descriptor Descriptor) bool {
|
||||
for _, candidate := range types {
|
||||
if descriptor.MediaType == candidate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func mediaTypePrefix(prefix string) func(Descriptor) bool {
|
||||
return func(descriptor Descriptor) bool {
|
||||
return strings.HasPrefix(descriptor.MediaType, prefix)
|
||||
}
|
||||
}
|
||||
|
||||
func extensions(values ...string) func(Descriptor) bool {
|
||||
return func(descriptor Descriptor) bool {
|
||||
extension := strings.ToLower(filepath.Ext(descriptor.Filename))
|
||||
for _, candidate := range values {
|
||||
if extension == candidate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func matchAny(matchers ...func(Descriptor) bool) func(Descriptor) bool {
|
||||
return func(descriptor Descriptor) bool {
|
||||
for _, matcher := range matchers {
|
||||
if matcher(descriptor) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func textMediaType(descriptor Descriptor) bool {
|
||||
switch descriptor.MediaType {
|
||||
case "application/json",
|
||||
"application/xml",
|
||||
"application/javascript",
|
||||
"application/x-javascript",
|
||||
"application/x-sh",
|
||||
"application/x-shellscript",
|
||||
"application/x-yaml",
|
||||
"application/yaml",
|
||||
"application/toml":
|
||||
return true
|
||||
default:
|
||||
return strings.HasPrefix(descriptor.MediaType, "text/")
|
||||
}
|
||||
}
|
||||
|
||||
// Catalog returns a copy; callers cannot reorder or disable the shared registry.
|
||||
func Catalog() []Adapter { return append([]Adapter(nil), projectionAdapters...) }
|
||||
@@ -0,0 +1,105 @@
|
||||
package kunstzeal
|
||||
|
||||
import (
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAdapterSelectionAndOrdering(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
mime, filename, id, kind string
|
||||
delivery Delivery
|
||||
needsText, window bool
|
||||
}{
|
||||
{"text/html", "scene.excalidraw", "html", "html", DeliveryHTML, false, false},
|
||||
{"application/json", "scene.EXCALIDRAW", "excalidraw", "excalidraw", DeliveryText, true, true},
|
||||
{"text/plain", "data.CSV", "csv", "csv", DeliveryText, true, true},
|
||||
{"application/zip", "archive", "archive", "archive", DeliveryDownload, false, false},
|
||||
{"application/octet-stream", "package.deb", "debian-package", "metadata", DeliveryDownload, false, false},
|
||||
{"text/plain", "mail.eml", "eml", "email", DeliveryText, false, false},
|
||||
{"application/octet-stream", "program.exe", "pe", "metadata", DeliveryDownload, false, false},
|
||||
{"application/json", "data.json", "json", "json", DeliveryText, false, true},
|
||||
{"text/x-python", "code.py", "text", "code", DeliveryText, true, true},
|
||||
{"image/png", "image", "image", "image", DeliveryRawMedia, false, false},
|
||||
{"audio/flac", "music", "audio", "audio", DeliveryRawMedia, false, false},
|
||||
{"video/mp4", "movie", "video", "video", DeliveryRawMedia, false, false},
|
||||
{"application/pdf", "manual", "pdf", "pdf", DeliveryRawFrame, false, false},
|
||||
{"application/octet-stream", "unknown", "download", "download", DeliveryDownload, false, false},
|
||||
} {
|
||||
got := SelectAdapter(Descriptor{MediaType: test.mime, Filename: test.filename, SizeBytes: InlineTextWindowThreshold + 1})
|
||||
if got.ID != test.id || got.Kind != test.kind || got.Delivery != test.delivery || got.NeedsText != test.needsText || got.WindowableText != test.window {
|
||||
t.Fatalf("%s/%s: %+v", test.mime, test.filename, got)
|
||||
}
|
||||
}
|
||||
if got := SelectAdapter(Descriptor{}); got.ID != "download" {
|
||||
t.Fatal(got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdapterCatalogIsCopyAndConcurrent(t *testing.T) {
|
||||
catalog := Catalog()
|
||||
ids := []string{}
|
||||
for _, a := range catalog {
|
||||
ids = append(ids, a.ID)
|
||||
}
|
||||
if strings.Join(ids, ",") != "html,excalidraw,csv,archive,debian-package,eml,pe,json,text,image,audio,video,pdf,download" {
|
||||
t.Fatal("catalog order changed", ids)
|
||||
}
|
||||
catalog[0].ID = "corrupt"
|
||||
catalog[0].Match = func(Descriptor) bool { return false }
|
||||
if Catalog()[0].ID != "html" || SelectAdapter(Descriptor{MediaType: "text/html"}).ID != "html" {
|
||||
t.Fatal("catalog mutation changed shared selection")
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := 0; j < 100; j++ {
|
||||
Catalog()[0].ID = "copy"
|
||||
if SelectAdapter(Descriptor{MediaType: "text/html"}).ID != "html" {
|
||||
t.Error("registry changed")
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestComponentBoundary(t *testing.T) {
|
||||
if Component != "kunstzeal" {
|
||||
t.Fatal("component identity changed")
|
||||
}
|
||||
files, err := filepath.Glob("*.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, file := range files {
|
||||
if strings.HasSuffix(file, "_test.go") {
|
||||
continue
|
||||
}
|
||||
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, spec := range ast.Imports {
|
||||
path, err := strconv.Unquote(spec.Path.Value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"pageswiki/internal", "database", "os/exec"} {
|
||||
if path == forbidden || strings.HasPrefix(path, forbidden+"/") {
|
||||
t.Errorf("%s imports %s: providers must not own application state or arbitrary process execution", file, path)
|
||||
}
|
||||
}
|
||||
if file == "adapters.go" && (strings.HasPrefix(path, "net") || path == "os") {
|
||||
t.Error("adapter selection acquired I/O")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package kunstzeal
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
var ErrDiagramCacheLimit = errors.New("diagram cache exceeds limit")
|
||||
|
||||
// ReadDiagramCache reads only a bounded derived artifact. filename must be
|
||||
// constructed by the application after validating kind/hash and authorizing
|
||||
// source provenance. A cache hit is not an access grant or a sanitized SVG.
|
||||
func ReadDiagramCache(filename string) ([]byte, error) {
|
||||
f, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
b, err := io.ReadAll(io.LimitReader(f, DiagramOutputLimit+1))
|
||||
if len(b) > DiagramOutputLimit {
|
||||
return nil, ErrDiagramCacheLimit
|
||||
}
|
||||
return b, err
|
||||
}
|
||||
|
||||
// WriteDiagramCache publishes a complete artifact with same-directory rename.
|
||||
// Concurrent readers observe the old or new file, never a truncated write.
|
||||
// This is disposable cache storage, not a durable canonical Document commit.
|
||||
func WriteDiagramCache(filename string, data []byte) error {
|
||||
if len(data) > DiagramOutputLimit {
|
||||
return ErrDiagramCacheLimit
|
||||
}
|
||||
dir := filepath.Dir(filename)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.CreateTemp(dir, ".diagram-*.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
defer f.Close()
|
||||
if _, err = f.Write(data); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = f.Chmod(0644); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(f.Name(), filename)
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package kunstzeal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDiagramCacheAtomicReplacement(t *testing.T) {
|
||||
filename := filepath.Join(t.TempDir(), "mermaid", "fixture.svg")
|
||||
a, b := bytes.Repeat([]byte("A"), 64<<10), bytes.Repeat([]byte("B"), 64<<10)
|
||||
if err := WriteDiagramCache(filename, a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
for n := 0; n < 20; n++ {
|
||||
if i%2 == 0 {
|
||||
data := a
|
||||
if n%2 == 0 {
|
||||
data = b
|
||||
}
|
||||
if err := WriteDiagramCache(filename, data); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
} else {
|
||||
data, err := ReadDiagramCache(filename)
|
||||
if err != nil || (!bytes.Equal(data, a) && !bytes.Equal(data, b)) {
|
||||
t.Errorf("partial cache: length=%d err=%v", len(data), err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
files, err := filepath.Glob(filepath.Join(filepath.Dir(filename), ".diagram-*.tmp"))
|
||||
if err != nil || len(files) != 0 {
|
||||
t.Fatal("temporary artifacts leaked", files, err)
|
||||
}
|
||||
info, err := os.Stat(filename)
|
||||
if err != nil || info.Mode().Perm() != 0644 {
|
||||
t.Fatal(info, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagramCacheBudgetsAndFailures(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
filename := filepath.Join(root, "fixture.svg")
|
||||
if err := WriteDiagramCache(filename, []byte("complete")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
large := bytes.Repeat([]byte("x"), DiagramOutputLimit+1)
|
||||
if err := WriteDiagramCache(filename, large); !errors.Is(err, ErrDiagramCacheLimit) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := ReadDiagramCache(filename)
|
||||
if err != nil || string(b) != "complete" {
|
||||
t.Fatal("failed write destroyed cache", err)
|
||||
}
|
||||
if err = os.WriteFile(filename, large, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b, err = ReadDiagramCache(filename); !errors.Is(err, ErrDiagramCacheLimit) || b != nil {
|
||||
t.Fatal("oversized cache returned", err)
|
||||
}
|
||||
if _, err = ReadDiagramCache(filepath.Join(root, "missing")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = WriteDiagramCache(root, []byte("cannot replace a directory")); err == nil {
|
||||
t.Fatal("rename failure hidden")
|
||||
}
|
||||
// Failed rename removes its temporary artifact too.
|
||||
files, err := filepath.Glob(filepath.Join(filepath.Dir(root), ".diagram-*.tmp"))
|
||||
if err != nil || len(files) != 0 {
|
||||
t.Fatal(files, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package kunstzeal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const Component = "kunstzeal"
|
||||
const DiagramSourceLimit = 1 << 20
|
||||
const DiagramOutputLimit = 4 << 20
|
||||
const DiagramInlineLimit = 32 << 10
|
||||
const DiagramTimeout = 15 * time.Second
|
||||
|
||||
var (
|
||||
ErrDiagramKind = errors.New("unsupported diagram type")
|
||||
ErrDiagramSource = errors.New("diagram source exceeds limit")
|
||||
ErrDiagramOutput = errors.New("diagram output unavailable or exceeds limit")
|
||||
ErrProviderConfig = errors.New("diagram renderer configuration invalid")
|
||||
ErrProviderUnavailable = errors.New("diagram renderer unavailable")
|
||||
ErrProviderFailed = errors.New("diagram renderer failed")
|
||||
)
|
||||
|
||||
// DiagramProvider does not authorize a request or trust its SVG output.
|
||||
// Callers must retain source access checks and the sandbox response policy.
|
||||
type DiagramProvider interface {
|
||||
Render(context.Context, string, string) ([]byte, error)
|
||||
}
|
||||
|
||||
func DiagramHash(kind, source, format string) string {
|
||||
sum := sha256.Sum256([]byte(kind + "\x00" + format + "\x00" + source))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func AllowedDiagramType(kind string) bool {
|
||||
switch kind {
|
||||
case "mermaid", "plantuml", "graphviz", "dot", "d2":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func NormalizeDiagramType(kind string) string {
|
||||
if kind == "dot" {
|
||||
return "graphviz"
|
||||
}
|
||||
return kind
|
||||
}
|
||||
|
||||
type HTTPClient interface {
|
||||
Do(*http.Request) (*http.Response, error)
|
||||
}
|
||||
|
||||
// Kroki receives an administrator-configured service address, never a URL from
|
||||
// document content. Private service-network endpoints are intentional here.
|
||||
// Client must honor context cancellation and reject redirects. No default
|
||||
// client is used, and provider failures never expose its URL or response body.
|
||||
type Kroki struct {
|
||||
Endpoint string
|
||||
Client HTTPClient
|
||||
}
|
||||
|
||||
func (k Kroki) Render(parent context.Context, kind, source string) ([]byte, error) {
|
||||
if err := parent.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kind = NormalizeDiagramType(kind)
|
||||
if !AllowedDiagramType(kind) {
|
||||
return nil, ErrDiagramKind
|
||||
}
|
||||
if len(source) > DiagramSourceLimit {
|
||||
return nil, ErrDiagramSource
|
||||
}
|
||||
u, err := url.Parse(k.Endpoint)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || k.Client == nil {
|
||||
return nil, ErrProviderConfig
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(parent, DiagramTimeout)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(k.Endpoint, "/")+"/"+kind+"/svg", strings.NewReader(source))
|
||||
if err != nil {
|
||||
return nil, ErrProviderConfig
|
||||
}
|
||||
req.Header.Set("Content-Type", "text/plain; charset=utf-8")
|
||||
resp, err := k.Client.Do(req)
|
||||
if err != nil {
|
||||
if resp != nil && resp.Body != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return nil, ErrProviderUnavailable
|
||||
}
|
||||
if resp == nil || resp.Body == nil {
|
||||
return nil, ErrProviderUnavailable
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, ErrProviderFailed
|
||||
}
|
||||
b, err := io.ReadAll(io.LimitReader(resp.Body, DiagramOutputLimit+1))
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if err != nil || len(b) > DiagramOutputLimit {
|
||||
return nil, ErrDiagramOutput
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package kunstzeal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var _ DiagramProvider = Kroki{}
|
||||
|
||||
type clientFixture func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f clientFixture) Do(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
type trackedReader struct {
|
||||
io.Reader
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (r *trackedReader) Close() error { r.closed = true; return nil }
|
||||
|
||||
func TestDiagramIdentityAndKinds(t *testing.T) {
|
||||
for _, kind := range []string{"mermaid", "plantuml", "graphviz", "dot", "d2"} {
|
||||
if !AllowedDiagramType(kind) {
|
||||
t.Fatal(kind)
|
||||
}
|
||||
for _, source := range []string{"", "graph LR; A-->B;", "日本語\x00source"} {
|
||||
sum := sha256.Sum256([]byte(kind + "\x00svg\x00" + source))
|
||||
if DiagramHash(kind, source, "svg") != hex.EncodeToString(sum[:]) {
|
||||
t.Fatal("cache identity changed")
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, kind := range []string{"", "nwdiag", "../mermaid", "Mermaid", "svg", "graphviz/svg"} {
|
||||
if AllowedDiagramType(kind) {
|
||||
t.Fatal("unknown kind", kind)
|
||||
}
|
||||
}
|
||||
if NormalizeDiagramType("dot") != "graphviz" || NormalizeDiagramType("mermaid") != "mermaid" {
|
||||
t.Fatal("normalization changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKrokiBoundedRequestAndOutput(t *testing.T) {
|
||||
for _, kind := range []string{"mermaid", "dot", "d2"} {
|
||||
body := &trackedReader{Reader: strings.NewReader("<svg>fixture</svg>")}
|
||||
k := Kroki{Endpoint: "http://renderer.example/prefix/", Client: clientFixture(func(r *http.Request) (*http.Response, error) {
|
||||
b, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.Method != "POST" || r.URL.String() != "http://renderer.example/prefix/"+NormalizeDiagramType(kind)+"/svg" || string(b) != "source" || r.Header.Get("Content-Type") != "text/plain; charset=utf-8" {
|
||||
t.Fatal("request contract changed")
|
||||
}
|
||||
if r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" {
|
||||
t.Fatal("credentials forwarded")
|
||||
}
|
||||
deadline, ok := r.Context().Deadline()
|
||||
if !ok || time.Until(deadline) > DiagramTimeout {
|
||||
t.Fatal("unbounded provider")
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Body: body}, nil
|
||||
})}
|
||||
b, err := k.Render(context.Background(), kind, "source")
|
||||
if err != nil || string(b) != "<svg>fixture</svg>" || !body.closed {
|
||||
t.Fatal(string(b), err, body.closed)
|
||||
}
|
||||
}
|
||||
for _, size := range []int{DiagramOutputLimit, DiagramOutputLimit + 1} {
|
||||
body := &trackedReader{Reader: strings.NewReader(strings.Repeat("x", size))}
|
||||
k := Kroki{Endpoint: "https://renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) { return &http.Response{StatusCode: 200, Body: body}, nil })}
|
||||
b, err := k.Render(context.Background(), "mermaid", strings.Repeat("x", DiagramSourceLimit))
|
||||
if size == DiagramOutputLimit {
|
||||
if err != nil || len(b) != size {
|
||||
t.Fatal(len(b), err)
|
||||
}
|
||||
} else if !errors.Is(err, ErrDiagramOutput) || b != nil {
|
||||
t.Fatal("oversized output escaped", err)
|
||||
}
|
||||
if !body.closed {
|
||||
t.Fatal("response leaked")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestKrokiFailsClosedBeforeIO(t *testing.T) {
|
||||
client := clientFixture(func(*http.Request) (*http.Response, error) { t.Fatal("invalid request performed I/O"); return nil, nil })
|
||||
for _, endpoint := range []string{"", "file:///etc/passwd", "https://user:secret@renderer.example", "https://renderer.example?token=secret", "https://renderer.example#fragment", "https://renderer.example?", "https:///path", "http://%zz"} {
|
||||
b, err := (Kroki{Endpoint: endpoint, Client: client}).Render(context.Background(), "mermaid", "source")
|
||||
if !errors.Is(err, ErrProviderConfig) || b != nil {
|
||||
t.Fatal("bad provider accepted", err)
|
||||
}
|
||||
}
|
||||
k := Kroki{Endpoint: "http://renderer.example", Client: client}
|
||||
if _, err := k.Render(context.Background(), "../../secret", "source"); !errors.Is(err, ErrDiagramKind) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Render(context.Background(), "mermaid", strings.Repeat("x", DiagramSourceLimit+1)); !errors.Is(err, ErrDiagramSource) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := (Kroki{Endpoint: k.Endpoint}).Render(context.Background(), "mermaid", "source"); !errors.Is(err, ErrProviderConfig) {
|
||||
t.Fatal("default HTTP client used", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := k.Render(ctx, "mermaid", "source"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKrokiFailuresAreOpaqueAndCloseBodies(t *testing.T) {
|
||||
for _, code := range []int{301, 302, 307, 308, 400, 401, 403, 404, 429, 500} {
|
||||
body := &trackedReader{Reader: strings.NewReader("PRIVATE SOURCE OR INTERNAL DIAGNOSTICS")}
|
||||
k := Kroki{Endpoint: "https://private-renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) { return &http.Response{StatusCode: code, Body: body}, nil })}
|
||||
b, err := k.Render(context.Background(), "mermaid", "PRIVATE SOURCE")
|
||||
if !errors.Is(err, ErrProviderFailed) || b != nil || !body.closed {
|
||||
t.Fatal(code, err, body.closed)
|
||||
}
|
||||
}
|
||||
for _, mode := range []string{"network", "nil-response", "nil-body", "read-error"} {
|
||||
k := Kroki{Endpoint: "https://private-renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) {
|
||||
switch mode {
|
||||
case "network":
|
||||
return nil, errors.New("https://private-renderer.example: secret")
|
||||
case "nil-response":
|
||||
return nil, nil
|
||||
case "nil-body":
|
||||
return &http.Response{StatusCode: 200}, nil
|
||||
default:
|
||||
return &http.Response{StatusCode: 200, Body: io.NopCloser(errorReader{})}, nil
|
||||
}
|
||||
})}
|
||||
b, err := k.Render(context.Background(), "mermaid", "PRIVATE SOURCE")
|
||||
if b != nil || err == nil || strings.Contains(err.Error(), "secret") || strings.Contains(err.Error(), "private-renderer") {
|
||||
t.Fatal(mode, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type errorReader struct{}
|
||||
|
||||
func (errorReader) Read([]byte) (int, error) { return 0, errors.New("secret body read failure") }
|
||||
|
||||
func TestKrokiCancellationDuringSendAndRead(t *testing.T) {
|
||||
for _, duringRead := range []bool{false, true} {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
body := &trackedReader{Reader: strings.NewReader("<svg/>")}
|
||||
k := Kroki{Endpoint: "https://renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) {
|
||||
cancel()
|
||||
if duringRead {
|
||||
return &http.Response{StatusCode: 200, Body: body}, nil
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Body: body}, errors.New("cancelled network")
|
||||
})}
|
||||
b, err := k.Render(ctx, "mermaid", "source")
|
||||
cancel()
|
||||
if !errors.Is(err, context.Canceled) || b != nil || !body.closed {
|
||||
t.Fatal(duringRead, err, body.closed)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package mestionora
|
||||
|
||||
import (
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestComponentBoundary(t *testing.T) {
|
||||
if Component != "mestionora" {
|
||||
t.Fatal("component identity changed")
|
||||
}
|
||||
files, err := filepath.Glob("*.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, file := range files {
|
||||
if strings.HasSuffix(file, "_test.go") {
|
||||
continue
|
||||
}
|
||||
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, spec := range ast.Imports {
|
||||
path, err := strconv.Unquote(spec.Path.Value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"pageswiki/internal", "net", "os"} {
|
||||
if path == forbidden || strings.HasPrefix(path, forbidden+"/") {
|
||||
t.Errorf("%s imports %s: search must not own HTTP, filesystem payloads or application policy", file, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, exists := reflect.TypeFor[*SQLite]().MethodByName("SearchPathsInDomain"); exists {
|
||||
t.Fatal("implicit allow-all belongs only in the legacy trusted-caller facade")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package mestionora
|
||||
|
||||
import (
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A display projection only: names, boundaries and manual titles remain stored
|
||||
// unchanged. Legacy generated Spots have no nested Spot semantics.
|
||||
func IsUnclassifiedCitySpot(name, boundary string) bool {
|
||||
return strings.HasSuffix(name, "-undefined") || strings.HasSuffix(boundary, "/_undefined")
|
||||
}
|
||||
|
||||
func CitySpotDisplayTitle(name, title, boundary, parentTitle string) string {
|
||||
if !IsUnclassifiedCitySpot(name, boundary) {
|
||||
if title != "" {
|
||||
return title
|
||||
}
|
||||
return name
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(title)) {
|
||||
case "", "undefined", "未定義", "未分類のpage", "unclassified pages", strings.ToLower(name):
|
||||
if parentTitle != "" {
|
||||
return parentTitle
|
||||
}
|
||||
if parent := path.Base(path.Dir(boundary)); parent != "/" && parent != "." {
|
||||
return parent
|
||||
}
|
||||
return "Spot"
|
||||
default:
|
||||
return title // A manual display title takes precedence.
|
||||
}
|
||||
}
|
||||
|
||||
func (r CitySearchResult) DisplayTitle() string {
|
||||
if r.Kind == "spot" {
|
||||
return CitySpotDisplayTitle(r.Name, r.Title, r.PathPrefix, r.StreetTitle)
|
||||
}
|
||||
if r.Title != "" {
|
||||
return r.Title
|
||||
}
|
||||
return r.Name
|
||||
}
|
||||
|
||||
func (r CitySearchResult) ContentPath() string {
|
||||
if r.NavigationPath != "" {
|
||||
return r.NavigationPath
|
||||
}
|
||||
return r.PathPrefix
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package mestionora
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCityContentExistenceUsesPathIndex(t *testing.T) {
|
||||
database := newSearchTestDB(t)
|
||||
for _, kind := range []string{"quartier", "street", "spot"} {
|
||||
rows, err := database.Query("EXPLAIN QUERY PLAN "+citySearchSQL(kind, ""), sql.Named("domain", "d"), sql.Named("scope", "/"), sql.Named("prefix", "/"), sql.Named("upper", "0"), sql.Named("query", "needle"), sql.Named("limit", 1000))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
indexed := false
|
||||
for rows.Next() {
|
||||
var id, parent, unused int
|
||||
var detail string
|
||||
if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(detail, "SEARCH cp USING") {
|
||||
indexed = true
|
||||
}
|
||||
if strings.Contains(detail, "SCAN cp") {
|
||||
t.Fatalf("unbounded Path lookup: %s", detail)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows.Close()
|
||||
if !indexed {
|
||||
t.Fatalf("no indexed portal lookup for %s", kind)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package mestionora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
CitySearchCandidateLimit = 1000 // Per locality kind, before authorization.
|
||||
CitySearchResultLimit = 30
|
||||
citySearchTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
// CitySearchResult describes a locality, not its contained Pages. Description
|
||||
// is a bounded plain-text excerpt around the match and is never trusted HTML.
|
||||
type CitySearchResult struct {
|
||||
UUID, Kind, Name, Title, Description, PathPrefix string
|
||||
QuartierTitle, StreetTitle, QuartierName string
|
||||
CreatedAtMS, UpdatedAtMS int64
|
||||
NameMatch, PathMatch, DescriptionMatch bool
|
||||
NavigationPath string
|
||||
HasPage bool
|
||||
}
|
||||
|
||||
type citySearchCandidate struct {
|
||||
CitySearchResult
|
||||
quartierPath, streetPath string
|
||||
}
|
||||
|
||||
// SearchCityInDomain searches locality metadata only. It does not construct a
|
||||
// CityMap, expand descendants, read Page bodies, or count hidden localities.
|
||||
// Scope selects a locality's own boundary, not an ancestor of the scope. Every
|
||||
// boundary in its parent chain must be readable before the result is exposed.
|
||||
func (s *SQLite) SearchCityInDomain(parent context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []CitySearchResult, searchErr error) {
|
||||
if err := options.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
query = strings.TrimSpace(query)
|
||||
if len(query) > 512 || len(strings.Fields(query)) > 16 {
|
||||
return nil, errors.New("search query is too long")
|
||||
}
|
||||
// File filters are intentionally not applied to locality metadata: a Spot
|
||||
// containing a PDF is not itself a PDF and must not claim to match one.
|
||||
if allowed == nil || options.FileType != "" || options.Extension != "" || (query == "" && !options.HasFilters()) {
|
||||
return nil, nil
|
||||
}
|
||||
if limit <= 0 || limit > CitySearchResultLimit {
|
||||
limit = CitySearchResultLimit
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(parent, citySearchTimeout)
|
||||
defer cancel()
|
||||
defer func() {
|
||||
if searchErr != nil && ctx.Err() != nil {
|
||||
searchErr = ctx.Err()
|
||||
}
|
||||
}()
|
||||
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
scope = normalizePath(scope)
|
||||
prefix, upper := strings.TrimRight(scope, "/")+"/", strings.TrimRight(scope, "/")+"0"
|
||||
args := []any{
|
||||
sql.Named("domain", domainUUID), sql.Named("scope", scope),
|
||||
sql.Named("prefix", prefix), sql.Named("upper", upper),
|
||||
sql.Named("query", query), sql.Named("limit", CitySearchCandidateLimit),
|
||||
}
|
||||
dateColumn := "n.updated_at_ms"
|
||||
if options.DateField == "created" {
|
||||
dateColumn = "n.created_at_ms"
|
||||
}
|
||||
dateWhere := ""
|
||||
if options.FromMS != nil {
|
||||
dateWhere += " AND " + dateColumn + ">=:from_ms"
|
||||
args = append(args, sql.Named("from_ms", *options.FromMS))
|
||||
}
|
||||
if options.BeforeMS != nil {
|
||||
dateWhere += " AND " + dateColumn + "<:before_ms"
|
||||
args = append(args, sql.Named("before_ms", *options.BeforeMS))
|
||||
}
|
||||
candidates := make([]citySearchCandidate, 0)
|
||||
for _, kind := range []string{"quartier", "street", "spot"} {
|
||||
rows, err := tx.QueryContext(ctx, citySearchSQL(kind, dateWhere), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var candidate citySearchCandidate
|
||||
candidate.Kind = kind
|
||||
if err := rows.Scan(&candidate.UUID, &candidate.Name, &candidate.Title, &candidate.Description,
|
||||
&candidate.PathPrefix, &candidate.CreatedAtMS, &candidate.UpdatedAtMS,
|
||||
&candidate.NameMatch, &candidate.PathMatch, &candidate.DescriptionMatch,
|
||||
&candidate.QuartierTitle, &candidate.StreetTitle, &candidate.quartierPath, &candidate.streetPath, &candidate.QuartierName,
|
||||
&candidate.NavigationPath, &candidate.HasPage); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
candidates = append(candidates, candidate)
|
||||
}
|
||||
err = rows.Err()
|
||||
closeErr := rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
}
|
||||
// Release the sole read connection before calling authorization, which may
|
||||
// itself need a database connection. HTTP normally passes its cached policy.
|
||||
if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) {
|
||||
return nil, err
|
||||
}
|
||||
sort.SliceStable(candidates, func(i, j int) bool {
|
||||
a, b := candidates[i], candidates[j]
|
||||
if a.Kind != b.Kind {
|
||||
return citySearchKindOrder(a.Kind) < citySearchKindOrder(b.Kind)
|
||||
}
|
||||
if a.NameMatch != b.NameMatch {
|
||||
return a.NameMatch
|
||||
}
|
||||
if a.PathMatch != b.PathMatch {
|
||||
return a.PathMatch
|
||||
}
|
||||
if a.PathPrefix != b.PathPrefix {
|
||||
return a.PathPrefix < b.PathPrefix
|
||||
}
|
||||
return a.UUID < b.UUID
|
||||
})
|
||||
readable := make(map[string]bool)
|
||||
canRead := func(path string) bool {
|
||||
if value, exists := readable[path]; exists {
|
||||
return value
|
||||
}
|
||||
value := allowed(path)
|
||||
readable[path] = value
|
||||
return value
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !canRead(candidate.PathPrefix) || (candidate.quartierPath != "" && !canRead(candidate.quartierPath)) || (candidate.streetPath != "" && !canRead(candidate.streetPath)) {
|
||||
continue
|
||||
}
|
||||
found = append(found, candidate.CitySearchResult)
|
||||
if len(found) == limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
func citySearchKindOrder(kind string) int {
|
||||
switch kind {
|
||||
case "quartier":
|
||||
return 0
|
||||
case "street":
|
||||
return 1
|
||||
default:
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
// All interpolated SQL fragments are package-owned constants. User input is
|
||||
// bound, so %, _, quotes and backslashes are literal substring characters.
|
||||
func citySearchSQL(kind, dateWhere string) string {
|
||||
from, description := "zones n", "n.description"
|
||||
navigationPath := "n.path_prefix"
|
||||
parentColumns := "'','','','',''"
|
||||
where := "n.kind='quartier'"
|
||||
switch kind {
|
||||
case "street":
|
||||
from += ` JOIN zones q ON q.uuid=n.parent_uuid AND q.domain_uuid=n.domain_uuid AND q.kind='quartier' AND q.state!='archived'`
|
||||
parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),'',q.path_prefix,'',q.name"
|
||||
where = "n.kind='street' AND " + citySearchAbsolutePathSQL("q.path_prefix")
|
||||
case "spot":
|
||||
navigationPath = `CASE WHEN substr(n.path_prefix,-11)='/_undefined' THEN st.path_prefix ELSE n.path_prefix END`
|
||||
from = `spots n JOIN street_spots ss ON ss.spot_uuid=n.uuid
|
||||
JOIN zones st ON st.uuid=ss.street_uuid AND st.domain_uuid=n.domain_uuid AND st.kind='street' AND st.state!='archived'
|
||||
JOIN zones q ON q.uuid=st.parent_uuid AND q.domain_uuid=st.domain_uuid AND q.kind='quartier' AND q.state!='archived'`
|
||||
description = "n.summary"
|
||||
parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),substr(COALESCE(NULLIF(st.title,''),st.name),1,256),q.path_prefix,st.path_prefix,q.name"
|
||||
where = citySearchAbsolutePathSQL("q.path_prefix") + " AND " + citySearchAbsolutePathSQL("st.path_prefix")
|
||||
}
|
||||
nameMatch := `(:query!='' AND (instr(lower(n.name),lower(:query))>0 OR instr(lower(n.title),lower(:query))>0))`
|
||||
pathMatch := `(:query!='' AND instr(lower(n.path_prefix),lower(:query))>0)`
|
||||
descriptionMatch := `(:query!='' AND instr(lower(` + description + `),lower(:query))>0)`
|
||||
return `SELECT n.uuid,substr(n.name,1,256),substr(n.title,1,256),
|
||||
substr(` + description + `,max(1,instr(lower(` + description + `),lower(:query))-160),512),
|
||||
n.path_prefix,n.created_at_ms,n.updated_at_ms,` + nameMatch + `,` + pathMatch + `,` + descriptionMatch + `,` + parentColumns + `,` + navigationPath + `,
|
||||
EXISTS(SELECT 1 FROM paths cp WHERE cp.domain_uuid=n.domain_uuid AND cp.path=(` + navigationPath + `))
|
||||
FROM ` + from + ` WHERE n.domain_uuid=:domain AND n.state!='archived' AND ` + where + `
|
||||
AND ` + citySearchAbsolutePathSQL("n.path_prefix") + `
|
||||
AND (n.path_prefix COLLATE BINARY=:scope OR (n.path_prefix COLLATE BINARY>=:prefix AND n.path_prefix COLLATE BINARY<:upper))
|
||||
AND (:query='' OR ` + nameMatch + ` OR ` + pathMatch + ` OR ` + descriptionMatch + `)` + dateWhere + `
|
||||
ORDER BY ` + nameMatch + ` DESC,` + pathMatch + ` DESC,n.path_prefix COLLATE BINARY,n.uuid LIMIT :limit`
|
||||
}
|
||||
|
||||
func citySearchAbsolutePathSQL(column string) string {
|
||||
// Keep authorization paths intact rather than truncating them. Malformed
|
||||
// legacy boundaries and impractically large metadata are not navigable hits.
|
||||
return `(substr(` + column + `,1,1)='/' AND length(` + column + `)<=4096 AND instr(` + column + `,char(0))=0)`
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package mestionora
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
)
|
||||
|
||||
func newSearchTestDB(t *testing.T) *sql.DB {
|
||||
t.Helper()
|
||||
database, err := db.Open(t.TempDir() + "/search.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
initSearchFixture(t, database)
|
||||
return database
|
||||
}
|
||||
|
||||
// Only the application columns/indexes consumed by search, not application
|
||||
// bootstrap, auth keys, job recovery or external calendar initialization.
|
||||
func initSearchFixture(t *testing.T, database *sql.DB) {
|
||||
t.Helper()
|
||||
_, err := database.Exec(`
|
||||
CREATE TABLE objects(uuid TEXT PRIMARY KEY, created_at INTEGER);
|
||||
CREATE TABLE revisions(uuid TEXT PRIMARY KEY,object_uuid TEXT,filename TEXT,body TEXT,content_type TEXT,document_hash TEXT,created_at_ms INTEGER,created_at INTEGER);
|
||||
CREATE TABLE paths(domain_uuid TEXT NOT NULL,path TEXT NOT NULL,object_uuid TEXT,PRIMARY KEY(domain_uuid,path));
|
||||
CREATE INDEX idx_paths_object_domain_page ON paths(object_uuid,domain_uuid,path);
|
||||
CREATE TABLE zones(uuid TEXT PRIMARY KEY,domain_uuid TEXT,kind TEXT,parent_uuid TEXT,name TEXT,title TEXT,description TEXT,path_prefix TEXT,state TEXT,created_at_ms INTEGER,updated_at_ms INTEGER);
|
||||
CREATE TABLE spots(uuid TEXT PRIMARY KEY,domain_uuid TEXT,name TEXT,title TEXT,summary TEXT,path_prefix TEXT,state TEXT,created_at_ms INTEGER,updated_at_ms INTEGER);
|
||||
CREATE TABLE street_spots(street_uuid TEXT,spot_uuid TEXT);
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := NewSQLite(database).EnsureSearchIndexSchema(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package mestionora
|
||||
|
||||
import (
|
||||
"golang.org/x/net/html"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// FirstHTMLHeading extracts untrusted title text. Callers bound input before
|
||||
// extraction and escape output at rendering time; this is not a sanitizer.
|
||||
func FirstHTMLHeading(source string) string {
|
||||
t := html.NewTokenizer(strings.NewReader(source))
|
||||
in := false
|
||||
var out strings.Builder
|
||||
for {
|
||||
kind := t.Next()
|
||||
if kind == html.ErrorToken {
|
||||
if t.Err() == io.EOF {
|
||||
break
|
||||
}
|
||||
break
|
||||
}
|
||||
tok := t.Token()
|
||||
if kind == html.StartTagToken && strings.EqualFold(tok.Data, "h1") {
|
||||
in = true
|
||||
continue
|
||||
}
|
||||
if kind == html.EndTagToken && strings.EqualFold(tok.Data, "h1") {
|
||||
break
|
||||
}
|
||||
if in && kind == html.TextToken {
|
||||
out.WriteString(tok.Data)
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(out.String())
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
package mestionora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const SearchCandidateLimit = 1000
|
||||
const SearchExcerptDocumentBytes = 64 << 10
|
||||
const SearchExcerptTotalBytes = 1 << 20
|
||||
const searchExcerptTimeout = 400 * time.Millisecond
|
||||
|
||||
const searchResultColumns = `p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.revision_uuid,''),COALESCE(r.created_at_ms/1000,0)`
|
||||
const searchPathBoundary = `(p.path=? OR (p.path>=? AND p.path<?))`
|
||||
const searchPathCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,0
|
||||
FROM paths p LEFT JOIN latest_revision_index r ON r.object_uuid=p.object_uuid
|
||||
WHERE p.domain_uuid=? AND ` + searchPathBoundary
|
||||
const searchPathQueryPredicate = ` AND (instr(lower(p.path),lower(?))>0 OR instr(lower(COALESCE(r.filename,'')),lower(?))>0)`
|
||||
const searchPathCandidatesSQL = searchPathCandidatesBaseSQL + searchPathQueryPredicate + ` ORDER BY p.path LIMIT ?`
|
||||
|
||||
// CROSS JOIN is intentional: revision_uuid is an UNINDEXED FTS payload column.
|
||||
// A Path-first plan re-runs the complete MATCH scan for every Path. Start with
|
||||
// MATCH exactly once, then seek current Revision metadata and Domain bindings
|
||||
// through their B-tree indexes. Keep scope filtering before the candidate cap.
|
||||
const searchFTSCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,revision_search.rowid
|
||||
FROM revision_search
|
||||
CROSS JOIN latest_revision_index r ON r.revision_uuid=revision_search.revision_uuid
|
||||
CROSS JOIN paths p ON p.object_uuid=r.object_uuid
|
||||
WHERE revision_search MATCH ? AND p.domain_uuid=? AND ` + searchPathBoundary
|
||||
const searchFTSCandidatesSQL = searchFTSCandidatesBaseSQL + ` LIMIT ?`
|
||||
|
||||
func searchBodyExcerptsSQL(count int) string {
|
||||
// UUID is not indexed by FTS5. Restrict the MATCH/snippet work using the
|
||||
// integer rowids collected from the same read snapshot, not payload UUIDs.
|
||||
return `SELECT revision_uuid,substr(snippet(revision_search,3,'','',' … ',36),1,4096)
|
||||
FROM revision_search WHERE revision_search MATCH ? AND rowid IN (` + strings.TrimRight(strings.Repeat("?,", count), ",") + `)`
|
||||
}
|
||||
|
||||
// Excerpts are optional enrichment. Even small documents can have expensive
|
||||
// prefix hit-position scans, so their time budget is separate from finding
|
||||
// results. A spent excerpt budget omits snippets, not already-authorized hits.
|
||||
func loadSearchBodyExcerpts(parent context.Context, tx *sql.Tx, expression string, ids []any, budget time.Duration) (excerpts map[string]string, err error) {
|
||||
ctx, cancel := context.WithTimeout(parent, budget)
|
||||
defer cancel()
|
||||
defer func() {
|
||||
if err != nil && ctx.Err() != nil {
|
||||
if parent.Err() != nil {
|
||||
err = parent.Err()
|
||||
} else {
|
||||
excerpts, err = nil, nil
|
||||
}
|
||||
}
|
||||
}()
|
||||
args := append([]any{"body : (" + expression + ")"}, ids...)
|
||||
rows, err := tx.QueryContext(ctx, searchBodyExcerptsSQL(len(ids)), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
excerpts = make(map[string]string)
|
||||
for rows.Next() {
|
||||
var id, excerpt string
|
||||
if err := rows.Scan(&id, &excerpt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
excerpts[id] = excerpt
|
||||
}
|
||||
return excerpts, rows.Err()
|
||||
}
|
||||
|
||||
type SearchResult struct {
|
||||
Path string
|
||||
ContentType string
|
||||
Filename string
|
||||
DocumentHash string
|
||||
RevisionUUID string
|
||||
Size int64 // Kept for API compatibility; search does not stat Document files.
|
||||
UpdatedAt int64
|
||||
PathMatch bool
|
||||
FilenameMatch bool
|
||||
BodyMatch bool
|
||||
Excerpt string // Untrusted source text, never HTML-safe markup.
|
||||
ExcerptOmitted bool // A body match remains visible when snippet work exceeds its budget.
|
||||
}
|
||||
|
||||
type PathSuggestion struct {
|
||||
SearchResult
|
||||
Title string
|
||||
}
|
||||
|
||||
// SearchPathsInDomainFiltered bounds SQL candidates before materialization and
|
||||
// authorizes each Path before loading an excerpt. It never computes a global hit
|
||||
// count or stats Document files. A narrow scope improves both recall and cost.
|
||||
// allowed runs inside a read snapshot. Prefer a preloaded request-local policy;
|
||||
// a callback doing DB reads requires spare pool capacity (not a one-connection
|
||||
// pool). Keeping policy I/O outside this snapshot is separate follow-up work.
|
||||
func (s *SQLite) SearchPathsInDomainFiltered(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]SearchResult, error) {
|
||||
return s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{})
|
||||
}
|
||||
|
||||
// SearchPathsInDomainWithOptions also permits filter-only searches. File/date
|
||||
// predicates apply before SQL candidate caps, while ACL checks stay ahead of all
|
||||
// snippet enrichment. Zero options preserve the original indexed query plans.
|
||||
func (s *SQLite) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, searchErr error) {
|
||||
return s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options, true)
|
||||
}
|
||||
|
||||
// SearchPathSuggestionsInDomain shares the current-Revision FTS and ACL path
|
||||
// with full search, but deliberately skips snippet generation. Typeahead calls
|
||||
// are frequent and only need a bounded display label, never a body excerpt.
|
||||
func (s *SQLite) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) {
|
||||
results, err := s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}, false)
|
||||
if err != nil || len(results) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
ids := make([]any, 0, len(results))
|
||||
seen := make(map[string]bool, len(results))
|
||||
for _, result := range results {
|
||||
if result.RevisionUUID != "" && strings.HasPrefix(strings.ToLower(result.ContentType), "text/html") && !seen[result.RevisionUUID] {
|
||||
ids = append(ids, result.RevisionUUID)
|
||||
seen[result.RevisionUUID] = true
|
||||
}
|
||||
}
|
||||
headings := make(map[string]string, len(ids))
|
||||
if len(ids) > 0 {
|
||||
rows, queryErr := s.db.QueryContext(ctx, `SELECT uuid,substr(CAST(body AS TEXT),1,65536) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
|
||||
if queryErr != nil {
|
||||
return nil, queryErr
|
||||
}
|
||||
for rows.Next() {
|
||||
var revision, source string
|
||||
if queryErr = rows.Scan(&revision, &source); queryErr != nil {
|
||||
rows.Close()
|
||||
return nil, queryErr
|
||||
}
|
||||
headings[revision] = FirstHTMLHeading(source)
|
||||
}
|
||||
queryErr = rows.Err()
|
||||
rows.Close()
|
||||
if queryErr != nil {
|
||||
return nil, queryErr
|
||||
}
|
||||
}
|
||||
out := make([]PathSuggestion, 0, len(results))
|
||||
for _, result := range results {
|
||||
out = append(out, PathSuggestion{SearchResult: result, Title: headings[result.RevisionUUID]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *SQLite) searchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions, loadExcerpts bool) (found []SearchResult, searchErr error) {
|
||||
if err := options.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scope, query = normalizePath(scope), strings.TrimSpace(query)
|
||||
if query == "" && !options.HasFilters() || allowed == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if len(query) > 512 || len(strings.Fields(query)) > 16 {
|
||||
return nil, errors.New("search query is too long")
|
||||
}
|
||||
if limit <= 0 || limit > 200 {
|
||||
limit = 100
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
defer func() {
|
||||
// SQLite can report an interrupted statement instead of wrapping the
|
||||
// context error. Preserve cancellation/deadline semantics for HTTP.
|
||||
if ctx.Err() != nil {
|
||||
found = nil
|
||||
searchErr = ctx.Err()
|
||||
}
|
||||
}()
|
||||
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
prefix := strings.TrimRight(scope, "/") + "/"
|
||||
upper := strings.TrimRight(scope, "/") + "0"
|
||||
results := make(map[string]SearchResult)
|
||||
ftsRows := make(map[string]int64)
|
||||
collect := func(statement string, body, filename bool, args ...any) error {
|
||||
rows, err := tx.QueryContext(ctx, statement, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var result SearchResult
|
||||
var rowID int64
|
||||
if err := rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt, &rowID); err != nil {
|
||||
return err
|
||||
}
|
||||
if !allowed(result.Path) {
|
||||
continue
|
||||
}
|
||||
if rowID != 0 {
|
||||
ftsRows[result.RevisionUUID] = rowID
|
||||
}
|
||||
result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query))
|
||||
result.FilenameMatch = filename || results[result.Path].FilenameMatch || query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query))
|
||||
result.BodyMatch = body || results[result.Path].BodyMatch
|
||||
results[result.Path] = result
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
filterSQL, filterArgs := searchOptionsPredicate(options)
|
||||
pathSQL := searchPathCandidatesBaseSQL
|
||||
pathArgs := []any{domainUUID, scope, prefix, upper}
|
||||
if query != "" {
|
||||
pathSQL += searchPathQueryPredicate
|
||||
pathArgs = append(pathArgs, query, query)
|
||||
}
|
||||
pathSQL += filterSQL + ` ORDER BY p.path LIMIT ?`
|
||||
pathArgs = append(pathArgs, filterArgs...)
|
||||
pathArgs = append(pathArgs, SearchCandidateLimit)
|
||||
if err := collect(pathSQL, false, false, pathArgs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expression := ftsPrefixExpression(query)
|
||||
if expression != "" {
|
||||
// No global ORDER BY or unbounded IN materialization: stop the FTS scan
|
||||
// after at most this many current, in-Domain, in-scope Path candidates.
|
||||
for _, field := range []string{"body", "filename"} {
|
||||
args := []any{field + " : (" + expression + ")", domainUUID, scope, prefix, upper}
|
||||
args = append(args, filterArgs...)
|
||||
args = append(args, SearchCandidateLimit)
|
||||
if err := collect(searchFTSCandidatesBaseSQL+filterSQL+` LIMIT ?`, field == "body", field == "filename", args...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]SearchResult, 0, len(results))
|
||||
for _, result := range results {
|
||||
out = append(out, result)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
// Direct name/Path matches precede body-only hits; all ties are stable.
|
||||
iName, jName := out[i].PathMatch || out[i].FilenameMatch, out[j].PathMatch || out[j].FilenameMatch
|
||||
if iName != jName {
|
||||
return iName
|
||||
}
|
||||
if len(out[i].Path) != len(out[j].Path) {
|
||||
return len(out[i].Path) < len(out[j].Path)
|
||||
}
|
||||
return out[i].Path < out[j].Path
|
||||
})
|
||||
if len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
if !loadExcerpts {
|
||||
return out, nil
|
||||
}
|
||||
// Only authorized, displayed results receive text excerpts. Batch the FTS
|
||||
// lookup so a broad query is not re-scanned once for every visible result.
|
||||
ids := make([]any, 0, len(out))
|
||||
seen := make(map[string]bool)
|
||||
for _, result := range out {
|
||||
if result.BodyMatch && !seen[result.RevisionUUID] {
|
||||
ids = append(ids, ftsRows[result.RevisionUUID])
|
||||
seen[result.RevisionUUID] = true
|
||||
}
|
||||
}
|
||||
if len(ids) != 0 {
|
||||
// snippet's output token/character limit does not bound its input work:
|
||||
// FTS5 may tokenize the entire matched document. Read byte lengths only
|
||||
// for authorized displayed rows before spending the snippet budget.
|
||||
rows, err := tx.QueryContext(ctx, `SELECT rowid,octet_length(body)
|
||||
FROM revision_search WHERE rowid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sizes := make(map[int64]int64)
|
||||
for rows.Next() {
|
||||
var rowID, size int64
|
||||
if err := rows.Scan(&rowID, &size); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
sizes[rowID] = size
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids = nil
|
||||
selected := make(map[string]bool)
|
||||
remaining := int64(SearchExcerptTotalBytes)
|
||||
for i := range out {
|
||||
if !out[i].BodyMatch {
|
||||
continue
|
||||
}
|
||||
revision := out[i].RevisionUUID
|
||||
include, decided := selected[revision]
|
||||
if !decided {
|
||||
rowID := ftsRows[revision]
|
||||
size, exists := sizes[rowID]
|
||||
include = exists && size >= 0 && size <= SearchExcerptDocumentBytes && size <= remaining
|
||||
if include {
|
||||
ids = append(ids, rowID)
|
||||
remaining -= size
|
||||
}
|
||||
selected[revision] = include
|
||||
}
|
||||
out[i].ExcerptOmitted = !include
|
||||
}
|
||||
}
|
||||
if len(ids) != 0 {
|
||||
excerpts, err := loadSearchBodyExcerpts(ctx, tx, expression, ids, searchExcerptTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range out {
|
||||
if out[i].BodyMatch {
|
||||
var present bool
|
||||
out[i].Excerpt, present = excerpts[out[i].RevisionUUID]
|
||||
out[i].ExcerptOmitted = !present
|
||||
}
|
||||
}
|
||||
}
|
||||
// A filename/Path-only hit still benefits from a short Page summary. Read
|
||||
// only a bounded prefix from displayed textual revisions, in one batch.
|
||||
ids, seen = nil, make(map[string]bool)
|
||||
for _, result := range out {
|
||||
media := strings.ToLower(result.ContentType)
|
||||
textual := strings.HasPrefix(media, "text/") || strings.HasPrefix(media, "application/json") || strings.HasPrefix(media, "application/xml") || strings.HasPrefix(media, "application/javascript") || strings.HasPrefix(media, "application/yaml") || strings.HasPrefix(media, "application/x-yaml")
|
||||
if !result.BodyMatch && textual && result.RevisionUUID != "" && !seen[result.RevisionUUID] {
|
||||
ids = append(ids, result.RevisionUUID)
|
||||
seen[result.RevisionUUID] = true
|
||||
}
|
||||
}
|
||||
if len(ids) != 0 {
|
||||
rows, err := tx.QueryContext(ctx, `SELECT uuid,substr(body,1,1024) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
excerpts := make(map[string]string)
|
||||
for rows.Next() {
|
||||
var id, excerpt string
|
||||
if err := rows.Scan(&id, &excerpt); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
excerpts[id] = excerpt
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range out {
|
||||
if !out[i].BodyMatch {
|
||||
out[i].Excerpt = excerpts[out[i].RevisionUUID]
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func ftsPrefixExpression(query string) string {
|
||||
words := strings.Fields(query)
|
||||
terms := make([]string, 0, len(words))
|
||||
for _, word := range words {
|
||||
word = strings.ReplaceAll(word, `"`, `""`)
|
||||
if word != "" {
|
||||
terms = append(terms, `"`+word+`"*`)
|
||||
}
|
||||
}
|
||||
return strings.Join(terms, " AND ")
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package mestionora
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var ErrInvalidSearchOptions = errors.New("invalid search options")
|
||||
|
||||
// SearchOptions applies to current Page metadata, not historical Revisions.
|
||||
// Dates are Unix milliseconds: FromMS is inclusive and BeforeMS is exclusive.
|
||||
// The created field refers to Object creation, not creation of a Path alias.
|
||||
// FileType and Extension are normalized lower-case values; Extension has no dot
|
||||
// prefix and may contain a compound suffix such as tar.gz.
|
||||
type SearchOptions struct {
|
||||
FileType string
|
||||
Extension string
|
||||
DateField string
|
||||
FromMS *int64
|
||||
BeforeMS *int64
|
||||
}
|
||||
|
||||
func (o SearchOptions) HasFilters() bool {
|
||||
return o.FileType != "" || o.Extension != "" || o.FromMS != nil || o.BeforeMS != nil
|
||||
}
|
||||
|
||||
func (o SearchOptions) Validate() error {
|
||||
switch o.FileType {
|
||||
case "", "html", "text", "image", "audio", "video", "pdf", "archive", "csv", "json", "code", "office", "other":
|
||||
default:
|
||||
return fmt.Errorf("%w: unknown file type", ErrInvalidSearchOptions)
|
||||
}
|
||||
switch o.DateField {
|
||||
case "", "updated", "created":
|
||||
default:
|
||||
return fmt.Errorf("%w: unknown date field", ErrInvalidSearchOptions)
|
||||
}
|
||||
if len(o.Extension) > 32 {
|
||||
return fmt.Errorf("%w: extension is too long", ErrInvalidSearchOptions)
|
||||
}
|
||||
for i, c := range o.Extension {
|
||||
alnum := c >= 'a' && c <= 'z' || c >= '0' && c <= '9'
|
||||
if !alnum && (i == 0 || !strings.ContainsRune("._+-", c)) {
|
||||
return fmt.Errorf("%w: invalid extension", ErrInvalidSearchOptions)
|
||||
}
|
||||
}
|
||||
if o.FromMS != nil && o.BeforeMS != nil && *o.FromMS >= *o.BeforeMS {
|
||||
return fmt.Errorf("%w: empty or reversed date range", ErrInvalidSearchOptions)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const searchMIMESQL = `lower(trim(substr(COALESCE(r.content_type,''),1,instr(COALESCE(r.content_type,'')||';',';')-1)))`
|
||||
|
||||
// The original filename wins over the Path suffix when present. Names remain
|
||||
// data: even extension patterns are bound arguments and do not become SQL.
|
||||
const searchFilenameSQL = `lower(CASE WHEN COALESCE(r.filename,'')<>'' THEN r.filename ELSE p.path END)`
|
||||
|
||||
func searchSuffixesSQL(suffixes ...string) string {
|
||||
clauses := make([]string, 0, len(suffixes))
|
||||
for _, suffix := range suffixes {
|
||||
// Only package-owned ASCII constants reach this helper. GLOB has no
|
||||
// escaping ambiguity for underscores, unlike LIKE.
|
||||
clauses = append(clauses, searchFilenameSQL+` GLOB '*.`+suffix+`'`)
|
||||
}
|
||||
return "(" + strings.Join(clauses, " OR ") + ")"
|
||||
}
|
||||
|
||||
// Specific MIME types are authoritative. Generic text/binary types may use a
|
||||
// known suffix (e.g. a Python source uploaded as text/plain); unrecognized,
|
||||
// specific MIME types stay Other instead of pretending to be a different type.
|
||||
// This uses metadata only and never hydrates a Document to sniff its bytes.
|
||||
var searchFileTypeSQL = `CASE
|
||||
WHEN ` + searchMIMESQL + ` IN ('text/html','application/xhtml+xml') THEN 'html'
|
||||
WHEN ` + searchMIMESQL + `='application/pdf' THEN 'pdf'
|
||||
WHEN ` + searchMIMESQL + ` LIKE 'image/%' THEN 'image'
|
||||
WHEN ` + searchMIMESQL + ` LIKE 'audio/%' THEN 'audio'
|
||||
WHEN ` + searchMIMESQL + ` LIKE 'video/%' THEN 'video'
|
||||
WHEN ` + searchMIMESQL + ` IN ('text/csv','application/csv','text/tab-separated-values') THEN 'csv'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/json','text/json') OR ` + searchMIMESQL + ` GLOB '*+json' THEN 'json'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/zip','application/x-zip-compressed','application/x-7z-compressed','application/vnd.rar','application/x-rar','application/x-rar-compressed','application/x-tar','application/gzip','application/x-gzip','application/x-xz','application/x-bzip2','application/zstd','application/x-zstd','application/x-compress','application/x-cpio') THEN 'archive'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/msword','application/vnd.ms-excel','application/vnd.ms-powerpoint','application/rtf','text/rtf') OR ` + searchMIMESQL + ` GLOB 'application/vnd.openxmlformats-officedocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.oasis.opendocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.ms-*.macroenabled.*' THEN 'office'
|
||||
WHEN ` + searchMIMESQL + ` IN ('text/markdown','text/x-markdown') THEN 'text'
|
||||
WHEN ` + searchMIMESQL + ` IN ('application/javascript','application/x-javascript','text/javascript','application/typescript','text/typescript','application/xml','text/xml','application/yaml','application/x-yaml','text/yaml','application/toml','application/x-sh','application/x-shellscript','application/x-python','application/x-python-code','application/x-httpd-php','application/sql','text/css') OR ` + searchMIMESQL + ` GLOB 'text/x-*' OR ` + searchMIMESQL + ` GLOB '*+xml' THEN 'code'
|
||||
WHEN ` + searchMIMESQL + ` NOT IN ('','text/plain','application/octet-stream','binary/octet-stream','application/x-empty') THEN CASE WHEN ` + searchMIMESQL + ` LIKE 'text/%' THEN 'text' ELSE 'other' END
|
||||
WHEN ` + searchSuffixesSQL("html", "htm", "xhtml") + ` THEN 'html'
|
||||
WHEN ` + searchSuffixesSQL("pdf") + ` THEN 'pdf'
|
||||
WHEN ` + searchSuffixesSQL("png", "jpg", "jpeg", "gif", "webp", "avif", "bmp", "svg", "tif", "tiff", "ico", "heic", "heif") + ` THEN 'image'
|
||||
WHEN ` + searchSuffixesSQL("mp3", "flac", "wav", "ogg", "oga", "opus", "m4a", "aac", "aiff", "alac", "mid", "midi") + ` THEN 'audio'
|
||||
WHEN ` + searchSuffixesSQL("mp4", "webm", "mkv", "mov", "avi", "m4v", "ogv", "mpeg", "mpg") + ` THEN 'video'
|
||||
WHEN ` + searchSuffixesSQL("zip", "7z", "rar", "tar", "gz", "tgz", "xz", "txz", "bz2", "tbz", "tbz2", "zst", "tzst", "cbz", "cbr", "cpio") + ` THEN 'archive'
|
||||
WHEN ` + searchSuffixesSQL("csv", "tsv") + ` THEN 'csv'
|
||||
WHEN ` + searchSuffixesSQL("json", "jsonl", "ndjson", "excalidraw", "ipynb") + ` THEN 'json'
|
||||
WHEN ` + searchSuffixesSQL("doc", "docx", "docm", "xls", "xlsx", "xlsm", "ppt", "pptx", "pptm", "odt", "ods", "odp", "rtf") + ` THEN 'office'
|
||||
WHEN ` + searchSuffixesSQL("py", "pyw", "sh", "bash", "zsh", "fish", "ps1", "bat", "cmd", "js", "mjs", "cjs", "jsx", "ts", "tsx", "go", "rs", "c", "h", "cpp", "hpp", "cc", "cs", "java", "kt", "kts", "swift", "rb", "php", "pl", "pm", "lua", "r", "sql", "css", "scss", "sass", "less", "xml", "xsl", "yaml", "yml", "toml", "ini", "conf", "vue", "svelte", "dart", "ex", "exs", "erl", "hs", "scala", "clj", "lisp", "el", "vim", "wasm") + ` THEN 'code'
|
||||
WHEN ` + searchMIMESQL + `='text/plain' OR ` + searchSuffixesSQL("txt", "text", "log", "md", "markdown", "rst", "adoc", "org") + ` THEN 'text'
|
||||
ELSE 'other' END`
|
||||
|
||||
// The returned predicate is inserted before each candidate stream's LIMIT,
|
||||
// including FTS-first streams. Filtering after their cap silently loses hits.
|
||||
func searchOptionsPredicate(options SearchOptions) (string, []any) {
|
||||
var predicate strings.Builder
|
||||
var args []any
|
||||
if options.FromMS != nil || options.BeforeMS != nil {
|
||||
dateSQL := "r.created_at_ms"
|
||||
if options.DateField == "created" {
|
||||
// Objects.uuid is a primary-key lookup. Avoid joining Objects in the
|
||||
// normal search plan when Object creation is not being filtered.
|
||||
predicate.WriteString(" AND EXISTS (SELECT 1 FROM objects o WHERE o.uuid=p.object_uuid")
|
||||
dateSQL = "o.created_at*1000"
|
||||
}
|
||||
if options.FromMS != nil {
|
||||
predicate.WriteString(" AND " + dateSQL + ">=?")
|
||||
args = append(args, *options.FromMS)
|
||||
}
|
||||
if options.BeforeMS != nil {
|
||||
predicate.WriteString(" AND " + dateSQL + "<?")
|
||||
args = append(args, *options.BeforeMS)
|
||||
}
|
||||
if options.DateField == "created" {
|
||||
predicate.WriteString(")")
|
||||
}
|
||||
}
|
||||
// Evaluate inexpensive date/suffix exclusions before the metadata category.
|
||||
if options.Extension != "" {
|
||||
predicate.WriteString(" AND " + searchFilenameSQL + " GLOB ?")
|
||||
args = append(args, "*."+options.Extension)
|
||||
}
|
||||
if options.FileType != "" {
|
||||
predicate.WriteString(" AND (" + searchFileTypeSQL + ")=?")
|
||||
args = append(args, options.FileType)
|
||||
}
|
||||
return predicate.String(), args
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package mestionora
|
||||
|
||||
import "database/sql"
|
||||
|
||||
// EnsureSearchIndexSchema installs an FTS5 index over immutable Revisions.
|
||||
// Paths stay outside the index because links can be renamed independently; the
|
||||
// search query joins matching latest Revisions to the current domain Paths.
|
||||
func (s *SQLite) EnsureSearchIndexSchema() error {
|
||||
var revisionsTable string
|
||||
if err := s.db.QueryRow(`
|
||||
SELECT name FROM sqlite_master
|
||||
WHERE type = 'table' AND name = 'revisions'
|
||||
`).Scan(&revisionsTable); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS revision_search USING fts5(
|
||||
revision_uuid UNINDEXED,
|
||||
object_uuid UNINDEXED,
|
||||
filename,
|
||||
body,
|
||||
tokenize = 'unicode61'
|
||||
)
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS revision_search_state (
|
||||
revision_uuid TEXT PRIMARY KEY
|
||||
)
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS latest_revision_index (
|
||||
object_uuid TEXT PRIMARY KEY,
|
||||
revision_uuid TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
filename TEXT NOT NULL,
|
||||
document_hash TEXT NOT NULL,
|
||||
created_at_ms INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS search_schema_state (
|
||||
name TEXT PRIMARY KEY
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_latest_revision_search_uuid ON latest_revision_index(revision_uuid)
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
var latestBackfilled int
|
||||
if err := s.db.QueryRow(`
|
||||
SELECT EXISTS(
|
||||
SELECT 1 FROM search_schema_state
|
||||
WHERE name = 'latest_revision_index_v1'
|
||||
)
|
||||
`).Scan(&latestBackfilled); err != nil {
|
||||
return err
|
||||
}
|
||||
if latestBackfilled == 0 {
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT OR REPLACE INTO latest_revision_index (
|
||||
object_uuid, revision_uuid, content_type, filename,
|
||||
document_hash, created_at_ms
|
||||
)
|
||||
SELECT
|
||||
object_uuid, uuid, COALESCE(content_type, ''),
|
||||
COALESCE(filename, ''), COALESCE(document_hash, ''),
|
||||
COALESCE(created_at_ms, created_at * 1000, 0)
|
||||
FROM (
|
||||
SELECT
|
||||
uuid, object_uuid, content_type, filename,
|
||||
document_hash, created_at_ms, created_at,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY object_uuid
|
||||
ORDER BY COALESCE(created_at_ms, created_at * 1000) DESC,
|
||||
rowid DESC
|
||||
) AS rn
|
||||
FROM revisions
|
||||
)
|
||||
WHERE rn = 1;
|
||||
INSERT OR IGNORE INTO search_schema_state (name)
|
||||
VALUES ('latest_revision_index_v1');
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Populate the cheap membership table once for installations that already
|
||||
// created revision_search before revision_search_state was introduced.
|
||||
var stateExists int
|
||||
if err := s.db.QueryRow(`
|
||||
SELECT EXISTS(SELECT 1 FROM revision_search_state LIMIT 1)
|
||||
`).Scan(&stateExists); err != nil {
|
||||
return err
|
||||
}
|
||||
if stateExists == 0 {
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT OR IGNORE INTO revision_search_state (revision_uuid)
|
||||
SELECT revision_uuid FROM revision_search
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Trigger definitions evolve with the projection schema; recreate them so
|
||||
// existing installations receive the current maintenance logic.
|
||||
if _, err := s.db.Exec(`
|
||||
DROP TRIGGER IF EXISTS revision_search_insert;
|
||||
DROP TRIGGER IF EXISTS revision_search_delete;
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
CREATE TRIGGER IF NOT EXISTS revision_search_insert
|
||||
AFTER INSERT ON revisions BEGIN
|
||||
INSERT INTO revision_search (
|
||||
revision_uuid, object_uuid, filename, body
|
||||
) VALUES (
|
||||
new.uuid,
|
||||
new.object_uuid,
|
||||
COALESCE(new.filename, ''),
|
||||
CASE
|
||||
WHEN lower(COALESCE(new.content_type, '')) LIKE 'text/%'
|
||||
OR lower(COALESCE(new.content_type, '')) LIKE 'application/json%'
|
||||
OR lower(COALESCE(new.content_type, '')) LIKE 'application/xml%'
|
||||
OR lower(COALESCE(new.content_type, '')) LIKE 'application/javascript%'
|
||||
OR lower(COALESCE(new.content_type, '')) LIKE 'application/x-yaml%'
|
||||
OR lower(COALESCE(new.content_type, '')) LIKE 'application/yaml%'
|
||||
THEN COALESCE(new.body, '')
|
||||
ELSE ''
|
||||
END
|
||||
);
|
||||
INSERT OR IGNORE INTO revision_search_state (revision_uuid)
|
||||
VALUES (new.uuid);
|
||||
INSERT INTO latest_revision_index (
|
||||
object_uuid, revision_uuid, content_type, filename,
|
||||
document_hash, created_at_ms
|
||||
) VALUES (
|
||||
new.object_uuid, new.uuid, COALESCE(new.content_type, ''),
|
||||
COALESCE(new.filename, ''), COALESCE(new.document_hash, ''),
|
||||
COALESCE(new.created_at_ms, new.created_at * 1000, 0)
|
||||
)
|
||||
ON CONFLICT(object_uuid) DO UPDATE SET
|
||||
revision_uuid = excluded.revision_uuid,
|
||||
content_type = excluded.content_type,
|
||||
filename = excluded.filename,
|
||||
document_hash = excluded.document_hash,
|
||||
created_at_ms = excluded.created_at_ms
|
||||
WHERE excluded.created_at_ms >= latest_revision_index.created_at_ms;
|
||||
END
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
CREATE TRIGGER IF NOT EXISTS revision_search_delete
|
||||
AFTER DELETE ON revisions BEGIN
|
||||
DELETE FROM revision_search WHERE revision_uuid = old.uuid;
|
||||
DELETE FROM revision_search_state WHERE revision_uuid = old.uuid;
|
||||
DELETE FROM latest_revision_index
|
||||
WHERE object_uuid = old.object_uuid
|
||||
AND revision_uuid = old.uuid;
|
||||
INSERT OR REPLACE INTO latest_revision_index (
|
||||
object_uuid, revision_uuid, content_type, filename,
|
||||
document_hash, created_at_ms
|
||||
)
|
||||
SELECT
|
||||
object_uuid, uuid, COALESCE(content_type, ''),
|
||||
COALESCE(filename, ''), COALESCE(document_hash, ''),
|
||||
COALESCE(created_at_ms, created_at * 1000, 0)
|
||||
FROM revisions
|
||||
WHERE object_uuid = old.object_uuid
|
||||
ORDER BY COALESCE(created_at_ms, created_at * 1000) DESC, rowid DESC
|
||||
LIMIT 1;
|
||||
END
|
||||
`); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := s.db.Exec(`
|
||||
INSERT INTO revision_search (
|
||||
revision_uuid, object_uuid, filename, body
|
||||
)
|
||||
SELECT
|
||||
r.uuid,
|
||||
r.object_uuid,
|
||||
COALESCE(r.filename, ''),
|
||||
CASE
|
||||
WHEN lower(COALESCE(r.content_type, '')) LIKE 'text/%'
|
||||
OR lower(COALESCE(r.content_type, '')) LIKE 'application/json%'
|
||||
OR lower(COALESCE(r.content_type, '')) LIKE 'application/xml%'
|
||||
OR lower(COALESCE(r.content_type, '')) LIKE 'application/javascript%'
|
||||
OR lower(COALESCE(r.content_type, '')) LIKE 'application/x-yaml%'
|
||||
OR lower(COALESCE(r.content_type, '')) LIKE 'application/yaml%'
|
||||
THEN COALESCE(r.body, '')
|
||||
ELSE ''
|
||||
END
|
||||
FROM revisions r
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM revision_search_state state
|
||||
WHERE state.revision_uuid = r.uuid
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.Exec(`
|
||||
INSERT OR IGNORE INTO revision_search_state (revision_uuid)
|
||||
SELECT uuid FROM revisions
|
||||
`)
|
||||
return err
|
||||
}
|
||||
+4
-13
@@ -1,4 +1,4 @@
|
||||
package store
|
||||
package mestionora
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -39,10 +39,7 @@ func TestSearchExcerptDeadlineIsOptionalButParentCancellationIsNot(t *testing.T)
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err := db.Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
New(database)
|
||||
initSearchFixture(t, database)
|
||||
tx, err := database.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -72,10 +69,7 @@ func TestSearchFTSPlanScansMatchesOnce(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err := db.Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
New(database)
|
||||
initSearchFixture(t, database)
|
||||
for _, field := range []string{"body", "filename"} {
|
||||
plan := searchQueryPlan(t, database, searchFTSCandidatesSQL,
|
||||
field+` : ("needle"*)`, "domain", "/", "/", "0", SearchCandidateLimit)
|
||||
@@ -96,10 +90,7 @@ func TestSearchFilteredFTSPlanKeepsMatchFirstAndIndexedSeeks(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err := db.Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
New(database)
|
||||
initSearchFixture(t, database)
|
||||
from, before := int64(1000), int64(2000)
|
||||
for _, options := range []SearchOptions{
|
||||
{FileType: "code"}, {Extension: "py"}, {FromMS: &from, BeforeMS: &before},
|
||||
@@ -0,0 +1,102 @@
|
||||
package mestionora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMissingPolicyDoesNotReadOrInitialize(t *testing.T) {
|
||||
// A nil pool makes accidental construction, search or maintenance I/O fail.
|
||||
engine := NewSQLite(nil)
|
||||
ctx := context.Background()
|
||||
if hits, err := engine.SearchPathsInDomainFiltered(ctx, "d", "/", "needle", 20, nil); err != nil || len(hits) != 0 {
|
||||
t.Fatalf("missing page policy: %v %v", hits, err)
|
||||
}
|
||||
if hits, err := engine.SearchPathSuggestionsInDomain(ctx, "d", "/", "needle", 8, nil); err != nil || len(hits) != 0 {
|
||||
t.Fatalf("missing suggestion policy: %v %v", hits, err)
|
||||
}
|
||||
if hits, err := engine.SearchCityInDomain(ctx, "d", "/", "needle", 30, nil, SearchOptions{}); err != nil || len(hits) != 0 {
|
||||
t.Fatalf("missing city policy: %v %v", hits, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeniedPagesDoNotLoadSummaryOrHeading(t *testing.T) {
|
||||
database := newSearchTestDB(t)
|
||||
_, err := database.Exec(`
|
||||
INSERT INTO objects(uuid,created_at) VALUES('o',1);
|
||||
INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('d','/needle','o');
|
||||
INSERT INTO revisions(uuid,object_uuid,filename,body,content_type,document_hash,created_at_ms,created_at)
|
||||
VALUES('r','o','needle.html','<h1>PRIVATE_HEADING</h1><p>PRIVATE_SUMMARY</p>','text/html','hash',1000,1);
|
||||
DROP TABLE revisions;
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine := NewSQLite(database)
|
||||
deny := func(string) bool { return false }
|
||||
ctx := context.Background()
|
||||
// Candidates still exist in the independent index. Any attempt to hydrate
|
||||
// denied summaries/headings now fails because their source table is absent.
|
||||
if hits, err := engine.SearchPathsInDomainFiltered(ctx, "d", "/", "needle", 20, deny); err != nil || len(hits) != 0 {
|
||||
t.Fatalf("denied summary loaded: %v %v", hits, err)
|
||||
}
|
||||
if hits, err := engine.SearchPathSuggestionsInDomain(ctx, "d", "/", "needle", 8, deny); err != nil || len(hits) != 0 {
|
||||
t.Fatalf("denied heading loaded: %v %v", hits, err)
|
||||
}
|
||||
allow := func(string) bool { return true }
|
||||
if _, err := engine.SearchPathsInDomainFiltered(ctx, "d", "/", "needle", 20, allow); err == nil {
|
||||
t.Fatal("fixture did not exercise summary loading")
|
||||
}
|
||||
if _, err := engine.SearchPathSuggestionsInDomain(ctx, "d", "/", "needle", 8, allow); err == nil {
|
||||
t.Fatal("fixture did not exercise heading loading")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancellationDuringPolicyNeverReturnsPartialSuccess(t *testing.T) {
|
||||
database := newSearchTestDB(t)
|
||||
if _, err := database.Exec(`INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('d','/needle','o')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, excerpts := range []bool{false, true} {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
allowed := func(string) bool { cancel(); return true }
|
||||
// Filter-only search has no later FTS statement to notice cancellation.
|
||||
hits, err := NewSQLite(database).searchPathsInDomainWithOptions(ctx, "d", "/", "", 20, allowed, SearchOptions{FileType: "other"}, excerpts)
|
||||
cancel()
|
||||
if !errors.Is(err, context.Canceled) || len(hits) != 0 {
|
||||
t.Fatalf("partial success after cancellation: %v %v", hits, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFTSExpressionTreatsOperatorsAsData(t *testing.T) {
|
||||
for input, want := range map[string]string{
|
||||
"": "", "alpha beta": `"alpha"* AND "beta"*`,
|
||||
`a"b OR`: `"a""b"* AND "OR"*`,
|
||||
`body:NEAR(x) -secret`: `"body:NEAR(x)"* AND "-secret"*`,
|
||||
"日本語": `"日本語"*`,
|
||||
} {
|
||||
if got := ftsPrefixExpression(input); got != want {
|
||||
t.Fatalf("%q: got %q want %q", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchDoesNotMaintainIndexOnReadOnlyConnection(t *testing.T) {
|
||||
database := newSearchTestDB(t)
|
||||
database.SetMaxOpenConns(1)
|
||||
if _, err := database.Exec(`INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('d','/needle','o'); PRAGMA query_only=ON`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine := NewSQLite(database)
|
||||
results, err := engine.SearchPathsInDomainFiltered(context.Background(), "d", "/", "needle", 20, func(string) bool { return true })
|
||||
if err != nil || len(results) != 1 {
|
||||
t.Fatalf("read-only search: %v %v", results, err)
|
||||
}
|
||||
// Schema maintenance is deliberately explicit, not performed by NewSQLite
|
||||
// or a query. An accidental write in those paths would fail the test above.
|
||||
if err := engine.EnsureSearchIndexSchema(); err == nil {
|
||||
t.Fatal("read-only fixture unexpectedly accepted maintenance")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
// Package mestionora owns bounded search, search projections and the SQLite
|
||||
// search index. It consumes read-policy decisions; it never grants permissions.
|
||||
// Search results and excerpts are untrusted text, not HTML-safe output.
|
||||
package mestionora
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const Component = "mestionora"
|
||||
|
||||
// SQLite borrows a metadata connection pool. Construction performs no I/O or
|
||||
// schema initialization. EnsureSearchIndexSchema is a separate startup step.
|
||||
// The caller owns the pool lifetime and provisions the application schema.
|
||||
type SQLite struct{ db *sql.DB }
|
||||
|
||||
func NewSQLite(db *sql.DB) *SQLite { return &SQLite{db: db} }
|
||||
|
||||
// Match the established Path scope rules, without filesystem path cleaning or
|
||||
// URL decoding. Percent/underscore characters remain literal search data.
|
||||
func normalizePath(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return "/"
|
||||
}
|
||||
if !strings.HasPrefix(value, "/") {
|
||||
value = "/" + value
|
||||
}
|
||||
if value != "/" {
|
||||
value = strings.TrimRight(value, "/")
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package ordoschnellic
|
||||
|
||||
import (
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestComponentBoundary(t *testing.T) {
|
||||
if Component != "ordoschnellic" || PushEventType != "notification.push.deliver" {
|
||||
t.Fatal("component/event identity changed")
|
||||
}
|
||||
files, err := filepath.Glob("*.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, file := range files {
|
||||
if strings.HasSuffix(file, "_test.go") {
|
||||
continue
|
||||
}
|
||||
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, spec := range ast.Imports {
|
||||
path, err := strconv.Unquote(spec.Path.Value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"pageswiki/internal", "database", "os"} {
|
||||
if path == forbidden || strings.HasPrefix(path, forbidden+"/") {
|
||||
t.Errorf("%s imports %s: delivery must not own application policy, DB transactions or files", file, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package ordoschnellic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Queue rechecks session/recipient eligibility in Claim. Empty Subscription
|
||||
// means the claimed item is no longer deliverable and should be finalized.
|
||||
// No outbound network operation runs inside a Queue transaction.
|
||||
type Queue interface {
|
||||
Cleanup(context.Context, time.Time) error
|
||||
Claim(context.Context, time.Time) (*PushDelivery, error)
|
||||
DeferFocused(context.Context, *PushDelivery, time.Time) error
|
||||
Finish(context.Context, *PushDelivery, Outcome, time.Time) error
|
||||
}
|
||||
|
||||
type Sender interface {
|
||||
Send(context.Context, *PushDelivery) Outcome
|
||||
}
|
||||
|
||||
type Stats struct{ Sent, Failed int }
|
||||
|
||||
type Dispatcher struct {
|
||||
Queue Queue
|
||||
Sender Sender
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (d Dispatcher) Run(parent context.Context) (Stats, error) {
|
||||
var stats Stats
|
||||
if d.Queue == nil || d.Sender == nil {
|
||||
return stats, errors.New("notification delivery dependencies are required")
|
||||
}
|
||||
if err := parent.Err(); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
now := d.Now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(parent, PushBatchTimeout)
|
||||
defer cancel()
|
||||
if err := d.Queue.Cleanup(ctx, now()); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
for i := 0; i < PushBatchLimit && ctx.Err() == nil; i++ {
|
||||
item, err := d.Queue.Claim(ctx, now())
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if item == nil {
|
||||
break
|
||||
}
|
||||
if item.Focused {
|
||||
if err := d.Queue.DeferFocused(ctx, item, now()); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
var outcome Outcome
|
||||
if item.Subscription != "" {
|
||||
outcome = d.Sender.Send(ctx, item)
|
||||
if outcome.Attempted {
|
||||
if outcome.Success {
|
||||
stats.Sent++
|
||||
} else {
|
||||
stats.Failed++
|
||||
}
|
||||
}
|
||||
}
|
||||
// Preserve completion when the caller or outbound request is cancelled.
|
||||
finishCtx, stop := context.WithTimeout(context.Background(), PushFinishTimeout)
|
||||
err = d.Queue.Finish(finishCtx, item, outcome, now())
|
||||
stop()
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
}
|
||||
return stats, parent.Err()
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package ordoschnellic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type queueFixture struct {
|
||||
items []*PushDelivery
|
||||
calls []string
|
||||
errAt string
|
||||
err error
|
||||
finished []Outcome
|
||||
finishContext func(context.Context)
|
||||
at time.Time
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (q *queueFixture) call(op string, at time.Time) error {
|
||||
q.t.Helper()
|
||||
if !at.Equal(q.at) {
|
||||
q.t.Fatalf("clock for %s: %v", op, at)
|
||||
}
|
||||
q.calls = append(q.calls, op)
|
||||
if op == q.errAt {
|
||||
return q.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (q *queueFixture) Cleanup(_ context.Context, at time.Time) error { return q.call("cleanup", at) }
|
||||
func (q *queueFixture) Claim(_ context.Context, at time.Time) (*PushDelivery, error) {
|
||||
if err := q.call("claim", at); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(q.items) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
d := q.items[0]
|
||||
q.items = q.items[1:]
|
||||
return d, nil
|
||||
}
|
||||
func (q *queueFixture) DeferFocused(_ context.Context, _ *PushDelivery, at time.Time) error {
|
||||
return q.call("defer", at)
|
||||
}
|
||||
func (q *queueFixture) Finish(ctx context.Context, _ *PushDelivery, o Outcome, at time.Time) error {
|
||||
if q.finishContext != nil {
|
||||
q.finishContext(ctx)
|
||||
}
|
||||
q.finished = append(q.finished, o)
|
||||
return q.call("finish", at)
|
||||
}
|
||||
|
||||
type senderFixture func(context.Context, *PushDelivery) Outcome
|
||||
|
||||
func (f senderFixture) Send(ctx context.Context, d *PushDelivery) Outcome { return f(ctx, d) }
|
||||
|
||||
func newQueueFixture(t *testing.T) *queueFixture {
|
||||
return &queueFixture{t: t, at: time.Unix(1800000000, 0), err: errors.New("queue failure")}
|
||||
}
|
||||
|
||||
func TestDispatcherBoundedBatchAndCounters(t *testing.T) {
|
||||
q := newQueueFixture(t)
|
||||
q.items = append(q.items, &PushDelivery{Focused: true}, &PushDelivery{})
|
||||
for i := 0; i < 25; i++ {
|
||||
q.items = append(q.items, &PushDelivery{Subscription: "valid"})
|
||||
}
|
||||
sends := 0
|
||||
sender := senderFixture(func(context.Context, *PushDelivery) Outcome {
|
||||
sends++
|
||||
return Outcome{Attempted: true, Success: sends%2 == 0, Retry: sends%2 != 0}
|
||||
})
|
||||
stats, err := (Dispatcher{Queue: q, Sender: sender, Now: func() time.Time { return q.at }}).Run(context.Background())
|
||||
if err != nil || stats != (Stats{Sent: 9, Failed: 9}) || sends != 18 || len(q.items) != 7 || len(q.finished) != 19 {
|
||||
t.Fatalf("stats=%+v sends=%d pending=%d finished=%d err=%v", stats, sends, len(q.items), len(q.finished), err)
|
||||
}
|
||||
if q.finished[0] != (Outcome{}) {
|
||||
t.Fatal("ineligible delivery was sent/retried")
|
||||
}
|
||||
if fmt.Sprint(q.calls[:5]) != "[cleanup claim defer claim finish]" {
|
||||
t.Fatal(q.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatcherStopsOnQueueErrors(t *testing.T) {
|
||||
for _, op := range []string{"cleanup", "claim", "defer", "finish"} {
|
||||
t.Run(op, func(t *testing.T) {
|
||||
q := newQueueFixture(t)
|
||||
q.errAt = op
|
||||
q.items = []*PushDelivery{{Subscription: "valid", Focused: op == "defer"}, {Subscription: "valid"}}
|
||||
sends := 0
|
||||
_, err := (Dispatcher{Queue: q, Sender: senderFixture(func(context.Context, *PushDelivery) Outcome { sends++; return Outcome{Attempted: true, Success: true} }), Now: func() time.Time { return q.at }}).Run(context.Background())
|
||||
if !errors.Is(err, q.err) || q.calls[len(q.calls)-1] != op || sends > 1 {
|
||||
t.Fatalf("calls=%v sends=%d err=%v", q.calls, sends, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatcherCancellationFinalizesClaim(t *testing.T) {
|
||||
q := newQueueFixture(t)
|
||||
q.items = []*PushDelivery{{Subscription: "valid"}, {Subscription: "valid"}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
q.finishContext = func(ctx context.Context) {
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal("completion inherited cancellation")
|
||||
}
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok || time.Until(deadline) > PushFinishTimeout {
|
||||
t.Fatal("completion is not bounded")
|
||||
}
|
||||
}
|
||||
stats, err := (Dispatcher{Queue: q, Now: func() time.Time { return q.at }, Sender: senderFixture(func(ctx context.Context, _ *PushDelivery) Outcome {
|
||||
if _, ok := ctx.Deadline(); !ok {
|
||||
t.Fatal("send has no deadline")
|
||||
}
|
||||
cancel()
|
||||
return Outcome{Attempted: true, Retry: true}
|
||||
})}).Run(ctx)
|
||||
if !errors.Is(err, context.Canceled) || stats.Failed != 1 || len(q.finished) != 1 || len(q.items) != 1 || !q.finished[0].Retry {
|
||||
t.Fatalf("%+v %v %+v", stats, err, q.finished)
|
||||
}
|
||||
q.calls = nil
|
||||
_, err = (Dispatcher{Queue: q, Sender: senderFixture(func(context.Context, *PushDelivery) Outcome { t.Fatal("cancelled batch sent"); return Outcome{} })}).Run(ctx)
|
||||
if !errors.Is(err, context.Canceled) || len(q.calls) != 0 {
|
||||
t.Fatal("cancelled batch accessed queue", err, q.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryPolicy(t *testing.T) {
|
||||
at := time.Unix(1800000000, 0)
|
||||
for _, attempt := range []int{-1, 0, 1, 2, 3, 4, 5, 1000000} {
|
||||
next, ok := RetryAt(attempt, at)
|
||||
if ok != (attempt >= 1 && attempt <= 4) {
|
||||
t.Fatalf("attempt %d", attempt)
|
||||
}
|
||||
if ok && next.Sub(at) != time.Duration(1<<attempt)*time.Minute {
|
||||
t.Fatal("backoff", attempt, next)
|
||||
}
|
||||
}
|
||||
for _, code := range []int{200, 201, 204, 301, 400, 401, 403, 404, 410, 429, 500, 503} {
|
||||
o := ClassifyPushStatus(code)
|
||||
if !o.Attempted || o.Success != (code >= 200 && code < 300) || o.Retry != (code == 429 || code >= 500) || o.Expired != (code == 404 || code == 410) {
|
||||
t.Fatal(code, o)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package ordoschnellic
|
||||
|
||||
import "time"
|
||||
|
||||
const (
|
||||
PushBatchLimit = 20
|
||||
PushBatchTimeout = 25 * time.Second
|
||||
PushFinishTimeout = 3 * time.Second
|
||||
PushLeaseDuration = time.Minute
|
||||
PushFocusDelay = time.Minute
|
||||
PushRetention = 24 * time.Hour
|
||||
PushAttemptLimit = 5
|
||||
)
|
||||
|
||||
// RetryAt preserves 2/4/8/16-minute backoff after attempts 1 through 4.
|
||||
// Exhausted or invalid attempts are terminal and cannot overflow a shift.
|
||||
func RetryAt(attempt int, now time.Time) (time.Time, bool) {
|
||||
if attempt < 1 || attempt >= PushAttemptLimit {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return now.Add(time.Duration(1<<attempt) * time.Minute), true
|
||||
}
|
||||
|
||||
func ClassifyPushStatus(status int) Outcome {
|
||||
return Outcome{Success: status >= 200 && status < 300, Retry: status == 429 || status >= 500, Expired: status == 404 || status == 410, Attempted: true}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
// Package ordoschnellic coordinates notification delivery. Business events,
|
||||
// recipient authorization, Inbox transactions and durable claims belong to its
|
||||
// callers. A transport acknowledgement is not proof of end-user receipt.
|
||||
package ordoschnellic
|
||||
|
||||
import "errors"
|
||||
|
||||
const Component = "ordoschnellic"
|
||||
const PushEventType = "notification.push.deliver"
|
||||
|
||||
type Notification struct {
|
||||
UUID string `json:"uuid"`
|
||||
DomainUUID string `json:"domain_uuid"`
|
||||
RecipientUUID string `json:"recipient_uuid"`
|
||||
EventUUID string `json:"event_uuid,omitempty"`
|
||||
Kind string `json:"kind"`
|
||||
Status string `json:"status"`
|
||||
RequestUUID string `json:"request_uuid,omitempty"`
|
||||
Href string `json:"href,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
CreatedAtMS int64 `json:"created_at_ms"`
|
||||
ReadAtMS int64 `json:"read_at_ms,omitempty"`
|
||||
}
|
||||
|
||||
var ErrNotificationInput = errors.New("invalid notification")
|
||||
|
||||
type PushDelivery struct {
|
||||
ID, Notification, Domain, Owner, Subscription string
|
||||
Focused bool
|
||||
Attempt int
|
||||
}
|
||||
|
||||
type Outcome struct {
|
||||
Success, Retry, Expired bool
|
||||
// Attempted includes key/encryption setup failures, but excludes invalid
|
||||
// stored subscriptions. This preserves the existing delivery counters.
|
||||
Attempted bool
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package ordoschnellic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdh"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
webpush "github.com/SherClockHolmes/webpush-go"
|
||||
)
|
||||
|
||||
type StoredBrowserPush struct {
|
||||
webpush.Subscription
|
||||
Origin string `json:"origin"`
|
||||
}
|
||||
|
||||
// Only browser push vendors are destinations, never arbitrary user-provided URLs.
|
||||
// DNS pinning/public-address checks at dial time provide a second SSRF boundary.
|
||||
func ValidPushSubscription(sub webpush.Subscription) bool {
|
||||
u, e := url.Parse(sub.Endpoint)
|
||||
if e != nil || u.Scheme != "https" || u.User != nil || u.Fragment != "" || u.Port() != "" || len(sub.Endpoint) > 4096 {
|
||||
return false
|
||||
}
|
||||
host := u.Hostname()
|
||||
if host != "fcm.googleapis.com" && host != "updates.push.services.mozilla.com" && host != "web.push.apple.com" && !strings.HasSuffix(host, ".push.services.mozilla.com") {
|
||||
return false
|
||||
}
|
||||
auth, e := base64.RawURLEncoding.DecodeString(sub.Keys.Auth)
|
||||
if e != nil || len(auth) != 16 {
|
||||
return false
|
||||
}
|
||||
key, e := base64.RawURLEncoding.DecodeString(sub.Keys.P256dh)
|
||||
if e != nil {
|
||||
return false
|
||||
}
|
||||
_, e = ecdh.P256().NewPublicKey(key)
|
||||
return e == nil
|
||||
}
|
||||
|
||||
type KeyProvider func(context.Context) (private, public string, err error)
|
||||
|
||||
// Client must enforce public-address DNS pinning, TLS, a bounded timeout and
|
||||
// no redirects. There is deliberately no default HTTP client fallback.
|
||||
type WebPush struct {
|
||||
Keys KeyProvider
|
||||
Client webpush.HTTPClient
|
||||
}
|
||||
|
||||
// wakePayload intentionally has no access to Inbox titles, bodies or tokens.
|
||||
func wakePayload(item *PushDelivery) []byte {
|
||||
payload, _ := json.Marshal(map[string]string{"uuid": item.Notification, "owner": item.Owner, "domain": item.Domain})
|
||||
return payload
|
||||
}
|
||||
|
||||
func (w WebPush) Send(ctx context.Context, item *PushDelivery) Outcome {
|
||||
if item == nil {
|
||||
return Outcome{}
|
||||
}
|
||||
var sub StoredBrowserPush
|
||||
if json.Unmarshal([]byte(item.Subscription), &sub) != nil || !ValidPushSubscription(sub.Subscription) {
|
||||
return Outcome{Expired: true}
|
||||
}
|
||||
if w.Keys == nil || w.Client == nil || ctx.Err() != nil {
|
||||
return Outcome{Retry: true, Attempted: true}
|
||||
}
|
||||
private, public, err := w.Keys(ctx)
|
||||
if err != nil {
|
||||
return Outcome{Retry: true, Attempted: true}
|
||||
}
|
||||
resp, err := webpush.SendNotificationWithContext(ctx, wakePayload(item), &sub.Subscription, &webpush.Options{HTTPClient: w.Client, Subscriber: sub.Origin, VAPIDPrivateKey: private, VAPIDPublicKey: public, TTL: 300, Urgency: webpush.UrgencyNormal, Topic: "pageswiki-inbox"})
|
||||
if err != nil {
|
||||
if resp != nil && resp.Body != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
return Outcome{Retry: true, Attempted: true}
|
||||
}
|
||||
if resp == nil {
|
||||
return Outcome{Retry: true, Attempted: true}
|
||||
}
|
||||
if resp.Body != nil {
|
||||
defer resp.Body.Close()
|
||||
}
|
||||
return ClassifyPushStatus(resp.StatusCode)
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package ordoschnellic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdh"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
webpush "github.com/SherClockHolmes/webpush-go"
|
||||
)
|
||||
|
||||
type httpFixture func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f httpFixture) Do(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
type trackedBody struct {
|
||||
io.Reader
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (b *trackedBody) Close() error { b.closed = true; return nil }
|
||||
|
||||
func subscriptionFixture(t *testing.T) StoredBrowserPush {
|
||||
t.Helper()
|
||||
key, err := ecdh.P256().GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return StoredBrowserPush{Subscription: webpush.Subscription{Endpoint: "https://fcm.googleapis.com/fcm/send/synthetic", Keys: webpush.Keys{Auth: base64.RawURLEncoding.EncodeToString(make([]byte, 16)), P256dh: base64.RawURLEncoding.EncodeToString(key.PublicKey().Bytes())}}, Origin: "https://wiki.example"}
|
||||
}
|
||||
func deliveryFixture(t *testing.T, sub StoredBrowserPush) *PushDelivery {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(sub)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &PushDelivery{Notification: "synthetic-notification", Owner: "synthetic-owner", Domain: "synthetic-domain", Subscription: string(b)}
|
||||
}
|
||||
|
||||
func TestWebPushPrivacyAndResponseHandling(t *testing.T) {
|
||||
private, public, err := webpush.GenerateVAPIDKeys()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := deliveryFixture(t, subscriptionFixture(t))
|
||||
var payload map[string]string
|
||||
if err := json.Unmarshal(wakePayload(d), &payload); err != nil || len(payload) != 3 || payload["uuid"] != d.Notification || payload["owner"] != d.Owner || payload["domain"] != d.Domain {
|
||||
t.Fatal("payload contract", payload, err)
|
||||
}
|
||||
for _, code := range []int{201, 302, 400, 404, 410, 429, 503} {
|
||||
body := &trackedBody{Reader: strings.NewReader("")}
|
||||
w := WebPush{Keys: func(context.Context) (string, string, error) { return private, public, nil }, Client: httpFixture(func(r *http.Request) (*http.Response, error) {
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(data), d.Notification) || strings.Contains(string(data), d.Owner) || r.Header.Get("Content-Encoding") != "aes128gcm" || r.Header.Get("TTL") != "300" || r.Header.Get("Topic") != "pageswiki-inbox" {
|
||||
t.Fatal("unprotected/changed payload")
|
||||
}
|
||||
return &http.Response{StatusCode: code, Body: body}, nil
|
||||
})}
|
||||
if o := w.Send(context.Background(), d); o != ClassifyPushStatus(code) || !body.closed {
|
||||
t.Fatal(code, o, body.closed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebPushFailsClosedWithoutNetwork(t *testing.T) {
|
||||
sub := subscriptionFixture(t)
|
||||
d := deliveryFixture(t, sub)
|
||||
for _, endpoint := range []string{"http://fcm.googleapis.com/x", "https://localhost/x", "https://127.0.0.1/x", "https://fcm.googleapis.com.evil.example/x", "https://fcm.googleapis.com:443/x", "https://user@fcm.googleapis.com/x", "https://web.push.apple.com/x#f"} {
|
||||
sub.Endpoint = endpoint
|
||||
w := WebPush{Keys: func(context.Context) (string, string, error) {
|
||||
t.Fatal("unsafe target requested keys")
|
||||
return "", "", nil
|
||||
}, Client: httpFixture(func(*http.Request) (*http.Response, error) { t.Fatal("unsafe target contacted"); return nil, nil })}
|
||||
if o := w.Send(context.Background(), deliveryFixture(t, sub)); !o.Expired || o.Attempted {
|
||||
t.Fatal(endpoint, o)
|
||||
}
|
||||
}
|
||||
for _, w := range []WebPush{{}, {Keys: func(context.Context) (string, string, error) {
|
||||
t.Fatal("no HTTP client must fail closed")
|
||||
return "", "", nil
|
||||
}}, {Client: httpFixture(func(*http.Request) (*http.Response, error) { t.Fatal("no keys must fail closed"); return nil, nil })}} {
|
||||
if o := w.Send(context.Background(), d); !o.Retry || !o.Attempted {
|
||||
t.Fatal(o)
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
w := WebPush{Keys: func(context.Context) (string, string, error) {
|
||||
t.Fatal("cancelled request loaded keys")
|
||||
return "", "", nil
|
||||
}, Client: httpFixture(func(*http.Request) (*http.Response, error) { t.Fatal("cancelled request sent"); return nil, nil })}
|
||||
if o := w.Send(ctx, d); !o.Retry {
|
||||
t.Fatal(o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebPushTransportFailure(t *testing.T) {
|
||||
private, public, err := webpush.GenerateVAPIDKeys()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := deliveryFixture(t, subscriptionFixture(t))
|
||||
for _, mode := range []string{"keys", "network", "nil-response"} {
|
||||
w := WebPush{Keys: func(context.Context) (string, string, error) {
|
||||
if mode == "keys" {
|
||||
return "", "", errors.New("secret must not escape")
|
||||
}
|
||||
return private, public, nil
|
||||
}, Client: httpFixture(func(*http.Request) (*http.Response, error) {
|
||||
if mode == "keys" {
|
||||
t.Fatal("sent after key failure")
|
||||
}
|
||||
if mode == "network" {
|
||||
return nil, errors.New("endpoint must not escape")
|
||||
}
|
||||
return nil, nil
|
||||
})}
|
||||
if o := w.Send(context.Background(), d); o != (Outcome{Retry: true, Attempted: true}) {
|
||||
t.Fatal(mode, o)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,9 @@
|
||||
package render
|
||||
|
||||
import (
|
||||
"github.com/gomarkdown/markdown"
|
||||
)
|
||||
import "pageswiki/internal/grammaratur"
|
||||
|
||||
// Markdown retains the renderer entry point while Grammaratur owns conversion.
|
||||
// It deliberately does not enable authoring fences or sanitize display HTML.
|
||||
func Markdown(input string) string {
|
||||
out := markdown.ToHTML(
|
||||
[]byte(input),
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
|
||||
return string(out)
|
||||
return grammaratur.MarkdownToHTML(input)
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
// Package schutzaria evaluates trusted, request-scoped authorization facts.
|
||||
// It does not authenticate callers, load grants, resolve aliases, or commit
|
||||
// publication. Callers must supply the correct Domain/principal snapshot.
|
||||
package schutzaria
|
||||
|
||||
import "strings"
|
||||
|
||||
const Component = "schutzaria"
|
||||
|
||||
// AccessLayer is the union of matching subjects at one locality. Layers at
|
||||
// the same Path and at ancestors intersect; Grant cannot override a denial.
|
||||
type AccessLayer struct {
|
||||
Path string
|
||||
Read, Write, Grant bool
|
||||
}
|
||||
|
||||
// LocalityAccess is immutable after construction and must not be cached
|
||||
// across requests. Admin bypasses caps only; it does not create base grants.
|
||||
type LocalityAccess struct {
|
||||
admin bool
|
||||
paths map[string][]AccessLayer
|
||||
}
|
||||
|
||||
func NewLocalityAccess(admin bool, layers []AccessLayer) *LocalityAccess {
|
||||
p := &LocalityAccess{admin: admin, paths: make(map[string][]AccessLayer)}
|
||||
for _, layer := range layers {
|
||||
p.paths[layer.Path] = append(p.paths[layer.Path], layer)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// RequiresPathChecks permits the existing Object-alias fast path only when
|
||||
// no local cap applies or the authenticated actor has recovery authority.
|
||||
// Foreign-Domain restrictions must be checked separately before this shortcut.
|
||||
func (p *LocalityAccess) RequiresPathChecks() bool {
|
||||
return p != nil && !p.admin && len(p.paths) != 0
|
||||
}
|
||||
|
||||
func (p *LocalityAccess) Decide(path, capability string, basis bool) bool {
|
||||
if !strings.HasPrefix(path, "/") {
|
||||
return false
|
||||
}
|
||||
if p == nil || p.admin {
|
||||
return basis
|
||||
}
|
||||
grant := false
|
||||
for at := path; ; {
|
||||
for _, layer := range p.paths[at] {
|
||||
allowed := layer.Write
|
||||
if capability == "read" {
|
||||
allowed = layer.Read
|
||||
}
|
||||
if !allowed {
|
||||
return false
|
||||
}
|
||||
if layer.Grant && (capability == "read" || capability == "write" || capability == "create" || capability == "upload") {
|
||||
grant = true
|
||||
}
|
||||
}
|
||||
if at == "/" {
|
||||
break
|
||||
}
|
||||
i := strings.LastIndex(at, "/")
|
||||
if i <= 0 {
|
||||
at = "/"
|
||||
} else {
|
||||
at = at[:i]
|
||||
}
|
||||
}
|
||||
return basis || grant
|
||||
}
|
||||
|
||||
// ReadPathPolicy contains only positive Path grants plus locality caps.
|
||||
// Object grants must never expose all semantic aliases. A nil/zero policy denies.
|
||||
type ReadPathPolicy struct {
|
||||
access *LocalityAccess
|
||||
all bool
|
||||
exact, subtrees map[string]bool
|
||||
}
|
||||
|
||||
// NewReadPathPolicy copies input slices; subsequent mutations cannot broaden
|
||||
// an existing snapshot. Paths are already normalized by the application.
|
||||
func NewReadPathPolicy(all bool, exact, subtrees []string, access *LocalityAccess) *ReadPathPolicy {
|
||||
p := &ReadPathPolicy{all: all, access: access, exact: make(map[string]bool), subtrees: make(map[string]bool)}
|
||||
for _, path := range exact {
|
||||
p.exact[path] = true
|
||||
}
|
||||
for _, path := range subtrees {
|
||||
p.subtrees[path] = true
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *ReadPathPolicy) Allows(path string) bool {
|
||||
if p == nil {
|
||||
return false
|
||||
}
|
||||
return p.access.Decide(path, "read", p.allowsBase(path))
|
||||
}
|
||||
|
||||
func (p *ReadPathPolicy) allowsBase(path string) bool {
|
||||
if p == nil || !strings.HasPrefix(path, "/") {
|
||||
return false
|
||||
}
|
||||
if p.all || p.exact[path] {
|
||||
return true
|
||||
}
|
||||
for at := strings.LastIndex(path, "/"); at > 0; at = strings.LastIndex(path[:at], "/") {
|
||||
if p.subtrees[path[:at]] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package schutzaria
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLocalityAccessIntersectionAndGrantLimits(t *testing.T) {
|
||||
for _, parentRead := range []bool{false, true} {
|
||||
for _, parentWrite := range []bool{false, true} {
|
||||
for _, childRead := range []bool{false, true} {
|
||||
for _, childWrite := range []bool{false, true} {
|
||||
for _, grant := range []bool{false, true} {
|
||||
for _, basis := range []bool{false, true} {
|
||||
p := NewLocalityAccess(false, []AccessLayer{{Path: "/life", Read: parentRead, Write: parentWrite}, {Path: "/life/home", Read: childRead, Write: childWrite, Grant: grant}})
|
||||
for _, cap := range []string{"read", "write", "create", "upload", "rename", "link", "delete", "manage"} {
|
||||
allowed := parentWrite && childWrite
|
||||
if cap == "read" {
|
||||
allowed = parentRead && childRead
|
||||
}
|
||||
positive := grant && (cap == "read" || cap == "write" || cap == "create" || cap == "upload")
|
||||
want := allowed && (basis || positive)
|
||||
if got := p.Decide("/life/home/page", cap, basis); got != want {
|
||||
t.Fatalf("parents=%v/%v children=%v/%v grant=%v base=%v cap=%s got=%v", parentRead, parentWrite, childRead, childWrite, grant, basis, cap, got)
|
||||
}
|
||||
}
|
||||
if got := p.Decide("/lifes/other", "read", basis); got != basis {
|
||||
t.Fatal("prefix boundary leaked")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Multiple locality nodes may share a boundary; one denial still wins.
|
||||
p := NewLocalityAccess(false, []AccessLayer{{Path: "/same", Read: true, Write: true, Grant: true}, {Path: "/same"}})
|
||||
if p.Decide("/same/child", "read", true) {
|
||||
t.Fatal("same-Path denial lost")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalityAdminAndAbsentCapsNeverCreateGrants(t *testing.T) {
|
||||
for _, p := range []*LocalityAccess{nil, NewLocalityAccess(false, nil), NewLocalityAccess(true, []AccessLayer{{Path: "/"}})} {
|
||||
for _, basis := range []bool{false, true} {
|
||||
if p.Decide("/a", "read", basis) != basis || p.Decide("relative", "read", basis) || p.Decide("", "read", basis) {
|
||||
t.Fatal("admin/absent cap created a grant or allowed non-Path")
|
||||
}
|
||||
}
|
||||
if p.RequiresPathChecks() {
|
||||
t.Fatal("unexpected alias checks")
|
||||
}
|
||||
}
|
||||
if !NewLocalityAccess(false, []AccessLayer{{Path: "/a"}}).RequiresPathChecks() {
|
||||
t.Fatal("Object alias checks skipped")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadPolicyLiteralScopesAndImmutability(t *testing.T) {
|
||||
exact := []string{"/exact", "/group_%"}
|
||||
subtrees := []string{"/group_%"}
|
||||
layers := []AccessLayer{{Path: "/group_%/private"}}
|
||||
access := NewLocalityAccess(false, layers)
|
||||
p := NewReadPathPolicy(false, exact, subtrees, access)
|
||||
exact[0] = "/leak"
|
||||
subtrees[0] = "/"
|
||||
layers[0] = AccessLayer{Path: "/group_%/private", Read: true, Grant: true}
|
||||
for path, want := range map[string]bool{"/exact": true, "/exact/child": false, "/group_%": true, "/group_%/a/b": true, "/group_%/private": false, "/group_%/private/child": false, "/groupXX/a": false, "/group_%2": false, "/leak": false, "/": false, "": false, "relative": false} {
|
||||
if got := p.Allows(path); got != want {
|
||||
t.Fatalf("path=%q got=%v want=%v", path, got, want)
|
||||
}
|
||||
}
|
||||
for _, deny := range []*ReadPathPolicy{nil, {}} {
|
||||
if deny.Allows("/anything") {
|
||||
t.Fatal("empty policy failed open")
|
||||
}
|
||||
}
|
||||
all := NewReadPathPolicy(true, nil, nil, access)
|
||||
if !all.Allows("/other") || all.Allows("/group_%/private") || all.Allows("relative") {
|
||||
t.Fatal("broad grant bypassed caps")
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 16; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := 0; j < 100; j++ {
|
||||
if !p.Allows("/exact") || p.Allows("/leak") {
|
||||
t.Error("snapshot changed")
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestGroupPermissionProfiles(t *testing.T) {
|
||||
wants := map[string]map[string]bool{
|
||||
"read": {"read": true},
|
||||
"read_write_once": {"read": true, "create": true, "upload": true},
|
||||
"read_write": {"read": true, "create": true, "write": true, "upload": true, "rename": true, "link": true},
|
||||
"upload_only": {"upload": true},
|
||||
"unknown": {},
|
||||
}
|
||||
for profile, want := range wants {
|
||||
for _, cap := range []string{"read", "create", "write", "upload", "rename", "link", "delete", "manage", ""} {
|
||||
for _, path := range []string{"/a_%", "/a_%/child"} {
|
||||
if GroupPermissionAllows("/a_%", profile, cap, path) != want[cap] {
|
||||
t.Fatal(profile, cap, path)
|
||||
}
|
||||
}
|
||||
if GroupPermissionAllows("/a_%", profile, cap, "/aXX/child") {
|
||||
t.Fatal("wildcard interpreted")
|
||||
}
|
||||
}
|
||||
}
|
||||
if p, err := NormalizeGroupPermission(" READ_WRITE_ONCE "); err != nil || p != "read_write_once" {
|
||||
t.Fatal(p, err)
|
||||
}
|
||||
for _, p := range []string{"", "delete", "group_admin", "read/write"} {
|
||||
if _, err := NormalizeGroupPermission(p); err == nil {
|
||||
t.Fatal("unknown profile", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package schutzaria
|
||||
|
||||
import "errors"
|
||||
|
||||
var ErrApprovalTransition = errors.New("invalid approval request transition")
|
||||
|
||||
func ValidApprovalDecision(decision string) bool {
|
||||
return decision == "approved" || decision == "rejected" || decision == "changes_requested"
|
||||
}
|
||||
|
||||
// ApprovalNextState preserves the existing request state machine only.
|
||||
// It does not verify reviewer authority, expiry, quorum, source revisions or
|
||||
// audience. The caller owns those checks and the transactional CAS/write.
|
||||
func ApprovalNextState(state string, currentGeneration, generation int64, decision string) (string, error) {
|
||||
if generation < 1 || state != "pending" || currentGeneration != generation || !ValidApprovalDecision(decision) {
|
||||
return "", ErrApprovalTransition
|
||||
}
|
||||
if decision == "changes_requested" {
|
||||
return "rejected", nil
|
||||
}
|
||||
return decision, nil
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package schutzaria
|
||||
|
||||
import (
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestComponentBoundary(t *testing.T) {
|
||||
if Component != "schutzaria" {
|
||||
t.Fatal("component identity changed")
|
||||
}
|
||||
files, err := filepath.Glob("*.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, file := range files {
|
||||
if strings.HasSuffix(file, "_test.go") {
|
||||
continue
|
||||
}
|
||||
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, spec := range ast.Imports {
|
||||
path, err := strconv.Unquote(spec.Path.Value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, prefix := range []string{"pageswiki/internal", "database", "net", "os", "time"} {
|
||||
if path == prefix || strings.HasPrefix(path, prefix+"/") {
|
||||
t.Errorf("%s imports %s: evaluation must not acquire state, perform I/O or read time", file, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package schutzaria
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var groupPermissions = map[string]map[string]bool{
|
||||
"read": {"read": true},
|
||||
"read_write_once": {"read": true, "create": true, "upload": true},
|
||||
"read_write": {"read": true, "create": true, "write": true, "upload": true, "rename": true, "link": true},
|
||||
"upload_only": {"upload": true},
|
||||
}
|
||||
|
||||
func NormalizeGroupPermission(value string) (string, error) {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
if _, ok := groupPermissions[value]; !ok {
|
||||
return "", errors.New("invalid group permission")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// GroupPermissionAllows evaluates a previously authenticated membership's
|
||||
// profile. Creation-only existence checks and principal state belong to callers.
|
||||
// This literal prefix rule deliberately preserves the existing stored scopes.
|
||||
func GroupPermissionAllows(scope, permission, capability, targetPath string) bool {
|
||||
inside := targetPath == scope || scope == "/" || strings.HasPrefix(targetPath, strings.TrimRight(scope, "/")+"/")
|
||||
return inside && groupPermissions[permission][capability]
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package schutzaria
|
||||
|
||||
import "strings"
|
||||
|
||||
type LocalityPolicy struct {
|
||||
Kind string `json:"kind"`
|
||||
UUID string `json:"uuid"`
|
||||
Policy string `json:"policy"`
|
||||
}
|
||||
|
||||
// EvaluateLocalityPublication combines a Domain-first chain's ceilings and
|
||||
// intent. Neither result is an anonymous-read grant or a reviewer approval.
|
||||
// The caller loads/validates topology and binds immutable revisions separately.
|
||||
func EvaluateLocalityPublication(chain []LocalityPolicy) (allows, published bool) {
|
||||
allows = true
|
||||
for i, p := range chain {
|
||||
if i == 0 && p.Policy == "inherit" {
|
||||
return false, false
|
||||
}
|
||||
switch p.Policy {
|
||||
case "authenticated_acl":
|
||||
allows = false
|
||||
case "public":
|
||||
if i > 0 {
|
||||
published = true
|
||||
}
|
||||
case "inherit":
|
||||
default:
|
||||
return false, false
|
||||
}
|
||||
}
|
||||
return allows, allows && published
|
||||
}
|
||||
|
||||
// PublicSemanticPath is a namespace eligibility check, never permission to
|
||||
// publish content. User-space publication has its own authenticated policy path.
|
||||
func PublicSemanticPath(path string) bool {
|
||||
if !strings.HasPrefix(path, "/") || strings.ContainsAny(path, "\\\x00\r\n") || strings.Contains(path, "//") {
|
||||
return false
|
||||
}
|
||||
for _, segment := range strings.Split(path, "/") {
|
||||
if segment == "." || segment == ".." {
|
||||
return false
|
||||
}
|
||||
}
|
||||
first := strings.Split(strings.TrimPrefix(path, "/"), "/")[0]
|
||||
switch first {
|
||||
case "_", "-", ":", "o", "r", "v", "l", "e", "s", "u", "g", "static", "health", "new":
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package schutzaria
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPublicationCeilings(t *testing.T) {
|
||||
values := []string{"public", "authenticated_acl", "inherit", "invalid"}
|
||||
for _, domain := range values {
|
||||
for _, quartier := range values {
|
||||
for _, street := range values {
|
||||
for _, spot := range values {
|
||||
chain := []LocalityPolicy{{Kind: "domain", Policy: domain}, {Kind: "quartier", Policy: quartier}, {Kind: "street", Policy: street}, {Kind: "spot", Policy: spot}}
|
||||
wantAllows := domain == "public" && quartier != "authenticated_acl" && street != "authenticated_acl" && spot != "authenticated_acl" && quartier != "invalid" && street != "invalid" && spot != "invalid"
|
||||
wantIntent := wantAllows && (quartier == "public" || street == "public" || spot == "public")
|
||||
a, p := EvaluateLocalityPublication(chain)
|
||||
if a != wantAllows || p != wantIntent {
|
||||
t.Fatalf("%+v got=%v/%v", chain, a, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Preserve the legacy neutral empty-chain result; it is never a grant.
|
||||
if a, p := EvaluateLocalityPublication(nil); !a || p {
|
||||
t.Fatal("empty chain semantics changed")
|
||||
}
|
||||
if a, p := EvaluateLocalityPublication([]LocalityPolicy{{Kind: "domain", Policy: "public"}}); !a || p {
|
||||
t.Fatal("Domain ceiling published content")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicNamespaceEligibility(t *testing.T) {
|
||||
for _, p := range []string{"/", "/life/home", "/blog/2026/11/01", "/日本語", "/user"} {
|
||||
if !PublicSemanticPath(p) {
|
||||
t.Fatal(p)
|
||||
}
|
||||
}
|
||||
for _, p := range []string{"", "relative", "/a/../b", "/a/./b", "/a//b", "/a\\b", "/a\x00b", "/a\nb", "/a\rb"} {
|
||||
if PublicSemanticPath(p) {
|
||||
t.Fatal(p)
|
||||
}
|
||||
}
|
||||
for _, root := range []string{"_", "-", ":", "o", "r", "v", "l", "e", "s", "u", "g", "static", "health", "new"} {
|
||||
if PublicSemanticPath("/"+root) || PublicSemanticPath("/"+root+"/child") {
|
||||
t.Fatal(root)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApprovalStateTransitionNotPublicationAuthorization(t *testing.T) {
|
||||
for _, state := range []string{"pending", "approved", "rejected", "withdrawn", "expired", "superseded", ""} {
|
||||
for _, decision := range []string{"approved", "rejected", "changes_requested", "unknown", ""} {
|
||||
for _, generation := range []int64{-1, 0, 1, 2} {
|
||||
next, err := ApprovalNextState(state, 1, generation, decision)
|
||||
valid := state == "pending" && generation == 1 && (decision == "approved" || decision == "rejected" || decision == "changes_requested")
|
||||
if valid {
|
||||
want := decision
|
||||
if want == "changes_requested" {
|
||||
want = "rejected"
|
||||
}
|
||||
if err != nil || next != want {
|
||||
t.Fatal(state, decision, next, err)
|
||||
}
|
||||
} else if !errors.Is(err, ErrApprovalTransition) || next != "" {
|
||||
t.Fatal(state, decision, generation, next, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserSpaceNarrowing(t *testing.T) {
|
||||
p := UserSpacePolicy{Username: "alice", RootVisibility: "domain", DescendantVisibility: "public", Overrides: []UserSpacePathPolicy{{Path: "/u/alice/work", Visibility: "domain"}, {Path: "/u/alice/work/private", Visibility: "private"}, {Path: "/u/alice/work/private/reopen", Visibility: "public"}}}
|
||||
for path, want := range map[string]string{"/u/alice": "domain", "/u/alice/blog": "public", "/u/alice/work": "domain", "/u/alice/work/page": "domain", "/u/alice/work/private": "private", "/u/alice/work/private/reopen/page": "private", "/u/bob/blog": "private", "/u/alice/_/edit": "private", "/u/alice/a/../blog": "private", "/u/alice/blog/": "private"} {
|
||||
if got := EffectiveUserSpaceVisibility(p, path); got != want {
|
||||
t.Fatal(path, got, want)
|
||||
}
|
||||
}
|
||||
// Existing semantics: root and descendants have independent settings.
|
||||
p.RootVisibility = "private"
|
||||
if EffectiveUserSpaceVisibility(p, "/u/alice/blog") != "public" {
|
||||
t.Fatal("root incorrectly became subtree cap")
|
||||
}
|
||||
if UserSpaceVisibilityRank("unknown") != -1 || UserSpaceDocumentPath("alice", "/u/alice/_", true) {
|
||||
t.Fatal("invalid user-space policy/path")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package schutzaria
|
||||
|
||||
import (
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type UserSpacePathPolicy struct {
|
||||
Path string `json:"path"`
|
||||
Visibility string `json:"visibility"`
|
||||
}
|
||||
|
||||
type UserSpacePolicy struct {
|
||||
PrincipalUUID string `json:"principal_uuid"`
|
||||
Username string `json:"username"`
|
||||
ProfileVisibility string `json:"profile_visibility"`
|
||||
AvatarVisibility string `json:"avatar_visibility"`
|
||||
RootVisibility string `json:"root_visibility"`
|
||||
DescendantVisibility string `json:"descendant_visibility"`
|
||||
Version int64 `json:"version"`
|
||||
DomainAllowsPublic bool `json:"domain_allows_public"`
|
||||
Overrides []UserSpacePathPolicy `json:"overrides"`
|
||||
}
|
||||
|
||||
func UserSpaceVisibilityRank(value string) int {
|
||||
switch value {
|
||||
case "private":
|
||||
return 0
|
||||
case "domain":
|
||||
return 1
|
||||
case "public":
|
||||
return 2
|
||||
default:
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
func UserSpaceDocumentPath(username, value string, descendant bool) bool {
|
||||
root := "/u/" + username
|
||||
if value != path.Clean(value) || strings.ContainsAny(value, "\\\x00\r\n") {
|
||||
return false
|
||||
}
|
||||
if descendant {
|
||||
if !strings.HasPrefix(value, root+"/") {
|
||||
return false
|
||||
}
|
||||
rest := strings.TrimPrefix(value, root+"/")
|
||||
return rest != "" && !strings.HasPrefix(rest, "_/") && rest != "_"
|
||||
}
|
||||
return value == root
|
||||
}
|
||||
|
||||
func EffectiveUserSpaceVisibility(p UserSpacePolicy, target string) string {
|
||||
root := "/u/" + p.Username
|
||||
if target == root {
|
||||
return p.RootVisibility
|
||||
}
|
||||
if !UserSpaceDocumentPath(p.Username, target, true) {
|
||||
return "private"
|
||||
}
|
||||
visibility := p.DescendantVisibility
|
||||
for _, item := range p.Overrides {
|
||||
if target == item.Path || strings.HasPrefix(target, strings.TrimRight(item.Path, "/")+"/") {
|
||||
if UserSpaceVisibilityRank(item.Visibility) < UserSpaceVisibilityRank(visibility) {
|
||||
visibility = item.Visibility
|
||||
}
|
||||
}
|
||||
}
|
||||
return visibility
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package seheweit
|
||||
|
||||
import "time"
|
||||
|
||||
const (
|
||||
ResourceEventID = "builtin.resource.sample"
|
||||
ResourceEventType = "resource.sample"
|
||||
RetentionDays = 7
|
||||
MaxHistoryPoints = 361
|
||||
)
|
||||
|
||||
// HistoryWindow describes the existing inclusive, epoch-aligned sample query.
|
||||
// Persistence chooses the most recently inserted sample in each bucket, not an
|
||||
// average, a peak or necessarily the largest timestamp in out-of-order data.
|
||||
type HistoryWindow struct {
|
||||
FromMS int64
|
||||
ToMS int64
|
||||
BucketMS int64
|
||||
}
|
||||
|
||||
func Window(hours int, now time.Time) HistoryWindow {
|
||||
if hours < 1 {
|
||||
hours = 1
|
||||
}
|
||||
if hours > RetentionDays*24 {
|
||||
hours = RetentionDays * 24
|
||||
}
|
||||
span := int64(time.Duration(hours) * time.Hour / time.Millisecond)
|
||||
bucket := span / (MaxHistoryPoints - 1)
|
||||
if bucket < int64(time.Minute/time.Millisecond) {
|
||||
bucket = int64(time.Minute / time.Millisecond)
|
||||
}
|
||||
return HistoryWindow{FromMS: now.UnixMilli() - span, ToMS: now.UnixMilli(), BucketMS: bucket}
|
||||
}
|
||||
|
||||
// RetentionCutoff is inclusive: only samples strictly older are expired.
|
||||
func RetentionCutoff(atMS int64) int64 {
|
||||
return atMS - int64(RetentionDays*24*time.Hour/time.Millisecond)
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package seheweit
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHistoryWindow(t *testing.T) {
|
||||
now := time.Date(2026, 11, 1, 12, 34, 56, 0, time.UTC)
|
||||
for _, tc := range []struct {
|
||||
hours, wantHours int
|
||||
bucket int64
|
||||
}{
|
||||
{-1, 1, 60000}, {0, 1, 60000}, {1, 1, 60000}, {24, 24, 240000}, {168, 168, 1680000}, {999, 168, 1680000},
|
||||
} {
|
||||
got := Window(tc.hours, now)
|
||||
if got.ToMS != now.UnixMilli() || got.FromMS != now.Add(-time.Duration(tc.wantHours)*time.Hour).UnixMilli() || got.BucketMS != tc.bucket {
|
||||
t.Fatalf("hours=%d %+v", tc.hours, got)
|
||||
}
|
||||
buckets := got.ToMS/got.BucketMS - got.FromMS/got.BucketMS + 1
|
||||
if buckets > MaxHistoryPoints {
|
||||
t.Fatalf("too many buckets: %d", buckets)
|
||||
}
|
||||
}
|
||||
if RetentionCutoff(now.UnixMilli()) != now.Add(-7*24*time.Hour).UnixMilli() {
|
||||
t.Fatal("retention changed")
|
||||
}
|
||||
if ResourceEventID != "builtin.resource.sample" || ResourceEventType != "resource.sample" {
|
||||
t.Fatal("persisted identifiers changed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
// Package seheweit collects instance observations without controlling the
|
||||
// observed workload. Authorization, queue execution and persistence belong to
|
||||
// its callers. It does not inspect document contents or contact remote services.
|
||||
package seheweit
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Identity survives multiple collectors, but not a process restart/PID reuse.
|
||||
var processID = uuid.NewString()
|
||||
|
||||
type ProcessIO struct {
|
||||
Scope string `json:"scope"`
|
||||
Available bool `json:"available"`
|
||||
ProcessID string `json:"process_id"`
|
||||
ReadBytes uint64 `json:"read_bytes"`
|
||||
WriteBytes uint64 `json:"write_bytes"`
|
||||
}
|
||||
|
||||
type CPU struct {
|
||||
LogicalCPUs int `json:"logical_cpus"`
|
||||
LoadAverage string `json:"load_average"`
|
||||
}
|
||||
|
||||
type Memory struct {
|
||||
HeapBytes uint64 `json:"heap_bytes"`
|
||||
GoSystemBytes uint64 `json:"go_system_bytes"`
|
||||
GCCycles uint32 `json:"gc_cycles"`
|
||||
Goroutines int `json:"goroutines"`
|
||||
}
|
||||
|
||||
type Storage struct {
|
||||
Path string `json:"path"`
|
||||
TotalBytes uint64 `json:"total_bytes"`
|
||||
UsedBytes uint64 `json:"used_bytes"`
|
||||
AvailableBytes uint64 `json:"available_bytes"`
|
||||
}
|
||||
|
||||
// Queue contains observations supplied by the application, not execution hooks.
|
||||
type Queue struct {
|
||||
Stats QueueStats `json:"stats"`
|
||||
ActiveWorkers int64 `json:"active_workers"`
|
||||
Paused bool `json:"paused"`
|
||||
}
|
||||
|
||||
type QueueStats struct {
|
||||
Queued int `json:"queued"`
|
||||
Running int `json:"running"`
|
||||
}
|
||||
|
||||
// Snapshot preserves the resource API and history JSON field names.
|
||||
// Sources are sampled sequentially, not in an atomic cross-source transaction.
|
||||
type Snapshot struct {
|
||||
TimestampMS int64 `json:"timestamp_ms"`
|
||||
IO ProcessIO `json:"io"`
|
||||
CPU CPU `json:"cpu"`
|
||||
Memory Memory `json:"memory"`
|
||||
Storage Storage `json:"storage"`
|
||||
Queue Queue `json:"queue"`
|
||||
}
|
||||
|
||||
type sources struct {
|
||||
now func() time.Time
|
||||
readFile func(string) ([]byte, error)
|
||||
statfs func(string, *syscall.Statfs_t) error
|
||||
memory func(*runtime.MemStats)
|
||||
cpus func() int
|
||||
goroutines func() int
|
||||
}
|
||||
|
||||
// Collect reads Go runtime counters, two bounded proc files and Statfs on the
|
||||
// configured document root. No directory walk, shell or background loop.
|
||||
func Collect(root string, queue Queue) Snapshot {
|
||||
return collect(root, queue, sources{
|
||||
now: time.Now, readFile: readProcFile, statfs: syscall.Statfs,
|
||||
memory: runtime.ReadMemStats, cpus: runtime.NumCPU, goroutines: runtime.NumGoroutine,
|
||||
})
|
||||
}
|
||||
|
||||
func collect(root string, queue Queue, src sources) Snapshot {
|
||||
var mem runtime.MemStats
|
||||
src.memory(&mem)
|
||||
if root == "" {
|
||||
root = "."
|
||||
}
|
||||
if absolute, err := filepath.Abs(root); err == nil {
|
||||
root = absolute
|
||||
}
|
||||
storage := Storage{Path: root}
|
||||
var fs syscall.Statfs_t
|
||||
if err := src.statfs(root, &fs); err == nil {
|
||||
storage = storageCounters(root, fs.Blocks, fs.Bavail, fs.Bsize)
|
||||
}
|
||||
loadAverage := ""
|
||||
if data, err := src.readFile("/proc/loadavg"); err == nil {
|
||||
fields := strings.Fields(string(data))
|
||||
if len(fields) >= 3 {
|
||||
loadAverage = strings.Join(fields[:3], " ")
|
||||
}
|
||||
}
|
||||
var ioData []byte
|
||||
if data, err := src.readFile("/proc/self/io"); err == nil {
|
||||
ioData = data
|
||||
}
|
||||
return Snapshot{
|
||||
TimestampMS: src.now().UnixMilli(), IO: ParseProcessIO(string(ioData)),
|
||||
CPU: CPU{LogicalCPUs: src.cpus(), LoadAverage: loadAverage},
|
||||
Memory: Memory{HeapBytes: mem.HeapAlloc, GoSystemBytes: mem.Sys, GCCycles: mem.NumGC, Goroutines: src.goroutines()},
|
||||
Storage: storage, Queue: queue,
|
||||
}
|
||||
}
|
||||
|
||||
func storageCounters(path string, blocks, available uint64, blockSize int64) Storage {
|
||||
result := Storage{Path: path}
|
||||
// Preserve the existing zero fallback rather than wrapping unsigned values.
|
||||
if blockSize <= 0 || available > blocks || blocks > math.MaxUint64/uint64(blockSize) {
|
||||
return result
|
||||
}
|
||||
result.TotalBytes = blocks * uint64(blockSize)
|
||||
result.AvailableBytes = available * uint64(blockSize)
|
||||
result.UsedBytes = result.TotalBytes - result.AvailableBytes
|
||||
return result
|
||||
}
|
||||
|
||||
const maxProcBytes = 16 << 10
|
||||
|
||||
func readProcFile(path string) ([]byte, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
data, err := io.ReadAll(io.LimitReader(f, maxProcBytes+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(data) > maxProcBytes {
|
||||
return nil, errors.New("resource counter file too large")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// ParseProcessIO uses actual storage bytes, not rchar/wchar (syscall traffic).
|
||||
// Partial, missing or invalid counters are unavailable, not a zero I/O rate.
|
||||
func ParseProcessIO(data string) ProcessIO {
|
||||
result := ProcessIO{Scope: "process", ProcessID: processID}
|
||||
read, write := false, false
|
||||
for _, line := range strings.Split(data, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != 2 || (fields[0] != "read_bytes:" && fields[0] != "write_bytes:") {
|
||||
continue
|
||||
}
|
||||
value, err := strconv.ParseUint(fields[1], 10, 64)
|
||||
if err != nil {
|
||||
return result
|
||||
}
|
||||
if fields[0] == "read_bytes:" {
|
||||
result.ReadBytes, read = value, true
|
||||
} else {
|
||||
result.WriteBytes, write = value, true
|
||||
}
|
||||
}
|
||||
result.Available = read && write
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package seheweit
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func fixtureSources(t *testing.T) sources {
|
||||
t.Helper()
|
||||
return sources{
|
||||
now: func() time.Time { return time.UnixMilli(123456789) },
|
||||
readFile: func(path string) ([]byte, error) {
|
||||
switch path {
|
||||
case "/proc/loadavg":
|
||||
return []byte("0.01 0.20 3.00 1/99 1\n"), nil
|
||||
case "/proc/self/io":
|
||||
return []byte("rchar: 999\nread_bytes: 10\nwrite_bytes: 20\n"), nil
|
||||
default:
|
||||
t.Fatalf("unexpected file read: %s", path)
|
||||
return nil, nil
|
||||
}
|
||||
},
|
||||
statfs: func(path string, fs *syscall.Statfs_t) error {
|
||||
if !filepath.IsAbs(path) {
|
||||
t.Fatalf("relative root %q", path)
|
||||
}
|
||||
fs.Blocks, fs.Bavail, fs.Bsize = 100, 30, 4096
|
||||
return nil
|
||||
},
|
||||
memory: func(m *runtime.MemStats) { m.HeapAlloc, m.Sys, m.NumGC = 1024, 4096, 9 },
|
||||
cpus: func() int { return 8 }, goroutines: func() int { return 17 },
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotWireCompatibility(t *testing.T) {
|
||||
queue := Queue{Stats: QueueStats{Queued: 2, Running: 1}, ActiveWorkers: 1, Paused: true}
|
||||
snapshot := collect(t.TempDir(), queue, fixtureSources(t))
|
||||
data, err := json.Marshal(snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Exact pre-extraction wire shape, including zeros/false (no omitempty).
|
||||
want := map[string]any{
|
||||
"timestamp_ms": int64(123456789),
|
||||
"io": map[string]any{"scope": "process", "available": true, "process_id": processID, "read_bytes": 10, "write_bytes": 20},
|
||||
"cpu": map[string]any{"logical_cpus": 8, "load_average": "0.01 0.20 3.00"},
|
||||
"memory": map[string]any{"heap_bytes": 1024, "go_system_bytes": 4096, "gc_cycles": 9, "goroutines": 17},
|
||||
"storage": map[string]any{"path": snapshot.Storage.Path, "total_bytes": 409600, "used_bytes": 286720, "available_bytes": 122880},
|
||||
"queue": map[string]any{"stats": map[string]int{"queued": 2, "running": 1}, "active_workers": 1, "paused": true},
|
||||
}
|
||||
wantData, _ := json.Marshal(want)
|
||||
var gotJSON, wantJSON any
|
||||
if err := json.Unmarshal(data, &gotJSON); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(wantData, &wantJSON); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(gotJSON, wantJSON) {
|
||||
t.Fatalf("wire changed: %s != %s", data, wantData)
|
||||
}
|
||||
zero, _ := json.Marshal(Snapshot{})
|
||||
if strings.Contains(string(zero), "null") || !strings.Contains(string(zero), `"paused":false`) {
|
||||
t.Fatalf("zero shape %s", zero)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectionFailureAndIsolation(t *testing.T) {
|
||||
src := fixtureSources(t)
|
||||
src.readFile = func(string) ([]byte, error) {
|
||||
return []byte("read_bytes: 5\nwrite_bytes: 6\n"), errors.New("unavailable")
|
||||
}
|
||||
src.statfs = func(_ string, fs *syscall.Statfs_t) error {
|
||||
fs.Blocks, fs.Bavail, fs.Bsize = 100, 20, 4096
|
||||
return errors.New("unavailable")
|
||||
}
|
||||
got := collect("", Queue{}, src)
|
||||
if !filepath.IsAbs(got.Storage.Path) || got.Storage.TotalBytes != 0 || got.IO.Available || got.IO.ReadBytes != 0 || got.CPU.LoadAverage != "" {
|
||||
t.Fatalf("failed source leaked partial values: %+v", got)
|
||||
}
|
||||
if got.Memory.HeapBytes != 1024 || got.CPU.LogicalCPUs != 8 || got.IO.ProcessID == "" {
|
||||
t.Fatalf("independent sources lost: %+v", got)
|
||||
}
|
||||
// Each call has independent values; the process identity alone is shared.
|
||||
for i := 0; i < 16; i++ {
|
||||
t.Run("parallel", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
other := collect("", Queue{}, fixtureSources(t))
|
||||
if other.IO.ProcessID != got.IO.ProcessID || other.Storage.TotalBytes != 409600 {
|
||||
t.Fatalf("identity/isolation %+v", other)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageInvalidCounters(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
blocks, available uint64
|
||||
size int64
|
||||
}{
|
||||
{10, 11, 4096}, {10, 1, -1}, {10, 1, 0}, {math.MaxUint64, 1, 4096},
|
||||
} {
|
||||
if got := storageCounters("/root", tc.blocks, tc.available, tc.size); got != (Storage{Path: "/root"}) {
|
||||
t.Fatalf("invalid storage wrapped: %+v", got)
|
||||
}
|
||||
}
|
||||
if got := storageCounters("/root", math.MaxUint64, 0, 1); got.UsedBytes != math.MaxUint64 {
|
||||
t.Fatalf("valid boundary %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessIOParser(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, text string
|
||||
available bool
|
||||
read, write uint64
|
||||
}{
|
||||
{"valid", "read_bytes: 12\nwrite_bytes: 34\nrchar: 999\ncancelled_write_bytes: 99", true, 12, 34},
|
||||
{"zero", "read_bytes: 0\nwrite_bytes: 0", true, 0, 0},
|
||||
{"empty", "", false, 0, 0},
|
||||
{"partial", "read_bytes: 12", false, 12, 0},
|
||||
{"negative", "read_bytes: -1\nwrite_bytes: 34", false, 0, 0},
|
||||
{"overflow", "read_bytes: 18446744073709551616\nwrite_bytes: 0", false, 0, 0},
|
||||
{"invalid", "read_bytes: 12\nwrite_bytes: bad", false, 12, 0},
|
||||
{"max", "read_bytes: 18446744073709551615\nwrite_bytes: 0", true, math.MaxUint64, 0},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := ParseProcessIO(tc.text)
|
||||
if got.Available != tc.available || got.ReadBytes != tc.read || got.WriteBytes != tc.write || got.Scope != "process" || got.ProcessID != processID {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcFileBound(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "counters")
|
||||
for _, size := range []int{maxProcBytes, maxProcBytes + 1} {
|
||||
if err := os.WriteFile(path, []byte(strings.Repeat("x", size)), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := readProcFile(path)
|
||||
if size == maxProcBytes && (err != nil || len(data) != size) {
|
||||
t.Fatalf("limit rejected: %v", err)
|
||||
}
|
||||
if size > maxProcBytes && (err == nil || data != nil) {
|
||||
t.Fatal("oversize file accepted")
|
||||
}
|
||||
}
|
||||
if _, err := readProcFile(path + "-missing"); err == nil {
|
||||
t.Fatal("missing file accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectLocalRuntime(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
before := time.Now().UnixMilli()
|
||||
got := Collect(root, Queue{Paused: true})
|
||||
if got.TimestampMS < before || got.TimestampMS > time.Now().UnixMilli() || got.Storage.Path != root || got.CPU.LogicalCPUs < 1 || got.Memory.Goroutines < 1 || !got.Queue.Paused || got.IO.ProcessID != processID {
|
||||
t.Fatalf("runtime snapshot %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"pageswiki/internal/kunstzeal"
|
||||
)
|
||||
|
||||
const adapterPluginsConfigKey = "projection_adapter_plugins"
|
||||
@@ -50,6 +52,7 @@ type builtinAdapterInfo struct {
|
||||
var adapterIDPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{1,63}$`)
|
||||
|
||||
func builtinAdapterManifest() []builtinAdapterInfo {
|
||||
projectionAdapters := kunstzeal.Catalog()
|
||||
out := make([]builtinAdapterInfo, 0, len(projectionAdapters))
|
||||
for _, adapter := range projectionAdapters {
|
||||
out = append(out, builtinAdapterInfo{
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"pageswiki/internal/compiler"
|
||||
"pageswiki/internal/grammaratur"
|
||||
"pageswiki/internal/render"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
@@ -63,7 +63,7 @@ func (s *Server) workspaceFile(session *store.AdapterSession, semanticPath strin
|
||||
body := revision.Body
|
||||
markdown := plugin.HTMLMarkdown && strings.HasPrefix(strings.ToLower(revision.ContentType), "text/html")
|
||||
if markdown {
|
||||
body, err = compiler.HTMLToMarkdown(body)
|
||||
body, err = grammaratur.HTMLToMarkdown(body)
|
||||
if err != nil {
|
||||
return adapterWorkspaceFile{}, err
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ func (s *Server) tryAnonymousProjection(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
func (s *Server) serveAnonymousRevision(w http.ResponseWriter, r *http.Request, semantic string, raw bool, v *store.Revision) bool {
|
||||
reader, err := s.store.OpenDocument(v.DocumentHash)
|
||||
reader, err := s.store.OpenDocumentContext(r.Context(), v.DocumentHash)
|
||||
if err != nil {
|
||||
http.Error(w, "published document unavailable", 503)
|
||||
return true
|
||||
@@ -156,6 +156,7 @@ func (s *Server) serveAnonymousRevision(w http.ResponseWriter, r *http.Request,
|
||||
return true
|
||||
}
|
||||
data["Text"] = string(body)
|
||||
data["CodeLanguage"] = codeLanguage(contentType, filename, string(body))
|
||||
}
|
||||
data["SocialMetadata"] = s.socialMetadataForRequest(r, "anonymous_view.gohtml", data)
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
|
||||
@@ -59,17 +59,26 @@ func (s *Server) handleSaveRevisionAPI(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
) {
|
||||
|
||||
var req struct {
|
||||
ObjectUUID string `json:"object_uuid"`
|
||||
Body string `json:"body"`
|
||||
Message string `json:"message"`
|
||||
Language string `json:"lang"`
|
||||
ContentType string `json:"content_type"`
|
||||
Filename string `json:"filename"`
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "private, no-store")
|
||||
if r.Method != http.MethodPost {
|
||||
w.Header().Set("Allow", "POST")
|
||||
api.WriteError(w, 405, "method_not_allowed", "POST required")
|
||||
return
|
||||
}
|
||||
|
||||
err := json.NewDecoder(r.Body).Decode(&req)
|
||||
var req struct {
|
||||
ObjectUUID string `json:"object_uuid"`
|
||||
Body string `json:"body"`
|
||||
Message string `json:"message"`
|
||||
Language string `json:"lang"`
|
||||
ContentType string `json:"content_type"`
|
||||
Filename string `json:"filename"`
|
||||
BaseRevision *string `json:"base_revision_uuid"`
|
||||
EditorPrincipal *string `json:"editor_principal_uuid"`
|
||||
}
|
||||
|
||||
err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4<<20)).Decode(&req)
|
||||
if err != nil {
|
||||
api.WriteError(
|
||||
w,
|
||||
@@ -80,6 +89,17 @@ func (s *Server) handleSaveRevisionAPI(
|
||||
return
|
||||
}
|
||||
domainUUID := s.domainUUIDForRequest(r)
|
||||
if req.EditorPrincipal != nil {
|
||||
principal, e := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r))
|
||||
id := ""
|
||||
if e == nil && principal != nil {
|
||||
id = principal.UUID
|
||||
}
|
||||
if id != *req.EditorPrincipal {
|
||||
api.WriteError(w, 401, "editor_identity_changed", "editor identity changed; reopen with the original account")
|
||||
return
|
||||
}
|
||||
}
|
||||
if mode, _ := s.store.GetConfigInDomain(domainUUID, "authorization_mode"); mode == "enforcing" {
|
||||
principal, authErr := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r))
|
||||
if authErr != nil {
|
||||
@@ -109,15 +129,16 @@ func (s *Server) handleSaveRevisionAPI(
|
||||
}
|
||||
}
|
||||
|
||||
revision, err :=
|
||||
s.store.SaveRevisionContent(
|
||||
req.ObjectUUID,
|
||||
[]byte(req.Body),
|
||||
req.Message,
|
||||
req.Language,
|
||||
req.ContentType,
|
||||
req.Filename,
|
||||
)
|
||||
var revision *store.Revision
|
||||
if req.BaseRevision != nil {
|
||||
revision, err = s.store.SaveRevisionContentIfHead(req.ObjectUUID, []byte(req.Body), req.Message, req.Language, req.ContentType, req.Filename, *req.BaseRevision)
|
||||
} else {
|
||||
revision, err = s.store.SaveRevisionContent(req.ObjectUUID, []byte(req.Body), req.Message, req.Language, req.ContentType, req.Filename)
|
||||
}
|
||||
if errors.Is(err, store.ErrRevisionConflict) {
|
||||
api.WriteError(w, http.StatusConflict, "revision_conflict", "document changed; reload the language head before saving")
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
status := http.StatusInternalServerError
|
||||
@@ -143,5 +164,6 @@ func (s *Server) handleSaveRevisionAPI(
|
||||
"lang": revision.Language,
|
||||
"content_type": revision.ContentType,
|
||||
"filename": revision.Filename,
|
||||
"body": revision.Body,
|
||||
})
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package server
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"mime"
|
||||
@@ -47,23 +46,20 @@ func (s *Server) handleArchiveEntry(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
if strings.EqualFold(filepath.Ext(revision.Filename), ".zip") {
|
||||
hydrated, err := s.store.GetRevision(revision.UUID)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
s.serveZIPEntry(w, r, hydrated.Body, entry)
|
||||
return
|
||||
}
|
||||
source, err := s.store.DocumentPath(revision.DocumentHash)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
s.serveZIPDocumentEntry(w, r, revision.DocumentHash, entry)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
||||
defer cancel()
|
||||
lease, err := s.store.MaterializeDocument(ctx, revision.DocumentHash, 1<<30)
|
||||
if err != nil {
|
||||
http.Error(w, "archive source unavailable or exceeds materialization limit", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
defer lease.Close()
|
||||
command := exec.CommandContext(ctx, "nice", "-n", "10",
|
||||
"7z", "x", "-so", "-y", "-spd", "-mmt1", "-p-", "--", source, entry)
|
||||
"7z", "x", "-so", "-y", "-spd", "-mmt1", "-p-", "--", lease.Path, entry)
|
||||
command.WaitDelay = time.Second
|
||||
stdout, err := command.StdoutPipe()
|
||||
if err != nil || command.Start() != nil {
|
||||
http.Error(w, "cannot extract archive entry", http.StatusUnprocessableEntity)
|
||||
@@ -85,16 +81,24 @@ func (s *Server) handleArchiveEntry(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) serveZIPEntry(
|
||||
func (s *Server) serveZIPDocumentEntry(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
body string,
|
||||
hash string,
|
||||
entry string,
|
||||
) {
|
||||
reader, err := zip.NewReader(
|
||||
bytes.NewReader([]byte(body)),
|
||||
int64(len(body)),
|
||||
)
|
||||
input, err := s.store.OpenDocumentContext(r.Context(), hash)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
defer input.Close()
|
||||
size, err := input.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
http.Error(w, "cannot determine archive size", http.StatusUnprocessableEntity)
|
||||
return
|
||||
}
|
||||
reader, err := zip.NewReader(input, size)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusUnprocessableEntity)
|
||||
return
|
||||
@@ -103,6 +107,10 @@ func (s *Server) serveZIPEntry(
|
||||
if file.Name != entry || file.FileInfo().IsDir() {
|
||||
continue
|
||||
}
|
||||
if file.UncompressedSize64 > 128<<20 {
|
||||
http.Error(w, "archive entry exceeds 128 MiB limit", http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
source, err := file.Open()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"pageswiki/internal/db"
|
||||
"pageswiki/internal/documentstore"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
func TestArchiveEntryReadsCanonicalDocumentWithoutRevisionHydration(t *testing.T) {
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err := db.Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := store.NewWithDocumentRoot(database, t.TempDir())
|
||||
domain, _ := st.GetDefaultDomain()
|
||||
object, err := st.CreatePageAtPathInDomain(domain.UUID, "/archive.zip")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var payload bytes.Buffer
|
||||
w := zip.NewWriter(&payload)
|
||||
file, err := w.Create("hello.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file.Write([]byte("hello 日本語"))
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
revision, err := st.SaveRevisionContent(object, payload.Bytes(), "test", "und", "application/zip", "archive.zip")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The canonical Document must work even after compatibility DB bodies
|
||||
// have been removed. This never touches production records.
|
||||
if _, err := database.Exec(`UPDATE revisions SET body='' WHERE uuid=?`, revision.UUID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := &Server{store: st}
|
||||
r := httptest.NewRequest(http.MethodGet, "/_/archive?path=/archive.zip&entry=hello.txt", nil)
|
||||
response := httptest.NewRecorder()
|
||||
s.serveZIPDocumentEntry(response, r, revision.DocumentHash, "hello.txt")
|
||||
if response.Code != http.StatusOK || response.Body.String() != "hello 日本語" {
|
||||
t.Fatalf("status=%d body=%q", response.Code, response.Body.String())
|
||||
}
|
||||
if response.Header().Get("Vary") != "Authorization, Cookie" {
|
||||
t.Fatal("lost authorization cache partition")
|
||||
}
|
||||
missing := httptest.NewRecorder()
|
||||
s.serveZIPDocumentEntry(missing, r, revision.DocumentHash, "missing")
|
||||
if missing.Code != http.StatusNotFound {
|
||||
t.Fatalf("missing status=%d", missing.Code)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(r.Context())
|
||||
cancel()
|
||||
cancelled := httptest.NewRecorder()
|
||||
s.serveZIPDocumentEntry(cancelled, r.WithContext(ctx), revision.DocumentHash, "hello.txt")
|
||||
if cancelled.Code == http.StatusOK {
|
||||
t.Fatal("cancelled request read Document")
|
||||
}
|
||||
}
|
||||
|
||||
// Measures the per-request cost of fetching a tiny entry from a large stored
|
||||
// ZIP. Setup is outside the timer; no DB body hydration or scratch copy occurs.
|
||||
func BenchmarkZIPDocumentEntry(b *testing.B) {
|
||||
var payload bytes.Buffer
|
||||
w := zip.NewWriter(&payload)
|
||||
file, err := w.Create("hello.txt")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
file.Write([]byte("hello"))
|
||||
padding, err := w.CreateHeader(&zip.FileHeader{Name: "padding.bin", Method: zip.Store})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
padding.Write(bytes.Repeat([]byte{0xaa}, 16<<20))
|
||||
if err := w.Close(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
root := b.TempDir()
|
||||
backend, err := documentstore.NewLocal(root)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(payload.Bytes())
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
if err := backend.PutIfAbsent(context.Background(), hash, int64(payload.Len()), bytes.NewReader(payload.Bytes())); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
s := &Server{store: &store.Store{DocumentRoot: root}}
|
||||
r := httptest.NewRequest(http.MethodGet, "/archive", nil)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
response := httptest.NewRecorder()
|
||||
s.serveZIPDocumentEntry(response, r, hash, "hello.txt")
|
||||
if response.Code != http.StatusOK || response.Body.String() != "hello" {
|
||||
b.Fatal("entry read failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewImageAndZIPWorkWithReaderOnlyLegacyStore(t *testing.T) {
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err := db.Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := store.New(database)
|
||||
domain, _ := st.GetDefaultDomain()
|
||||
s := &Server{store: st, previewRoot: t.TempDir()}
|
||||
var picture bytes.Buffer
|
||||
im := image.NewRGBA(image.Rect(0, 0, 10, 10))
|
||||
im.Set(0, 0, color.White)
|
||||
if err := png.Encode(&picture, im); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var archive bytes.Buffer
|
||||
w := zip.NewWriter(&archive)
|
||||
for _, name := range []string{"1.png", "2.png", "3.png"} {
|
||||
f, err := w.Create(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Write(picture.Bytes())
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
kind, filename string
|
||||
body []byte
|
||||
}{
|
||||
{"image", "picture.png", picture.Bytes()}, {"archive", "comic.zip", archive.Bytes()},
|
||||
} {
|
||||
object, err := st.CreatePageAtPathInDomain(domain.UUID, "/"+tc.filename)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
revision, err := st.SaveRevisionContent(object, tc.body, "test", "und", "application/octet-stream", tc.filename)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.generatePreview(previewEventPayload{DocumentHash: revision.DocumentHash, Kind: tc.kind, Filename: tc.filename}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info, err := os.Stat(s.previewCachePath(revision.DocumentHash, tc.kind)); err != nil || info.Size() == 0 {
|
||||
t.Fatalf("preview: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectionMetadataMaterializesAndCleansSource(t *testing.T) {
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
if err := db.Migrate(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := store.NewWithDocumentRoot(database, t.TempDir())
|
||||
domain, _ := st.GetDefaultDomain()
|
||||
object, err := st.CreatePageAtPathInDomain(domain.UUID, "/test.pdf")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
revision, err := st.SaveRevisionContent(object, []byte("synthetic PDF fixture"), "test", "und", "application/pdf", "test.pdf")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Synthetic tools validate only the storage handoff, not PDF rendering.
|
||||
bin := t.TempDir()
|
||||
for name, script := range map[string]string{
|
||||
"nice": "#!/bin/sh\nshift\nshift\nexec \"$@\"\n",
|
||||
"pdfinfo": "#!/bin/sh\ntest -r \"$3\" || exit 1\nprintf '%s' \"$3\" > \"$PAGESWIKI_TEST_SOURCE\"\nprintf 'Pages: 3\\n'\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(bin, name), []byte(script), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
logFile := filepath.Join(t.TempDir(), "source")
|
||||
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
t.Setenv("PAGESWIKI_TEST_SOURCE", logFile)
|
||||
s := &Server{store: st}
|
||||
result, err := s.inspectProjectionMetadata(metadataProjectionPayload{DocumentHash: revision.DocumentHash, Kind: "pdf", Version: metadataProjectionVersion})
|
||||
if err != nil || result.PDFPages != 3 {
|
||||
t.Fatalf("result=%+v err=%v", result, err)
|
||||
}
|
||||
source, err := os.ReadFile(logFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(source), "pageswiki-document-") || strings.HasPrefix(string(source), st.DocumentRoot+string(os.PathSeparator)) {
|
||||
t.Fatalf("tool received canonical path: %s", source)
|
||||
}
|
||||
if _, err := os.Stat(string(source)); !os.IsNotExist(err) {
|
||||
t.Fatalf("scratch leaked: %v", err)
|
||||
}
|
||||
reader, err := st.OpenDocument(revision.DocumentHash)
|
||||
if err != nil {
|
||||
t.Fatal("canonical document removed", err)
|
||||
}
|
||||
reader.Close()
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"log"
|
||||
"net/http"
|
||||
"pageswiki/internal/store"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -34,6 +35,17 @@ func (s *Server) handleAuthAdmin(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
panel := "auth"
|
||||
switch r.FormValue("action") {
|
||||
case "save_session_policy":
|
||||
panel = "auth-sessions"
|
||||
minutes, err := strconv.Atoi(r.FormValue("session_minutes"))
|
||||
if err != nil || minutes < 5 || minutes > store.MaxAuthSessionMinutes {
|
||||
http.Error(w, "session lifetime must be between 5 and 43200 minutes", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := s.store.SetAuthSessionLifetimeMinutes(domainUUID, minutes); err != nil {
|
||||
http.Error(w, "could not save session policy", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
case "revoke_session":
|
||||
panel = "auth-sessions"
|
||||
if err := s.store.RevokeAuthSessionByID(domainUUID, r.FormValue("session_uuid")); err != nil {
|
||||
|
||||
@@ -62,7 +62,7 @@ func requestIsHTTPS(r *http.Request) bool {
|
||||
}
|
||||
|
||||
func (s *Server) setAuthCookie(w http.ResponseWriter, r *http.Request, token string) {
|
||||
http.SetCookie(w, &http.Cookie{Name: authCookieName, Value: token, Path: "/", HttpOnly: true, Secure: requestIsHTTPS(r), SameSite: http.SameSiteLaxMode, MaxAge: 24 * 60 * 60})
|
||||
http.SetCookie(w, &http.Cookie{Name: authCookieName, Value: token, Path: "/", HttpOnly: true, Secure: requestIsHTTPS(r), SameSite: http.SameSiteLaxMode, MaxAge: s.store.AuthSessionLifetimeMinutes(s.domainUUIDForRequest(r)) * 60})
|
||||
}
|
||||
|
||||
func (s *Server) authTokenFromRequest(r *http.Request) string {
|
||||
@@ -205,7 +205,7 @@ func (s *Server) handleAuth(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
if err == nil {
|
||||
if strings.Contains(r.Header.Get("Accept"), "application/json") {
|
||||
writeEventJSON(w, http.StatusOK, map[string]any{"access_token": token, "token_type": "Bearer", "expires_in": 86400})
|
||||
writeEventJSON(w, http.StatusOK, map[string]any{"access_token": token, "token_type": "Bearer", "expires_in": s.store.AuthSessionLifetimeMinutes(domainUUID) * 60})
|
||||
return
|
||||
}
|
||||
s.setAuthCookie(w, r, token)
|
||||
|
||||
@@ -245,7 +245,7 @@ func (s *Server) handleLDAPLogin(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
if strings.Contains(r.Header.Get("Accept"), "application/json") {
|
||||
writeEventJSON(w, http.StatusOK, map[string]any{"access_token": token, "token_type": "Bearer", "expires_in": 86400})
|
||||
writeEventJSON(w, http.StatusOK, map[string]any{"access_token": token, "token_type": "Bearer", "expires_in": s.store.AuthSessionLifetimeMinutes(domainUUID) * 60})
|
||||
return
|
||||
}
|
||||
s.setAuthCookie(w, r, token)
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSessionPolicyAdminUIAndCookie(t *testing.T) {
|
||||
s, token, domain := configFixture(t)
|
||||
form := url.Values{"action": {"save_session_policy"}, "session_minutes": {"60"}}
|
||||
w := configRequest(s, token, "POST", "/-/sys/config/auth", form)
|
||||
if w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/-/sys/config?panel=auth-sessions&saved=1" {
|
||||
t.Fatalf("policy save %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
if s.store.AuthSessionLifetimeMinutes(domain) != 60 {
|
||||
t.Fatal("policy not saved")
|
||||
}
|
||||
w = configRequest(s, token, "GET", "/-/sys/config?panel=auth-sessions", nil)
|
||||
if w.Code != 200 || !strings.Contains(w.Body.String(), `name="session_minutes" value="60"`) {
|
||||
t.Fatal("policy control missing")
|
||||
}
|
||||
for _, bad := range []string{"", "abc", "0", "99999999999999999999999"} {
|
||||
form.Set("session_minutes", bad)
|
||||
if w = configRequest(s, token, "POST", "/-/sys/config/auth", form); w.Code != 400 {
|
||||
t.Fatalf("bad lifetime %q accepted", bad)
|
||||
}
|
||||
}
|
||||
user, err := s.store.CreatePasswordAccount(domain, "reader", "a long test password", "Reader", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reader, err := s.store.CreateAuthSession(domain, user.UUID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
form.Set("session_minutes", "120")
|
||||
if w = configRequest(s, reader, "POST", "/-/sys/config/auth", form); w.Code != 403 {
|
||||
t.Fatalf("non-admin policy change: %d", w.Code)
|
||||
}
|
||||
w = httptest.NewRecorder()
|
||||
r := httptest.NewRequest("GET", "https://localhost/", nil)
|
||||
s.setAuthCookie(w, r, "fixture-token")
|
||||
cookies := w.Result().Cookies()
|
||||
if len(cookies) != 1 || cookies[0].MaxAge != 3600 || !cookies[0].HttpOnly || !cookies[0].Secure {
|
||||
t.Fatalf("cookie policy %#v", cookies)
|
||||
}
|
||||
}
|
||||
@@ -58,6 +58,14 @@ func (s *Server) handleBrowser(
|
||||
if !strings.HasPrefix(currentPath, "/") {
|
||||
currentPath = "/" + currentPath
|
||||
}
|
||||
// Browser navigation is scoped to this semantic Path, not a Domain-wide
|
||||
// read grant. Filter each child separately below as well.
|
||||
if !s.requestCanReadPath(r, currentPath) {
|
||||
http.Error(w, "forbidden", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "private, no-store")
|
||||
w.Header().Add("Vary", "Cookie, Authorization")
|
||||
|
||||
entries, err :=
|
||||
s.store.ListBrowserEntriesInDomain(
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package server
|
||||
|
||||
func init() {
|
||||
for lang, entries := range map[string]map[string]string{
|
||||
"en": {
|
||||
"command.label": "Command · Tab: complete Path · Esc: close", "command.close": "Close command mode", "command.unavailable": "Path completion unavailable. Sign in or retry.", "command.empty": "No matching readable Paths.", "command.more": "More matches exist. Type a longer Path prefix.",
|
||||
"auth.session_minutes": "Session lifetime (minutes)", "auth.session_policy_help": "5–43200 minutes (30 days); default 1440 (24 hours). Applies to new browser/API logins, including OIDC/LDAP. Existing sessions and CalDAV app passwords are unchanged. This is an absolute expiry, not an inactivity timeout.",
|
||||
"browser.format": "Document format", "browser.wiki": "Wiki (default editor format)", "browser.plain": "PlainText (text/plain)",
|
||||
"theme.quick": "Theme", "theme.default": "Domain default", "theme.local": "This browser only", "theme.failed": "Could not load the theme.",
|
||||
"browser.address": "Path", "browser.go": "Go", "browser.up": "Parent folder",
|
||||
"browser.command_help": "Enter /path or a relative path · cd .. · vi filename · view filename",
|
||||
"browser.new": "New document", "browser.folder": "Open virtual folder", "browser.name": "Name or Path",
|
||||
"browser.create_help": "Open the editor to create a document. If the Path already exists, edit that document.",
|
||||
"browser.folder_help": "You can enter an empty Path without creating a folder, then create or upload documents there.",
|
||||
"browser.invalid": "Enter a Wiki Path, not an external URL or a path containing control characters or backslashes.",
|
||||
"browser.empty": "No visible items. Create a document, upload a file, or enter a different Path.",
|
||||
},
|
||||
"ja": {
|
||||
"command.label": "コマンド · Tab: Path補完 · Esc: 閉じる", "command.close": "コマンドモードを閉じる", "command.unavailable": "Path補完を取得できません。ログイン状態を確認して再試行してください。", "command.empty": "閲覧可能な一致するPathがありません。", "command.more": "候補がほかにもあります。Pathをさらに入力してください。",
|
||||
"auth.session_minutes": "セッションの有効期間(分)", "auth.session_policy_help": "5〜43200分(30日)。既定は1440分(24時間)。OIDC・LDAPを含む新規ブラウザ/APIログインに適用します。既存セッションとCalDAVのアプリパスワードは変更しません。無操作時間ではなくログインからの絶対期限です。",
|
||||
"browser.format": "文書形式", "browser.wiki": "Wiki(既定の編集形式)", "browser.plain": "PlainText(text/plain)",
|
||||
"theme.quick": "テーマ", "theme.default": "ドメインの既定", "theme.local": "このブラウザだけに適用", "theme.failed": "テーマを読み込めませんでした。",
|
||||
"browser.address": "Path", "browser.go": "移動", "browser.up": "親フォルダ",
|
||||
"browser.command_help": "/path または相対パス · cd .. · vi ファイル名 · view ファイル名",
|
||||
"browser.new": "新規文書", "browser.folder": "仮想フォルダを開く", "browser.name": "名前またはPath",
|
||||
"browser.create_help": "エディタを開いて文書を作成します。既存のPathでは、その文書を編集します。",
|
||||
"browser.folder_help": "空のPathにもフォルダを作らず移動できます。移動先で文書の作成やアップロードができます。",
|
||||
"browser.invalid": "WikiのPathを入力してください。外部URL・制御文字・バックスラッシュは使用できません。",
|
||||
"browser.empty": "表示できる項目はありません。文書の作成、ファイルのアップロード、別のPathへの移動ができます。",
|
||||
},
|
||||
} {
|
||||
for key, value := range entries {
|
||||
translations[lang][key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
func TestCitySearchContentDestinations(t *testing.T) {
|
||||
s, token, domain := searchServerFixture(t)
|
||||
street := seedSearchCity(t, s, token, domain)
|
||||
createUserSpacePage(t, s, domain, "/city?#%", "<h1>Portal</h1>")
|
||||
if _, err := s.store.CreateSpotAtPath(domain, "test", street, "nebula-way-undefined", "Undefined", "nebula", "/city?#%/street/_undefined"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
principal, err := s.store.PrincipalForSession(domain, token)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.store.DB.Exec(`INSERT INTO auth_capability_grants(uuid,role_uuid,capability,target_type,target_id,created_at_ms)
|
||||
SELECT 'entrance-grant',b.role_uuid,'read','path','/city?#%/street/_undefined',1 FROM auth_role_bindings b JOIN domain_memberships m ON m.uuid=b.membership_uuid WHERE m.domain_uuid=? AND m.principal_uuid=? LIMIT 1`, domain, principal.UUID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.mux.HandleFunc("/_/api/v1/search/suggest", s.handleSearchSuggestAPI)
|
||||
s.mux.HandleFunc("/v/browser/", s.handleBrowser)
|
||||
s.mux.HandleFunc("/", s.handleSemanticRoute)
|
||||
w := configRequest(s, token, "GET", "/_/api/v1/search/suggest?q=nebula", nil)
|
||||
var payload struct {
|
||||
Items []searchSuggestionItem `json:"items"`
|
||||
}
|
||||
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &payload) != nil {
|
||||
t.Fatalf("suggest: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
foundPortal, foundEntrance := false, false
|
||||
for _, item := range payload.Items {
|
||||
if strings.HasPrefix(item.Href, "/v/city/") || strings.Contains(item.Href, "_undefined") {
|
||||
t.Fatalf("management/virtual link: %+v", item)
|
||||
}
|
||||
if item.Kind == "quartier" {
|
||||
foundPortal = item.Href == "/city%3F%23%25"
|
||||
}
|
||||
if item.Kind == "spot" && item.Label == "Nebula Street" {
|
||||
foundEntrance = item.Href == "/v/browser/city%3F%23%25/street"
|
||||
}
|
||||
}
|
||||
if !foundPortal || !foundEntrance {
|
||||
t.Fatalf("destinations missing: %+v", payload.Items)
|
||||
}
|
||||
for _, target := range []struct{ url, marker string }{
|
||||
{"/city%3F%23%25", "<h1>Portal</h1>"},
|
||||
{"/v/browser/city%3F%23%25/street", "browser-layout"},
|
||||
} {
|
||||
opened := configRequest(s, token, "GET", target.url, nil)
|
||||
if opened.Code != 200 || !strings.Contains(opened.Body.String(), target.marker) || strings.Contains(opened.Body.String(), `class="city-node-detail"`) {
|
||||
t.Fatalf("destination %s: %d", target.url, opened.Code)
|
||||
}
|
||||
}
|
||||
if denied := configRequest(s, token, "GET", "/v/browser/private-city", nil); denied.Code != 403 {
|
||||
t.Fatalf("unreadable folder: %d", denied.Code)
|
||||
}
|
||||
w = configRequest(s, token, "GET", "/v/search?q=nebula", nil)
|
||||
if w.Code != 200 || !strings.Contains(w.Body.String(), `href="/city%3F%23%25"`) || strings.Contains(w.Body.String(), `href="/v/city/`) {
|
||||
t.Fatalf("search: %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCitySearchViewAndSuggestionsAgree(t *testing.T) {
|
||||
for _, hasPage := range []bool{false, true} {
|
||||
result := store.CitySearchResult{Kind: "spot", Name: "notes-undefined", Title: "Undefined", PathPrefix: "/日本語 #?%/_undefined", NavigationPath: "/日本語 #?%", StreetTitle: "Notes", HasPage: hasPage}
|
||||
view := citySearchResultView(result, "notes", time.UTC)
|
||||
suggestion := citySuggestion(result, "notes")
|
||||
if view.Href != suggestion.Href || view.Href != view.PathHref || suggestion.Label != "Notes" {
|
||||
t.Fatalf("inconsistent: %+v %+v", view, suggestion)
|
||||
}
|
||||
if strings.Contains(view.Href, "#") || strings.Contains(view.Href, "?") || strings.Contains(view.Href, "_undefined") {
|
||||
t.Fatal(view.Href)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -80,7 +80,7 @@ func (s *Server) handleCityMap(w http.ResponseWriter, r *http.Request) {
|
||||
if detail.Street != nil {
|
||||
streetHref = cityDetailHref("street", detail.Street.Name, detail.Quartier.Name, detail.Street.UUID)
|
||||
}
|
||||
s.renderWithContext(w, r, "city_node.gohtml", map[string]any{"Title": detail.Title, "Node": detail, "Pages": pageViews, "QuartierHref": quartierHref, "StreetHref": streetHref, "MapHref": "/v/city?" + kind + "=" + detail.UUID, "CanPlan": canPlan, "Locale": s.getLocale(domainUUID), "GovernanceEndpoint": s.governanceForUI(r, kind, detail.UUID)})
|
||||
s.renderWithContext(w, r, "city_node.gohtml", map[string]any{"Title": detail.DisplayTitle(), "Node": detail, "Pages": pageViews, "QuartierHref": quartierHref, "StreetHref": streetHref, "MapHref": "/v/city?" + kind + "=" + detail.UUID, "CanPlan": canPlan, "Locale": s.getLocale(domainUUID), "GovernanceEndpoint": s.governanceForUI(r, kind, detail.UUID)})
|
||||
return
|
||||
}
|
||||
s.renderWithContext(w, r, "city_map.gohtml", map[string]any{"Title": translate(s.getLocale(domainUUID), "city.map"), "CanPlan": canPlan, "Locale": s.getLocale(domainUUID), "GovernanceEndpoint": s.governanceForUI(r, "domain", domainUUID)})
|
||||
|
||||
@@ -34,6 +34,19 @@ func cityPageHref(path string) string {
|
||||
return (&url.URL{Path: path}).EscapedPath()
|
||||
}
|
||||
|
||||
// Search is a reading entry point, not a locality management shortcut. Both
|
||||
// title and Path links use this projection; no additional per-hit DB reads.
|
||||
func cityContentHref(result store.CitySearchResult) string {
|
||||
path := result.ContentPath()
|
||||
if !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") {
|
||||
return "/v/browser"
|
||||
}
|
||||
if result.HasPage {
|
||||
return cityPageHref(path)
|
||||
}
|
||||
return cityPageHref("/v/browser" + path)
|
||||
}
|
||||
|
||||
type cityPageView struct {
|
||||
store.CityPagePlacement
|
||||
Href string
|
||||
|
||||
@@ -29,6 +29,11 @@ func filterCityProjection(city store.CityMap, allowed func(string) bool) store.C
|
||||
continue
|
||||
}
|
||||
spot.DetailHref = cityDetailHref("spot", spot.Name, quartier.Name, spot.UUID)
|
||||
parentTitle := street.Title
|
||||
if parentTitle == "" {
|
||||
parentTitle = street.Name
|
||||
}
|
||||
spot.DisplayTitle = store.CitySpotDisplayTitle(spot.Name, spot.Title, spot.PathPrefix, parentTitle)
|
||||
pages := []store.CityPagePlacement{}
|
||||
unique := map[string]bool{}
|
||||
spot.PortalPath, spot.PortalPageUUID = "", ""
|
||||
@@ -113,7 +118,7 @@ func (s *Server) pageSpotAppearancesForRequest(r *http.Request, objectUUID strin
|
||||
out := []store.CitySpotAppearance{}
|
||||
allowed := s.pathReadFilter(r)
|
||||
for _, spot := range spots {
|
||||
if allowed(spot.PathPrefix) {
|
||||
if allowed(spot.PathPrefix) && allowed(spot.StreetPath) && allowed(spot.QuartierPath) {
|
||||
out = append(out, spot)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,6 +49,19 @@ func TestCityReadPortalPriorityAfterFiltering(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCityProjectionDisplayTitleDoesNotMutateCachedIdentity(t *testing.T) {
|
||||
spot := store.CitySpot{UUID: "spot", Name: "home-undefined", Title: "Undefined", PathPrefix: "/life/home/_undefined"}
|
||||
city := store.CityMap{Quartiers: []store.CityQuartier{{PathPrefix: "/life", Streets: []store.CityStreet{{Title: "Home", PathPrefix: "/life/home", Spots: []store.CitySpot{spot}}}}}}
|
||||
projected := filterCityProjection(city, func(string) bool { return true })
|
||||
got := projected.Quartiers[0].Streets[0].Spots[0]
|
||||
if got.DisplayTitle != "Home" || got.Title != "Undefined" || got.Name != spot.Name || got.PathPrefix != spot.PathPrefix {
|
||||
t.Fatalf("projection: %+v", got)
|
||||
}
|
||||
if city.Quartiers[0].Streets[0].Spots[0].DisplayTitle != "" {
|
||||
t.Fatal("shared navigation cache mutated")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCityReadProjectionHidesPagesLocalitiesPortalsAndRelations(t *testing.T) {
|
||||
s, _, domain := configFixture(t)
|
||||
s.mux.HandleFunc("/_/api/v1/city/", s.handleCityAPI)
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCommandCompletionReadACL(t *testing.T) {
|
||||
s, token, _ := searchServerFixture(t)
|
||||
s.mux.HandleFunc("/_/api/v1/path-picker", s.handlePathPickerAPI)
|
||||
w := configRequest(s, token, "GET", "/_/api/v1/path-picker?path=/docs&mode=children", nil)
|
||||
if w.Code != 200 || strings.Contains(w.Body.String(), "hidden") || !strings.Contains(w.Body.String(), "nebula guide?#%") {
|
||||
t.Fatalf("ACL completion: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandPathCompletion(t *testing.T) {
|
||||
s, token, domain := configFixture(t)
|
||||
s.mux.HandleFunc("/_/api/v1/path-picker", s.handlePathPickerAPI)
|
||||
createUserSpacePage(t, s, domain, "/manual", "<h1>Parent document and folder</h1>")
|
||||
for i := 0; i < 260; i++ {
|
||||
if _, err := s.store.CreatePageAtPathInDomain(domain, fmt.Sprintf("/manual/a%03d", i)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
createUserSpacePage(t, s, domain, "/manual/日本語 #?", "plain")
|
||||
w := configRequest(s, token, "GET", "/_/api/v1/path-picker?path=/manual&mode=children&prefix=日本語", nil)
|
||||
var result struct {
|
||||
Path string `json:"path"`
|
||||
Entries []pathPickerEntry `json:"entries"`
|
||||
}
|
||||
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &result) != nil || result.Path != "/manual" || len(result.Entries) != 1 || result.Entries[0].Name != "日本語 #?" {
|
||||
t.Fatalf("completion: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
w = configRequest(s, "", "GET", "/_/api/v1/path-picker?path=/manual&mode=children", nil)
|
||||
if w.Code == 200 {
|
||||
t.Fatal("anonymous API listed private paths")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandAndHighlightChromium(t *testing.T) {
|
||||
if os.Getenv("PAGESWIKI_BROWSER_TEST") != "1" {
|
||||
t.Skip("opt-in Chromium")
|
||||
}
|
||||
s, token, domain, _ := editorFixture(t)
|
||||
s.mux.HandleFunc("/v/browser/", s.handleBrowser)
|
||||
s.mux.HandleFunc("/_/api/v1/path-picker", s.handlePathPickerAPI)
|
||||
s.mux.HandleFunc("/notes", s.handleSemanticRoute)
|
||||
s.mux.HandleFunc("/", s.handleHome)
|
||||
createUserSpacePage(t, s, domain, "/notes", `<h1>Code sample</h1><pre><code class="language-python"># comment
|
||||
def hello():
|
||||
return "<script>literal</script>"</code></pre>`)
|
||||
createUserSpacePage(t, s, domain, "/notes/alpha", "<h1>Alpha</h1>")
|
||||
createUserSpacePage(t, s, domain, "/notes/alpine", "<h1>Alpine</h1>")
|
||||
createUserSpacePage(t, s, domain, "/日本語 #?/child", "<h1>Unicode</h1>")
|
||||
script, err := os.ReadFile("internal/server/testdata/command-highlight.js")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
r.Host = "localhost"
|
||||
r.Header.Set("Authorization", "Bearer "+token)
|
||||
if r.URL.Path == "/fixture" {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write([]byte("<!doctype html><body style='margin:0'><script>" + string(script) + "</script></body>"))
|
||||
return
|
||||
}
|
||||
s.withAuthorization(s.mux).ServeHTTP(w, r)
|
||||
}))
|
||||
defer server.Close()
|
||||
for _, width := range []string{"1440", "375"} {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
shot := ""
|
||||
if base := os.Getenv("PAGESWIKI_COMMAND_SCREENSHOT"); base != "" {
|
||||
shot = base + "-" + width + ".png"
|
||||
}
|
||||
out, err := exec.CommandContext(ctx, "node", "internal/server/testdata/vim-browser-smoke.mjs", server.URL+"/fixture", t.TempDir(), shot, width, "950").CombinedOutput()
|
||||
cancel()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v\n%s", width, err, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"pageswiki/internal/compiler"
|
||||
"pageswiki/internal/grammaratur"
|
||||
)
|
||||
|
||||
func (s *Server) compileMarkdownFromPath(
|
||||
@@ -46,9 +46,11 @@ func (s *Server) transpileMarkdownFromPath(r *http.Request, path string, mode ma
|
||||
html := latest.Body
|
||||
if mode == markdownProjection {
|
||||
html = s.expandMacrosForRequest(r, html, path, s.domainUUIDForRequest(r))
|
||||
} else {
|
||||
return grammaratur.HTMLToMarkdownAuthoring(html)
|
||||
}
|
||||
|
||||
return compiler.HTMLToMarkdown(
|
||||
return grammaratur.HTMLToMarkdown(
|
||||
html,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1088,6 +1088,7 @@ func (s *Server) handleConfig(
|
||||
"CampusTokens": campusTokens,
|
||||
"AuthPrincipals": authPrincipals,
|
||||
"AuthSessions": authSessions,
|
||||
"AuthSessionMinutes": s.store.AuthSessionLifetimeMinutes(s.domainUUIDForRequest(r)),
|
||||
"OIDCConfig": oidcConfig,
|
||||
"SMTPConfig": smtpConfig,
|
||||
"LDAPConfig": ldapConfig,
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package server
|
||||
|
||||
func init() {
|
||||
for lang, entries := range map[string]map[string]string{
|
||||
"en": {
|
||||
"contact.title": "Contact details", "contact.phone": "Phone number",
|
||||
"contact.help": "Private by default, independently of your profile. A tap opens the phone app. Leave blank to remove. Use 3–15 digits with an optional leading +, spaces, hyphens or parentheses; extensions and service codes are not supported.",
|
||||
"contact.visibility_help": "The profile audience remains the upper limit. Anonymous access also requires the domain to permit public profiles. Administrators do not automatically see private numbers.",
|
||||
"contact.confirm": "I explicitly agree to make this phone number public.",
|
||||
"contact.invalid_phone": "Enter a phone number with 3–15 digits and optional leading +; only spaces, hyphens and parentheses may separate digits.",
|
||||
},
|
||||
"ja": {
|
||||
"contact.title": "連絡先", "contact.phone": "電話番号",
|
||||
"contact.help": "プロフィールとは別に、既定で非公開です。タップすると電話アプリを開きます。空欄で削除。3〜15桁の数字と先頭の+、空白・ハイフン・括弧を使用できます。内線やサービスコードは未対応です。",
|
||||
"contact.visibility_help": "プロフィールの公開範囲を上限とします。匿名公開にはドメイン側の許可も必要です。管理者でも非公開の番号は表示されません。",
|
||||
"contact.confirm": "この電話番号をインターネットに公開することに同意します。",
|
||||
"contact.invalid_phone": "電話番号は3〜15桁の数字と先頭の+で指定してください。区切りには空白・ハイフン・括弧のみ使用できます。",
|
||||
},
|
||||
} {
|
||||
for key, value := range entries {
|
||||
translations[lang][key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,225 +1,20 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
import "pageswiki/internal/kunstzeal"
|
||||
|
||||
type projectionDelivery string
|
||||
type projectionDelivery = kunstzeal.Delivery
|
||||
type contentDescriptor = kunstzeal.Descriptor
|
||||
type projectionAdapter = kunstzeal.Adapter
|
||||
|
||||
const (
|
||||
deliveryHTML projectionDelivery = "html"
|
||||
deliveryText projectionDelivery = "text"
|
||||
deliveryRawMedia projectionDelivery = "raw-media"
|
||||
deliveryRawFrame projectionDelivery = "raw-frame"
|
||||
deliveryDownload projectionDelivery = "download"
|
||||
deliveryHTML = kunstzeal.DeliveryHTML
|
||||
deliveryText = kunstzeal.DeliveryText
|
||||
deliveryRawMedia = kunstzeal.DeliveryRawMedia
|
||||
deliveryRawFrame = kunstzeal.DeliveryRawFrame
|
||||
deliveryDownload = kunstzeal.DeliveryDownload
|
||||
inlineTextWindowThreshold = kunstzeal.InlineTextWindowThreshold
|
||||
)
|
||||
|
||||
type contentDescriptor struct {
|
||||
MediaType string
|
||||
Filename string
|
||||
SizeBytes int64
|
||||
}
|
||||
|
||||
type projectionAdapter struct {
|
||||
ID string
|
||||
Kind string
|
||||
Delivery projectionDelivery
|
||||
NeedsText bool
|
||||
WindowableText bool
|
||||
Match func(contentDescriptor) bool
|
||||
}
|
||||
|
||||
const inlineTextWindowThreshold = int64(2 << 20)
|
||||
|
||||
var projectionAdapters = []projectionAdapter{
|
||||
{
|
||||
ID: "html",
|
||||
Kind: "html",
|
||||
Delivery: deliveryHTML,
|
||||
Match: mediaTypes("text/html"),
|
||||
},
|
||||
{
|
||||
ID: "excalidraw",
|
||||
Kind: "excalidraw",
|
||||
Delivery: deliveryText,
|
||||
NeedsText: true,
|
||||
WindowableText: true,
|
||||
Match: extensions(".excalidraw"),
|
||||
},
|
||||
{
|
||||
ID: "csv",
|
||||
Kind: "csv",
|
||||
Delivery: deliveryText,
|
||||
NeedsText: true,
|
||||
WindowableText: true,
|
||||
Match: matchAny(
|
||||
mediaTypes("text/csv", "application/csv"),
|
||||
extensions(".csv"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "archive",
|
||||
Kind: "archive",
|
||||
Delivery: deliveryDownload,
|
||||
Match: matchAny(
|
||||
mediaTypes(
|
||||
"application/zip", "application/x-zip-compressed",
|
||||
"application/x-rar-compressed", "application/vnd.rar",
|
||||
"application/x-7z-compressed", "application/x-tar",
|
||||
"application/gzip", "application/x-xz",
|
||||
),
|
||||
extensions(".zip", ".rar", ".7z", ".tar", ".gz", ".tgz", ".xz", ".txz"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "debian-package",
|
||||
Kind: "metadata",
|
||||
Delivery: deliveryDownload,
|
||||
Match: matchAny(
|
||||
mediaTypes(
|
||||
"application/vnd.debian.binary-package",
|
||||
"application/x-debian-package",
|
||||
),
|
||||
extensions(".deb"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "eml",
|
||||
Kind: "email",
|
||||
Delivery: deliveryText,
|
||||
Match: matchAny(
|
||||
mediaTypes("message/rfc822"),
|
||||
extensions(".eml"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "pe",
|
||||
Kind: "metadata",
|
||||
Delivery: deliveryDownload,
|
||||
Match: extensions(".exe", ".dll", ".sys"),
|
||||
},
|
||||
{
|
||||
ID: "json",
|
||||
Kind: "json",
|
||||
Delivery: deliveryText,
|
||||
WindowableText: true,
|
||||
Match: matchAny(mediaTypes("application/json", "application/ld+json"), extensions(".json", ".jsonl")),
|
||||
},
|
||||
{
|
||||
ID: "text",
|
||||
Kind: "code",
|
||||
Delivery: deliveryText,
|
||||
NeedsText: true,
|
||||
WindowableText: true,
|
||||
Match: textMediaType,
|
||||
},
|
||||
{
|
||||
ID: "image",
|
||||
Kind: "image",
|
||||
Delivery: deliveryRawMedia,
|
||||
Match: matchAny(
|
||||
mediaTypePrefix("image/"),
|
||||
extensions(".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif", ".svg"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "audio",
|
||||
Kind: "audio",
|
||||
Delivery: deliveryRawMedia,
|
||||
Match: matchAny(
|
||||
mediaTypePrefix("audio/"),
|
||||
extensions(".mp3", ".flac", ".wav", ".ogg", ".m4a", ".aac", ".opus"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "video",
|
||||
Kind: "video",
|
||||
Delivery: deliveryRawMedia,
|
||||
Match: matchAny(
|
||||
mediaTypePrefix("video/"),
|
||||
extensions(".mp4", ".webm", ".mkv", ".mov", ".m4v", ".ogv"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "pdf",
|
||||
Kind: "pdf",
|
||||
Delivery: deliveryRawFrame,
|
||||
Match: matchAny(
|
||||
mediaTypes("application/pdf"),
|
||||
extensions(".pdf"),
|
||||
),
|
||||
},
|
||||
{
|
||||
ID: "download",
|
||||
Kind: "download",
|
||||
Delivery: deliveryDownload,
|
||||
Match: func(contentDescriptor) bool { return true },
|
||||
},
|
||||
}
|
||||
|
||||
func selectProjectionAdapter(descriptor contentDescriptor) projectionAdapter {
|
||||
for _, adapter := range projectionAdapters {
|
||||
if adapter.Match(descriptor) {
|
||||
return adapter
|
||||
}
|
||||
}
|
||||
return projectionAdapters[len(projectionAdapters)-1]
|
||||
}
|
||||
|
||||
func mediaTypes(types ...string) func(contentDescriptor) bool {
|
||||
return func(descriptor contentDescriptor) bool {
|
||||
for _, candidate := range types {
|
||||
if descriptor.MediaType == candidate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func mediaTypePrefix(prefix string) func(contentDescriptor) bool {
|
||||
return func(descriptor contentDescriptor) bool {
|
||||
return strings.HasPrefix(descriptor.MediaType, prefix)
|
||||
}
|
||||
}
|
||||
|
||||
func extensions(values ...string) func(contentDescriptor) bool {
|
||||
return func(descriptor contentDescriptor) bool {
|
||||
extension := strings.ToLower(filepath.Ext(descriptor.Filename))
|
||||
for _, candidate := range values {
|
||||
if extension == candidate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func matchAny(matchers ...func(contentDescriptor) bool) func(contentDescriptor) bool {
|
||||
return func(descriptor contentDescriptor) bool {
|
||||
for _, matcher := range matchers {
|
||||
if matcher(descriptor) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func textMediaType(descriptor contentDescriptor) bool {
|
||||
switch descriptor.MediaType {
|
||||
case "application/json",
|
||||
"application/xml",
|
||||
"application/javascript",
|
||||
"application/x-javascript",
|
||||
"application/x-sh",
|
||||
"application/x-shellscript",
|
||||
"application/x-yaml",
|
||||
"application/yaml",
|
||||
"application/toml":
|
||||
return true
|
||||
default:
|
||||
return strings.HasPrefix(descriptor.MediaType, "text/")
|
||||
}
|
||||
return kunstzeal.SelectAdapter(descriptor)
|
||||
}
|
||||
@@ -14,7 +14,7 @@ func (s *Server) windowedContentViewForRequest(r *http.Request, objectUUID, sema
|
||||
if err != nil || metadata.UUID == "" {
|
||||
return contentView{}, false, err
|
||||
}
|
||||
size, err := s.store.DocumentSize(metadata.DocumentHash)
|
||||
size, err := s.store.DocumentSizeContext(r.Context(), metadata.DocumentHash)
|
||||
if err != nil {
|
||||
return contentView{}, false, nil
|
||||
}
|
||||
@@ -30,7 +30,7 @@ func (s *Server) windowedContentViewForRequest(r *http.Request, objectUUID, sema
|
||||
if size <= inlineTextWindowThreshold || !adapter.WindowableText {
|
||||
return contentView{}, false, nil
|
||||
}
|
||||
window, err := s.store.ReadDocumentWindow(metadata.DocumentHash, 0, 128<<10)
|
||||
window, err := s.store.ReadDocumentWindowContext(r.Context(), metadata.DocumentHash, 0, 128<<10)
|
||||
if err != nil {
|
||||
return contentView{}, false, err
|
||||
}
|
||||
|
||||
Loaded 100 of 302 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user