Refactor architecture into eight components and consolidate reliability work
Build and publish container images / images (push) Successful in 9m34s
Release container images / release (push) Successful in 36s

Extract Grammaratur, Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit behind compatible application contracts.

Include accumulated editor resilience, navigation and theme UX, remote CLI, storage/schema and DR groundwork, tests and design/operations documentation.

Verified: uncached full Go suite; go vet ./...; race tests for all eight components; 11 targeted Chromium tests; Node suites; 19 Python CLI tests; reproducible editor build. No push, release tag or deployment in this checkpoint.
This commit is contained in:
Anya Tokugawa committed 2026-11-01 15:23:31 +09:00
1 parent 9dd425f765
commit d5d21b7841
302 files changed
+20037 -3492

No files matched your search

+15
View File
@@ -0,0 +1,15 @@
package compiler
import "pageswiki/internal/grammaratur"
// HTMLToMarkdownAuthoring delegates loss-preserving conversion to Grammaratur.
// Deprecated: use grammaratur.HTMLToMarkdownAuthoring in new application code.
func HTMLToMarkdownAuthoring(source string) (string, error) {
return grammaratur.HTMLToMarkdownAuthoring(source)
}
// MarkdownAuthoringHTML preserves the existing authoring entry point.
// Deprecated: use grammaratur.MarkdownAuthoringHTML in new application code.
func MarkdownAuthoringHTML(source string) string {
return grammaratur.MarkdownAuthoringHTML(source)
}
+4 -20
View File
@@ -1,24 +1,8 @@
package compiler
import (
"crypto/sha256"
"encoding/hex"
)
import "pageswiki/internal/kunstzeal"
func DiagramHash(
diagramType string,
source string,
format string,
) string {
sum :=
sha256.Sum256(
[]byte(
diagramType + "\x00" + format + "\x00" + source,
),
)
return hex.EncodeToString(
sum[:],
)
// DiagramHash preserves the legacy key format through the rendering component.
func DiagramHash(diagramType, source, format string) string {
return kunstzeal.DiagramHash(diagramType, source, format)
}
+4 -4
View File
@@ -1,14 +1,14 @@
package compiler
import (
htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
)
import "pageswiki/internal/grammaratur"
// HTMLToMarkdown preserves the legacy conversion contract (without protection
// fences). Deprecated: use grammaratur.HTMLToMarkdown in new application code.
func HTMLToMarkdown(
html string,
) (string, error) {
return htmltomarkdown.ConvertString(
return grammaratur.HTMLToMarkdown(
html,
)
}
+17 -1
View File
@@ -1,9 +1,12 @@
package db
import (
"context"
"database/sql"
"fmt"
"net/url"
"path/filepath"
"time"
_ "modernc.org/sqlite"
)
@@ -24,7 +27,20 @@ func Open(path string) (*sql.DB, error) {
},
}.Encode(),
}).String()
return sql.Open("sqlite", dsn)
database, err := sql.Open("sqlite", dsn)
if err != nil {
return nil, err
}
// sql.Open is lazy. Establish WAL once before exposing the pool, otherwise
// simultaneous first borrowers race the initial journal_mode transition in
// their connection pragmas (SQLITE_BUSY before BEGIN IMMEDIATE is reached).
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := database.PingContext(ctx); err != nil {
database.Close()
return nil, fmt.Errorf("initialize SQLite connection: %w", err)
}
return database, nil
}
func Init(db *sql.DB) error {
+87 -90
View File
@@ -4,6 +4,7 @@ import (
"crypto/sha256"
"database/sql"
"encoding/hex"
"fmt"
)
func Migrate(
@@ -96,54 +97,21 @@ func Migrate(
}
}
alterStmts := []string{
`
ALTER TABLE paths
ADD COLUMN created_at INTEGER
`,
`
ALTER TABLE objects
ADD COLUMN created_at INTEGER
`,
`
ALTER TABLE revisions
ADD COLUMN created_at INTEGER
`,
`
ALTER TABLE revisions
ADD COLUMN document_hash TEXT
`,
`
ALTER TABLE revisions
ADD COLUMN lang TEXT NOT NULL DEFAULT 'und'
`,
`
ALTER TABLE revisions
ADD COLUMN created_at_ms INTEGER
`,
`
ALTER TABLE revisions
ADD COLUMN content_type TEXT NOT NULL DEFAULT 'text/html; charset=utf-8'
`,
`
ALTER TABLE revisions
ADD COLUMN filename TEXT NOT NULL DEFAULT ''
`,
`
ALTER TABLE tags
ADD COLUMN created_at INTEGER
`,
columns := []struct{ table, name, declaration string }{
{"paths", "created_at", "INTEGER"},
{"objects", "created_at", "INTEGER"},
{"revisions", "created_at", "INTEGER"},
{"revisions", "document_hash", "TEXT"},
{"revisions", "lang", "TEXT NOT NULL DEFAULT 'und'"},
{"revisions", "created_at_ms", "INTEGER"},
{"revisions", "content_type", "TEXT NOT NULL DEFAULT 'text/html; charset=utf-8'"},
{"revisions", "filename", "TEXT NOT NULL DEFAULT ''"},
{"tags", "created_at", "INTEGER"},
}
for _, stmt := range alterStmts {
_, _ = db.Exec(stmt)
for _, column := range columns {
if err := ensureSQLiteColumn(db, column.table, column.name, column.declaration); err != nil {
return err
}
}
if err := migrateRevisionMetadata(db); err != nil {
@@ -185,54 +153,47 @@ func migrateRevisionMetadata(db *sql.DB) error {
return nil
}
rows, err := db.Query(`
SELECT
uuid,
COALESCE(body, '')
FROM revisions
WHERE document_hash IS NULL
OR document_hash = ''
`)
if err != nil {
return err
}
type legacyRevision struct {
uuid string
body string
}
var revisions []legacyRevision
for rows.Next() {
var revision legacyRevision
if err := rows.Scan(&revision.uuid, &revision.body); err != nil {
// Fetch only identifiers in batches, then hydrate one legacy body at a
// time. The old implementation accumulated all payloads in memory.
lastUUID := ""
for {
rows, err := db.Query(`SELECT uuid FROM revisions WHERE uuid>? AND (document_hash IS NULL OR document_hash='') ORDER BY uuid LIMIT 128`, lastUUID)
if err != nil {
return err
}
var ids []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
rows.Close()
return err
}
ids = append(ids, id)
}
if err := rows.Err(); err != nil {
rows.Close()
return err
}
revisions = append(revisions, revision)
}
if err := rows.Close(); err != nil {
return err
}
for _, revision := range revisions {
sum := sha256.Sum256([]byte(revision.body))
documentHash := hex.EncodeToString(sum[:])
if _, err := db.Exec(`
UPDATE revisions
SET document_hash = ?
WHERE uuid = ?
`, documentHash, revision.uuid); err != nil {
if err := rows.Close(); err != nil {
return err
}
for _, id := range ids {
var body string
if err := db.QueryRow(`SELECT COALESCE(body,'') FROM revisions WHERE uuid=?`, id).Scan(&body); err != nil {
return err
}
sum := sha256.Sum256([]byte(body))
if _, err := db.Exec(`UPDATE revisions SET document_hash=? WHERE uuid=? AND (document_hash IS NULL OR document_hash='')`, hex.EncodeToString(sum[:]), id); err != nil {
return err
}
lastUUID = id
}
if len(ids) < 128 {
break
}
}
if _, err = db.Exec(`
if _, err := db.Exec(`
UPDATE revisions
SET
lang = COALESCE(NULLIF(lang, ''), 'und'),
@@ -247,7 +208,7 @@ func migrateRevisionMetadata(db *sql.DB) error {
return err
}
if _, err = db.Exec(`
if _, err := db.Exec(`
CREATE INDEX IF NOT EXISTS idx_revisions_object_created
ON revisions (
object_uuid,
@@ -257,9 +218,45 @@ func migrateRevisionMetadata(db *sql.DB) error {
return err
}
_, err = db.Exec(`
_, err := db.Exec(`
INSERT OR IGNORE INTO schema_migrations (name, applied_at)
VALUES ('revision_metadata_v1', strftime('%s','now'))
`)
return err
}
func sqliteColumnExists(database *sql.DB, table, column string) (bool, error) {
// All identifiers originate in the fixed legacy bootstrap above.
rows, err := database.Query(fmt.Sprintf("PRAGMA table_info(%q)", table))
if err != nil {
return false, err
}
defer rows.Close()
found := false
for rows.Next() {
var cid, notNull, primaryKey int
var name, kind string
var defaultValue any
if err := rows.Scan(&cid, &name, &kind, &notNull, &defaultValue, &primaryKey); err != nil {
return false, err
}
found = found || name == column
}
return found, rows.Err()
}
func ensureSQLiteColumn(database *sql.DB, table, column, declaration string) error {
exists, err := sqliteColumnExists(database, table, column)
if err != nil || exists {
return err
}
_, err = database.Exec(fmt.Sprintf("ALTER TABLE %q ADD COLUMN %q %s", table, column, declaration))
if err != nil {
// Only tolerate another bootstrap adding the exact column concurrently.
if exists, checkErr := sqliteColumnExists(database, table, column); checkErr == nil && exists {
return nil
}
return fmt.Errorf("add column %s.%s: %w", table, column, err)
}
return nil
}
+139
View File
@@ -0,0 +1,139 @@
package db
import (
"context"
"database/sql"
"fmt"
"sort"
)
type MigrationPlanStep struct {
Version int `json:"version"`
Name string `json:"name"`
Statements int `json:"statements"`
AddColumns []SQLiteColumn `json:"add_columns,omitempty"`
}
type MigrationInspection struct {
Component string `json:"component"`
State string `json:"state"` // current, pending, blocked, unknown
AppliedVersion int `json:"applied_version"`
LatestVersion int `json:"latest_version"`
Pending []MigrationPlanStep `json:"pending,omitempty"`
ContractChecked bool `json:"contract_checked"`
Issues []string `json:"issues,omitempty"`
}
// InspectSQLiteMigrations takes a read transaction and never applies DDL/DML.
// Pending definitions are a plan, not a successful rehearsal: data-dependent
// DDL and assertions on not-yet-created objects require a disposable DB copy.
func InspectSQLiteMigrations(ctx context.Context, database *sql.DB, plans map[string][]SQLMigration) ([]MigrationInspection, error) {
tx, err := database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return nil, err
}
defer tx.Rollback()
var ledger bool
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_migration_steps')`).Scan(&ledger); err != nil {
return nil, err
}
type record struct {
version int
name, checksum string
}
history := map[string][]record{}
if ledger {
rows, err := tx.QueryContext(ctx, `SELECT component,version,name,checksum FROM schema_migration_steps ORDER BY component,version`)
if err != nil {
return nil, err
}
for rows.Next() {
var component string
var r record
if err := rows.Scan(&component, &r.version, &r.name, &r.checksum); err != nil {
rows.Close()
return nil, err
}
history[component] = append(history[component], r)
}
if err := rows.Err(); err != nil {
rows.Close()
return nil, err
}
if err := rows.Close(); err != nil {
return nil, err
}
}
names := make([]string, 0, len(plans)+len(history))
for name := range plans {
names = append(names, name)
}
for name := range history {
if _, ok := plans[name]; !ok {
names = append(names, name)
}
}
sort.Strings(names)
result := make([]MigrationInspection, 0, len(names))
for _, name := range names {
if err := ctx.Err(); err != nil {
return nil, err
}
steps, known := plans[name]
item := MigrationInspection{Component: name, State: "current", LatestVersion: len(steps)}
for i, step := range steps {
if step.Version != i+1 || step.Name == "" || (len(step.Statements) == 0 && len(step.AddColumns) == 0 && len(step.Assertions) == 0) {
return nil, fmt.Errorf("invalid migration definition %s/%d", name, step.Version)
}
}
if !known {
item.State = "unknown"
item.Issues = []string{"component is not registered in this binary"}
}
for i, record := range history[name] {
item.AppliedVersion = record.version
if known && (record.version != i+1 || record.version > len(steps) || record.name != steps[record.version-1].Name || record.checksum != migrationChecksum(steps[record.version-1])) {
item.State = "blocked"
item.Issues = append(item.Issues, fmt.Sprintf("migration history mismatch at version %d", record.version))
}
}
if item.State == "current" {
for _, step := range steps[len(history[name]):] {
item.Pending = append(item.Pending, MigrationPlanStep{Version: step.Version, Name: step.Name, Statements: len(step.Statements), AddColumns: step.AddColumns})
}
if len(item.Pending) > 0 {
item.State = "pending"
}
if len(steps) > 0 && len(steps[len(steps)-1].Assertions) > 0 {
item.ContractChecked = true
for i, assertion := range steps[len(steps)-1].Assertions {
rows, err := tx.QueryContext(ctx, assertion)
if err != nil {
item.Issues = append(item.Issues, fmt.Sprintf("assertion %d: %v", i+1, err))
continue
}
violated := rows.Next()
err = rows.Err()
closeErr := rows.Close()
if err != nil {
return nil, err
}
if closeErr != nil {
return nil, closeErr
}
if violated {
item.Issues = append(item.Issues, fmt.Sprintf("assertion %d failed", i+1))
}
}
if len(item.Issues) > 0 && item.State == "current" {
item.State = "blocked"
}
}
}
result = append(result, item)
}
if err := ctx.Err(); err != nil {
return nil, err
}
return result, tx.Commit()
}
+121
View File
@@ -0,0 +1,121 @@
package db
import (
"context"
"errors"
"path/filepath"
"testing"
)
func TestInspectMigrationsReadOnlyLifecycle(t *testing.T) {
path := filepath.Join(t.TempDir(), "inspect.db")
writer, err := Open(path)
if err != nil {
t.Fatal(err)
}
defer writer.Close()
if err := writer.Ping(); err != nil {
t.Fatal(err)
}
reader, err := OpenReadOnly(path)
if err != nil {
t.Fatal(err)
}
defer reader.Close()
steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample(id INTEGER PRIMARY KEY, value TEXT)`, `INSERT INTO sample VALUES(1,'keep')`}, Assertions: []string{`SELECT id,value FROM sample LIMIT 0`}}}
plans := map[string][]SQLMigration{"sample": steps}
inspect := func(state string, issues bool) {
t.Helper()
got, err := InspectSQLiteMigrations(context.Background(), reader, plans)
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].State != state || (len(got[0].Issues) > 0) != issues || !got[0].ContractChecked {
t.Fatalf("report: %+v", got)
}
}
inspect("pending", true)
var count int
if err := writer.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name IN ('sample','schema_migration_steps')`).Scan(&count); err != nil || count != 0 {
t.Fatalf("inspection created tables: %d %v", count, err)
}
if _, err := reader.Exec(`CREATE TABLE forbidden(id INTEGER)`); err == nil {
t.Fatal("connection is writable")
}
// Keep the writer open: the reader must see committed WAL contents too.
if err := ApplySQLiteMigrations(context.Background(), writer, "sample", steps); err != nil {
t.Fatal(err)
}
inspect("current", false)
if _, err := writer.Exec(`ALTER TABLE sample RENAME COLUMN value TO lost`); err != nil {
t.Fatal(err)
}
inspect("blocked", true)
var value string
if err := writer.QueryRow(`SELECT lost FROM sample WHERE id=1`).Scan(&value); err != nil || value != "keep" {
t.Fatalf("inspection changed data: %q %v", value, err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := InspectSQLiteMigrations(ctx, reader, plans); !errors.Is(err, context.Canceled) {
t.Fatalf("cancel: %v", err)
}
}
func TestInspectMigrationsRejectHistoryMismatch(t *testing.T) {
for _, tc := range []struct{ name, mutation string }{
{"checksum", `UPDATE schema_migration_steps SET checksum='changed'`},
{"name", `UPDATE schema_migration_steps SET name='changed'`},
{"zero", `UPDATE schema_migration_steps SET version=0`},
{"negative", `UPDATE schema_migration_steps SET version=-1`},
{"newer", `UPDATE schema_migration_steps SET version=2`},
} {
t.Run(tc.name, func(t *testing.T) {
database, err := Open(filepath.Join(t.TempDir(), "inspect.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample(id INTEGER)`}}}
if err := ApplySQLiteMigrations(context.Background(), database, "sample", steps); err != nil {
t.Fatal(err)
}
if _, err := database.Exec(tc.mutation); err != nil {
t.Fatal(err)
}
got, err := InspectSQLiteMigrations(context.Background(), database, map[string][]SQLMigration{"sample": steps})
if err != nil || len(got) != 1 || got[0].State != "blocked" {
t.Fatalf("report=%+v err=%v", got, err)
}
})
}
}
func TestInspectMigrationsUnknownAndPendingPlan(t *testing.T) {
database, err := Open(filepath.Join(t.TempDir(), "inspect.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample(id INTEGER)`}}}
if err := ApplySQLiteMigrations(context.Background(), database, "sample", steps); err != nil {
t.Fatal(err)
}
got, err := InspectSQLiteMigrations(context.Background(), database, nil)
if err != nil || len(got) != 1 || got[0].State != "unknown" {
t.Fatalf("unknown=%+v %v", got, err)
}
steps = append(steps, SQLMigration{Version: 2, Name: "add_value", AddColumns: []SQLiteColumn{{Table: "sample", Name: "value", Declaration: "TEXT NOT NULL DEFAULT ''"}}, Assertions: []string{`SELECT value FROM sample LIMIT 0`}})
got, err = InspectSQLiteMigrations(context.Background(), database, map[string][]SQLMigration{"sample": steps})
if err != nil || len(got) != 1 {
t.Fatalf("pending=%+v %v", got, err)
}
item := got[0]
if item.State != "pending" || item.AppliedVersion != 1 || item.LatestVersion != 2 || len(item.Pending) != 1 || len(item.Pending[0].AddColumns) != 1 || len(item.Issues) == 0 {
t.Fatalf("pending=%+v", item)
}
var count int
if err := database.QueryRow(`SELECT count(*) FROM pragma_table_info('sample') WHERE name='value'`).Scan(&count); err != nil || count != 0 {
t.Fatalf("pending DDL applied: %d %v", count, err)
}
}
+188
View File
@@ -0,0 +1,188 @@
package db
import (
"context"
"crypto/sha256"
"database/sql"
"database/sql/driver"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
"time"
)
// SQLMigration is append-only once released. Edit a new version, never an
// applied statement. This runner is SQLite-specific; other DBs need their own
// locking/DDL implementations rather than a driver-name substitution.
type SQLMigration struct {
Version int
Name string
Statements []string
// Omit empty new fields to preserve checksums of previously released steps.
AddColumns []SQLiteColumn `json:",omitempty"`
// Each assertion must execute successfully and return no rows. The latest
// version defines the whole current contract; older assertions are superseded
// to permit deliberate column removal in a later, immutable migration.
Assertions []string `json:",omitempty"`
}
type SQLiteColumn struct {
Table string
Name string
Declaration string
}
func migrationChecksum(step SQLMigration) string {
encoded, _ := json.Marshal(step)
sum := sha256.Sum256(encoded)
return hex.EncodeToString(sum[:])
}
// ApplySQLiteMigrations serializes processes on a dedicated connection with
// BEGIN IMMEDIATE. DDL, data statements and ledger updates commit together.
// The legacy schema_migrations table remains untouched for old installations.
func ApplySQLiteMigrations(ctx context.Context, database *sql.DB, component string, steps []SQLMigration) error {
if component == "" {
return fmt.Errorf("migration component is required")
}
for i, step := range steps {
if step.Version != i+1 || step.Name == "" || (len(step.Statements) == 0 && len(step.AddColumns) == 0 && len(step.Assertions) == 0) {
return fmt.Errorf("invalid migration sequence for %s", component)
}
}
conn, err := database.Conn(ctx)
if err != nil {
return fmt.Errorf("migration %s acquire connection: %w", component, err)
}
defer conn.Close()
if _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil {
return fmt.Errorf("migration %s begin immediate: %w", component, err)
}
committed := false
defer func() {
if committed {
return
}
cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if _, err := conn.ExecContext(cleanup, "ROLLBACK"); err != nil {
// Never return a connection with uncertain transaction state to
// the pool. A failed/ambiguous COMMIT also takes this path.
_ = conn.Raw(func(any) error { return driver.ErrBadConn })
}
}()
if _, err := conn.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_migration_steps (
component TEXT NOT NULL, version INTEGER NOT NULL, name TEXT NOT NULL,
checksum TEXT NOT NULL, applied_at_ms INTEGER NOT NULL,
PRIMARY KEY(component, version))`); err != nil {
return err
}
rows, err := conn.QueryContext(ctx, `SELECT version,name,checksum FROM schema_migration_steps WHERE component=? ORDER BY version`, component)
if err != nil {
return err
}
applied := 0
for rows.Next() {
var version int
var name, checksum string
if err := rows.Scan(&version, &name, &checksum); err != nil {
rows.Close()
return err
}
if version != applied+1 || version > len(steps) || steps[version-1].Name != name || migrationChecksum(steps[version-1]) != checksum {
rows.Close()
return fmt.Errorf("migration history mismatch: component=%s version=%d (changed, missing or newer migration)", component, version)
}
applied++
}
if err := rows.Err(); err != nil {
rows.Close()
return err
}
if err := rows.Close(); err != nil {
return err
}
for _, step := range steps[applied:] {
for i, statement := range step.Statements {
if _, err := conn.ExecContext(ctx, statement); err != nil {
return fmt.Errorf("migration %s/%d %s statement %d: %w", component, step.Version, step.Name, i+1, err)
}
}
for _, column := range step.AddColumns {
if err := addSQLiteColumn(ctx, conn, column); err != nil {
return fmt.Errorf("migration %s/%d column %s.%s: %w", component, step.Version, column.Table, column.Name, err)
}
}
if _, err := conn.ExecContext(ctx, `INSERT INTO schema_migration_steps(component,version,name,checksum,applied_at_ms) VALUES(?,?,?,?,?)`, component, step.Version, step.Name, migrationChecksum(step), time.Now().UnixMilli()); err != nil {
return err
}
}
if len(steps) > 0 {
step := steps[len(steps)-1]
for i, assertion := range step.Assertions {
rows, err := conn.QueryContext(ctx, assertion)
if err != nil {
return fmt.Errorf("schema contract %s/%d assertion %d: %w", component, step.Version, i+1, err)
}
violated := rows.Next()
err = rows.Err()
closeErr := rows.Close()
if err != nil {
return err
}
if closeErr != nil {
return closeErr
}
if violated {
return fmt.Errorf("schema contract %s/%d assertion %d failed", component, step.Version, i+1)
}
}
}
_, err = conn.ExecContext(ctx, "COMMIT")
committed = err == nil
return err
}
func addSQLiteColumn(ctx context.Context, conn *sql.Conn, column SQLiteColumn) error {
// Identifiers are trusted migration definitions, but validate before SQL
// interpolation so accidental punctuation cannot change the statement.
for _, identifier := range []string{column.Table, column.Name} {
if identifier == "" || strings.IndexFunc(identifier, func(r rune) bool {
return !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_')
}) >= 0 {
return fmt.Errorf("invalid schema identifier")
}
}
if strings.TrimSpace(column.Declaration) == "" {
return fmt.Errorf("column declaration is required")
}
rows, err := conn.QueryContext(ctx, `SELECT name FROM pragma_table_info(?)`, column.Table)
if err != nil {
return err
}
found, tableExists := false, false
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
rows.Close()
return err
}
found, tableExists = found || name == column.Name, true
}
if err := rows.Err(); err != nil {
rows.Close()
return err
}
if err := rows.Close(); err != nil {
return err
}
if !tableExists {
return fmt.Errorf("table %s is missing", column.Table)
}
if found {
return nil
}
_, err = conn.ExecContext(ctx, fmt.Sprintf("ALTER TABLE %q ADD COLUMN %q %s", column.Table, column.Name, column.Declaration))
return err
}
+150
View File
@@ -0,0 +1,150 @@
package db
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"path/filepath"
"strings"
"sync"
"testing"
)
func TestSQLMigrationLedgerConcurrencyAndChecksum(t *testing.T) {
database, err := Open(filepath.Join(t.TempDir(), "migration.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample (id INTEGER PRIMARY KEY)`, `INSERT INTO sample VALUES (1)`}}}
var wg sync.WaitGroup
errors := make(chan error, 6)
for i := 0; i < 6; i++ {
wg.Add(1)
go func() {
defer wg.Done()
errors <- ApplySQLiteMigrations(context.Background(), database, "test", steps)
}()
}
wg.Wait()
close(errors)
for err := range errors {
if err != nil {
t.Fatal(err)
}
}
var count int
if err := database.QueryRow(`SELECT count(*) FROM sample`).Scan(&count); err != nil || count != 1 {
t.Fatalf("count=%d err=%v", count, err)
}
changed := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample (id TEXT)`}}}
if err := ApplySQLiteMigrations(context.Background(), database, "test", changed); err == nil || !strings.Contains(err.Error(), "history mismatch") {
t.Fatalf("changed migration accepted: %v", err)
}
if err := ApplySQLiteMigrations(context.Background(), database, "test", nil); err == nil {
t.Fatal("downgrade accepted")
}
if err := ApplySQLiteMigrations(context.Background(), database, "other", []SQLMigration{{Version: 2, Name: "gap", Statements: []string{`SELECT 1`}}}); err == nil {
t.Fatal("version gap accepted")
}
}
func TestSQLMigrationFailureRollsBackDDLAndLedger(t *testing.T) {
database, err := Open(filepath.Join(t.TempDir(), "migration.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
steps := []SQLMigration{{Version: 1, Name: "broken", Statements: []string{`CREATE TABLE partial (id INTEGER)`, `NOT VALID SQL`}}}
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err == nil {
t.Fatal("DDL failure ignored")
}
var count int
if err := database.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name IN ('partial','schema_migration_steps')`).Scan(&count); err != nil || count != 0 {
t.Fatalf("partial migration remains count=%d err=%v", count, err)
}
steps[0].Statements = steps[0].Statements[:1]
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil {
t.Fatal("retry after rollback", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if err := ApplySQLiteMigrations(ctx, database, "test", steps); err == nil {
t.Fatal("cancel ignored")
}
}
func TestLegacyRevisionHashMigrationSpansBoundedBatches(t *testing.T) {
database, err := Open(filepath.Join(t.TempDir(), "legacy.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
if _, err := database.Exec(`CREATE TABLE revisions (uuid TEXT PRIMARY KEY,object_uuid TEXT,body TEXT,message TEXT)`); err != nil {
t.Fatal(err)
}
for i := 0; i < 260; i++ {
if _, err := database.Exec(`INSERT INTO revisions(uuid,body) VALUES(?,?)`, fmt.Sprintf("revision-%04d", i), "body"); err != nil {
t.Fatal(err)
}
}
if err := Migrate(database); err != nil {
t.Fatal(err)
}
var count int
if err := database.QueryRow(`SELECT count(*) FROM revisions WHERE length(document_hash)=64 AND lang='und' AND created_at_ms=0`).Scan(&count); err != nil || count != 260 {
t.Fatalf("migrated=%d err=%v", count, err)
}
if err := ensureSQLiteColumn(database, "does_not_exist", "created_at", "INTEGER"); err == nil {
t.Fatal("ALTER failure swallowed")
}
}
func TestMigrationChecksumPreservesLegacyFormat(t *testing.T) {
step := SQLMigration{Version: 1, Name: "legacy", Statements: []string{"SELECT 1"}}
sum := sha256.Sum256([]byte(`{"Version":1,"Name":"legacy","Statements":["SELECT 1"]}`))
if got := migrationChecksum(step); got != hex.EncodeToString(sum[:]) {
t.Fatalf("legacy checksum changed: %s", got)
}
}
func TestMigrationConditionalColumnsAndAssertionsAreAtomic(t *testing.T) {
database, err := Open(filepath.Join(t.TempDir(), "schema.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
if _, err := database.Exec(`CREATE TABLE sample(id INTEGER PRIMARY KEY); INSERT INTO sample VALUES(1)`); err != nil {
t.Fatal(err)
}
steps := []SQLMigration{{Version: 1, Name: "adopt", AddColumns: []SQLiteColumn{{Table: "sample", Name: "value", Declaration: "TEXT NOT NULL DEFAULT ''"}}, Assertions: []string{`SELECT missing FROM sample LIMIT 0`}}}
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err == nil {
t.Fatal("invalid schema accepted")
}
var count int
if err := database.QueryRow(`SELECT count(*) FROM pragma_table_info('sample') WHERE name='value'`).Scan(&count); err != nil || count != 0 {
t.Fatalf("ALTER was not rolled back: %d %v", count, err)
}
steps[0].Assertions = []string{`SELECT id,value FROM sample LIMIT 0`}
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil {
t.Fatal(err)
}
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil {
t.Fatal("repeated apply", err)
}
var value string
if err := database.QueryRow(`SELECT value FROM sample WHERE id=1`).Scan(&value); err != nil || value != "" {
t.Fatalf("legacy row lost: %q %v", value, err)
}
steps = append(steps, SQLMigration{Version: 2, Name: "remove_value", Statements: []string{`ALTER TABLE sample DROP COLUMN value`}, Assertions: []string{`SELECT id FROM sample LIMIT 0`}})
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil {
t.Fatal("new contract must supersede old contract", err)
}
if _, err := database.Exec(`DROP TABLE sample`); err != nil {
t.Fatal(err)
}
if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err == nil {
t.Fatal("recorded history masked missing table")
}
}
+25
View File
@@ -0,0 +1,25 @@
package db
import (
"database/sql"
"net/url"
"path/filepath"
)
// OpenReadOnly deliberately does not migrate, set journal_mode, or create a DB.
func OpenReadOnly(path string) (*sql.DB, error) {
path, err := filepath.Abs(path)
if err != nil {
return nil, err
}
dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: url.Values{"mode": {"ro"}, "_pragma": {"query_only(1)", "busy_timeout(5000)"}}.Encode()}).String()
database, err := sql.Open("sqlite", dsn)
if err != nil {
return nil, err
}
if err = database.Ping(); err != nil {
database.Close()
return nil, err
}
return database, nil
}
@@ -0,0 +1,176 @@
package documentstore_test
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
"pageswiki/internal/documentstore"
"pageswiki/internal/geduldh"
)
// These assignments intentionally fail compilation if aliases become wrappers
// with different type identities or constructor signatures.
var (
_ func(string) (*geduldh.Local, error) = documentstore.NewLocal
_ func(documentstore.Source, documentstore.MaterializeOptions) (*documentstore.Materializer, error) = geduldh.NewMaterializer
_ documentstore.Store = (*geduldh.Local)(nil)
_ geduldh.Store = (*documentstore.Local)(nil)
)
func TestGeduldhLegacyLayoutAndSharedIdentity(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
body := []byte("existing Document 日本語\x00\xff")
sum := sha256.Sum256(body)
hash := hex.EncodeToString(sum[:])
// Seed the pre-extraction layout independently of either Key implementation.
filename := filepath.Join(root, "sha256", hash[:2], hash)
if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, body, 0644); err != nil {
t.Fatal(err)
}
before, err := os.Stat(filename)
if err != nil {
t.Fatal(err)
}
old, err := documentstore.NewLocal(root)
if err != nil {
t.Fatal(err)
}
current, err := geduldh.NewLocal(root)
if err != nil {
t.Fatal(err)
}
for _, backend := range []geduldh.Store{old, current} {
// The old normalization rules and payload bytes stay unchanged.
inputHash := " " + strings.ToUpper(hash) + " "
info, err := backend.Stat(ctx, inputHash)
if err != nil || info.Hash != hash || info.Size != int64(len(body)) {
t.Fatalf("stat=%+v err=%v", info, err)
}
reader, err := backend.Open(ctx, inputHash)
if err != nil {
t.Fatal(err)
}
func() {
defer reader.Close()
if _, err := reader.Seek(2, io.SeekStart); err != nil {
t.Fatal(err)
}
buf := make([]byte, 4)
if _, err := reader.ReadAt(buf, 5); err != nil || !bytes.Equal(buf, body[5:9]) {
t.Fatalf("ReadAt=%q err=%v", buf, err)
}
// ReaderAt must not move the sequential stream cursor (ZIP and Range).
data, err := io.ReadAll(reader)
if err != nil || !bytes.Equal(data, body[2:]) {
t.Fatalf("stream=%q err=%v", data, err)
}
}()
if err := backend.PutIfAbsent(ctx, hash, int64(len(body)), bytes.NewReader(body)); err != nil {
t.Fatal(err)
}
if err := backend.Verify(ctx, hash); err != nil {
t.Fatal(err)
}
}
after, err := os.Stat(filename)
if err != nil || !os.SameFile(before, after) {
t.Fatalf("canonical inode replaced: %v", err)
}
oldKey, err := documentstore.Key(hash)
if err != nil {
t.Fatal(err)
}
newKey, err := geduldh.Key(hash)
if err != nil || oldKey != newKey || oldKey != "sha256/"+hash[:2]+"/"+hash {
t.Fatalf("key changed: %s %s %v", oldKey, newKey, err)
}
if documentstore.ErrInvalidHash != geduldh.ErrInvalidHash || documentstore.ErrHashMismatch != geduldh.ErrHashMismatch || documentstore.ErrSizeMismatch != geduldh.ErrSizeMismatch || documentstore.ErrMaterializeLimit != geduldh.ErrMaterializeLimit {
t.Fatal("legacy error identity changed")
}
if err := os.WriteFile(filename, []byte("corrupt"), 0644); err != nil {
t.Fatal(err)
}
for _, backend := range []geduldh.Store{old, current} {
if err := backend.PutIfAbsent(ctx, hash, int64(len(body)), bytes.NewReader(body)); !errors.Is(err, documentstore.ErrHashMismatch) {
t.Fatalf("corruption repaired or error changed: %v", err)
}
}
corrupt, err := os.ReadFile(filename)
if err != nil || string(corrupt) != "corrupt" {
t.Fatalf("corrupt canonical overwritten: %v", err)
}
}
func TestGeduldhCompatibilityMaterializer(t *testing.T) {
root, scratch := t.TempDir(), t.TempDir()
backend, err := geduldh.NewLocal(root)
if err != nil {
t.Fatal(err)
}
body := []byte("shared scratch bytes")
sum := sha256.Sum256(body)
hash := hex.EncodeToString(sum[:])
ctx := context.Background()
if err := backend.PutIfAbsent(ctx, hash, int64(len(body)), bytes.NewReader(body)); err != nil {
t.Fatal(err)
}
var materializer *geduldh.Materializer
materializer, err = documentstore.NewMaterializer(backend, documentstore.MaterializeOptions{TempDir: scratch, MaxBytes: 1024, Concurrent: 1, Timeout: time.Second})
if err != nil {
t.Fatal(err)
}
var first *documentstore.Lease
first, err = materializer.Acquire(ctx, hash, 1024)
if err != nil {
t.Fatal(err)
}
defer first.Close()
second, err := materializer.Acquire(ctx, hash, 1024)
if err != nil {
t.Fatal(err)
}
defer second.Close()
if first.Path != second.Path {
t.Fatal("legacy/current leases no longer share the same copy")
}
first.Close()
data, err := os.ReadFile(second.Path)
if err != nil || !bytes.Equal(data, body) {
t.Fatalf("active lease lost: %v", err)
}
second.Close()
if _, err := os.Stat(second.Path); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("scratch not reclaimed: %v", err)
}
if err := backend.Verify(ctx, hash); err != nil {
t.Fatal("lease cleanup affected canonical content", err)
}
}
func TestGeduldhConstructorsDoNotProvisionRoots(t *testing.T) {
root := filepath.Join(t.TempDir(), "missing")
for _, create := range []func(string) (*geduldh.Local, error){documentstore.NewLocal, geduldh.NewLocal} {
if _, err := create(root); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(root); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("constructor wrote to storage root: %v", err)
}
if _, err := create(" "); err == nil {
t.Fatal("empty root accepted")
}
}
}
+34
View File
@@ -0,0 +1,34 @@
// Package documentstore preserves the original Document Store entry points.
// Payload storage and scratch materialization are implemented by Geduldh.
// New consumers should use pageswiki/internal/geduldh directly.
package documentstore
import "pageswiki/internal/geduldh"
// Aliases preserve interface/type identity, method sets and error matching.
// This package must not acquire its own persistence or validation logic.
type (
Info = geduldh.Info
Reader = geduldh.Reader
Store = geduldh.Store
Local = geduldh.Local
Source = geduldh.Source
MaterializeOptions = geduldh.MaterializeOptions
Materializer = geduldh.Materializer
Lease = geduldh.Lease
)
var (
ErrInvalidHash = geduldh.ErrInvalidHash
ErrHashMismatch = geduldh.ErrHashMismatch
ErrSizeMismatch = geduldh.ErrSizeMismatch
ErrMaterializeLimit = geduldh.ErrMaterializeLimit
)
func Key(hash string) (string, error) { return geduldh.Key(hash) }
func NewLocal(root string) (*Local, error) { return geduldh.NewLocal(root) }
func NewMaterializer(source Source, options MaterializeOptions) (*Materializer, error) {
return geduldh.NewMaterializer(source, options)
}
+294
View File
@@ -0,0 +1,294 @@
package dr
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
"testing"
"time"
"pageswiki/internal/db"
)
func fixture(t *testing.T) string {
t.Helper()
root := t.TempDir()
database, err := db.Open(filepath.Join(root, "pages.db"))
if err != nil {
t.Fatal(err)
}
if _, err = database.Exec("CREATE TABLE revisions(document_hash TEXT); INSERT INTO revisions(rowid,document_hash) VALUES(42,'')"); err != nil {
t.Fatal(err)
}
if err = database.Close(); err != nil {
t.Fatal(err)
}
if _, err = Init(root); err != nil {
t.Fatal(err)
}
if err = os.WriteFile(filepath.Join(root, "auth.key"), make([]byte, 32), 0600); err != nil {
t.Fatal(err)
}
return root
}
func TestSnapshotRestorePromote(t *testing.T) {
root := fixture(t)
ctx := context.Background()
dest := filepath.Join(t.TempDir(), "snapshot")
body := []byte("synthetic DR test document")
sum := sha256.Sum256(body)
hash := hex.EncodeToString(sum[:])
path := filepath.Join(root, "documents", "sha256", hash[:2], hash)
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, body, 0600); err != nil {
t.Fatal(err)
}
database, err := db.Open(filepath.Join(root, "pages.db"))
if err != nil {
t.Fatal(err)
}
// Hold a connection open to keep committed WAL pages present. No writer is
// active during snapshot; this covers recovery after an uncheckpointed stop.
if _, err = database.Exec("PRAGMA wal_autocheckpoint=0; UPDATE revisions SET document_hash=? WHERE rowid=42", hash); err != nil {
t.Fatal(err)
}
defer database.Close()
if err = os.MkdirAll(filepath.Join(root, "previews"), 0700); err != nil {
t.Fatal(err)
}
os.WriteFile(filepath.Join(root, "previews", "ignored"), []byte("cache"), 0600)
m, digest, err := Snapshot(ctx, root, dest)
if err != nil {
t.Fatal(err)
}
if _, err = Verify(ctx, dest, digest); err != nil {
t.Fatal(err)
}
if _, err = Verify(ctx, dest, digest); err != nil {
t.Fatalf("verification must not mutate snapshot: %v", err)
}
restored := filepath.Join(t.TempDir(), "standby")
node, err := Restore(ctx, dest, digest, restored)
if err != nil {
t.Fatal(err)
}
if node.Role != "standby" || node.NodeID == node.PrimaryNodeID || node.SnapshotID != m.ID || !node.Readable(time.Now()) {
t.Fatalf("bad standby: %+v", node)
}
if _, err = os.Stat(filepath.Join(restored, "previews")); !os.IsNotExist(err) {
t.Fatal("copied cache")
}
ro, err := db.OpenReadOnly(filepath.Join(restored, "pages.db"))
if err != nil {
t.Fatal(err)
}
defer ro.Close()
var got string
if err = ro.QueryRow("SELECT document_hash FROM revisions WHERE rowid=42").Scan(&got); err != nil || got != hash {
t.Fatalf("WAL or rowid lost: %s %v", got, err)
}
if _, err = ro.Exec("DELETE FROM revisions"); err == nil {
t.Fatal("standby accepted write")
}
shared, err := Lock(restored, false)
if err != nil {
t.Fatal(err)
}
if _, err = Promote(restored, node.PrimaryNodeID, "test fence", true); err == nil {
t.Fatal("promoted running standby")
}
shared.Close()
if _, err = Promote(restored, "wrong", "test fence", true); err == nil {
t.Fatal("accepted wrong primary")
}
if _, err = Promote(restored, node.PrimaryNodeID, "test fence", false); err == nil {
t.Fatal("accepted implicit data loss")
}
promoted, err := Promote(restored, node.PrimaryNodeID, "test fence", true)
if err != nil {
t.Fatal(err)
}
if promoted.Role != "primary" || promoted.Epoch != node.Epoch+1 || len(promoted.History) != 1 {
t.Fatalf("bad promotion: %+v", promoted)
}
if promoted.WorkersEnabled {
t.Fatal("promotion replayed jobs without review")
}
if _, err = ResumeWorkers(restored, false); err == nil {
t.Fatal("resumed jobs without review")
}
resumed, err := ResumeWorkers(restored, true)
if err != nil {
t.Fatal(err)
}
if !resumed.WorkersEnabled || resumed.WorkersResumedAt.IsZero() {
t.Fatal("worker resumption not audited")
}
if _, err = Restore(ctx, dest, digest, restored); err == nil {
t.Fatal("overwrote existing restore")
}
}
func TestSnapshotCalendarReferencesAndRestoreSafety(t *testing.T) {
root := fixture(t)
dir := filepath.Join(root, "documents", "calendars", "synthetic-domain", "synthetic-calendar")
if err := os.MkdirAll(dir, 0700); err != nil {
t.Fatal(err)
}
body := []byte("synthetic opaque monthly generation; no external calendar data")
sum := sha256.Sum256(body)
hash := hex.EncodeToString(sum[:])
metadata, _ := json.Marshal(map[string]any{"months": map[string]string{"2026-09": hash}})
if err := os.WriteFile(filepath.Join(dir, "calendar.json"), metadata, 0600); err != nil {
t.Fatal(err)
}
dest := filepath.Join(t.TempDir(), "snapshot")
if _, _, err := Snapshot(context.Background(), root, dest); err == nil {
t.Fatal("accepted missing calendar month")
}
if err := os.WriteFile(filepath.Join(dir, "2026-09."+hash+".ics"), body, 0600); err != nil {
t.Fatal(err)
}
_, digest, err := Snapshot(context.Background(), root, dest)
if err != nil {
t.Fatal(err)
}
if _, err = Verify(context.Background(), dest, digest); err != nil {
t.Fatal(err)
}
if _, _, err = Snapshot(context.Background(), root, filepath.Join(root, "inside")); err == nil {
t.Fatal("snapshot inside source accepted")
}
if _, err = Restore(context.Background(), dest, digest, root); err == nil {
t.Fatal("restore overwrote existing data")
}
if _, err = Restore(context.Background(), dest, digest, filepath.Join(dest, "nested")); err == nil {
t.Fatal("restore corrupted source snapshot")
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err = Verify(ctx, dest, digest); err == nil {
t.Fatal("verification ignored cancellation")
}
}
func TestSnapshotRefusesActiveMissingAndSymlink(t *testing.T) {
root := fixture(t)
dest := filepath.Join(t.TempDir(), "snapshot")
ctx := context.Background()
shared, err := Lock(root, false)
if err != nil {
t.Fatal(err)
}
if _, _, err = Snapshot(ctx, root, dest); err == nil {
t.Fatal("snapshotted running instance")
}
shared.Close()
database, err := db.Open(filepath.Join(root, "pages.db"))
if err != nil {
t.Fatal(err)
}
_, err = database.Exec("UPDATE revisions SET document_hash=?", string(make([]byte, 64)))
if err != nil {
t.Fatal(err)
}
database.Close()
if _, _, err = Snapshot(ctx, root, dest); err == nil {
t.Fatal("accepted missing canonical document")
}
if _, err = os.Stat(dest); !os.IsNotExist(err) {
t.Fatal("published failed snapshot")
}
if err = os.Symlink(t.TempDir(), filepath.Join(root, "themes")); err != nil {
t.Fatal(err)
}
if _, _, err = Snapshot(ctx, root, dest); err == nil {
t.Fatal("followed source symlink")
}
}
func TestSnapshotTamperAndPathValidation(t *testing.T) {
root := fixture(t)
dest := filepath.Join(t.TempDir(), "snapshot")
ctx := context.Background()
_, digest, err := Snapshot(ctx, root, dest)
if err != nil {
t.Fatal(err)
}
if _, err = Verify(ctx, dest, ""); err == nil {
t.Fatal("accepted unauthenticated manifest")
}
manifestPath := filepath.Join(dest, "manifest.json")
encoded, err := os.ReadFile(manifestPath)
if err != nil {
t.Fatal(err)
}
var m Manifest
if err = json.Unmarshal(encoded, &m); err != nil {
t.Fatal(err)
}
m.Files[0].Path = "../outside"
altered, _ := json.Marshal(m)
sum := sha256.Sum256(altered)
os.WriteFile(manifestPath, altered, 0600)
if _, err = Verify(ctx, dest, digest); err == nil {
t.Fatal("accepted changed manifest")
}
if _, err = Verify(ctx, dest, hex.EncodeToString(sum[:])); err == nil {
t.Fatal("accepted traversal even with matching manifest digest")
}
os.WriteFile(manifestPath, encoded, 0600)
os.WriteFile(filepath.Join(dest, "data", "auth.key"), []byte("tampered"), 0600)
if _, err = Verify(ctx, dest, digest); err == nil {
t.Fatal("accepted payload tamper")
}
for _, path := range []string{"/etc/passwd", "documents/../auth.key", "documents\\foo", "dr-node.json", "pages.db-wal", "cache/foo"} {
if canonical(path) {
t.Errorf("accepted %q", path)
}
}
}
func TestConfigFailsClosedAndExpiry(t *testing.T) {
root := fixture(t)
c, err := Load(root)
if err != nil {
t.Fatal(err)
}
c.Role = "standby"
if err = Save(root, *c); err == nil {
t.Fatal("accepted inconsistent role")
}
os.WriteFile(filepath.Join(root, ConfigFile), []byte(`{"role":"broken"}`), 0600)
if _, err = Load(root); err == nil {
t.Fatal("invalid config accepted")
}
c.Role = "standby"
c.CapturedAt = time.Now().Add(-2 * time.Hour)
c.MaxReadAgeSeconds = 3600
if c.Readable(time.Now()) {
t.Fatal("stale snapshot is readable")
}
c.CapturedAt = time.Now().Add(time.Hour)
if c.Readable(time.Now()) {
t.Fatal("future snapshot is readable")
}
if err = RequireRole(nil, "standby"); err == nil {
t.Fatal("empty standby volume would start writable")
}
if err = RequireRole(c, "primary"); err == nil {
t.Fatal("mismatched deployment role accepted")
}
if err = RequireRole(c, "standby"); err != nil {
t.Fatal(err)
}
if err = RequireRole(nil, ""); err != nil {
t.Fatal("legacy startup broken")
}
}
+70
View File
@@ -0,0 +1,70 @@
package dr
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"io"
"os"
"path/filepath"
"testing"
"pageswiki/internal/db"
"pageswiki/internal/geduldh"
)
func TestGeduldhSnapshotRestoreCompatibility(t *testing.T) {
root := fixture(t)
documents := filepath.Join(root, "documents")
if err := os.MkdirAll(documents, 0755); err != nil {
t.Fatal(err)
}
backend, err := geduldh.NewLocal(documents)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
body := []byte("Geduldh DR compatibility 日本語\x00\xff")
sum := sha256.Sum256(body)
hash := hex.EncodeToString(sum[:])
if err := backend.PutIfAbsent(ctx, hash, int64(len(body)), bytes.NewReader(body)); err != nil {
t.Fatal(err)
}
database, err := db.Open(filepath.Join(root, "pages.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
if _, err := database.Exec(`UPDATE revisions SET document_hash=? WHERE rowid=42`, hash); err != nil {
t.Fatal(err)
}
snapshot := filepath.Join(t.TempDir(), "snapshot")
_, digest, err := Snapshot(ctx, root, snapshot)
if err != nil {
t.Fatal(err)
}
if _, err := Verify(ctx, snapshot, digest); err != nil {
t.Fatal(err)
}
restored := filepath.Join(t.TempDir(), "standby")
if _, err := Restore(ctx, snapshot, digest, restored); err != nil {
t.Fatal(err)
}
recovered, err := geduldh.NewLocal(filepath.Join(restored, "documents"))
if err != nil {
t.Fatal(err)
}
if err := recovered.Verify(ctx, hash); err != nil {
t.Fatal(err)
}
reader, err := recovered.Open(ctx, hash)
if err != nil {
t.Fatal(err)
}
defer reader.Close()
data, err := io.ReadAll(reader)
if err != nil || !bytes.Equal(data, body) {
t.Fatalf("restored payload changed: %v", err)
}
}
+273
View File
@@ -0,0 +1,273 @@
// Package dr implements offline, operator-fenced recovery, not leader election.
package dr
import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/google/uuid"
)
const ConfigFile = "dr-node.json"
const LockFile = ".pageswiki-instance.lock"
// RequireRole is an optional deployment guard against mounting an empty/wrong
// data volume on a DR host and accidentally starting a new writable instance.
func RequireRole(c *Config, expected string) error {
if expected == "" {
return nil
}
if expected != "primary" && expected != "standby" {
return errors.New("PAGESWIKI_DR_EXPECT_ROLE must be primary or standby")
}
if c == nil || c.Role != expected {
return errors.New("DR role does not match PAGESWIKI_DR_EXPECT_ROLE; refusing startup")
}
return nil
}
type Promotion struct {
At time.Time `json:"at"`
PreviousNode string `json:"previous_node"`
FenceReference string `json:"fence_reference"`
SnapshotID string `json:"snapshot_id"`
Epoch uint64 `json:"epoch"`
}
// Config is node-local and MUST NOT be overwritten by a primary's snapshot.
// Epoch is an audit generation only; it is NOT a distributed fencing token.
type Config struct {
Version int `json:"version"`
ClusterID string `json:"cluster_id"`
NodeID string `json:"node_id"`
PrimaryNodeID string `json:"primary_node_id"`
Role string `json:"role"`
Epoch uint64 `json:"epoch"`
SnapshotID string `json:"snapshot_id,omitempty"`
CapturedAt time.Time `json:"captured_at,omitempty"`
MaxReadAgeSeconds int64 `json:"max_read_age_seconds"`
WorkersEnabled bool `json:"workers_enabled"`
WorkersResumedAt time.Time `json:"workers_resumed_at,omitempty"`
History []Promotion `json:"promotion_history,omitempty"`
}
func (c Config) Validate() error {
for _, id := range []string{c.ClusterID, c.NodeID, c.PrimaryNodeID} {
u, err := uuid.Parse(id)
if err != nil || u == uuid.Nil || u.String() != id {
return errors.New("DR identity must be a canonical nonzero UUID")
}
}
if c.Version != 1 || c.Epoch == 0 || c.MaxReadAgeSeconds < 1 || c.MaxReadAgeSeconds > 86400 {
return errors.New("invalid DR version, epoch or max_read_age_seconds (1..86400)")
}
if c.Role != "primary" && c.Role != "standby" {
return errors.New("DR role must be primary or standby")
}
if (c.Role == "primary") != (c.PrimaryNodeID == c.NodeID) {
return errors.New("DR role and primary identity disagree")
}
if c.Role == "standby" && (c.SnapshotID == "" || c.CapturedAt.IsZero()) {
return errors.New("standby requires a verified snapshot")
}
if c.Role == "standby" && c.WorkersEnabled {
return errors.New("standby cannot enable background workers")
}
if c.SnapshotID != "" {
u, err := uuid.Parse(c.SnapshotID)
if err != nil || u == uuid.Nil || u.String() != c.SnapshotID {
return errors.New("invalid snapshot identity")
}
}
return nil
}
func (c Config) Readable(now time.Time) bool {
return c.Role == "standby" && !c.CapturedAt.After(now.Add(time.Minute)) && now.Sub(c.CapturedAt) <= time.Duration(c.MaxReadAgeSeconds)*time.Second
}
func (c Config) ReadUntil() time.Time {
if c.CapturedAt.After(time.Now().Add(time.Minute)) {
return time.Time{}
}
return c.CapturedAt.Add(time.Duration(c.MaxReadAgeSeconds) * time.Second)
}
func Load(dataDir string) (*Config, error) {
f, err := os.Open(filepath.Join(dataDir, ConfigFile))
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
if err != nil {
return nil, err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return nil, err
}
if !info.Mode().IsRegular() || info.Size() > 1<<20 {
return nil, errors.New("invalid or oversized DR configuration")
}
var c Config
d := json.NewDecoder(io.LimitReader(f, 1<<20))
d.DisallowUnknownFields()
if err = d.Decode(&c); err != nil {
return nil, err
}
if err = d.Decode(new(any)); err != io.EOF {
return nil, errors.New("trailing DR configuration data")
}
return &c, c.Validate()
}
func Save(dataDir string, c Config) error {
if err := c.Validate(); err != nil {
return err
}
b, err := json.MarshalIndent(c, "", " ")
if err != nil {
return err
}
f, err := os.CreateTemp(dataDir, ".dr-node-")
if err != nil {
return err
}
defer os.Remove(f.Name())
if _, err = f.Write(append(b, '\n')); err != nil {
f.Close()
return err
}
if err = f.Sync(); err != nil {
f.Close()
return err
}
if err = f.Close(); err != nil {
return err
}
if err = os.Rename(f.Name(), filepath.Join(dataDir, ConfigFile)); err != nil {
return err
}
return syncDir(dataDir)
}
// Lock is shared for a running server/CLI, exclusive for offline maintenance.
// Advisory locking cannot fence older binaries or an independent SQLite client.
func Lock(dataDir string, exclusive bool) (*os.File, error) {
// Shared locks also work with a read-only standby mount.
path := filepath.Join(dataDir, LockFile)
f, err := os.Open(path)
if exclusive || errors.Is(err, os.ErrNotExist) {
if f != nil {
f.Close()
}
f, err = os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600)
}
if err != nil {
return nil, err
}
mode := syscall.LOCK_SH
if exclusive {
mode = syscall.LOCK_EX
}
if err = syscall.Flock(int(f.Fd()), mode|syscall.LOCK_NB); err != nil {
f.Close()
return nil, fmt.Errorf("DR data directory is in use; stop all writers/readers before maintenance: %w", err)
}
return f, nil
}
func Init(dataDir string) (*Config, error) {
l, err := Lock(dataDir, true)
if err != nil {
return nil, err
}
defer l.Close()
if c, err := Load(dataDir); err != nil || c != nil {
return nil, errors.New("DR configuration exists or cannot be read; refusing initialization")
}
id := uuid.NewString()
c := Config{Version: 1, ClusterID: uuid.NewString(), NodeID: id, PrimaryNodeID: id, Role: "primary", Epoch: 1, MaxReadAgeSeconds: 3600, WorkersEnabled: true}
return &c, Save(dataDir, c)
}
// Promote records an operator's fencing attestation. It does not contact,
// power off, or independently verify the former primary. No automatic promotion.
func Promote(dataDir, fencedNode, reference string, acceptLoss bool) (*Config, error) {
l, err := Lock(dataDir, true)
if err != nil {
return nil, err
}
defer l.Close()
c, err := Load(dataDir)
if err != nil {
return nil, err
}
if c == nil || c.Role != "standby" {
return nil, errors.New("only an offline standby can be promoted")
}
if err = checkAuthKey(dataDir); err != nil {
return nil, err
}
if fencedNode != c.PrimaryNodeID || !acceptLoss || len(strings.TrimSpace(reference)) < 4 || len(reference) > 1024 {
return nil, errors.New("promotion requires exact former primary ID, fencing reference and explicit acceptance of snapshot data loss")
}
if c.Epoch == ^uint64(0) {
return nil, errors.New("DR epoch exhausted")
}
c.Epoch++
c.History = append(c.History, Promotion{At: time.Now().UTC(), PreviousNode: fencedNode, FenceReference: strings.TrimSpace(reference), SnapshotID: c.SnapshotID, Epoch: c.Epoch})
c.Role, c.PrimaryNodeID = "primary", c.NodeID
// Replaying pre-snapshot jobs can duplicate notifications or external writes.
// Resume them only after an operator reviews the restored queue.
c.WorkersEnabled = false
return c, Save(dataDir, *c)
}
func checkAuthKey(dataDir string) error {
if strings.TrimSpace(os.Getenv("PAGESWIKI_AUTH_KEY")) != "" {
return nil
}
key, err := os.ReadFile(filepath.Join(dataDir, "auth.key"))
if err != nil || len(key) != 32 {
return errors.New("original auth.key or PAGESWIKI_AUTH_KEY is required; refusing implicit key regeneration")
}
return nil
}
func ResumeWorkers(dataDir string, reviewed bool) (*Config, error) {
if !reviewed {
return nil, errors.New("review restored jobs and federation state before resuming workers")
}
l, err := Lock(dataDir, true)
if err != nil {
return nil, err
}
defer l.Close()
c, err := Load(dataDir)
if err != nil {
return nil, err
}
if c == nil || c.Role != "primary" {
return nil, errors.New("only an offline primary can resume workers")
}
c.WorkersEnabled = true
c.WorkersResumedAt = time.Now().UTC()
return c, Save(dataDir, *c)
}
func syncDir(path string) error {
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
return f.Sync()
}
+579
View File
@@ -0,0 +1,579 @@
package dr
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/google/uuid"
"golang.org/x/sys/unix"
"modernc.org/sqlite"
"pageswiki/internal/db"
)
type File struct {
Path string `json:"path"`
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
}
type Manifest struct {
Version int `json:"version"`
ID string `json:"id"`
ClusterID string `json:"cluster_id"`
PrimaryNodeID string `json:"primary_node_id"`
Epoch uint64 `json:"epoch"`
CapturedAt time.Time `json:"captured_at"`
CompletedAt time.Time `json:"completed_at"`
Scope string `json:"scope"`
Files []File `json:"files"`
}
// Include canonical stores, auth/signing keys and managed-campus enrollment.
// Caches, temporary uploads, editor workspaces and node-local DR state are NOT
// recovery authorities. Extra mounted storage/environment secrets are external.
func canonical(path string) bool {
if !filepath.IsLocal(path) || filepath.ToSlash(path) != path || strings.Contains(path, "\\") || path != filepath.Clean(path) {
return false
}
root := strings.Split(path, "/")[0]
switch root {
case "documents", "themes", "federation":
return true
case "pages.db", "auth.key", "instance-ed25519.key":
return path == root
}
return false
}
func fileHash(path string) (File, error) {
f, err := os.Open(path)
if err != nil {
return File{}, err
}
defer f.Close()
i, err := f.Stat()
if err != nil || !i.Mode().IsRegular() {
return File{}, errors.New("snapshot entry is not a regular file")
}
h := sha256.New()
n, err := io.CopyBuffer(h, f, make([]byte, 256<<10))
return File{Size: n, SHA256: hex.EncodeToString(h.Sum(nil))}, err
}
func copyFile(source, target string) error {
i, err := os.Lstat(source)
if err != nil {
return err
}
if !i.Mode().IsRegular() {
return fmt.Errorf("refusing nonregular source: %s", source)
}
in, err := os.Open(source)
if err != nil {
return err
}
defer in.Close()
opened, err := in.Stat()
if err != nil || !os.SameFile(i, opened) {
return errors.New("source changed during snapshot")
}
if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil {
return err
}
out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return err
}
_, err = io.CopyBuffer(out, in, make([]byte, 256<<10))
if err == nil {
err = out.Sync()
}
closeErr := out.Close()
if err == nil {
err = closeErr
}
return err
}
// Backup API preserves rowids (unlike VACUUM INTO) and includes committed WAL.
func backupDB(ctx context.Context, source, target string) error {
database, err := db.OpenReadOnly(source)
if err != nil {
return err
}
defer database.Close()
conn, err := database.Conn(ctx)
if err != nil {
return err
}
defer conn.Close()
err = conn.Raw(func(raw any) error {
b, err := raw.(interface {
NewBackup(string) (*sqlite.Backup, error)
}).NewBackup((&url.URL{Scheme: "file", Path: target}).String())
if err != nil {
return err
}
for {
if err = ctx.Err(); err != nil {
b.Finish()
return err
}
more, e := b.Step(256)
if e != nil {
b.Finish()
return e
}
if !more {
return b.Finish()
}
}
})
if err != nil {
return err
}
// Do not leave a WAL-mode snapshot that creates sidecars during verification.
dst, err := sql.Open("sqlite", (&url.URL{Scheme: "file", Path: target, RawQuery: "mode=rw"}).String())
if err != nil {
return err
}
_, err = dst.ExecContext(ctx, "PRAGMA journal_mode=DELETE")
closeErr := dst.Close()
if err != nil {
return err
}
if closeErr != nil {
return closeErr
}
// Finish leaves a standalone DB; fsync before publishing the generation.
f, err := os.OpenFile(target, os.O_RDWR, 0600)
if err != nil {
return err
}
defer f.Close()
if err = f.Chmod(0600); err != nil {
return err
}
return f.Sync()
}
// Snapshot requires every writer stopped, including older binaries and CLIs.
// The lock enforces this for upgraded PagesWiki processes, not arbitrary tools.
func Snapshot(ctx context.Context, dataDir, target string) (*Manifest, string, error) {
source, err := filepath.EvalSymlinks(dataDir)
if err != nil {
return nil, "", err
}
source, err = filepath.Abs(source)
if err != nil {
return nil, "", err
}
target, err = filepath.Abs(target)
if err != nil {
return nil, "", err
}
parent, err := filepath.EvalSymlinks(filepath.Dir(target))
if err != nil {
return nil, "", err
}
target = filepath.Join(parent, filepath.Base(target))
if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) {
return nil, "", errors.New("snapshot destination must be outside the data directory")
}
l, err := Lock(source, true)
if err != nil {
return nil, "", err
}
defer l.Close()
c, err := Load(source)
if err != nil {
return nil, "", err
}
if c == nil || c.Role != "primary" {
return nil, "", errors.New("initialize primary DR configuration before snapshot")
}
if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
return nil, "", errors.New("snapshot destination must not exist")
}
stage, err := os.MkdirTemp(parent, ".dr-snapshot-")
if err != nil {
return nil, "", err
}
defer os.RemoveAll(stage)
m := &Manifest{Version: 1, ID: uuid.NewString(), ClusterID: c.ClusterID, PrimaryNodeID: c.NodeID, Epoch: c.Epoch, CapturedAt: time.Now().UTC(), Scope: "instance-offline-v1"}
root := filepath.Join(stage, "data")
if err = os.Mkdir(root, 0700); err != nil {
return nil, "", err
}
info, err := os.Lstat(filepath.Join(source, "pages.db"))
if err != nil {
return nil, "", err
}
if !info.Mode().IsRegular() {
return nil, "", errors.New("snapshot database must be a regular file, not a symlink")
}
if err = checkAuthKey(source); err != nil {
return nil, "", err
}
if err = backupDB(ctx, filepath.Join(source, "pages.db"), filepath.Join(root, "pages.db")); err != nil {
return nil, "", err
}
for _, name := range []string{"documents", "themes", "federation", "auth.key", "instance-ed25519.key"} {
base := filepath.Join(source, name)
if _, e := os.Lstat(base); errors.Is(e, os.ErrNotExist) {
continue
} else if e != nil {
return nil, "", e
}
err = filepath.WalkDir(base, func(path string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if err := ctx.Err(); err != nil {
return err
}
if entry.Type()&os.ModeSymlink != 0 {
return fmt.Errorf("snapshot refuses symlink: %s", path)
}
rel, _ := filepath.Rel(source, path)
if entry.IsDir() {
return os.MkdirAll(filepath.Join(root, rel), 0700)
}
return copyFile(path, filepath.Join(root, rel))
})
if err != nil {
return nil, "", err
}
}
err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, e error) error {
if e != nil {
return e
}
if entry.IsDir() {
return syncDir(path)
}
if len(m.Files) >= 100000 {
return errors.New("snapshot exceeds 100000 files; partitioned snapshots require a later format")
}
f, e := fileHash(path)
if e != nil {
return e
}
f.Path, _ = filepath.Rel(root, path)
m.Files = append(m.Files, f)
return nil
})
if err != nil {
return nil, "", err
}
m.CompletedAt = time.Now().UTC()
encoded, err := json.MarshalIndent(m, "", " ")
if err != nil {
return nil, "", err
}
if err = os.WriteFile(filepath.Join(stage, "manifest.json"), encoded, 0600); err != nil {
return nil, "", err
}
manifestFile, err := os.OpenFile(filepath.Join(stage, "manifest.json"), os.O_RDWR, 0600)
if err != nil {
return nil, "", err
}
err = manifestFile.Sync()
manifestFile.Close()
if err != nil {
return nil, "", err
}
hash := sha256.Sum256(encoded)
if _, err = Verify(ctx, stage, hex.EncodeToString(hash[:])); err != nil {
return nil, "", err
}
if err = syncDir(stage); err != nil {
return nil, "", err
}
if err = publish(stage, target); err != nil {
return nil, "", err
}
return m, hex.EncodeToString(hash[:]), syncDir(parent)
}
func Verify(ctx context.Context, root, expectedHash string) (*Manifest, error) {
if len(expectedHash) != 64 {
return nil, errors.New("expected manifest SHA-256 is required from a trusted channel")
}
root, err := filepath.Abs(root)
if err != nil {
return nil, err
}
info, err := os.Lstat(filepath.Join(root, "manifest.json"))
if err != nil {
return nil, err
}
if !info.Mode().IsRegular() || info.Size() > 64<<20 {
return nil, errors.New("invalid or oversized snapshot manifest")
}
encoded, err := os.ReadFile(filepath.Join(root, "manifest.json"))
if err != nil {
return nil, err
}
sum := sha256.Sum256(encoded)
if hex.EncodeToString(sum[:]) != expectedHash {
return nil, errors.New("manifest SHA-256 mismatch")
}
var m Manifest
if err = json.Unmarshal(encoded, &m); err != nil {
return nil, err
}
if m.Version != 1 || m.Scope != "instance-offline-v1" || m.Epoch == 0 || len(m.Files) == 0 || len(m.Files) > 100000 || m.CapturedAt.IsZero() || m.CompletedAt.Before(m.CapturedAt) || m.CompletedAt.After(time.Now().Add(time.Minute)) {
return nil, errors.New("invalid snapshot manifest")
}
for _, id := range []string{m.ID, m.ClusterID, m.PrimaryNodeID} {
u, e := uuid.Parse(id)
if e != nil || u == uuid.Nil || u.String() != id {
return nil, errors.New("invalid snapshot identity")
}
}
seen := map[string]bool{}
hashes := map[string]string{}
for _, item := range m.Files {
if err = ctx.Err(); err != nil {
return nil, err
}
if !canonical(item.Path) || seen[item.Path] {
return nil, errors.New("invalid/duplicate snapshot path")
}
seen[item.Path] = true
hashes[item.Path] = item.SHA256
path := filepath.Join(root, "data", item.Path)
// Refuse symlinks in every component, including parent directories.
for p := path; p != filepath.Dir(root) && p != "."; p = filepath.Dir(p) {
info, e := os.Lstat(p)
if e != nil {
return nil, e
}
if info.Mode()&os.ModeSymlink != 0 {
return nil, errors.New("snapshot contains symlink")
}
if p == root || filepath.Dir(p) == p {
break
}
}
actual, e := fileHash(path)
if e != nil {
return nil, e
}
if actual.Size != item.Size || actual.SHA256 != item.SHA256 {
return nil, fmt.Errorf("snapshot payload mismatch: %s", item.Path)
}
if strings.HasPrefix(item.Path, "documents/sha256/") && filepath.Base(item.Path) != actual.SHA256 {
return nil, errors.New("CAS filename does not match payload hash")
}
}
if !seen["pages.db"] {
return nil, errors.New("snapshot database missing")
}
if err = verifyCalendarFiles(root, hashes); err != nil {
return nil, err
}
err = filepath.WalkDir(filepath.Join(root, "data"), func(path string, entry fs.DirEntry, e error) error {
if e != nil {
return e
}
if entry.IsDir() {
return nil
}
rel, _ := filepath.Rel(filepath.Join(root, "data"), path)
if !seen[rel] {
return errors.New("unlisted snapshot payload")
}
return nil
})
if err != nil {
return nil, err
}
database, err := db.OpenReadOnly(filepath.Join(root, "data", "pages.db"))
if err != nil {
return nil, err
}
defer database.Close()
var integrity string
if err = database.QueryRowContext(ctx, "PRAGMA quick_check").Scan(&integrity); err != nil {
return nil, err
}
if integrity != "ok" {
return nil, errors.New("snapshot database integrity check failed")
}
if err = verifyReferences(ctx, database, seen); err != nil {
return nil, err
}
return &m, nil
}
// Restore never merges into or overwrites a running/existing data directory.
// It publishes a new generation with a fresh physical node ID, always standby.
func Restore(ctx context.Context, snapshot, expectedHash, target string) (*Config, error) {
m, err := Verify(ctx, snapshot, expectedHash)
if err != nil {
return nil, err
}
target, err = filepath.Abs(target)
if err != nil {
return nil, err
}
parent, err := filepath.EvalSymlinks(filepath.Dir(target))
if err != nil {
return nil, err
}
target = filepath.Join(parent, filepath.Base(target))
source, err := filepath.EvalSymlinks(snapshot)
if err != nil {
return nil, err
}
source, err = filepath.Abs(source)
if err != nil {
return nil, err
}
if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) {
return nil, errors.New("restore destination must be outside the snapshot")
}
if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
return nil, errors.New("restore target must not exist; use a new generation")
}
stage, err := os.MkdirTemp(filepath.Dir(target), ".dr-restore-")
if err != nil {
return nil, err
}
defer os.RemoveAll(stage)
for _, item := range m.Files {
if err = ctx.Err(); err != nil {
return nil, err
}
if err = copyFile(filepath.Join(snapshot, "data", item.Path), filepath.Join(stage, item.Path)); err != nil {
return nil, err
}
actual, e := fileHash(filepath.Join(stage, item.Path))
if e != nil {
return nil, e
}
if actual.Size != item.Size || actual.SHA256 != item.SHA256 {
return nil, errors.New("snapshot changed during restore")
}
}
c := &Config{Version: 1, ClusterID: m.ClusterID, NodeID: uuid.NewString(), PrimaryNodeID: m.PrimaryNodeID, Role: "standby", Epoch: m.Epoch, SnapshotID: m.ID, CapturedAt: m.CapturedAt, MaxReadAgeSeconds: 3600}
if err = Save(stage, *c); err != nil {
return nil, err
}
if err = os.MkdirAll(filepath.Join(stage, "documents", "sha256"), 0700); err != nil {
return nil, err
}
// Create the lock before a read-only data mount is applied.
l, err := Lock(stage, true)
if err != nil {
return nil, err
}
l.Close()
err = filepath.WalkDir(stage, func(path string, entry fs.DirEntry, e error) error {
if e != nil {
return e
}
if entry.IsDir() {
return syncDir(path)
}
return nil
})
if err != nil {
return nil, err
}
if err = publish(stage, target); err != nil {
return nil, err
}
return c, syncDir(filepath.Dir(target))
}
// Linux deployment: atomically publish without replacing even an empty directory.
func publish(stage, target string) error {
return unix.Renameat2(unix.AT_FDCWD, stage, unix.AT_FDCWD, target, unix.RENAME_NOREPLACE)
}
func verifyReferences(ctx context.Context, database *sql.DB, seen map[string]bool) error {
// Only authoritative references, not derived projection/preview caches. Some
// installations predate optional modules, hence the explicit table check.
for _, ref := range [][2]string{{"revisions", "document_hash"}, {"assets", "head_document_hash"}, {"asset_head_history", "document_hash"}, {"organizer_items", "document_hash"}, {"organizer_revisions", "document_hash"}} {
var exists bool
if err := database.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?)", ref[0]).Scan(&exists); err != nil {
return err
}
if !exists {
continue
}
rows, err := database.QueryContext(ctx, "SELECT DISTINCT "+ref[1]+" FROM "+ref[0]+" WHERE "+ref[1]+" IS NOT NULL AND "+ref[1]+"<>''")
if err != nil {
return err
}
for rows.Next() {
var hash string
if err = rows.Scan(&hash); err != nil {
rows.Close()
return err
}
if len(hash) != 64 || !seen["documents/sha256/"+hash[:2]+"/"+hash] {
rows.Close()
return fmt.Errorf("snapshot lacks canonical payload referenced by %s", ref[0])
}
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
}
return nil
}
// Verify file-based calendar authority without parsing event bodies or making
// external calendar requests. Derived calendar indexes may be repaired only
// after promotion, never by a standby read request.
func verifyCalendarFiles(root string, hashes map[string]string) error {
for path := range hashes {
if !strings.HasPrefix(path, "documents/calendars/") || filepath.Base(path) != "calendar.json" {
continue
}
filename := filepath.Join(root, "data", path)
info, err := os.Stat(filename)
if err != nil {
return err
}
if info.Size() > 16<<20 {
return errors.New("calendar metadata exceeds snapshot verification limit")
}
encoded, err := os.ReadFile(filename)
if err != nil {
return err
}
var metadata struct {
Months map[string]string `json:"months"`
}
if err = json.Unmarshal(encoded, &metadata); err != nil {
return errors.New("invalid calendar metadata in snapshot")
}
for month, hash := range metadata.Months {
if _, err = time.Parse("2006-01", month); err != nil {
return errors.New("invalid calendar month in snapshot")
}
if len(hash) != 64 || hashes[filepath.ToSlash(filepath.Join(filepath.Dir(path), month+"."+hash+".ics"))] != hash {
return errors.New("snapshot lacks a valid calendar monthly generation")
}
}
}
return nil
}
+40
View File
@@ -0,0 +1,40 @@
package dregarnuhr
import (
"go/parser"
"go/token"
"path/filepath"
"strconv"
"strings"
"testing"
)
func TestComponentBoundary(t *testing.T) {
if Component != "dregarnuhr" {
t.Fatal("component identity changed")
}
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, file := range files {
if strings.HasSuffix(file, "_test.go") {
continue
}
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
if err != nil {
t.Fatal(err)
}
for _, spec := range ast.Imports {
path, err := strconv.Unquote(spec.Path.Value)
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"pageswiki/internal", "database", "net", "os"} {
if path == forbidden || strings.HasPrefix(path, forbidden+"/") {
t.Errorf("%s imports %s: scheduler core must not own persistence or business I/O", file, path)
}
}
}
}
}
+20
View File
@@ -0,0 +1,20 @@
package dregarnuhr
import "time"
type Timer interface {
C() <-chan time.Time
Stop() bool
}
type Clock interface {
Now() time.Time
NewTimer(time.Duration) Timer
}
type SystemClock struct{}
func (SystemClock) Now() time.Time { return time.Now() }
func (SystemClock) NewTimer(d time.Duration) Timer { return systemTimer{time.NewTimer(d)} }
type systemTimer struct{ *time.Timer }
func (t systemTimer) C() <-chan time.Time { return t.Timer.C }
+65
View File
@@ -0,0 +1,65 @@
package dregarnuhr
import (
"sync"
"testing"
"time"
)
type fakeClock struct {
mu sync.Mutex
now time.Time
timers map[*fakeTimer]time.Time
created chan struct{}
}
type fakeTimer struct {
clock *fakeClock
ch chan time.Time
}
func newFakeClock() *fakeClock {
return &fakeClock{now: time.Unix(100000, 0), timers: map[*fakeTimer]time.Time{}, created: make(chan struct{}, 100)}
}
func (c *fakeClock) Now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.now }
func (c *fakeClock) NewTimer(delay time.Duration) Timer {
c.mu.Lock()
defer c.mu.Unlock()
t := &fakeTimer{clock: c, ch: make(chan time.Time, 1)}
if delay <= 0 {
t.ch <- c.now
} else {
c.timers[t] = c.now.Add(delay)
}
c.created <- struct{}{}
return t
}
func (t *fakeTimer) C() <-chan time.Time { return t.ch }
func (t *fakeTimer) Stop() bool {
t.clock.mu.Lock()
defer t.clock.mu.Unlock()
_, ok := t.clock.timers[t]
delete(t.clock.timers, t)
return ok
}
func (c *fakeClock) advance(d time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
c.now = c.now.Add(d)
for timer, at := range c.timers {
if !at.After(c.now) {
timer.ch <- c.now
delete(c.timers, timer)
}
}
}
func receive[T any](t *testing.T, ch <-chan T) T {
t.Helper()
select {
case v := <-ch:
return v
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for test worker")
var zero T
return zero
}
}
+104
View File
@@ -0,0 +1,104 @@
package dregarnuhr
import (
"context"
"errors"
"fmt"
"time"
)
type Trigger struct {
Name string
After, Every time.Duration
Run func(context.Context, time.Time) error
}
type Loop struct {
Clock Clock
Every time.Duration
Wake <-chan struct{}
OnWake func(context.Context, time.Time) error
Triggers []Trigger
Step func(context.Context, time.Time) error
Report func(string, error)
}
// Run coalesces overdue ticks (no catch-up storm), stops its timer on
// cancellation, and invokes caller-supplied hooks. It knows no event types.
func (l Loop) Run(ctx context.Context) error {
if l.Every <= 0 || l.Step == nil {
return errors.New("loop requires positive cadence and step")
}
clock := l.Clock
if clock == nil {
clock = SystemClock{}
}
now := clock.Now()
next := now.Add(l.Every)
due := make([]time.Time, len(l.Triggers))
seen := map[string]bool{}
for i, t := range l.Triggers {
if t.Name == "" || seen[t.Name] || t.Run == nil || t.After < 0 || t.Every < 0 || (t.After == 0 && t.Every == 0) {
return fmt.Errorf("invalid trigger %q", t.Name)
}
seen[t.Name] = true
delay := t.After
if delay == 0 {
delay = t.Every
}
due[i] = now.Add(delay)
}
wake := l.Wake
call := func(name string, fn func(context.Context, time.Time) error, now time.Time) {
if fn != nil {
if err := fn(ctx, now); err != nil && l.Report != nil {
l.Report(name, err)
}
}
}
for ctx.Err() == nil {
at := next
for _, d := range due {
if !d.IsZero() && d.Before(at) {
at = d
}
}
timer := clock.NewTimer(max(time.Duration(0), at.Sub(clock.Now())))
woken := false
select {
case <-ctx.Done():
timer.Stop()
return ctx.Err()
case _, ok := <-wake:
if !ok {
wake = nil
timer.Stop()
continue
}
woken = true
case <-timer.C():
}
timer.Stop()
if ctx.Err() != nil {
return ctx.Err()
}
now = clock.Now()
if woken {
call("wake", l.OnWake, now)
}
for i, t := range l.Triggers {
if !due[i].IsZero() && !due[i].After(now) {
call(t.Name, t.Run, now)
if t.Every > 0 {
due[i] = now.Add(t.Every)
} else {
due[i] = time.Time{}
}
}
}
call("step", l.Step, now)
if !next.After(now) {
next = now.Add(l.Every)
}
}
return ctx.Err()
}
+77
View File
@@ -0,0 +1,77 @@
package dregarnuhr
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
)
func TestLoopFakeClockWakeAndCoalescing(t *testing.T) {
clock := newFakeClock()
wake := make(chan struct{}, 1)
steps := make(chan time.Time, 20)
reports := make(chan string, 10)
var once, periodic, awakened atomic.Int64
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
loop := Loop{Clock: clock, Every: time.Second, Wake: wake,
OnWake: func(context.Context, time.Time) error { awakened.Add(1); return nil },
Triggers: []Trigger{
{Name: "once", After: 5 * time.Second, Run: func(context.Context, time.Time) error { once.Add(1); return nil }},
{Name: "periodic", Every: 10 * time.Second, Run: func(context.Context, time.Time) error { periodic.Add(1); return errors.New("synthetic") }},
},
Step: func(_ context.Context, at time.Time) error { steps <- at; return nil },
Report: func(stage string, _ error) { reports <- stage },
}
done := make(chan error, 1)
go func() { done <- loop.Run(ctx) }()
receive(t, clock.created)
wake <- struct{}{}
receive(t, steps)
receive(t, clock.created)
if awakened.Load() != 1 || once.Load() != 0 {
t.Fatal("wake changed timer deadlines")
}
clock.advance(5 * time.Second)
receive(t, steps)
receive(t, clock.created)
if once.Load() != 1 || periodic.Load() != 0 {
t.Fatal("initial cadence")
}
clock.advance(time.Hour)
receive(t, steps)
receive(t, clock.created)
if once.Load() != 1 || periodic.Load() != 1 {
t.Fatal("catch-up storm", once.Load(), periodic.Load())
}
if receive(t, reports) != "periodic" {
t.Fatal("hook failure missing")
}
close(wake)
receive(t, clock.created) // closed wake channel must not spin
clock.advance(10 * time.Second)
receive(t, steps)
receive(t, clock.created)
if periodic.Load() != 2 {
t.Fatal("periodic stopped after error")
}
cancel()
if !errors.Is(receive(t, done), context.Canceled) {
t.Fatal("loop did not stop")
}
clock.mu.Lock()
defer clock.mu.Unlock()
if len(clock.timers) != 0 {
t.Fatal("timer leaked")
}
}
func TestLoopRejectsInvalidHooks(t *testing.T) {
for _, l := range []Loop{{}, {Every: time.Second, Step: func(context.Context, time.Time) error { return nil }, Triggers: []Trigger{{Name: "invalid"}}}} {
if l.Run(context.Background()) == nil {
t.Fatal("invalid loop accepted")
}
}
}
+58
View File
@@ -0,0 +1,58 @@
package dregarnuhr
import (
"errors"
"time"
)
// CanStart retains a single background worker under foreground load, avoiding
// starvation by long-lived requests while protecting foreground capacity.
func CanStart(s Settings, active int64, foreground bool) bool {
return !s.Paused && active < int64(s.Concurrency) && (!foreground || active == 0)
}
// RetryDelay doubles per completed attempt and saturates without overflowing.
func RetryDelay(baseSeconds, attempt int) time.Duration {
if baseSeconds < 1 {
baseSeconds = 1
}
if baseSeconds >= 86400 {
return 24 * time.Hour
}
delay := time.Duration(baseSeconds) * time.Second
for i := 1; i < attempt; i++ {
if delay >= 12*time.Hour {
return 24 * time.Hour
}
delay *= 2
}
return delay
}
type Completion struct {
Status, Message, Level string
RetryAt time.Time
}
// Complete is independent of DB access. The repository persists this decision.
func Complete(job Job, runErr error, baseSeconds int, now time.Time) Completion {
out := Completion{Status: "succeeded", Level: "info"}
if runErr == nil {
return out
}
out.Message = runErr.Error()
switch {
case errors.Is(runErr, ErrSkipped):
out.Status = "skipped"
case job.CancelRequested:
out.Status = "cancelled"
case job.Attempts < job.MaxAttempts:
out.Status = "queued"
out.Level = "warning"
out.RetryAt = now.Add(RetryDelay(baseSeconds, job.Attempts))
default:
out.Status = "failed"
out.Level = "error"
}
return out
}
+63
View File
@@ -0,0 +1,63 @@
package dregarnuhr
import (
"errors"
"fmt"
"math"
"testing"
"time"
)
func TestRetryDelay(t *testing.T) {
for _, tc := range []struct {
base, attempt int
want time.Duration
}{
{30, 1, 30 * time.Second}, {30, 2, time.Minute}, {30, 3, 2 * time.Minute},
{30, 1000000, 24 * time.Hour}, {86401, 1, 24 * time.Hour}, {math.MaxInt, math.MaxInt, 24 * time.Hour}, {0, 1, time.Second},
} {
if got := RetryDelay(tc.base, tc.attempt); got != tc.want {
t.Errorf("%+v got=%s", tc, got)
}
}
}
func TestCompletionPolicy(t *testing.T) {
now := time.Unix(1000, 0)
for _, tc := range []struct {
name string
job Job
err error
status, level string
}{
{"success", Job{Attempts: 1, MaxAttempts: 3}, nil, "succeeded", "info"},
{"retry", Job{Attempts: 2, MaxAttempts: 3}, errors.New("failed"), "queued", "warning"},
{"exhausted", Job{Attempts: 3, MaxAttempts: 3}, errors.New("failed"), "failed", "error"},
{"cancel", Job{Attempts: 1, MaxAttempts: 3, CancelRequested: true}, errors.New("cancelled"), "cancelled", "info"},
{"skip-before-cancel", Job{Attempts: 1, MaxAttempts: 3, CancelRequested: true}, fmt.Errorf("wrapped: %w", ErrSkipped), "skipped", "info"},
} {
t.Run(tc.name, func(t *testing.T) {
got := Complete(tc.job, tc.err, 30, now)
if got.Status != tc.status || got.Level != tc.level {
t.Fatalf("%+v", got)
}
if tc.status == "queued" && !got.RetryAt.Equal(now.Add(time.Minute)) {
t.Fatal(got)
}
if tc.status != "queued" && !got.RetryAt.IsZero() {
t.Fatal("terminal job retried")
}
})
}
}
func TestAdmissionPolicy(t *testing.T) {
s := Settings{Concurrency: 4}
if !CanStart(s, 0, true) || CanStart(s, 1, true) || !CanStart(s, 3, false) || CanStart(s, 4, false) {
t.Fatal("admission policy changed")
}
s.Paused = true
if CanStart(s, 0, false) {
t.Fatal("paused worker admitted")
}
}
+49
View File
@@ -0,0 +1,49 @@
package dregarnuhr
import (
"context"
"errors"
"fmt"
"strings"
"sync"
)
type Handler func(context.Context, *Job) error
// Registry holds existing event identifiers, not new provider or consumer IDs.
type Registry struct {
mu sync.RWMutex
handlers map[string]Handler
}
func (r *Registry) Register(kind string, handler Handler) error {
if strings.TrimSpace(kind) == "" || handler == nil {
return errors.New("event type and handler are required")
}
r.mu.Lock()
defer r.mu.Unlock()
if r.handlers == nil {
r.handlers = make(map[string]Handler)
}
if _, ok := r.handlers[kind]; ok {
return fmt.Errorf("duplicate event type %q", kind)
}
r.handlers[kind] = handler
return nil
}
func (r *Registry) Run(ctx context.Context, job *Job) error {
if job == nil {
return errors.New("job is required")
}
if err := ctx.Err(); err != nil {
return err
}
r.mu.RLock()
handler := r.handlers[job.Type]
r.mu.RUnlock()
if handler == nil {
return fmt.Errorf("unknown event type %q", job.Type)
}
return handler(ctx, job)
}
+127
View File
@@ -0,0 +1,127 @@
package dregarnuhr
import (
"context"
"errors"
"sync"
"sync/atomic"
"time"
)
// Queue owns atomic priority-ordered claims and persisted completion. ErrNoJob
// means no eligible job. The initial adapter is single-writer-process, not HA.
type Queue interface {
Settings(context.Context) (Settings, error)
Claim(context.Context, time.Time) (*Job, error)
Cancellation(context.Context, string) (bool, error)
Started(context.Context, *Job) error
Finish(context.Context, *Job, error, time.Time) error
}
type RunnerOptions struct {
Clock Clock
// Active is an optional compatibility counter for an existing monitor.
Active *atomic.Int64
Foreground func() bool
Wake func()
Report func(stage string, job *Job, err error)
}
type Runner struct {
queue Queue
registry *Registry
opts RunnerOptions
mu sync.Mutex
}
func NewRunner(queue Queue, registry *Registry, opts RunnerOptions) *Runner {
if opts.Clock == nil {
opts.Clock = SystemClock{}
}
if opts.Active == nil {
opts.Active = new(atomic.Int64)
}
return &Runner{queue: queue, registry: registry, opts: opts}
}
func (r *Runner) report(stage string, job *Job, err error) {
if err != nil && r.opts.Report != nil {
r.opts.Report(stage, job, err)
}
}
func (r *Runner) Running() int64 { return r.opts.Active.Load() }
// Dispatch serializes admission plus claim across concurrent callers. Each
// invocation is bounded even if workers finish immediately and refill slots.
func (r *Runner) Dispatch(ctx context.Context) {
r.mu.Lock()
defer r.mu.Unlock()
for n := 0; n < 32 && ctx.Err() == nil; n++ {
settings, err := r.queue.Settings(ctx)
if err != nil {
r.report("settings", nil, err)
return
}
foreground := r.opts.Foreground != nil && r.opts.Foreground()
if !CanStart(settings, r.Running(), foreground) {
return
}
job, err := r.queue.Claim(ctx, r.opts.Clock.Now())
if errors.Is(err, ErrNoJob) || errors.Is(err, context.Canceled) {
return
}
if err != nil {
r.report("claim", nil, err)
return
}
if job == nil {
r.report("claim", nil, errors.New("queue returned nil job"))
return
}
r.opts.Active.Add(1)
r.report("start-log", job, r.queue.Started(ctx, job))
go r.execute(ctx, job)
}
}
func (r *Runner) execute(ctx context.Context, job *Job) {
defer func() {
r.opts.Active.Add(-1)
if r.opts.Wake != nil {
r.opts.Wake()
}
}()
runErr := r.invoke(ctx, job)
// Completion must still be attempted when the worker context is cancelled.
finishCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
r.report("finish", job, r.queue.Finish(finishCtx, job, runErr, r.opts.Clock.Now()))
r.report("execute", job, runErr)
}
func (r *Runner) invoke(ctx context.Context, job *Job) (err error) {
defer func() {
if recover() != nil {
err = errors.New("event handler panicked")
}
}()
cancelled, err := r.queue.Cancellation(ctx, job.UUID)
if err != nil {
return err
}
if cancelled {
return errors.New("cancelled before execution")
}
if err = r.registry.Run(ctx, job); err != nil {
return err
}
cancelled, err = r.queue.Cancellation(ctx, job.UUID)
if err != nil {
return err
}
if cancelled {
return errors.New("cancelled during execution")
}
return nil
}
+185
View File
@@ -0,0 +1,185 @@
package dregarnuhr
import (
"context"
"errors"
"fmt"
"sync"
"testing"
"time"
)
type finishedJob struct {
id string
err error
at time.Time
}
type fakeQueue struct {
mu sync.Mutex
settings Settings
jobs []Job
claimed int
cancelled bool
cancelErr error
finished chan finishedJob
claimAt time.Time
}
func newFakeQueue(concurrency int) *fakeQueue {
q := &fakeQueue{settings: Settings{Concurrency: concurrency}, finished: make(chan finishedJob, 100)}
for n := 0; n < 64; n++ {
q.jobs = append(q.jobs, Job{UUID: fmt.Sprint(n), Type: "synthetic"})
}
return q
}
func (q *fakeQueue) Settings(context.Context) (Settings, error) {
q.mu.Lock()
defer q.mu.Unlock()
return q.settings, nil
}
func (q *fakeQueue) Claim(_ context.Context, at time.Time) (*Job, error) {
q.mu.Lock()
defer q.mu.Unlock()
if len(q.jobs) == 0 {
return nil, ErrNoJob
}
j := q.jobs[0]
q.jobs = q.jobs[1:]
q.claimed++
q.claimAt = at
return &j, nil
}
func (q *fakeQueue) Cancellation(context.Context, string) (bool, error) {
q.mu.Lock()
defer q.mu.Unlock()
return q.cancelled, q.cancelErr
}
func (q *fakeQueue) Started(context.Context, *Job) error { return nil }
func (q *fakeQueue) Finish(ctx context.Context, j *Job, err error, at time.Time) error {
if ctx.Err() != nil {
return ctx.Err()
}
q.finished <- finishedJob{j.UUID, err, at}
return nil
}
func TestRunnerConcurrentAdmission(t *testing.T) {
for _, foreground := range []bool{false, true} {
t.Run(fmt.Sprint(foreground), func(t *testing.T) {
q := newFakeQueue(4)
reg := &Registry{}
block := make(chan struct{})
started := make(chan struct{}, 4)
wakes := make(chan struct{}, 4)
clock := newFakeClock()
defer close(block)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if err := reg.Register("synthetic", func(ctx context.Context, _ *Job) error {
started <- struct{}{}
select {
case <-ctx.Done():
return ctx.Err()
case <-block:
return nil
}
}); err != nil {
t.Fatal(err)
}
r := NewRunner(q, reg, RunnerOptions{Clock: clock, Foreground: func() bool { return foreground }, Wake: func() { wakes <- struct{}{} }})
var wg sync.WaitGroup
for n := 0; n < 16; n++ {
wg.Add(1)
go func() { defer wg.Done(); r.Dispatch(ctx) }()
}
wg.Wait()
want := 4
if foreground {
want = 1
}
for n := 0; n < want; n++ {
receive(t, started)
}
if r.Running() != int64(want) || q.claimed != want || !q.claimAt.Equal(clock.Now()) {
t.Fatalf("active=%d claimed=%d want=%d", r.Running(), q.claimed, want)
}
cancel()
for n := 0; n < want; n++ {
receive(t, q.finished)
receive(t, wakes)
}
if r.Running() != 0 {
t.Fatal("worker slots leaked")
}
})
}
}
func TestRunnerOutcomesAndCancellation(t *testing.T) {
for _, mode := range []string{"success", "cancel-before", "cancel-after", "cancellation-read-error", "panic", "unknown", "failure"} {
t.Run(mode, func(t *testing.T) {
q := newFakeQueue(1)
q.jobs = q.jobs[:1]
q.cancelled = mode == "cancel-before"
if mode == "cancellation-read-error" {
q.cancelErr = errors.New("database unavailable")
}
reg := &Registry{}
called := false
if mode != "unknown" {
if err := reg.Register("synthetic", func(context.Context, *Job) error {
called = true
if mode == "panic" {
panic("sensitive panic payload")
}
if mode == "failure" {
return errors.New("handler failure")
}
if mode == "cancel-after" {
q.mu.Lock()
q.cancelled = true
q.mu.Unlock()
}
return nil
}); err != nil {
t.Fatal(err)
}
}
wakes := make(chan struct{}, 1)
clock := newFakeClock()
r := NewRunner(q, reg, RunnerOptions{Clock: clock, Wake: func() { wakes <- struct{}{} }})
r.Dispatch(context.Background())
out := receive(t, q.finished)
receive(t, wakes)
if (out.err == nil) != (mode == "success") {
t.Fatal(mode, out.err)
}
if (mode == "cancel-before" || mode == "cancellation-read-error") && called {
t.Fatal("handler ran without cancellation check")
}
if mode == "panic" && out.err.Error() != "event handler panicked" {
t.Fatal("panic payload leaked")
}
if r.Running() != 0 || !out.at.Equal(clock.Now()) {
t.Fatal("completion/clock failure")
}
})
}
}
func TestRegistryAndPausedRunner(t *testing.T) {
r := &Registry{}
h := func(context.Context, *Job) error { return nil }
if r.Register("", h) == nil || r.Register("nil", nil) == nil {
t.Fatal("invalid registration")
}
if r.Register("synthetic", h) != nil || r.Register("synthetic", h) == nil {
t.Fatal("duplicate replaced")
}
q := newFakeQueue(3)
q.settings.Paused = true
NewRunner(q, r, RunnerOptions{}).Dispatch(context.Background())
if q.claimed != 0 {
t.Fatal("paused claim")
}
}
+40
View File
@@ -0,0 +1,40 @@
// Package dregarnuhr owns in-process scheduling, worker admission and retry
// policy. Queue provides persistence; Registry provides business operations.
// Atomic claims are not cross-instance expiring leases.
package dregarnuhr
import "errors"
const Component = "dregarnuhr"
var (
ErrNoJob = errors.New("no due job")
ErrSkipped = errors.New("event is not applicable")
)
type Job struct {
UUID string `json:"uuid"`
EventUUID string `json:"event_uuid,omitempty"`
Type string `json:"event_type"`
Key string `json:"event_key"`
Payload string `json:"payload,omitempty"`
Status string `json:"status"`
Priority int `json:"priority"`
Attempts int `json:"attempts"`
MaxAttempts int `json:"max_attempts"`
RunAtMS int64 `json:"run_at_ms"`
StartedAtMS int64 `json:"started_at_ms,omitempty"`
FinishedAtMS int64 `json:"finished_at_ms,omitempty"`
CancelRequested bool `json:"cancel_requested"`
Error string `json:"error,omitempty"`
CreatedAt int64 `json:"created_at_ms"`
UpdatedAt int64 `json:"updated_at_ms"`
}
type Settings struct {
Paused bool `json:"paused"`
Concurrency int `json:"concurrency"`
MaxDepth int `json:"max_depth"`
RetryBaseSeconds int `json:"retry_base_seconds"`
RetentionHours int `json:"retention_hours"`
}
+50
View File
@@ -0,0 +1,50 @@
package geduldh
import (
"go/parser"
"go/token"
"path/filepath"
"reflect"
"strconv"
"strings"
"testing"
)
func TestComponentBoundary(t *testing.T) {
if Component != "geduldh" {
t.Fatal("component identity changed")
}
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, file := range files {
if strings.HasSuffix(file, "_test.go") {
continue
}
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
if err != nil {
t.Fatal(err)
}
for _, spec := range ast.Imports {
path, err := strconv.Unquote(spec.Path.Value)
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"pageswiki/internal", "database", "net/http"} {
if path == forbidden || strings.HasPrefix(path, forbidden+"/") {
t.Errorf("%s imports %s: payload storage must not own metadata, HTTP, authorization or the compatibility facade", file, path)
}
}
}
}
// Ordinary payload access cannot delete objects or make retention decisions.
contract := reflect.TypeFor[Store]()
var methods []string
for i := 0; i < contract.NumMethod(); i++ {
methods = append(methods, contract.Method(i).Name)
}
if want := []string{"Open", "PutIfAbsent", "Stat", "Verify"}; !reflect.DeepEqual(methods, want) {
t.Fatalf("Document Store contract changed: %v", methods)
}
}
+264
View File
@@ -0,0 +1,264 @@
package geduldh
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"math"
"os"
"path"
"path/filepath"
"strings"
)
// Local requires hard-link no-replace publication and directory fsync support.
// A mounted NFS/SMB/SSHFS share is not certified just because it has a path.
// Failed persistence primitives return errors; there is no unsafe rename fallback.
type Local struct{ root string }
var _ Store = (*Local)(nil)
// NewLocal has no I/O side effects. The root must be provisioned by the caller.
func NewLocal(root string) (*Local, error) {
if strings.TrimSpace(root) == "" {
return nil, errors.New("Document root is required")
}
absolute, err := filepath.Abs(root)
if err != nil {
return nil, err
}
return &Local{root: absolute}, nil
}
func (s *Local) Open(ctx context.Context, hash string) (Reader, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
key, err := Key(hash)
if err != nil {
return nil, err
}
root, err := os.OpenRoot(s.root)
if err != nil {
return nil, err
}
defer root.Close()
file, err := root.Open(key)
if err != nil {
return nil, err
}
info, err := file.Stat()
if err != nil || !info.Mode().IsRegular() {
file.Close()
if err != nil {
return nil, err
}
return nil, errors.New("Document is not a regular file")
}
return &localReader{file: file, ctx: ctx}, nil
}
func (s *Local) Stat(ctx context.Context, hash string) (Info, error) {
reader, err := s.Open(ctx, hash)
if err != nil {
return Info{}, err
}
defer reader.Close()
info, err := reader.(*localReader).file.Stat()
if err != nil {
return Info{}, err
}
return Info{Hash: strings.ToLower(strings.TrimSpace(hash)), Size: info.Size()}, nil
}
func verify(ctx context.Context, hash string, reader io.Reader) error {
sum := sha256.New()
if _, err := io.CopyBuffer(sum, contextReader{ctx, reader}, make([]byte, 64<<10)); err != nil {
return err
}
if hex.EncodeToString(sum.Sum(nil)) != strings.ToLower(strings.TrimSpace(hash)) {
return ErrHashMismatch
}
return ctx.Err()
}
func (s *Local) Verify(ctx context.Context, hash string) error {
reader, err := s.Open(ctx, hash)
if err != nil {
return err
}
defer reader.Close()
return verify(ctx, hash, reader)
}
func (s *Local) PutIfAbsent(ctx context.Context, hash string, size int64, input io.Reader) error {
if err := ctx.Err(); err != nil {
return err
}
key, err := Key(hash)
if err != nil {
return err
}
if size < 0 || size == math.MaxInt64 {
return ErrSizeMismatch
}
if input == nil {
return errors.New("Document reader is required")
}
root, err := os.OpenRoot(s.root)
if err != nil {
return err
}
defer root.Close()
// The common retry/backfill path must not rewrite or stage existing blobs.
// Still validate the supplied bytes, then verify the stored content.
if existing, err := root.Open(key); err == nil {
defer existing.Close()
info, err := existing.Stat()
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return errors.New("Document is not a regular file")
}
if err := copyValidated(ctx, hash, size, input, io.Discard); err != nil {
return err
}
if err := verify(ctx, hash, existing); err != nil {
return err
}
return syncPublishedDirectories(root, path.Dir(key))
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
dir := path.Dir(key)
if err := root.MkdirAll(dir, 0755); err != nil {
return err
}
var nonce [16]byte
if _, err := rand.Read(nonce[:]); err != nil {
return err
}
tmp := dir + "/.document-" + hex.EncodeToString(nonce[:])
file, err := root.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return err
}
defer root.Remove(tmp)
defer file.Close()
if err := copyValidated(ctx, hash, size, input, file); err != nil {
return err
}
if err := file.Chmod(0644); err != nil {
return err
}
if err := file.Sync(); err != nil {
return err
}
if err := file.Close(); err != nil {
return err
}
// Link is atomic and fails if the destination exists. Rename would silently
// replace an existing payload between an existence check and publication.
if err := root.Link(tmp, key); err != nil {
if !errors.Is(err, os.ErrExist) {
return err
}
existing, err := root.Open(key)
if err != nil {
return err
}
defer existing.Close()
info, err := existing.Stat()
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return errors.New("Document is not a regular file")
}
if err := verify(ctx, hash, existing); err != nil {
return fmt.Errorf("existing Document: %w", err)
}
}
// Persist the new entry and newly created shard directories before allowing
// a metadata transaction to reference the payload. Errors are not swallowed.
if err := syncPublishedDirectories(root, dir); err != nil {
return err
}
return ctx.Err()
}
func syncPublishedDirectories(root *os.Root, shard string) error {
for _, name := range []string{shard, "sha256", "."} {
directory, err := root.Open(name)
if err != nil {
return err
}
err = directory.Sync()
closeErr := directory.Close()
if err != nil {
return err
}
if closeErr != nil {
return closeErr
}
}
return nil
}
func copyValidated(ctx context.Context, hash string, size int64, input io.Reader, output io.Writer) error {
sum := sha256.New()
n, err := io.CopyBuffer(io.MultiWriter(output, sum), contextReader{ctx, io.LimitReader(input, size+1)}, make([]byte, 64<<10))
if err != nil {
return err
}
if n != size {
return ErrSizeMismatch
}
if hex.EncodeToString(sum.Sum(nil)) != strings.ToLower(strings.TrimSpace(hash)) {
return ErrHashMismatch
}
return ctx.Err()
}
type contextReader struct {
ctx context.Context
reader io.Reader
}
func (r contextReader) Read(p []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.reader.Read(p)
}
type localReader struct {
file *os.File
ctx context.Context
}
func (r *localReader) Close() error { return r.file.Close() }
func (r *localReader) Read(p []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.file.Read(p)
}
func (r *localReader) ReadAt(p []byte, offset int64) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.file.ReadAt(p, offset)
}
func (r *localReader) Seek(offset int64, whence int) (int64, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.file.Seek(offset, whence)
}
+264
View File
@@ -0,0 +1,264 @@
package geduldh
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"
)
func digest(data []byte) string { sum := sha256.Sum256(data); return hex.EncodeToString(sum[:]) }
func local(t *testing.T) (*Local, string) {
t.Helper()
root := t.TempDir()
s, err := NewLocal(root)
if err != nil {
t.Fatal(err)
}
return s, root
}
func TestLocalRoundTripAndRange(t *testing.T) {
for _, payload := range [][]byte{nil, []byte("text 日本語"), {0, 0xff, 1, 0, 3}} {
s, _ := local(t)
ctx := context.Background()
hash := digest(payload)
if err := s.PutIfAbsent(ctx, hash, int64(len(payload)), bytes.NewReader(payload)); err != nil {
t.Fatal(err)
}
info, err := s.Stat(ctx, hash)
if err != nil || info.Hash != hash || info.Size != int64(len(payload)) {
t.Fatalf("info=%+v err=%v", info, err)
}
reader, err := s.Open(ctx, hash)
if err != nil {
t.Fatal(err)
}
data, err := io.ReadAll(reader)
if err != nil || !bytes.Equal(data, payload) {
t.Fatalf("data=%v err=%v", data, err)
}
if size, err := reader.Seek(0, io.SeekEnd); err != nil || size != int64(len(payload)) {
t.Fatalf("size=%d err=%v", size, err)
}
if len(payload) > 2 {
buf := make([]byte, 2)
if _, err := reader.ReadAt(buf, 1); err != nil || !bytes.Equal(buf, payload[1:3]) {
t.Fatalf("range=%v err=%v", buf, err)
}
}
reader.Close()
if err := s.Verify(ctx, hash); err != nil {
t.Fatal(err)
}
}
}
func TestLocalRejectsIncompleteOrCorruptInput(t *testing.T) {
payload := []byte("good")
for _, tc := range []struct {
name string
data []byte
size int64
want error
}{
{"hash", []byte("evil"), 4, ErrHashMismatch},
{"short", []byte("goo"), 4, ErrSizeMismatch},
{"long", []byte("good!"), 4, ErrSizeMismatch},
{"negative", payload, -1, ErrSizeMismatch},
} {
t.Run(tc.name, func(t *testing.T) {
s, root := local(t)
err := s.PutIfAbsent(context.Background(), digest(payload), tc.size, bytes.NewReader(tc.data))
if !errors.Is(err, tc.want) {
t.Fatalf("got %v want %v", err, tc.want)
}
if _, err := s.Stat(context.Background(), digest(payload)); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("published bad content: %v", err)
}
assertNoTemps(t, root)
})
}
}
func TestLocalExistingAndConcurrentWritesNeverReplace(t *testing.T) {
s, root := local(t)
payload := []byte("immutable payload")
hash := digest(payload)
ctx := context.Background()
var wg sync.WaitGroup
errorsCh := make(chan error, 12)
for range 12 {
wg.Add(1)
go func() {
defer wg.Done()
errorsCh <- s.PutIfAbsent(ctx, hash, int64(len(payload)), bytes.NewReader(payload))
}()
}
wg.Wait()
close(errorsCh)
for err := range errorsCh {
if err != nil {
t.Fatal(err)
}
}
key, _ := Key(hash)
filename := filepath.Join(root, filepath.FromSlash(key))
before, _ := os.Stat(filename)
if err := s.PutIfAbsent(ctx, hash, int64(len(payload)), bytes.NewReader(payload)); err != nil {
t.Fatal(err)
}
after, _ := os.Stat(filename)
if !os.SameFile(before, after) {
t.Fatal("existing inode replaced")
}
if err := os.WriteFile(filename, []byte("corrupt"), 0644); err != nil {
t.Fatal(err)
}
if err := s.Verify(ctx, hash); !errors.Is(err, ErrHashMismatch) {
t.Fatalf("verify=%v", err)
}
if err := s.PutIfAbsent(ctx, hash, int64(len(payload)), bytes.NewReader(payload)); !errors.Is(err, ErrHashMismatch) {
t.Fatalf("put=%v", err)
}
content, _ := os.ReadFile(filename)
if string(content) != "corrupt" {
t.Fatal("corruption silently overwritten")
}
assertNoTemps(t, root)
}
func TestLocalCancellationAndReaderFailure(t *testing.T) {
s, root := local(t)
ctx, cancel := context.WithCancel(context.Background())
cancel()
hash := digest([]byte("data"))
if err := s.PutIfAbsent(ctx, hash, 4, strings.NewReader("data")); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if _, err := s.Open(ctx, hash); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
failure := errors.New("source failed")
if err := s.PutIfAbsent(context.Background(), hash, 4, failingReader{failure}); !errors.Is(err, failure) {
t.Fatal(err)
}
if _, err := s.Stat(context.Background(), hash); !errors.Is(err, os.ErrNotExist) {
t.Fatal(err)
}
if err := s.PutIfAbsent(context.Background(), hash, 4, strings.NewReader("data")); err != nil {
t.Fatal(err)
}
ctx, cancel = context.WithCancel(context.Background())
r, err := s.Open(ctx, hash)
if err != nil {
t.Fatal(err)
}
defer r.Close()
cancel()
if _, err := r.Read(make([]byte, 1)); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if _, err := r.ReadAt(make([]byte, 1), 0); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
assertNoTemps(t, root)
}
type failingReader struct{ err error }
func (r failingReader) Read([]byte) (int, error) { return 0, r.err }
func TestLocalStreamsWithBoundedReadsAndMidStreamCancellation(t *testing.T) {
s, root := local(t)
payload := bytes.Repeat([]byte("large document\x00"), 100000)
hash := digest(payload)
input := &boundedReader{reader: bytes.NewReader(payload)}
if err := s.PutIfAbsent(context.Background(), hash, int64(len(payload)), input); err != nil {
t.Fatal(err)
}
if input.reads < 2 {
t.Fatal("expected multiple bounded reads")
}
// Cancellation while staging must never publish the final content address.
s2, root2 := local(t)
ctx, cancel := context.WithCancel(context.Background())
input = &boundedReader{reader: bytes.NewReader(payload), cancel: cancel}
if err := s2.PutIfAbsent(ctx, hash, int64(len(payload)), input); !errors.Is(err, context.Canceled) {
t.Fatalf("put=%v", err)
}
if _, err := s2.Stat(context.Background(), hash); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("published canceled upload: %v", err)
}
assertNoTemps(t, root)
assertNoTemps(t, root2)
}
type boundedReader struct {
reader io.Reader
reads int
cancel context.CancelFunc
}
func (r *boundedReader) Read(p []byte) (int, error) {
if len(p) > 64<<10 {
return 0, errors.New("unbounded input read")
}
r.reads++
n, err := r.reader.Read(p)
if r.cancel != nil {
r.cancel()
}
return n, err
}
func TestLocalKeysAndSymlinkEscape(t *testing.T) {
for _, hash := range []string{"", "../../etc/passwd", strings.Repeat("g", 64), strings.Repeat("a", 63)} {
if _, err := Key(hash); !errors.Is(err, ErrInvalidHash) {
t.Fatal(err)
}
}
hash := digest([]byte("data"))
key, err := Key(" " + strings.ToUpper(hash) + " ")
if err != nil || key != "sha256/"+hash[:2]+"/"+hash {
t.Fatalf("key=%q err=%v", key, err)
}
s, root := local(t)
outside := t.TempDir()
if err := os.Symlink(outside, filepath.Join(root, "sha256")); err != nil {
t.Skip(err)
}
if err := s.PutIfAbsent(context.Background(), hash, 4, strings.NewReader("data")); err == nil {
t.Fatal("escaped storage root")
}
if _, err := s.Open(context.Background(), hash); err == nil {
t.Fatal("opened escaped path")
}
entries, err := os.ReadDir(outside)
if err != nil || len(entries) != 0 {
t.Fatalf("outside modified: %v %v", entries, err)
}
}
func assertNoTemps(t *testing.T, root string) {
t.Helper()
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if strings.HasPrefix(entry.Name(), ".document-") {
t.Errorf("temporary file left: %s", path)
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
+219
View File
@@ -0,0 +1,219 @@
package geduldh
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
)
var ErrMaterializeLimit = errors.New("Document materialization capacity exceeded")
// Source deliberately excludes mutation. Authorization precedes Acquire.
type Source interface {
Stat(context.Context, string) (Info, error)
Open(context.Context, string) (Reader, error)
}
type MaterializeOptions struct {
// TempDir must be node-local scratch, never canonical storage. Empty uses TMPDIR.
TempDir string
MaxBytes int64
Concurrent int
// MaxEntries bounds queued/leased objects too, including empty payloads.
// Zero selects 64.
MaxEntries int
Timeout time.Duration
}
// Materializer shares in-flight/leased copies only. The last lease removes the
// copy; it is not a persistent cache and needs no idle cache eviction goroutine.
// Limits include both copies being downloaded and copies in use by tools.
type Materializer struct {
source Source
options MaterializeOptions
slots chan struct{}
mu sync.Mutex
entries map[string]*materializedEntry
reserved int64
active int
}
type materializedEntry struct {
hash string
size int64
refs int
ready chan struct{}
cancel context.CancelFunc
finished bool
dir, path string
err error
}
// Lease pins an immutable scratch copy until Close. Tools must finish before
// Close; never give its path to a detached process. Close is idempotent.
type Lease struct {
Path string
Size int64
once sync.Once
release func()
}
func (l *Lease) Close() { l.once.Do(l.release) }
func NewMaterializer(source Source, options MaterializeOptions) (*Materializer, error) {
if source == nil || options.MaxBytes <= 0 || options.Concurrent <= 0 || options.Timeout <= 0 {
return nil, errors.New("invalid Document materializer configuration")
}
if options.MaxEntries == 0 {
options.MaxEntries = 64
}
if options.MaxEntries < 0 {
return nil, errors.New("invalid materializer entry limit")
}
return &Materializer{source: source, options: options, slots: make(chan struct{}, options.Concurrent), entries: make(map[string]*materializedEntry)}, nil
}
func (m *Materializer) Acquire(ctx context.Context, hash string, maxBytes int64) (*Lease, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if _, err := Key(hash); err != nil {
return nil, err
}
hash = strings.ToLower(strings.TrimSpace(hash))
info, err := m.source.Stat(ctx, hash)
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
if info.Size < 0 || maxBytes < 0 || info.Size > maxBytes || info.Size > m.options.MaxBytes {
return nil, ErrMaterializeLimit
}
m.mu.Lock()
e := m.entries[hash]
if e != nil && e.size != info.Size {
m.mu.Unlock()
return nil, ErrSizeMismatch
}
if e == nil {
if info.Size > m.options.MaxBytes-m.reserved || m.active >= m.options.MaxEntries {
m.mu.Unlock()
return nil, ErrMaterializeLimit
}
// One caller leaving must not cancel another caller's transfer. The
// shared deadline is propagated to backend operations. Backends must
// honor cancellation; it cannot interrupt an uninterruptible OS syscall.
loadCtx, cancel := context.WithTimeout(context.Background(), m.options.Timeout)
e = &materializedEntry{hash: hash, size: info.Size, ready: make(chan struct{}), cancel: cancel}
m.entries[hash] = e
m.reserved += info.Size
m.active++
go m.load(loadCtx, e)
}
e.refs++
m.mu.Unlock()
select {
case <-ctx.Done():
m.release(e)
return nil, ctx.Err()
case <-e.ready:
if err := ctx.Err(); err != nil {
m.release(e)
return nil, err
}
if e.err != nil {
m.release(e)
return nil, e.err
}
return &Lease{Path: e.path, Size: e.size, release: func() { m.release(e) }}, nil
}
}
func (m *Materializer) load(ctx context.Context, e *materializedEntry) {
defer e.cancel()
var dir, filename string
var err error
select {
case m.slots <- struct{}{}:
dir, filename, err = m.copy(ctx, e.hash, e.size)
<-m.slots
case <-ctx.Done():
err = ctx.Err()
}
m.mu.Lock()
e.dir, e.path, e.err, e.finished = dir, filename, err, true
close(e.ready)
if e.refs == 0 {
m.remove(e)
}
m.mu.Unlock()
}
func (m *Materializer) release(e *materializedEntry) {
m.mu.Lock()
defer m.mu.Unlock()
e.refs--
if e.refs == 0 {
e.cancel()
// A later caller must not join an already-cancelled download.
if m.entries[e.hash] == e {
delete(m.entries, e.hash)
}
if e.finished {
m.remove(e)
}
}
}
func (m *Materializer) remove(e *materializedEntry) {
if m.entries[e.hash] == e {
delete(m.entries, e.hash)
}
// Failed removal keeps the byte reservation: do not pretend disk space
// was reclaimed. Recovery of crash leftovers is an operator/scratch task.
if e.dir != "" {
if err := os.RemoveAll(e.dir); err != nil {
return
}
}
m.reserved -= e.size
m.active--
}
func (m *Materializer) copy(ctx context.Context, hash string, size int64) (dir, filename string, err error) {
reader, err := m.source.Open(ctx, hash)
if err != nil {
return "", "", err
}
defer reader.Close()
dir, err = os.MkdirTemp(m.options.TempDir, "pageswiki-document-*")
if err != nil {
return "", "", err
}
// Failed partial copies are retained in dir until load performs cleanup.
filename = filepath.Join(dir, hash)
file, err := os.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return dir, "", err
}
err = copyValidated(ctx, hash, size, reader, file)
if err == nil {
err = file.Chmod(0400)
}
closeErr := file.Close()
if err == nil {
err = closeErr
}
if err != nil {
return dir, "", fmt.Errorf("materialize Document: %w", err)
}
// Scratch is reproducible, not durable canonical storage: no fsync here.
return dir, filename, ctx.Err()
}
+283
View File
@@ -0,0 +1,283 @@
package geduldh
import (
"bytes"
"context"
"errors"
"os"
"path/filepath"
"sync/atomic"
"testing"
"time"
)
type materializeSource struct {
body []byte
size int64
gate chan struct{}
started chan struct{}
opens atomic.Int64
}
func (s *materializeSource) Stat(ctx context.Context, hash string) (Info, error) {
return Info{Hash: hash, Size: s.size}, ctx.Err()
}
func (s *materializeSource) Open(ctx context.Context, hash string) (Reader, error) {
s.opens.Add(1)
if s.started != nil {
s.started <- struct{}{}
}
if s.gate != nil {
select {
case <-s.gate:
case <-ctx.Done():
return nil, ctx.Err()
}
}
return materializeReader{bytes.NewReader(s.body)}, ctx.Err()
}
type materializeReader struct{ *bytes.Reader }
func (materializeReader) Close() error { return nil }
func newTestMaterializer(t *testing.T, source Source, quota int64) (*Materializer, string) {
t.Helper()
dir := t.TempDir()
m, err := NewMaterializer(source, MaterializeOptions{TempDir: dir, MaxBytes: quota, Concurrent: 1, Timeout: 5 * time.Second})
if err != nil {
t.Fatal(err)
}
return m, dir
}
func waitMaterializer(t *testing.T, m *Materializer, predicate func() bool) {
t.Helper()
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
m.mu.Lock()
ok := predicate()
m.mu.Unlock()
if ok {
return
}
time.Sleep(time.Millisecond)
}
t.Fatal("materializer condition did not become true")
}
func TestMaterializerSharesPinnedCopiesAndCleansLastLease(t *testing.T) {
body := []byte("materialized 日本語")
s := &materializeSource{body: body, size: int64(len(body))}
m, root := newTestMaterializer(t, s, 1024)
a, err := m.Acquire(context.Background(), digest(body), 1024)
if err != nil {
t.Fatal(err)
}
b, err := m.Acquire(context.Background(), digest(body), 1024)
if err != nil {
t.Fatal(err)
}
if a.Path != b.Path || s.opens.Load() != 1 {
t.Fatal("copy was not shared")
}
data, err := os.ReadFile(a.Path)
if err != nil || !bytes.Equal(data, body) {
t.Fatalf("body=%q err=%v", data, err)
}
info, _ := os.Stat(a.Path)
if info.Mode().Perm() != 0400 {
t.Fatalf("permissions=%v", info.Mode())
}
dirInfo, _ := os.Stat(filepath.Dir(a.Path))
if dirInfo.Mode().Perm() != 0700 {
t.Fatalf("directory permissions=%v", dirInfo.Mode())
}
a.Close()
a.Close()
if _, err := os.Stat(b.Path); err != nil {
t.Fatal("active lease lost its file", err)
}
b.Close()
items, _ := os.ReadDir(root)
if len(items) != 0 || m.reserved != 0 || m.active != 0 {
t.Fatal("scratch was not reclaimed")
}
}
func TestMaterializerCoalescesDownloadsAndCancelsOnlyLastWaiter(t *testing.T) {
body := []byte("shared")
s := &materializeSource{body: body, size: int64(len(body)), gate: make(chan struct{}), started: make(chan struct{}, 2)}
m, _ := newTestMaterializer(t, s, 1024)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
errA := make(chan error, 1)
go func() {
lease, err := m.Acquire(ctx, digest(body), 1024)
if lease != nil {
lease.Close()
}
errA <- err
}()
<-s.started
leaseB := make(chan *Lease, 1)
errB := make(chan error, 1)
go func() {
lease, err := m.Acquire(context.Background(), digest(body), 1024)
leaseB <- lease
errB <- err
}()
waitMaterializer(t, m, func() bool { return m.entries[digest(body)].refs == 2 })
cancel()
if err := <-errA; !errors.Is(err, context.Canceled) {
t.Fatalf("first waiter: %v", err)
}
close(s.gate)
b := <-leaseB
if err := <-errB; err != nil {
t.Fatal(err)
}
if s.opens.Load() != 1 {
t.Fatal("concurrent transfer not coalesced")
}
b.Close()
s.gate = make(chan struct{})
ctx, cancel = context.WithCancel(context.Background())
go func() {
lease, err := m.Acquire(ctx, digest(body), 1024)
if lease != nil {
lease.Close()
}
errA <- err
}()
<-s.started
cancel()
if err := <-errA; !errors.Is(err, context.Canceled) {
t.Fatalf("last waiter: %v", err)
}
waitMaterializer(t, m, func() bool { return m.reserved == 0 && m.active == 0 })
}
func TestMaterializerRejectsCorruptionSizeAndCapacity(t *testing.T) {
for _, tc := range []struct {
name string
body []byte
size int64
hash string
want error
}{
{"hash", []byte("bad"), 3, digest([]byte("yes")), ErrHashMismatch},
{"short", []byte("x"), 2, digest([]byte("x")), ErrSizeMismatch},
{"long", []byte("xyz"), 2, digest([]byte("xyz")), ErrSizeMismatch},
{"oversize", []byte("xyz"), 3, digest([]byte("xyz")), ErrMaterializeLimit},
} {
t.Run(tc.name, func(t *testing.T) {
s := &materializeSource{body: tc.body, size: tc.size}
m, root := newTestMaterializer(t, s, 10)
limit := int64(10)
if tc.name == "oversize" {
limit = 2
}
lease, err := m.Acquire(context.Background(), tc.hash, limit)
if lease != nil || !errors.Is(err, tc.want) {
t.Fatalf("lease=%v err=%v", lease, err)
}
items, _ := os.ReadDir(root)
if len(items) != 0 || m.reserved != 0 {
t.Fatal("failed input leaked scratch")
}
})
}
s := &materializeSource{body: []byte("x"), size: 1}
m, _ := newTestMaterializer(t, s, 1)
lease, err := m.Acquire(context.Background(), digest(s.body), 1)
if err != nil {
t.Fatal(err)
}
defer lease.Close()
if _, err := m.Acquire(context.Background(), digest([]byte("y")), 1); !errors.Is(err, ErrMaterializeLimit) {
t.Fatalf("quota: %v", err)
}
if s.opens.Load() != 1 {
t.Fatal("over-quota transfer started")
}
}
func TestMaterializerTimeoutAndEntryBound(t *testing.T) {
s := &materializeSource{body: []byte("x"), size: 1, gate: make(chan struct{}), started: make(chan struct{}, 3)}
m, err := NewMaterializer(s, MaterializeOptions{TempDir: t.TempDir(), MaxBytes: 100, Concurrent: 1, MaxEntries: 1, Timeout: 50 * time.Millisecond})
if err != nil {
t.Fatal(err)
}
done := make(chan error, 1)
go func() { _, err := m.Acquire(context.Background(), digest(s.body), 10); done <- err }()
<-s.started
if _, err := m.Acquire(context.Background(), digest([]byte("y")), 10); !errors.Is(err, ErrMaterializeLimit) {
t.Fatalf("entry bound: %v", err)
}
if err := <-done; !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("timeout: %v", err)
}
waitMaterializer(t, m, func() bool { return m.active == 0 })
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := m.Acquire(ctx, digest(s.body), 10); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
}
func TestMaterializerConcurrencyBoundAndCanonicalIsolation(t *testing.T) {
s := &materializeSource{body: []byte("x"), size: 1, gate: make(chan struct{}), started: make(chan struct{}, 3)}
m, _ := newTestMaterializer(t, s, 100)
done := make(chan error, 2)
for _, data := range []string{"x", "y"} {
go func(data string) {
lease, err := m.Acquire(context.Background(), digest([]byte(data)), 100)
if lease != nil {
lease.Close()
}
done <- err
}(data)
}
<-s.started
waitMaterializer(t, m, func() bool { return m.active == 2 })
if s.opens.Load() != 1 {
t.Fatal("concurrency limit exceeded")
}
close(s.gate)
for i := 0; i < 2; i++ {
if err := <-done; err != nil && !errors.Is(err, ErrHashMismatch) {
t.Fatal(err)
}
}
waitMaterializer(t, m, func() bool { return m.active == 0 })
canonical, root := local(t)
body := []byte("canonical")
hash := digest(body)
if err := canonical.PutIfAbsent(context.Background(), hash, int64(len(body)), bytes.NewReader(body)); err != nil {
t.Fatal(err)
}
m, _ = newTestMaterializer(t, canonical, 100)
lease, err := m.Acquire(context.Background(), hash, 100)
if err != nil {
t.Fatal(err)
}
defer lease.Close()
key, _ := Key(hash)
sourceInfo, _ := os.Stat(filepath.Join(root, filepath.FromSlash(key)))
tempInfo, _ := os.Stat(lease.Path)
if os.SameFile(sourceInfo, tempInfo) {
t.Fatal("canonical inode exposed to external tools")
}
if err := os.Chmod(lease.Path, 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(lease.Path, []byte("modified!"), 0600); err != nil {
t.Fatal(err)
}
if err := canonical.Verify(context.Background(), hash); err != nil {
t.Fatal("scratch mutation affected canonical data", err)
}
}
+58
View File
@@ -0,0 +1,58 @@
// Package geduldh stores immutable, content-addressed Document payloads.
// Authorization and reference/retention decisions belong to the application,
// not to a storage backend. A hash or storage key is never an access token.
package geduldh
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"strings"
)
const Component = "geduldh"
var (
ErrInvalidHash = errors.New("invalid Document SHA-256")
ErrHashMismatch = errors.New("Document payload does not match SHA-256")
ErrSizeMismatch = errors.New("Document payload does not match declared size")
)
type Info struct {
Hash string
Size int64
}
// Reader supports bounded and seekable access, including HTTP Range and ZIP.
// A remote backend must not download the entire object merely to open it.
type Reader interface {
io.ReadSeekCloser
io.ReaderAt
}
// Store is the immutable payload boundary, not a filesystem abstraction.
// PutIfAbsent must verify the size and SHA-256 before publishing a complete
// object, never replace an existing object, and reject corrupt existing bytes.
// A successful write must precede the metadata transaction which references it.
// Delete is deliberately absent: GC requires reference and retention protection.
type Store interface {
Stat(context.Context, string) (Info, error)
Open(context.Context, string) (Reader, error)
PutIfAbsent(context.Context, string, int64, io.Reader) error
Verify(context.Context, string) error
}
// Key preserves the existing on-disk layout and is also suitable for S3 keys.
// Backend placement never changes Page/Revision identity or public URLs.
func Key(hash string) (string, error) {
hash = strings.ToLower(strings.TrimSpace(hash))
if len(hash) != sha256.Size*2 {
return "", ErrInvalidHash
}
if _, err := hex.DecodeString(hash); err != nil {
return "", ErrInvalidHash
}
return "sha256/" + hash[:2] + "/" + hash, nil
}
+130
View File
@@ -0,0 +1,130 @@
package grammaratur
import (
"bytes"
"encoding/json"
"sort"
"strings"
"github.com/google/uuid"
"golang.org/x/net/html"
)
// HTMLToMarkdownAuthoring keeps source HTML when conversion loses structure,
// attributes, comments or text. This is not a sanitization boundary.
func HTMLToMarkdownAuthoring(source string) (string, error) {
converted, err := HTMLToMarkdown(source)
if err != nil {
return "", err
}
if htmlStructure(source) != htmlStructure(MarkdownToHTML(converted)) {
return rawHTMLBlock(source), nil
}
return converted, nil
}
// {=html} is an explicit authoring fence, not an ordinary code sample. The
// closing delimiter is longer than any run in the source, so arbitrary HTML
// (including Markdown examples) cannot accidentally terminate it.
func rawHTMLBlock(source string) string {
longest, run := 2, 0
for _, c := range source {
if c == '`' {
run++
if run > longest {
longest = run
}
} else {
run = 0
}
}
fence := strings.Repeat("`", longest+1)
return fence + "{=html}\n" + source + "\n" + fence
}
// MarkdownAuthoringHTML preserves protected HTML byte-for-byte while rendering
// ordinary Markdown. Callers must still apply projection sanitization for UI.
func MarkdownAuthoringHTML(source string) string {
lines := strings.SplitAfter(source, "\n")
var markdown strings.Builder
type rawBlock struct{ token, html string }
var blocks []rawBlock
var ordinary byte
ordinaryLen := 0
for i := 0; i < len(lines); i++ {
line := strings.TrimSuffix(strings.TrimSuffix(lines[i], "\n"), "\r")
char, n := byte(0), 0
if len(line) > 0 && (line[0] == '`' || line[0] == '~') {
char = line[0]
for n < len(line) && line[n] == char {
n++
}
}
if ordinary != 0 {
markdown.WriteString(lines[i])
if char == ordinary && n >= ordinaryLen && strings.TrimSpace(line[n:]) == "" {
ordinary = 0
}
continue
}
if char == '`' && n >= 3 && strings.TrimSpace(line[n:]) == "{=html}" {
end := i + 1
for ; end < len(lines); end++ {
close := strings.TrimSpace(lines[end])
if len(close) >= n && strings.Trim(close, "`") == "" {
break
}
}
if end < len(lines) {
raw := strings.TrimSuffix(strings.Join(lines[i+1:end], ""), "\n")
token := "PAGESWIKIRAW" + strings.ReplaceAll(uuid.NewString(), "-", "")
blocks = append(blocks, rawBlock{token, raw})
markdown.WriteString("\n\n" + token + "\n\n")
i = end
continue
}
}
if n >= 3 {
ordinary, ordinaryLen = char, n
}
markdown.WriteString(lines[i])
}
if len(blocks) == 1 && strings.TrimSpace(markdown.String()) == blocks[0].token {
return blocks[0].html
}
out := MarkdownToHTML(markdown.String())
for _, block := range blocks {
out = strings.Replace(out, "<p>"+block.token+"</p>", block.html, 1)
}
return out
}
func htmlStructure(source string) string {
n, err := html.Parse(strings.NewReader(source))
if err != nil {
return source
}
var out bytes.Buffer
var visit func(*html.Node, bool)
visit = func(n *html.Node, literal bool) {
literal = literal || n.Data == "pre" || n.Data == "code" || n.Data == "script" || n.Data == "style"
data := n.Data
if n.Type == html.TextNode && !literal {
if strings.TrimSpace(data) == "" && (n.Parent == nil || n.Parent.Data == "body" || n.Parent.Data == "html") {
return
}
// Inline whitespace can separate words (including around an inline
// element). Never normalize it away when deciding conversion safety.
}
attrs := append([]html.Attribute(nil), n.Attr...)
sort.Slice(attrs, func(i, j int) bool { return attrs[i].Namespace+":"+attrs[i].Key < attrs[j].Namespace+":"+attrs[j].Key })
b, _ := json.Marshal([]any{n.Type, n.Namespace, data, attrs})
out.Write(b)
for c := n.FirstChild; c != nil; c = c.NextSibling {
visit(c, literal)
}
out.WriteByte(')')
}
visit(n, false)
return out.String()
}
+67
View File
@@ -0,0 +1,67 @@
package grammaratur
import (
"strings"
"testing"
)
func TestAuthoringRoundTrip(t *testing.T) {
for _, source := range []string{
"<h1>Heading</h1><p>Hello <strong>world</strong>.</p>",
"<p>{{children:/guide}}</p>",
"<figure data-extra=\"kept\"><img src=\"/photo\" alt=\"Photo\"><figcaption>A caption</figcaption></figure>",
"<!-- keep comment --><p id=\"anchor\" class=\"custom\">text</p>",
"<table><tr><td rowspan=\"2\">a</td><td>b</td></tr><tr><td>c</td></tr></table>",
"<p>a <em>b</em> c</p>",
"<pre><code class=\"language-python\"> print('x')\n</code></pre>",
"<custom-element data-x=\"1\">opaque</custom-element>",
"<div class=\"custom\">\n<p>one</p>\n\n<p>two</p>\n</div>",
"<script>const x = '<b>untouched</b>';</script>",
} {
t.Run(source, func(t *testing.T) {
md, err := HTMLToMarkdownAuthoring(source)
if err != nil {
t.Fatal(err)
}
if htmlStructure(MarkdownAuthoringHTML(md)) != htmlStructure(source) {
t.Fatalf("lossy conversion: %q", md)
}
})
}
md, err := HTMLToMarkdownAuthoring("<h1>Hello</h1>")
if err != nil || md != "# Hello" {
t.Fatalf("simple Markdown conversion: %q %v", md, err)
}
for _, source := range []string{`<p data-custom="yes">Hello</p>`, `<!-- saved --><h1>Title</h1>`, `<p>a <em>b</em> c</p>`} {
md, err := HTMLToMarkdownAuthoring(source)
if err != nil {
t.Fatal(err)
}
if htmlStructure(MarkdownAuthoringHTML(md)) != htmlStructure(source) {
t.Fatal("whitespace/attributes lost")
}
}
}
func TestAuthoringProtectedHTML(t *testing.T) {
for _, raw := range []string{"<figure data-x='a'>a</figure>", "<pre>```\n{=html}\n````</pre>\n", "<script>const raw = `x`</script>", "<p>line\r\n</p>"} {
md := rawHTMLBlock(raw)
if got := MarkdownAuthoringHTML(md); got != raw {
t.Fatalf("raw bytes changed: %q want %q", got, raw)
}
mixed := MarkdownAuthoringHTML("# Added\n\n" + md + "\n\nTail")
if !strings.Contains(mixed, raw) || !strings.Contains(mixed, "<h1>Added</h1>") || !strings.Contains(mixed, "<p>Tail</p>") {
t.Fatalf("mixed blocks: %q", mixed)
}
}
code := "````text\n```{=html}\n<script>unsafe()</script>\n```\n````"
if strings.Contains(MarkdownAuthoringHTML(code), "<script>") {
t.Fatal("HTML escaped out of ordinary code sample")
}
}
func TestAuthoringWhitespaceComparisonIsConservative(t *testing.T) {
if htmlStructure("<p>a <em>b</em> c</p>") == htmlStructure("<p>a<em>b</em>c</p>") {
t.Fatal("word boundaries ignored")
}
}
+40
View File
@@ -0,0 +1,40 @@
package grammaratur
import (
"go/parser"
"go/token"
"path/filepath"
"strconv"
"strings"
"testing"
)
func TestComponentBoundary(t *testing.T) {
if Component != "grammaratur" {
t.Fatal("component identity changed")
}
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, file := range files {
if strings.HasSuffix(file, "_test.go") {
continue
}
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
if err != nil {
t.Fatal(err)
}
for _, spec := range ast.Imports {
path, err := strconv.Unquote(spec.Path.Value)
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"pageswiki/internal/", "database/", "net", "os"} {
if path == forbidden || strings.HasPrefix(path, strings.TrimSuffix(forbidden, "/")+"/") {
t.Errorf("%s imports %s: Grammaratur must not own I/O, persistence or application policy", file, path)
}
}
}
}
}
@@ -0,0 +1,72 @@
package grammaratur_test
import (
"strings"
"testing"
htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
"github.com/gomarkdown/markdown"
"pageswiki/internal/compiler"
"pageswiki/internal/grammaratur"
"pageswiki/internal/render"
)
func TestLegacyMarkdownCompatibility(t *testing.T) {
for _, source := range []string{
"", "# Heading\n\n本文", "{{children:/guide}}", "- one\n- two\n",
"```python\nprint('hello')\n```", "[link](/path)\n\n![image](/image)",
"<p data-custom='keep'>raw HTML</p>",
"```{=html}\n<figure data-x='1'>source</figure>\n```",
} {
t.Run(source, func(t *testing.T) {
want := string(markdown.ToHTML([]byte(source), nil, nil))
if got := grammaratur.MarkdownToHTML(source); got != want {
t.Fatalf("dialect changed: %q want %q", got, want)
}
if got := render.Markdown(source); got != want {
t.Fatalf("render entry changed: %q want %q", got, want)
}
})
}
}
func TestLegacyHTMLAndAuthoringCompatibility(t *testing.T) {
for _, source := range []string{"", "<h1>Heading</h1>", "<p>{{children}}</p>", "<!--retain--><figure data-custom='keep'><figcaption>Caption</figcaption></figure>", "<script>never execute</script>"} {
want, err := htmltomarkdown.ConvertString(source)
if err != nil {
t.Fatal(err)
}
for _, convert := range []func(string) (string, error){grammaratur.HTMLToMarkdown, compiler.HTMLToMarkdown} {
got, err := convert(source)
if err != nil || got != want {
t.Fatalf("ordinary conversion changed: %q %v", got, err)
}
}
protected, err := grammaratur.HTMLToMarkdownAuthoring(source)
if err != nil {
t.Fatal(err)
}
compat, err := compiler.HTMLToMarkdownAuthoring(source)
if err != nil || protected != compat {
t.Fatal("authoring facade changed", err)
}
if compiler.MarkdownAuthoringHTML(compat) != grammaratur.MarkdownAuthoringHTML(protected) {
t.Fatal("protected render facade changed")
}
}
}
func TestAuthoringIsExplicitAndNotASanitizer(t *testing.T) {
const source = "```{=html}\n<script>untrusted()</script>\n```"
if strings.Contains(grammaratur.MarkdownToHTML(source), "<script>") {
t.Fatal("ordinary conversion opted into authoring")
}
if got := grammaratur.MarkdownAuthoringHTML(source); got != "<script>untrusted()</script>" {
t.Fatalf("authoring removed source: %q", got)
}
// Authorization, macro expansion and sanitization stay in the projection
// layer. Grammaratur returns source, not HTML trusted for a browser sink.
if got := grammaratur.MarkdownToHTML("{{children}}"); !strings.Contains(got, "{{children}}") {
t.Fatal("macro unexpectedly expanded")
}
}
+16
View File
@@ -0,0 +1,16 @@
// Package grammaratur owns document-format conversion and authoring fidelity.
//
// Ordinary conversion and protected authoring conversion are distinct contracts:
// callers must opt into {=html} fences. Neither path authorizes content, expands
// macros, sanitizes a display projection, persists revisions, or fetches URLs.
// Those responsibilities remain with the caller. In particular, preserving
// source HTML never grants permission to execute its scripts or event handlers.
//
// Grammaratur is an in-process component, not a service or a new public API.
// The name denotes the words/format-conversion responsibility in PagesWiki's
// architecture; see docs/design/ja/component-names.md.
package grammaratur
// Component is the stable component label for diagnostics and instrumentation.
// It is not a replacement for API routes, provider IDs or persisted settings.
const Component = "grammaratur"
+18
View File
@@ -0,0 +1,18 @@
package grammaratur
import (
htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
"github.com/gomarkdown/markdown"
)
// HTMLToMarkdown is the existing ordinary conversion, used by integrations
// which do not understand protected authoring blocks. It can be lossy.
func HTMLToMarkdown(source string) (string, error) {
return htmltomarkdown.ConvertString(source)
}
// MarkdownToHTML preserves the existing Markdown dialect and renderer options.
// It neither interprets authoring fences nor sanitizes the returned HTML.
func MarkdownToHTML(source string) string {
return string(markdown.ToHTML([]byte(source), nil, nil))
}
+230
View File
@@ -0,0 +1,230 @@
// Package kunstzeal selects projections and executes bounded rendering providers.
// Authentication, source provenance and canonical storage remain caller-owned.
package kunstzeal
import (
"path/filepath"
"strings"
)
type Delivery string
const (
DeliveryHTML Delivery = "html"
DeliveryText Delivery = "text"
DeliveryRawMedia Delivery = "raw-media"
DeliveryRawFrame Delivery = "raw-frame"
DeliveryDownload Delivery = "download"
)
type Descriptor struct {
MediaType string
Filename string
SizeBytes int64
}
type Adapter struct {
ID string
Kind string
Delivery Delivery
NeedsText bool
WindowableText bool
Match func(Descriptor) bool
}
const InlineTextWindowThreshold = int64(2 << 20)
var projectionAdapters = []Adapter{
{
ID: "html",
Kind: "html",
Delivery: DeliveryHTML,
Match: mediaTypes("text/html"),
},
{
ID: "excalidraw",
Kind: "excalidraw",
Delivery: DeliveryText,
NeedsText: true,
WindowableText: true,
Match: extensions(".excalidraw"),
},
{
ID: "csv",
Kind: "csv",
Delivery: DeliveryText,
NeedsText: true,
WindowableText: true,
Match: matchAny(
mediaTypes("text/csv", "application/csv"),
extensions(".csv"),
),
},
{
ID: "archive",
Kind: "archive",
Delivery: DeliveryDownload,
Match: matchAny(
mediaTypes(
"application/zip", "application/x-zip-compressed",
"application/x-rar-compressed", "application/vnd.rar",
"application/x-7z-compressed", "application/x-tar",
"application/gzip", "application/x-xz",
),
extensions(".zip", ".rar", ".7z", ".tar", ".gz", ".tgz", ".xz", ".txz"),
),
},
{
ID: "debian-package",
Kind: "metadata",
Delivery: DeliveryDownload,
Match: matchAny(
mediaTypes(
"application/vnd.debian.binary-package",
"application/x-debian-package",
),
extensions(".deb"),
),
},
{
ID: "eml",
Kind: "email",
Delivery: DeliveryText,
Match: matchAny(
mediaTypes("message/rfc822"),
extensions(".eml"),
),
},
{
ID: "pe",
Kind: "metadata",
Delivery: DeliveryDownload,
Match: extensions(".exe", ".dll", ".sys"),
},
{
ID: "json",
Kind: "json",
Delivery: DeliveryText,
WindowableText: true,
Match: matchAny(mediaTypes("application/json", "application/ld+json"), extensions(".json", ".jsonl")),
},
{
ID: "text",
Kind: "code",
Delivery: DeliveryText,
NeedsText: true,
WindowableText: true,
Match: textMediaType,
},
{
ID: "image",
Kind: "image",
Delivery: DeliveryRawMedia,
Match: matchAny(
mediaTypePrefix("image/"),
extensions(".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif", ".svg"),
),
},
{
ID: "audio",
Kind: "audio",
Delivery: DeliveryRawMedia,
Match: matchAny(
mediaTypePrefix("audio/"),
extensions(".mp3", ".flac", ".wav", ".ogg", ".m4a", ".aac", ".opus"),
),
},
{
ID: "video",
Kind: "video",
Delivery: DeliveryRawMedia,
Match: matchAny(
mediaTypePrefix("video/"),
extensions(".mp4", ".webm", ".mkv", ".mov", ".m4v", ".ogv"),
),
},
{
ID: "pdf",
Kind: "pdf",
Delivery: DeliveryRawFrame,
Match: matchAny(
mediaTypes("application/pdf"),
extensions(".pdf"),
),
},
{
ID: "download",
Kind: "download",
Delivery: DeliveryDownload,
Match: func(Descriptor) bool { return true },
},
}
func SelectAdapter(descriptor Descriptor) Adapter {
for _, adapter := range projectionAdapters {
if adapter.Match(descriptor) {
return adapter
}
}
return projectionAdapters[len(projectionAdapters)-1]
}
func mediaTypes(types ...string) func(Descriptor) bool {
return func(descriptor Descriptor) bool {
for _, candidate := range types {
if descriptor.MediaType == candidate {
return true
}
}
return false
}
}
func mediaTypePrefix(prefix string) func(Descriptor) bool {
return func(descriptor Descriptor) bool {
return strings.HasPrefix(descriptor.MediaType, prefix)
}
}
func extensions(values ...string) func(Descriptor) bool {
return func(descriptor Descriptor) bool {
extension := strings.ToLower(filepath.Ext(descriptor.Filename))
for _, candidate := range values {
if extension == candidate {
return true
}
}
return false
}
}
func matchAny(matchers ...func(Descriptor) bool) func(Descriptor) bool {
return func(descriptor Descriptor) bool {
for _, matcher := range matchers {
if matcher(descriptor) {
return true
}
}
return false
}
}
func textMediaType(descriptor Descriptor) bool {
switch descriptor.MediaType {
case "application/json",
"application/xml",
"application/javascript",
"application/x-javascript",
"application/x-sh",
"application/x-shellscript",
"application/x-yaml",
"application/yaml",
"application/toml":
return true
default:
return strings.HasPrefix(descriptor.MediaType, "text/")
}
}
// Catalog returns a copy; callers cannot reorder or disable the shared registry.
func Catalog() []Adapter { return append([]Adapter(nil), projectionAdapters...) }
+105
View File
@@ -0,0 +1,105 @@
package kunstzeal
import (
"go/parser"
"go/token"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
)
func TestAdapterSelectionAndOrdering(t *testing.T) {
for _, test := range []struct {
mime, filename, id, kind string
delivery Delivery
needsText, window bool
}{
{"text/html", "scene.excalidraw", "html", "html", DeliveryHTML, false, false},
{"application/json", "scene.EXCALIDRAW", "excalidraw", "excalidraw", DeliveryText, true, true},
{"text/plain", "data.CSV", "csv", "csv", DeliveryText, true, true},
{"application/zip", "archive", "archive", "archive", DeliveryDownload, false, false},
{"application/octet-stream", "package.deb", "debian-package", "metadata", DeliveryDownload, false, false},
{"text/plain", "mail.eml", "eml", "email", DeliveryText, false, false},
{"application/octet-stream", "program.exe", "pe", "metadata", DeliveryDownload, false, false},
{"application/json", "data.json", "json", "json", DeliveryText, false, true},
{"text/x-python", "code.py", "text", "code", DeliveryText, true, true},
{"image/png", "image", "image", "image", DeliveryRawMedia, false, false},
{"audio/flac", "music", "audio", "audio", DeliveryRawMedia, false, false},
{"video/mp4", "movie", "video", "video", DeliveryRawMedia, false, false},
{"application/pdf", "manual", "pdf", "pdf", DeliveryRawFrame, false, false},
{"application/octet-stream", "unknown", "download", "download", DeliveryDownload, false, false},
} {
got := SelectAdapter(Descriptor{MediaType: test.mime, Filename: test.filename, SizeBytes: InlineTextWindowThreshold + 1})
if got.ID != test.id || got.Kind != test.kind || got.Delivery != test.delivery || got.NeedsText != test.needsText || got.WindowableText != test.window {
t.Fatalf("%s/%s: %+v", test.mime, test.filename, got)
}
}
if got := SelectAdapter(Descriptor{}); got.ID != "download" {
t.Fatal(got)
}
}
func TestAdapterCatalogIsCopyAndConcurrent(t *testing.T) {
catalog := Catalog()
ids := []string{}
for _, a := range catalog {
ids = append(ids, a.ID)
}
if strings.Join(ids, ",") != "html,excalidraw,csv,archive,debian-package,eml,pe,json,text,image,audio,video,pdf,download" {
t.Fatal("catalog order changed", ids)
}
catalog[0].ID = "corrupt"
catalog[0].Match = func(Descriptor) bool { return false }
if Catalog()[0].ID != "html" || SelectAdapter(Descriptor{MediaType: "text/html"}).ID != "html" {
t.Fatal("catalog mutation changed shared selection")
}
var wg sync.WaitGroup
for i := 0; i < 8; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 100; j++ {
Catalog()[0].ID = "copy"
if SelectAdapter(Descriptor{MediaType: "text/html"}).ID != "html" {
t.Error("registry changed")
}
}
}()
}
wg.Wait()
}
func TestComponentBoundary(t *testing.T) {
if Component != "kunstzeal" {
t.Fatal("component identity changed")
}
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, file := range files {
if strings.HasSuffix(file, "_test.go") {
continue
}
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
if err != nil {
t.Fatal(err)
}
for _, spec := range ast.Imports {
path, err := strconv.Unquote(spec.Path.Value)
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"pageswiki/internal", "database", "os/exec"} {
if path == forbidden || strings.HasPrefix(path, forbidden+"/") {
t.Errorf("%s imports %s: providers must not own application state or arbitrary process execution", file, path)
}
}
if file == "adapters.go" && (strings.HasPrefix(path, "net") || path == "os") {
t.Error("adapter selection acquired I/O")
}
}
}
}
+55
View File
@@ -0,0 +1,55 @@
package kunstzeal
import (
"errors"
"io"
"os"
"path/filepath"
)
var ErrDiagramCacheLimit = errors.New("diagram cache exceeds limit")
// ReadDiagramCache reads only a bounded derived artifact. filename must be
// constructed by the application after validating kind/hash and authorizing
// source provenance. A cache hit is not an access grant or a sanitized SVG.
func ReadDiagramCache(filename string) ([]byte, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
b, err := io.ReadAll(io.LimitReader(f, DiagramOutputLimit+1))
if len(b) > DiagramOutputLimit {
return nil, ErrDiagramCacheLimit
}
return b, err
}
// WriteDiagramCache publishes a complete artifact with same-directory rename.
// Concurrent readers observe the old or new file, never a truncated write.
// This is disposable cache storage, not a durable canonical Document commit.
func WriteDiagramCache(filename string, data []byte) error {
if len(data) > DiagramOutputLimit {
return ErrDiagramCacheLimit
}
dir := filepath.Dir(filename)
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
f, err := os.CreateTemp(dir, ".diagram-*.tmp")
if err != nil {
return err
}
defer os.Remove(f.Name())
defer f.Close()
if _, err = f.Write(data); err != nil {
return err
}
if err = f.Chmod(0644); err != nil {
return err
}
if err = f.Close(); err != nil {
return err
}
return os.Rename(f.Name(), filename)
}
+84
View File
@@ -0,0 +1,84 @@
package kunstzeal
import (
"bytes"
"errors"
"os"
"path/filepath"
"sync"
"testing"
)
func TestDiagramCacheAtomicReplacement(t *testing.T) {
filename := filepath.Join(t.TempDir(), "mermaid", "fixture.svg")
a, b := bytes.Repeat([]byte("A"), 64<<10), bytes.Repeat([]byte("B"), 64<<10)
if err := WriteDiagramCache(filename, a); err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
for i := 0; i < 8; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
for n := 0; n < 20; n++ {
if i%2 == 0 {
data := a
if n%2 == 0 {
data = b
}
if err := WriteDiagramCache(filename, data); err != nil {
t.Error(err)
}
} else {
data, err := ReadDiagramCache(filename)
if err != nil || (!bytes.Equal(data, a) && !bytes.Equal(data, b)) {
t.Errorf("partial cache: length=%d err=%v", len(data), err)
return
}
}
}
}(i)
}
wg.Wait()
files, err := filepath.Glob(filepath.Join(filepath.Dir(filename), ".diagram-*.tmp"))
if err != nil || len(files) != 0 {
t.Fatal("temporary artifacts leaked", files, err)
}
info, err := os.Stat(filename)
if err != nil || info.Mode().Perm() != 0644 {
t.Fatal(info, err)
}
}
func TestDiagramCacheBudgetsAndFailures(t *testing.T) {
root := t.TempDir()
filename := filepath.Join(root, "fixture.svg")
if err := WriteDiagramCache(filename, []byte("complete")); err != nil {
t.Fatal(err)
}
large := bytes.Repeat([]byte("x"), DiagramOutputLimit+1)
if err := WriteDiagramCache(filename, large); !errors.Is(err, ErrDiagramCacheLimit) {
t.Fatal(err)
}
b, err := ReadDiagramCache(filename)
if err != nil || string(b) != "complete" {
t.Fatal("failed write destroyed cache", err)
}
if err = os.WriteFile(filename, large, 0644); err != nil {
t.Fatal(err)
}
if b, err = ReadDiagramCache(filename); !errors.Is(err, ErrDiagramCacheLimit) || b != nil {
t.Fatal("oversized cache returned", err)
}
if _, err = ReadDiagramCache(filepath.Join(root, "missing")); !errors.Is(err, os.ErrNotExist) {
t.Fatal(err)
}
if err = WriteDiagramCache(root, []byte("cannot replace a directory")); err == nil {
t.Fatal("rename failure hidden")
}
// Failed rename removes its temporary artifact too.
files, err := filepath.Glob(filepath.Join(filepath.Dir(root), ".diagram-*.tmp"))
if err != nil || len(files) != 0 {
t.Fatal(files, err)
}
}
+116
View File
@@ -0,0 +1,116 @@
package kunstzeal
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"net/http"
"net/url"
"strings"
"time"
)
const Component = "kunstzeal"
const DiagramSourceLimit = 1 << 20
const DiagramOutputLimit = 4 << 20
const DiagramInlineLimit = 32 << 10
const DiagramTimeout = 15 * time.Second
var (
ErrDiagramKind = errors.New("unsupported diagram type")
ErrDiagramSource = errors.New("diagram source exceeds limit")
ErrDiagramOutput = errors.New("diagram output unavailable or exceeds limit")
ErrProviderConfig = errors.New("diagram renderer configuration invalid")
ErrProviderUnavailable = errors.New("diagram renderer unavailable")
ErrProviderFailed = errors.New("diagram renderer failed")
)
// DiagramProvider does not authorize a request or trust its SVG output.
// Callers must retain source access checks and the sandbox response policy.
type DiagramProvider interface {
Render(context.Context, string, string) ([]byte, error)
}
func DiagramHash(kind, source, format string) string {
sum := sha256.Sum256([]byte(kind + "\x00" + format + "\x00" + source))
return hex.EncodeToString(sum[:])
}
func AllowedDiagramType(kind string) bool {
switch kind {
case "mermaid", "plantuml", "graphviz", "dot", "d2":
return true
}
return false
}
func NormalizeDiagramType(kind string) string {
if kind == "dot" {
return "graphviz"
}
return kind
}
type HTTPClient interface {
Do(*http.Request) (*http.Response, error)
}
// Kroki receives an administrator-configured service address, never a URL from
// document content. Private service-network endpoints are intentional here.
// Client must honor context cancellation and reject redirects. No default
// client is used, and provider failures never expose its URL or response body.
type Kroki struct {
Endpoint string
Client HTTPClient
}
func (k Kroki) Render(parent context.Context, kind, source string) ([]byte, error) {
if err := parent.Err(); err != nil {
return nil, err
}
kind = NormalizeDiagramType(kind)
if !AllowedDiagramType(kind) {
return nil, ErrDiagramKind
}
if len(source) > DiagramSourceLimit {
return nil, ErrDiagramSource
}
u, err := url.Parse(k.Endpoint)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || k.Client == nil {
return nil, ErrProviderConfig
}
ctx, cancel := context.WithTimeout(parent, DiagramTimeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(k.Endpoint, "/")+"/"+kind+"/svg", strings.NewReader(source))
if err != nil {
return nil, ErrProviderConfig
}
req.Header.Set("Content-Type", "text/plain; charset=utf-8")
resp, err := k.Client.Do(req)
if err != nil {
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
if ctx.Err() != nil {
return nil, ctx.Err()
}
return nil, ErrProviderUnavailable
}
if resp == nil || resp.Body == nil {
return nil, ErrProviderUnavailable
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, ErrProviderFailed
}
b, err := io.ReadAll(io.LimitReader(resp.Body, DiagramOutputLimit+1))
if ctx.Err() != nil {
return nil, ctx.Err()
}
if err != nil || len(b) > DiagramOutputLimit {
return nil, ErrDiagramOutput
}
return b, nil
}
+167
View File
@@ -0,0 +1,167 @@
package kunstzeal
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"net/http"
"strings"
"testing"
"time"
)
var _ DiagramProvider = Kroki{}
type clientFixture func(*http.Request) (*http.Response, error)
func (f clientFixture) Do(r *http.Request) (*http.Response, error) { return f(r) }
type trackedReader struct {
io.Reader
closed bool
}
func (r *trackedReader) Close() error { r.closed = true; return nil }
func TestDiagramIdentityAndKinds(t *testing.T) {
for _, kind := range []string{"mermaid", "plantuml", "graphviz", "dot", "d2"} {
if !AllowedDiagramType(kind) {
t.Fatal(kind)
}
for _, source := range []string{"", "graph LR; A-->B;", "日本語\x00source"} {
sum := sha256.Sum256([]byte(kind + "\x00svg\x00" + source))
if DiagramHash(kind, source, "svg") != hex.EncodeToString(sum[:]) {
t.Fatal("cache identity changed")
}
}
}
for _, kind := range []string{"", "nwdiag", "../mermaid", "Mermaid", "svg", "graphviz/svg"} {
if AllowedDiagramType(kind) {
t.Fatal("unknown kind", kind)
}
}
if NormalizeDiagramType("dot") != "graphviz" || NormalizeDiagramType("mermaid") != "mermaid" {
t.Fatal("normalization changed")
}
}
func TestKrokiBoundedRequestAndOutput(t *testing.T) {
for _, kind := range []string{"mermaid", "dot", "d2"} {
body := &trackedReader{Reader: strings.NewReader("<svg>fixture</svg>")}
k := Kroki{Endpoint: "http://renderer.example/prefix/", Client: clientFixture(func(r *http.Request) (*http.Response, error) {
b, err := io.ReadAll(r.Body)
if err != nil {
t.Fatal(err)
}
if r.Method != "POST" || r.URL.String() != "http://renderer.example/prefix/"+NormalizeDiagramType(kind)+"/svg" || string(b) != "source" || r.Header.Get("Content-Type") != "text/plain; charset=utf-8" {
t.Fatal("request contract changed")
}
if r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" {
t.Fatal("credentials forwarded")
}
deadline, ok := r.Context().Deadline()
if !ok || time.Until(deadline) > DiagramTimeout {
t.Fatal("unbounded provider")
}
return &http.Response{StatusCode: 200, Body: body}, nil
})}
b, err := k.Render(context.Background(), kind, "source")
if err != nil || string(b) != "<svg>fixture</svg>" || !body.closed {
t.Fatal(string(b), err, body.closed)
}
}
for _, size := range []int{DiagramOutputLimit, DiagramOutputLimit + 1} {
body := &trackedReader{Reader: strings.NewReader(strings.Repeat("x", size))}
k := Kroki{Endpoint: "https://renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) { return &http.Response{StatusCode: 200, Body: body}, nil })}
b, err := k.Render(context.Background(), "mermaid", strings.Repeat("x", DiagramSourceLimit))
if size == DiagramOutputLimit {
if err != nil || len(b) != size {
t.Fatal(len(b), err)
}
} else if !errors.Is(err, ErrDiagramOutput) || b != nil {
t.Fatal("oversized output escaped", err)
}
if !body.closed {
t.Fatal("response leaked")
}
}
}
func TestKrokiFailsClosedBeforeIO(t *testing.T) {
client := clientFixture(func(*http.Request) (*http.Response, error) { t.Fatal("invalid request performed I/O"); return nil, nil })
for _, endpoint := range []string{"", "file:///etc/passwd", "https://user:secret@renderer.example", "https://renderer.example?token=secret", "https://renderer.example#fragment", "https://renderer.example?", "https:///path", "http://%zz"} {
b, err := (Kroki{Endpoint: endpoint, Client: client}).Render(context.Background(), "mermaid", "source")
if !errors.Is(err, ErrProviderConfig) || b != nil {
t.Fatal("bad provider accepted", err)
}
}
k := Kroki{Endpoint: "http://renderer.example", Client: client}
if _, err := k.Render(context.Background(), "../../secret", "source"); !errors.Is(err, ErrDiagramKind) {
t.Fatal(err)
}
if _, err := k.Render(context.Background(), "mermaid", strings.Repeat("x", DiagramSourceLimit+1)); !errors.Is(err, ErrDiagramSource) {
t.Fatal(err)
}
if _, err := (Kroki{Endpoint: k.Endpoint}).Render(context.Background(), "mermaid", "source"); !errors.Is(err, ErrProviderConfig) {
t.Fatal("default HTTP client used", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := k.Render(ctx, "mermaid", "source"); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
}
func TestKrokiFailuresAreOpaqueAndCloseBodies(t *testing.T) {
for _, code := range []int{301, 302, 307, 308, 400, 401, 403, 404, 429, 500} {
body := &trackedReader{Reader: strings.NewReader("PRIVATE SOURCE OR INTERNAL DIAGNOSTICS")}
k := Kroki{Endpoint: "https://private-renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) { return &http.Response{StatusCode: code, Body: body}, nil })}
b, err := k.Render(context.Background(), "mermaid", "PRIVATE SOURCE")
if !errors.Is(err, ErrProviderFailed) || b != nil || !body.closed {
t.Fatal(code, err, body.closed)
}
}
for _, mode := range []string{"network", "nil-response", "nil-body", "read-error"} {
k := Kroki{Endpoint: "https://private-renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) {
switch mode {
case "network":
return nil, errors.New("https://private-renderer.example: secret")
case "nil-response":
return nil, nil
case "nil-body":
return &http.Response{StatusCode: 200}, nil
default:
return &http.Response{StatusCode: 200, Body: io.NopCloser(errorReader{})}, nil
}
})}
b, err := k.Render(context.Background(), "mermaid", "PRIVATE SOURCE")
if b != nil || err == nil || strings.Contains(err.Error(), "secret") || strings.Contains(err.Error(), "private-renderer") {
t.Fatal(mode, err)
}
}
}
type errorReader struct{}
func (errorReader) Read([]byte) (int, error) { return 0, errors.New("secret body read failure") }
func TestKrokiCancellationDuringSendAndRead(t *testing.T) {
for _, duringRead := range []bool{false, true} {
ctx, cancel := context.WithCancel(context.Background())
body := &trackedReader{Reader: strings.NewReader("<svg/>")}
k := Kroki{Endpoint: "https://renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) {
cancel()
if duringRead {
return &http.Response{StatusCode: 200, Body: body}, nil
}
return &http.Response{StatusCode: 200, Body: body}, errors.New("cancelled network")
})}
b, err := k.Render(ctx, "mermaid", "source")
cancel()
if !errors.Is(err, context.Canceled) || b != nil || !body.closed {
t.Fatal(duringRead, err, body.closed)
}
}
}
+44
View File
@@ -0,0 +1,44 @@
package mestionora
import (
"go/parser"
"go/token"
"path/filepath"
"reflect"
"strconv"
"strings"
"testing"
)
func TestComponentBoundary(t *testing.T) {
if Component != "mestionora" {
t.Fatal("component identity changed")
}
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, file := range files {
if strings.HasSuffix(file, "_test.go") {
continue
}
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
if err != nil {
t.Fatal(err)
}
for _, spec := range ast.Imports {
path, err := strconv.Unquote(spec.Path.Value)
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"pageswiki/internal", "net", "os"} {
if path == forbidden || strings.HasPrefix(path, forbidden+"/") {
t.Errorf("%s imports %s: search must not own HTTP, filesystem payloads or application policy", file, path)
}
}
}
}
if _, exists := reflect.TypeFor[*SQLite]().MethodByName("SearchPathsInDomain"); exists {
t.Fatal("implicit allow-all belongs only in the legacy trusted-caller facade")
}
}
+50
View File
@@ -0,0 +1,50 @@
package mestionora
import (
"path"
"strings"
)
// A display projection only: names, boundaries and manual titles remain stored
// unchanged. Legacy generated Spots have no nested Spot semantics.
func IsUnclassifiedCitySpot(name, boundary string) bool {
return strings.HasSuffix(name, "-undefined") || strings.HasSuffix(boundary, "/_undefined")
}
func CitySpotDisplayTitle(name, title, boundary, parentTitle string) string {
if !IsUnclassifiedCitySpot(name, boundary) {
if title != "" {
return title
}
return name
}
switch strings.ToLower(strings.TrimSpace(title)) {
case "", "undefined", "未定義", "未分類のpage", "unclassified pages", strings.ToLower(name):
if parentTitle != "" {
return parentTitle
}
if parent := path.Base(path.Dir(boundary)); parent != "/" && parent != "." {
return parent
}
return "Spot"
default:
return title // A manual display title takes precedence.
}
}
func (r CitySearchResult) DisplayTitle() string {
if r.Kind == "spot" {
return CitySpotDisplayTitle(r.Name, r.Title, r.PathPrefix, r.StreetTitle)
}
if r.Title != "" {
return r.Title
}
return r.Name
}
func (r CitySearchResult) ContentPath() string {
if r.NavigationPath != "" {
return r.NavigationPath
}
return r.PathPrefix
}
@@ -0,0 +1,38 @@
package mestionora
import (
"database/sql"
"strings"
"testing"
)
func TestCityContentExistenceUsesPathIndex(t *testing.T) {
database := newSearchTestDB(t)
for _, kind := range []string{"quartier", "street", "spot"} {
rows, err := database.Query("EXPLAIN QUERY PLAN "+citySearchSQL(kind, ""), sql.Named("domain", "d"), sql.Named("scope", "/"), sql.Named("prefix", "/"), sql.Named("upper", "0"), sql.Named("query", "needle"), sql.Named("limit", 1000))
if err != nil {
t.Fatal(err)
}
indexed := false
for rows.Next() {
var id, parent, unused int
var detail string
if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
t.Fatal(err)
}
if strings.Contains(detail, "SEARCH cp USING") {
indexed = true
}
if strings.Contains(detail, "SCAN cp") {
t.Fatalf("unbounded Path lookup: %s", detail)
}
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
rows.Close()
if !indexed {
t.Fatalf("no indexed portal lookup for %s", kind)
}
}
}
+212
View File
@@ -0,0 +1,212 @@
package mestionora
import (
"context"
"database/sql"
"errors"
"sort"
"strings"
"time"
)
const (
CitySearchCandidateLimit = 1000 // Per locality kind, before authorization.
CitySearchResultLimit = 30
citySearchTimeout = 2 * time.Second
)
// CitySearchResult describes a locality, not its contained Pages. Description
// is a bounded plain-text excerpt around the match and is never trusted HTML.
type CitySearchResult struct {
UUID, Kind, Name, Title, Description, PathPrefix string
QuartierTitle, StreetTitle, QuartierName string
CreatedAtMS, UpdatedAtMS int64
NameMatch, PathMatch, DescriptionMatch bool
NavigationPath string
HasPage bool
}
type citySearchCandidate struct {
CitySearchResult
quartierPath, streetPath string
}
// SearchCityInDomain searches locality metadata only. It does not construct a
// CityMap, expand descendants, read Page bodies, or count hidden localities.
// Scope selects a locality's own boundary, not an ancestor of the scope. Every
// boundary in its parent chain must be readable before the result is exposed.
func (s *SQLite) SearchCityInDomain(parent context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []CitySearchResult, searchErr error) {
if err := options.Validate(); err != nil {
return nil, err
}
query = strings.TrimSpace(query)
if len(query) > 512 || len(strings.Fields(query)) > 16 {
return nil, errors.New("search query is too long")
}
// File filters are intentionally not applied to locality metadata: a Spot
// containing a PDF is not itself a PDF and must not claim to match one.
if allowed == nil || options.FileType != "" || options.Extension != "" || (query == "" && !options.HasFilters()) {
return nil, nil
}
if limit <= 0 || limit > CitySearchResultLimit {
limit = CitySearchResultLimit
}
ctx, cancel := context.WithTimeout(parent, citySearchTimeout)
defer cancel()
defer func() {
if searchErr != nil && ctx.Err() != nil {
searchErr = ctx.Err()
}
}()
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return nil, err
}
defer tx.Rollback()
scope = normalizePath(scope)
prefix, upper := strings.TrimRight(scope, "/")+"/", strings.TrimRight(scope, "/")+"0"
args := []any{
sql.Named("domain", domainUUID), sql.Named("scope", scope),
sql.Named("prefix", prefix), sql.Named("upper", upper),
sql.Named("query", query), sql.Named("limit", CitySearchCandidateLimit),
}
dateColumn := "n.updated_at_ms"
if options.DateField == "created" {
dateColumn = "n.created_at_ms"
}
dateWhere := ""
if options.FromMS != nil {
dateWhere += " AND " + dateColumn + ">=:from_ms"
args = append(args, sql.Named("from_ms", *options.FromMS))
}
if options.BeforeMS != nil {
dateWhere += " AND " + dateColumn + "<:before_ms"
args = append(args, sql.Named("before_ms", *options.BeforeMS))
}
candidates := make([]citySearchCandidate, 0)
for _, kind := range []string{"quartier", "street", "spot"} {
rows, err := tx.QueryContext(ctx, citySearchSQL(kind, dateWhere), args...)
if err != nil {
return nil, err
}
for rows.Next() {
var candidate citySearchCandidate
candidate.Kind = kind
if err := rows.Scan(&candidate.UUID, &candidate.Name, &candidate.Title, &candidate.Description,
&candidate.PathPrefix, &candidate.CreatedAtMS, &candidate.UpdatedAtMS,
&candidate.NameMatch, &candidate.PathMatch, &candidate.DescriptionMatch,
&candidate.QuartierTitle, &candidate.StreetTitle, &candidate.quartierPath, &candidate.streetPath, &candidate.QuartierName,
&candidate.NavigationPath, &candidate.HasPage); err != nil {
rows.Close()
return nil, err
}
candidates = append(candidates, candidate)
}
err = rows.Err()
closeErr := rows.Close()
if err != nil {
return nil, err
}
if closeErr != nil {
return nil, closeErr
}
}
// Release the sole read connection before calling authorization, which may
// itself need a database connection. HTTP normally passes its cached policy.
if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) {
return nil, err
}
sort.SliceStable(candidates, func(i, j int) bool {
a, b := candidates[i], candidates[j]
if a.Kind != b.Kind {
return citySearchKindOrder(a.Kind) < citySearchKindOrder(b.Kind)
}
if a.NameMatch != b.NameMatch {
return a.NameMatch
}
if a.PathMatch != b.PathMatch {
return a.PathMatch
}
if a.PathPrefix != b.PathPrefix {
return a.PathPrefix < b.PathPrefix
}
return a.UUID < b.UUID
})
readable := make(map[string]bool)
canRead := func(path string) bool {
if value, exists := readable[path]; exists {
return value
}
value := allowed(path)
readable[path] = value
return value
}
for _, candidate := range candidates {
if err := ctx.Err(); err != nil {
return nil, err
}
if !canRead(candidate.PathPrefix) || (candidate.quartierPath != "" && !canRead(candidate.quartierPath)) || (candidate.streetPath != "" && !canRead(candidate.streetPath)) {
continue
}
found = append(found, candidate.CitySearchResult)
if len(found) == limit {
break
}
}
if err := ctx.Err(); err != nil {
return nil, err
}
return found, nil
}
func citySearchKindOrder(kind string) int {
switch kind {
case "quartier":
return 0
case "street":
return 1
default:
return 2
}
}
// All interpolated SQL fragments are package-owned constants. User input is
// bound, so %, _, quotes and backslashes are literal substring characters.
func citySearchSQL(kind, dateWhere string) string {
from, description := "zones n", "n.description"
navigationPath := "n.path_prefix"
parentColumns := "'','','','',''"
where := "n.kind='quartier'"
switch kind {
case "street":
from += ` JOIN zones q ON q.uuid=n.parent_uuid AND q.domain_uuid=n.domain_uuid AND q.kind='quartier' AND q.state!='archived'`
parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),'',q.path_prefix,'',q.name"
where = "n.kind='street' AND " + citySearchAbsolutePathSQL("q.path_prefix")
case "spot":
navigationPath = `CASE WHEN substr(n.path_prefix,-11)='/_undefined' THEN st.path_prefix ELSE n.path_prefix END`
from = `spots n JOIN street_spots ss ON ss.spot_uuid=n.uuid
JOIN zones st ON st.uuid=ss.street_uuid AND st.domain_uuid=n.domain_uuid AND st.kind='street' AND st.state!='archived'
JOIN zones q ON q.uuid=st.parent_uuid AND q.domain_uuid=st.domain_uuid AND q.kind='quartier' AND q.state!='archived'`
description = "n.summary"
parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),substr(COALESCE(NULLIF(st.title,''),st.name),1,256),q.path_prefix,st.path_prefix,q.name"
where = citySearchAbsolutePathSQL("q.path_prefix") + " AND " + citySearchAbsolutePathSQL("st.path_prefix")
}
nameMatch := `(:query!='' AND (instr(lower(n.name),lower(:query))>0 OR instr(lower(n.title),lower(:query))>0))`
pathMatch := `(:query!='' AND instr(lower(n.path_prefix),lower(:query))>0)`
descriptionMatch := `(:query!='' AND instr(lower(` + description + `),lower(:query))>0)`
return `SELECT n.uuid,substr(n.name,1,256),substr(n.title,1,256),
substr(` + description + `,max(1,instr(lower(` + description + `),lower(:query))-160),512),
n.path_prefix,n.created_at_ms,n.updated_at_ms,` + nameMatch + `,` + pathMatch + `,` + descriptionMatch + `,` + parentColumns + `,` + navigationPath + `,
EXISTS(SELECT 1 FROM paths cp WHERE cp.domain_uuid=n.domain_uuid AND cp.path=(` + navigationPath + `))
FROM ` + from + ` WHERE n.domain_uuid=:domain AND n.state!='archived' AND ` + where + `
AND ` + citySearchAbsolutePathSQL("n.path_prefix") + `
AND (n.path_prefix COLLATE BINARY=:scope OR (n.path_prefix COLLATE BINARY>=:prefix AND n.path_prefix COLLATE BINARY<:upper))
AND (:query='' OR ` + nameMatch + ` OR ` + pathMatch + ` OR ` + descriptionMatch + `)` + dateWhere + `
ORDER BY ` + nameMatch + ` DESC,` + pathMatch + ` DESC,n.path_prefix COLLATE BINARY,n.uuid LIMIT :limit`
}
func citySearchAbsolutePathSQL(column string) string {
// Keep authorization paths intact rather than truncating them. Malformed
// legacy boundaries and impractically large metadata are not navigable hits.
return `(substr(` + column + `,1,1)='/' AND length(` + column + `)<=4096 AND instr(` + column + `,char(0))=0)`
}
+40
View File
@@ -0,0 +1,40 @@
package mestionora
import (
"database/sql"
"testing"
"pageswiki/internal/db"
)
func newSearchTestDB(t *testing.T) *sql.DB {
t.Helper()
database, err := db.Open(t.TempDir() + "/search.db")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { database.Close() })
initSearchFixture(t, database)
return database
}
// Only the application columns/indexes consumed by search, not application
// bootstrap, auth keys, job recovery or external calendar initialization.
func initSearchFixture(t *testing.T, database *sql.DB) {
t.Helper()
_, err := database.Exec(`
CREATE TABLE objects(uuid TEXT PRIMARY KEY, created_at INTEGER);
CREATE TABLE revisions(uuid TEXT PRIMARY KEY,object_uuid TEXT,filename TEXT,body TEXT,content_type TEXT,document_hash TEXT,created_at_ms INTEGER,created_at INTEGER);
CREATE TABLE paths(domain_uuid TEXT NOT NULL,path TEXT NOT NULL,object_uuid TEXT,PRIMARY KEY(domain_uuid,path));
CREATE INDEX idx_paths_object_domain_page ON paths(object_uuid,domain_uuid,path);
CREATE TABLE zones(uuid TEXT PRIMARY KEY,domain_uuid TEXT,kind TEXT,parent_uuid TEXT,name TEXT,title TEXT,description TEXT,path_prefix TEXT,state TEXT,created_at_ms INTEGER,updated_at_ms INTEGER);
CREATE TABLE spots(uuid TEXT PRIMARY KEY,domain_uuid TEXT,name TEXT,title TEXT,summary TEXT,path_prefix TEXT,state TEXT,created_at_ms INTEGER,updated_at_ms INTEGER);
CREATE TABLE street_spots(street_uuid TEXT,spot_uuid TEXT);
`)
if err != nil {
t.Fatal(err)
}
if err := NewSQLite(database).EnsureSearchIndexSchema(); err != nil {
t.Fatal(err)
}
}
+36
View File
@@ -0,0 +1,36 @@
package mestionora
import (
"golang.org/x/net/html"
"io"
"strings"
)
// FirstHTMLHeading extracts untrusted title text. Callers bound input before
// extraction and escape output at rendering time; this is not a sanitizer.
func FirstHTMLHeading(source string) string {
t := html.NewTokenizer(strings.NewReader(source))
in := false
var out strings.Builder
for {
kind := t.Next()
if kind == html.ErrorToken {
if t.Err() == io.EOF {
break
}
break
}
tok := t.Token()
if kind == html.StartTagToken && strings.EqualFold(tok.Data, "h1") {
in = true
continue
}
if kind == html.EndTagToken && strings.EqualFold(tok.Data, "h1") {
break
}
if in && kind == html.TextToken {
out.WriteString(tok.Data)
}
}
return strings.TrimSpace(out.String())
}
+377
View File
@@ -0,0 +1,377 @@
package mestionora
import (
"context"
"database/sql"
"errors"
"sort"
"strings"
"time"
)
const SearchCandidateLimit = 1000
const SearchExcerptDocumentBytes = 64 << 10
const SearchExcerptTotalBytes = 1 << 20
const searchExcerptTimeout = 400 * time.Millisecond
const searchResultColumns = `p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.revision_uuid,''),COALESCE(r.created_at_ms/1000,0)`
const searchPathBoundary = `(p.path=? OR (p.path>=? AND p.path<?))`
const searchPathCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,0
FROM paths p LEFT JOIN latest_revision_index r ON r.object_uuid=p.object_uuid
WHERE p.domain_uuid=? AND ` + searchPathBoundary
const searchPathQueryPredicate = ` AND (instr(lower(p.path),lower(?))>0 OR instr(lower(COALESCE(r.filename,'')),lower(?))>0)`
const searchPathCandidatesSQL = searchPathCandidatesBaseSQL + searchPathQueryPredicate + ` ORDER BY p.path LIMIT ?`
// CROSS JOIN is intentional: revision_uuid is an UNINDEXED FTS payload column.
// A Path-first plan re-runs the complete MATCH scan for every Path. Start with
// MATCH exactly once, then seek current Revision metadata and Domain bindings
// through their B-tree indexes. Keep scope filtering before the candidate cap.
const searchFTSCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,revision_search.rowid
FROM revision_search
CROSS JOIN latest_revision_index r ON r.revision_uuid=revision_search.revision_uuid
CROSS JOIN paths p ON p.object_uuid=r.object_uuid
WHERE revision_search MATCH ? AND p.domain_uuid=? AND ` + searchPathBoundary
const searchFTSCandidatesSQL = searchFTSCandidatesBaseSQL + ` LIMIT ?`
func searchBodyExcerptsSQL(count int) string {
// UUID is not indexed by FTS5. Restrict the MATCH/snippet work using the
// integer rowids collected from the same read snapshot, not payload UUIDs.
return `SELECT revision_uuid,substr(snippet(revision_search,3,'','',' … ',36),1,4096)
FROM revision_search WHERE revision_search MATCH ? AND rowid IN (` + strings.TrimRight(strings.Repeat("?,", count), ",") + `)`
}
// Excerpts are optional enrichment. Even small documents can have expensive
// prefix hit-position scans, so their time budget is separate from finding
// results. A spent excerpt budget omits snippets, not already-authorized hits.
func loadSearchBodyExcerpts(parent context.Context, tx *sql.Tx, expression string, ids []any, budget time.Duration) (excerpts map[string]string, err error) {
ctx, cancel := context.WithTimeout(parent, budget)
defer cancel()
defer func() {
if err != nil && ctx.Err() != nil {
if parent.Err() != nil {
err = parent.Err()
} else {
excerpts, err = nil, nil
}
}
}()
args := append([]any{"body : (" + expression + ")"}, ids...)
rows, err := tx.QueryContext(ctx, searchBodyExcerptsSQL(len(ids)), args...)
if err != nil {
return nil, err
}
defer rows.Close()
excerpts = make(map[string]string)
for rows.Next() {
var id, excerpt string
if err := rows.Scan(&id, &excerpt); err != nil {
return nil, err
}
excerpts[id] = excerpt
}
return excerpts, rows.Err()
}
type SearchResult struct {
Path string
ContentType string
Filename string
DocumentHash string
RevisionUUID string
Size int64 // Kept for API compatibility; search does not stat Document files.
UpdatedAt int64
PathMatch bool
FilenameMatch bool
BodyMatch bool
Excerpt string // Untrusted source text, never HTML-safe markup.
ExcerptOmitted bool // A body match remains visible when snippet work exceeds its budget.
}
type PathSuggestion struct {
SearchResult
Title string
}
// SearchPathsInDomainFiltered bounds SQL candidates before materialization and
// authorizes each Path before loading an excerpt. It never computes a global hit
// count or stats Document files. A narrow scope improves both recall and cost.
// allowed runs inside a read snapshot. Prefer a preloaded request-local policy;
// a callback doing DB reads requires spare pool capacity (not a one-connection
// pool). Keeping policy I/O outside this snapshot is separate follow-up work.
func (s *SQLite) SearchPathsInDomainFiltered(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]SearchResult, error) {
return s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{})
}
// SearchPathsInDomainWithOptions also permits filter-only searches. File/date
// predicates apply before SQL candidate caps, while ACL checks stay ahead of all
// snippet enrichment. Zero options preserve the original indexed query plans.
func (s *SQLite) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, searchErr error) {
return s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options, true)
}
// SearchPathSuggestionsInDomain shares the current-Revision FTS and ACL path
// with full search, but deliberately skips snippet generation. Typeahead calls
// are frequent and only need a bounded display label, never a body excerpt.
func (s *SQLite) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) {
results, err := s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}, false)
if err != nil || len(results) == 0 {
return nil, err
}
ids := make([]any, 0, len(results))
seen := make(map[string]bool, len(results))
for _, result := range results {
if result.RevisionUUID != "" && strings.HasPrefix(strings.ToLower(result.ContentType), "text/html") && !seen[result.RevisionUUID] {
ids = append(ids, result.RevisionUUID)
seen[result.RevisionUUID] = true
}
}
headings := make(map[string]string, len(ids))
if len(ids) > 0 {
rows, queryErr := s.db.QueryContext(ctx, `SELECT uuid,substr(CAST(body AS TEXT),1,65536) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
if queryErr != nil {
return nil, queryErr
}
for rows.Next() {
var revision, source string
if queryErr = rows.Scan(&revision, &source); queryErr != nil {
rows.Close()
return nil, queryErr
}
headings[revision] = FirstHTMLHeading(source)
}
queryErr = rows.Err()
rows.Close()
if queryErr != nil {
return nil, queryErr
}
}
out := make([]PathSuggestion, 0, len(results))
for _, result := range results {
out = append(out, PathSuggestion{SearchResult: result, Title: headings[result.RevisionUUID]})
}
return out, nil
}
func (s *SQLite) searchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions, loadExcerpts bool) (found []SearchResult, searchErr error) {
if err := options.Validate(); err != nil {
return nil, err
}
scope, query = normalizePath(scope), strings.TrimSpace(query)
if query == "" && !options.HasFilters() || allowed == nil {
return nil, nil
}
if len(query) > 512 || len(strings.Fields(query)) > 16 {
return nil, errors.New("search query is too long")
}
if limit <= 0 || limit > 200 {
limit = 100
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
defer func() {
// SQLite can report an interrupted statement instead of wrapping the
// context error. Preserve cancellation/deadline semantics for HTTP.
if ctx.Err() != nil {
found = nil
searchErr = ctx.Err()
}
}()
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return nil, err
}
defer tx.Rollback()
prefix := strings.TrimRight(scope, "/") + "/"
upper := strings.TrimRight(scope, "/") + "0"
results := make(map[string]SearchResult)
ftsRows := make(map[string]int64)
collect := func(statement string, body, filename bool, args ...any) error {
rows, err := tx.QueryContext(ctx, statement, args...)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var result SearchResult
var rowID int64
if err := rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt, &rowID); err != nil {
return err
}
if !allowed(result.Path) {
continue
}
if rowID != 0 {
ftsRows[result.RevisionUUID] = rowID
}
result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query))
result.FilenameMatch = filename || results[result.Path].FilenameMatch || query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query))
result.BodyMatch = body || results[result.Path].BodyMatch
results[result.Path] = result
}
return rows.Err()
}
filterSQL, filterArgs := searchOptionsPredicate(options)
pathSQL := searchPathCandidatesBaseSQL
pathArgs := []any{domainUUID, scope, prefix, upper}
if query != "" {
pathSQL += searchPathQueryPredicate
pathArgs = append(pathArgs, query, query)
}
pathSQL += filterSQL + ` ORDER BY p.path LIMIT ?`
pathArgs = append(pathArgs, filterArgs...)
pathArgs = append(pathArgs, SearchCandidateLimit)
if err := collect(pathSQL, false, false, pathArgs...); err != nil {
return nil, err
}
expression := ftsPrefixExpression(query)
if expression != "" {
// No global ORDER BY or unbounded IN materialization: stop the FTS scan
// after at most this many current, in-Domain, in-scope Path candidates.
for _, field := range []string{"body", "filename"} {
args := []any{field + " : (" + expression + ")", domainUUID, scope, prefix, upper}
args = append(args, filterArgs...)
args = append(args, SearchCandidateLimit)
if err := collect(searchFTSCandidatesBaseSQL+filterSQL+` LIMIT ?`, field == "body", field == "filename", args...); err != nil {
return nil, err
}
}
}
out := make([]SearchResult, 0, len(results))
for _, result := range results {
out = append(out, result)
}
sort.Slice(out, func(i, j int) bool {
// Direct name/Path matches precede body-only hits; all ties are stable.
iName, jName := out[i].PathMatch || out[i].FilenameMatch, out[j].PathMatch || out[j].FilenameMatch
if iName != jName {
return iName
}
if len(out[i].Path) != len(out[j].Path) {
return len(out[i].Path) < len(out[j].Path)
}
return out[i].Path < out[j].Path
})
if len(out) > limit {
out = out[:limit]
}
if !loadExcerpts {
return out, nil
}
// Only authorized, displayed results receive text excerpts. Batch the FTS
// lookup so a broad query is not re-scanned once for every visible result.
ids := make([]any, 0, len(out))
seen := make(map[string]bool)
for _, result := range out {
if result.BodyMatch && !seen[result.RevisionUUID] {
ids = append(ids, ftsRows[result.RevisionUUID])
seen[result.RevisionUUID] = true
}
}
if len(ids) != 0 {
// snippet's output token/character limit does not bound its input work:
// FTS5 may tokenize the entire matched document. Read byte lengths only
// for authorized displayed rows before spending the snippet budget.
rows, err := tx.QueryContext(ctx, `SELECT rowid,octet_length(body)
FROM revision_search WHERE rowid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
if err != nil {
return nil, err
}
sizes := make(map[int64]int64)
for rows.Next() {
var rowID, size int64
if err := rows.Scan(&rowID, &size); err != nil {
rows.Close()
return nil, err
}
sizes[rowID] = size
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
ids = nil
selected := make(map[string]bool)
remaining := int64(SearchExcerptTotalBytes)
for i := range out {
if !out[i].BodyMatch {
continue
}
revision := out[i].RevisionUUID
include, decided := selected[revision]
if !decided {
rowID := ftsRows[revision]
size, exists := sizes[rowID]
include = exists && size >= 0 && size <= SearchExcerptDocumentBytes && size <= remaining
if include {
ids = append(ids, rowID)
remaining -= size
}
selected[revision] = include
}
out[i].ExcerptOmitted = !include
}
}
if len(ids) != 0 {
excerpts, err := loadSearchBodyExcerpts(ctx, tx, expression, ids, searchExcerptTimeout)
if err != nil {
return nil, err
}
for i := range out {
if out[i].BodyMatch {
var present bool
out[i].Excerpt, present = excerpts[out[i].RevisionUUID]
out[i].ExcerptOmitted = !present
}
}
}
// A filename/Path-only hit still benefits from a short Page summary. Read
// only a bounded prefix from displayed textual revisions, in one batch.
ids, seen = nil, make(map[string]bool)
for _, result := range out {
media := strings.ToLower(result.ContentType)
textual := strings.HasPrefix(media, "text/") || strings.HasPrefix(media, "application/json") || strings.HasPrefix(media, "application/xml") || strings.HasPrefix(media, "application/javascript") || strings.HasPrefix(media, "application/yaml") || strings.HasPrefix(media, "application/x-yaml")
if !result.BodyMatch && textual && result.RevisionUUID != "" && !seen[result.RevisionUUID] {
ids = append(ids, result.RevisionUUID)
seen[result.RevisionUUID] = true
}
}
if len(ids) != 0 {
rows, err := tx.QueryContext(ctx, `SELECT uuid,substr(body,1,1024) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...)
if err != nil {
return nil, err
}
excerpts := make(map[string]string)
for rows.Next() {
var id, excerpt string
if err := rows.Scan(&id, &excerpt); err != nil {
rows.Close()
return nil, err
}
excerpts[id] = excerpt
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
for i := range out {
if !out[i].BodyMatch {
out[i].Excerpt = excerpts[out[i].RevisionUUID]
}
}
}
return out, nil
}
func ftsPrefixExpression(query string) string {
words := strings.Fields(query)
terms := make([]string, 0, len(words))
for _, word := range words {
word = strings.ReplaceAll(word, `"`, `""`)
if word != "" {
terms = append(terms, `"`+word+`"*`)
}
}
return strings.Join(terms, " AND ")
}
+135
View File
@@ -0,0 +1,135 @@
package mestionora
import (
"errors"
"fmt"
"strings"
)
var ErrInvalidSearchOptions = errors.New("invalid search options")
// SearchOptions applies to current Page metadata, not historical Revisions.
// Dates are Unix milliseconds: FromMS is inclusive and BeforeMS is exclusive.
// The created field refers to Object creation, not creation of a Path alias.
// FileType and Extension are normalized lower-case values; Extension has no dot
// prefix and may contain a compound suffix such as tar.gz.
type SearchOptions struct {
FileType string
Extension string
DateField string
FromMS *int64
BeforeMS *int64
}
func (o SearchOptions) HasFilters() bool {
return o.FileType != "" || o.Extension != "" || o.FromMS != nil || o.BeforeMS != nil
}
func (o SearchOptions) Validate() error {
switch o.FileType {
case "", "html", "text", "image", "audio", "video", "pdf", "archive", "csv", "json", "code", "office", "other":
default:
return fmt.Errorf("%w: unknown file type", ErrInvalidSearchOptions)
}
switch o.DateField {
case "", "updated", "created":
default:
return fmt.Errorf("%w: unknown date field", ErrInvalidSearchOptions)
}
if len(o.Extension) > 32 {
return fmt.Errorf("%w: extension is too long", ErrInvalidSearchOptions)
}
for i, c := range o.Extension {
alnum := c >= 'a' && c <= 'z' || c >= '0' && c <= '9'
if !alnum && (i == 0 || !strings.ContainsRune("._+-", c)) {
return fmt.Errorf("%w: invalid extension", ErrInvalidSearchOptions)
}
}
if o.FromMS != nil && o.BeforeMS != nil && *o.FromMS >= *o.BeforeMS {
return fmt.Errorf("%w: empty or reversed date range", ErrInvalidSearchOptions)
}
return nil
}
const searchMIMESQL = `lower(trim(substr(COALESCE(r.content_type,''),1,instr(COALESCE(r.content_type,'')||';',';')-1)))`
// The original filename wins over the Path suffix when present. Names remain
// data: even extension patterns are bound arguments and do not become SQL.
const searchFilenameSQL = `lower(CASE WHEN COALESCE(r.filename,'')<>'' THEN r.filename ELSE p.path END)`
func searchSuffixesSQL(suffixes ...string) string {
clauses := make([]string, 0, len(suffixes))
for _, suffix := range suffixes {
// Only package-owned ASCII constants reach this helper. GLOB has no
// escaping ambiguity for underscores, unlike LIKE.
clauses = append(clauses, searchFilenameSQL+` GLOB '*.`+suffix+`'`)
}
return "(" + strings.Join(clauses, " OR ") + ")"
}
// Specific MIME types are authoritative. Generic text/binary types may use a
// known suffix (e.g. a Python source uploaded as text/plain); unrecognized,
// specific MIME types stay Other instead of pretending to be a different type.
// This uses metadata only and never hydrates a Document to sniff its bytes.
var searchFileTypeSQL = `CASE
WHEN ` + searchMIMESQL + ` IN ('text/html','application/xhtml+xml') THEN 'html'
WHEN ` + searchMIMESQL + `='application/pdf' THEN 'pdf'
WHEN ` + searchMIMESQL + ` LIKE 'image/%' THEN 'image'
WHEN ` + searchMIMESQL + ` LIKE 'audio/%' THEN 'audio'
WHEN ` + searchMIMESQL + ` LIKE 'video/%' THEN 'video'
WHEN ` + searchMIMESQL + ` IN ('text/csv','application/csv','text/tab-separated-values') THEN 'csv'
WHEN ` + searchMIMESQL + ` IN ('application/json','text/json') OR ` + searchMIMESQL + ` GLOB '*+json' THEN 'json'
WHEN ` + searchMIMESQL + ` IN ('application/zip','application/x-zip-compressed','application/x-7z-compressed','application/vnd.rar','application/x-rar','application/x-rar-compressed','application/x-tar','application/gzip','application/x-gzip','application/x-xz','application/x-bzip2','application/zstd','application/x-zstd','application/x-compress','application/x-cpio') THEN 'archive'
WHEN ` + searchMIMESQL + ` IN ('application/msword','application/vnd.ms-excel','application/vnd.ms-powerpoint','application/rtf','text/rtf') OR ` + searchMIMESQL + ` GLOB 'application/vnd.openxmlformats-officedocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.oasis.opendocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.ms-*.macroenabled.*' THEN 'office'
WHEN ` + searchMIMESQL + ` IN ('text/markdown','text/x-markdown') THEN 'text'
WHEN ` + searchMIMESQL + ` IN ('application/javascript','application/x-javascript','text/javascript','application/typescript','text/typescript','application/xml','text/xml','application/yaml','application/x-yaml','text/yaml','application/toml','application/x-sh','application/x-shellscript','application/x-python','application/x-python-code','application/x-httpd-php','application/sql','text/css') OR ` + searchMIMESQL + ` GLOB 'text/x-*' OR ` + searchMIMESQL + ` GLOB '*+xml' THEN 'code'
WHEN ` + searchMIMESQL + ` NOT IN ('','text/plain','application/octet-stream','binary/octet-stream','application/x-empty') THEN CASE WHEN ` + searchMIMESQL + ` LIKE 'text/%' THEN 'text' ELSE 'other' END
WHEN ` + searchSuffixesSQL("html", "htm", "xhtml") + ` THEN 'html'
WHEN ` + searchSuffixesSQL("pdf") + ` THEN 'pdf'
WHEN ` + searchSuffixesSQL("png", "jpg", "jpeg", "gif", "webp", "avif", "bmp", "svg", "tif", "tiff", "ico", "heic", "heif") + ` THEN 'image'
WHEN ` + searchSuffixesSQL("mp3", "flac", "wav", "ogg", "oga", "opus", "m4a", "aac", "aiff", "alac", "mid", "midi") + ` THEN 'audio'
WHEN ` + searchSuffixesSQL("mp4", "webm", "mkv", "mov", "avi", "m4v", "ogv", "mpeg", "mpg") + ` THEN 'video'
WHEN ` + searchSuffixesSQL("zip", "7z", "rar", "tar", "gz", "tgz", "xz", "txz", "bz2", "tbz", "tbz2", "zst", "tzst", "cbz", "cbr", "cpio") + ` THEN 'archive'
WHEN ` + searchSuffixesSQL("csv", "tsv") + ` THEN 'csv'
WHEN ` + searchSuffixesSQL("json", "jsonl", "ndjson", "excalidraw", "ipynb") + ` THEN 'json'
WHEN ` + searchSuffixesSQL("doc", "docx", "docm", "xls", "xlsx", "xlsm", "ppt", "pptx", "pptm", "odt", "ods", "odp", "rtf") + ` THEN 'office'
WHEN ` + searchSuffixesSQL("py", "pyw", "sh", "bash", "zsh", "fish", "ps1", "bat", "cmd", "js", "mjs", "cjs", "jsx", "ts", "tsx", "go", "rs", "c", "h", "cpp", "hpp", "cc", "cs", "java", "kt", "kts", "swift", "rb", "php", "pl", "pm", "lua", "r", "sql", "css", "scss", "sass", "less", "xml", "xsl", "yaml", "yml", "toml", "ini", "conf", "vue", "svelte", "dart", "ex", "exs", "erl", "hs", "scala", "clj", "lisp", "el", "vim", "wasm") + ` THEN 'code'
WHEN ` + searchMIMESQL + `='text/plain' OR ` + searchSuffixesSQL("txt", "text", "log", "md", "markdown", "rst", "adoc", "org") + ` THEN 'text'
ELSE 'other' END`
// The returned predicate is inserted before each candidate stream's LIMIT,
// including FTS-first streams. Filtering after their cap silently loses hits.
func searchOptionsPredicate(options SearchOptions) (string, []any) {
var predicate strings.Builder
var args []any
if options.FromMS != nil || options.BeforeMS != nil {
dateSQL := "r.created_at_ms"
if options.DateField == "created" {
// Objects.uuid is a primary-key lookup. Avoid joining Objects in the
// normal search plan when Object creation is not being filtered.
predicate.WriteString(" AND EXISTS (SELECT 1 FROM objects o WHERE o.uuid=p.object_uuid")
dateSQL = "o.created_at*1000"
}
if options.FromMS != nil {
predicate.WriteString(" AND " + dateSQL + ">=?")
args = append(args, *options.FromMS)
}
if options.BeforeMS != nil {
predicate.WriteString(" AND " + dateSQL + "<?")
args = append(args, *options.BeforeMS)
}
if options.DateField == "created" {
predicate.WriteString(")")
}
}
// Evaluate inexpensive date/suffix exclusions before the metadata category.
if options.Extension != "" {
predicate.WriteString(" AND " + searchFilenameSQL + " GLOB ?")
args = append(args, "*."+options.Extension)
}
if options.FileType != "" {
predicate.WriteString(" AND (" + searchFileTypeSQL + ")=?")
args = append(args, options.FileType)
}
return predicate.String(), args
}
+211
View File
@@ -0,0 +1,211 @@
package mestionora
import "database/sql"
// EnsureSearchIndexSchema installs an FTS5 index over immutable Revisions.
// Paths stay outside the index because links can be renamed independently; the
// search query joins matching latest Revisions to the current domain Paths.
func (s *SQLite) EnsureSearchIndexSchema() error {
var revisionsTable string
if err := s.db.QueryRow(`
SELECT name FROM sqlite_master
WHERE type = 'table' AND name = 'revisions'
`).Scan(&revisionsTable); err != nil {
if err == sql.ErrNoRows {
return nil
}
return err
}
if _, err := s.db.Exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS revision_search USING fts5(
revision_uuid UNINDEXED,
object_uuid UNINDEXED,
filename,
body,
tokenize = 'unicode61'
)
`); err != nil {
return err
}
if _, err := s.db.Exec(`
CREATE TABLE IF NOT EXISTS revision_search_state (
revision_uuid TEXT PRIMARY KEY
)
`); err != nil {
return err
}
if _, err := s.db.Exec(`
CREATE TABLE IF NOT EXISTS latest_revision_index (
object_uuid TEXT PRIMARY KEY,
revision_uuid TEXT NOT NULL,
content_type TEXT NOT NULL,
filename TEXT NOT NULL,
document_hash TEXT NOT NULL,
created_at_ms INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS search_schema_state (
name TEXT PRIMARY KEY
);
CREATE INDEX IF NOT EXISTS idx_latest_revision_search_uuid ON latest_revision_index(revision_uuid)
`); err != nil {
return err
}
var latestBackfilled int
if err := s.db.QueryRow(`
SELECT EXISTS(
SELECT 1 FROM search_schema_state
WHERE name = 'latest_revision_index_v1'
)
`).Scan(&latestBackfilled); err != nil {
return err
}
if latestBackfilled == 0 {
if _, err := s.db.Exec(`
INSERT OR REPLACE INTO latest_revision_index (
object_uuid, revision_uuid, content_type, filename,
document_hash, created_at_ms
)
SELECT
object_uuid, uuid, COALESCE(content_type, ''),
COALESCE(filename, ''), COALESCE(document_hash, ''),
COALESCE(created_at_ms, created_at * 1000, 0)
FROM (
SELECT
uuid, object_uuid, content_type, filename,
document_hash, created_at_ms, created_at,
ROW_NUMBER() OVER (
PARTITION BY object_uuid
ORDER BY COALESCE(created_at_ms, created_at * 1000) DESC,
rowid DESC
) AS rn
FROM revisions
)
WHERE rn = 1;
INSERT OR IGNORE INTO search_schema_state (name)
VALUES ('latest_revision_index_v1');
`); err != nil {
return err
}
}
// Populate the cheap membership table once for installations that already
// created revision_search before revision_search_state was introduced.
var stateExists int
if err := s.db.QueryRow(`
SELECT EXISTS(SELECT 1 FROM revision_search_state LIMIT 1)
`).Scan(&stateExists); err != nil {
return err
}
if stateExists == 0 {
if _, err := s.db.Exec(`
INSERT OR IGNORE INTO revision_search_state (revision_uuid)
SELECT revision_uuid FROM revision_search
`); err != nil {
return err
}
}
// Trigger definitions evolve with the projection schema; recreate them so
// existing installations receive the current maintenance logic.
if _, err := s.db.Exec(`
DROP TRIGGER IF EXISTS revision_search_insert;
DROP TRIGGER IF EXISTS revision_search_delete;
`); err != nil {
return err
}
if _, err := s.db.Exec(`
CREATE TRIGGER IF NOT EXISTS revision_search_insert
AFTER INSERT ON revisions BEGIN
INSERT INTO revision_search (
revision_uuid, object_uuid, filename, body
) VALUES (
new.uuid,
new.object_uuid,
COALESCE(new.filename, ''),
CASE
WHEN lower(COALESCE(new.content_type, '')) LIKE 'text/%'
OR lower(COALESCE(new.content_type, '')) LIKE 'application/json%'
OR lower(COALESCE(new.content_type, '')) LIKE 'application/xml%'
OR lower(COALESCE(new.content_type, '')) LIKE 'application/javascript%'
OR lower(COALESCE(new.content_type, '')) LIKE 'application/x-yaml%'
OR lower(COALESCE(new.content_type, '')) LIKE 'application/yaml%'
THEN COALESCE(new.body, '')
ELSE ''
END
);
INSERT OR IGNORE INTO revision_search_state (revision_uuid)
VALUES (new.uuid);
INSERT INTO latest_revision_index (
object_uuid, revision_uuid, content_type, filename,
document_hash, created_at_ms
) VALUES (
new.object_uuid, new.uuid, COALESCE(new.content_type, ''),
COALESCE(new.filename, ''), COALESCE(new.document_hash, ''),
COALESCE(new.created_at_ms, new.created_at * 1000, 0)
)
ON CONFLICT(object_uuid) DO UPDATE SET
revision_uuid = excluded.revision_uuid,
content_type = excluded.content_type,
filename = excluded.filename,
document_hash = excluded.document_hash,
created_at_ms = excluded.created_at_ms
WHERE excluded.created_at_ms >= latest_revision_index.created_at_ms;
END
`); err != nil {
return err
}
if _, err := s.db.Exec(`
CREATE TRIGGER IF NOT EXISTS revision_search_delete
AFTER DELETE ON revisions BEGIN
DELETE FROM revision_search WHERE revision_uuid = old.uuid;
DELETE FROM revision_search_state WHERE revision_uuid = old.uuid;
DELETE FROM latest_revision_index
WHERE object_uuid = old.object_uuid
AND revision_uuid = old.uuid;
INSERT OR REPLACE INTO latest_revision_index (
object_uuid, revision_uuid, content_type, filename,
document_hash, created_at_ms
)
SELECT
object_uuid, uuid, COALESCE(content_type, ''),
COALESCE(filename, ''), COALESCE(document_hash, ''),
COALESCE(created_at_ms, created_at * 1000, 0)
FROM revisions
WHERE object_uuid = old.object_uuid
ORDER BY COALESCE(created_at_ms, created_at * 1000) DESC, rowid DESC
LIMIT 1;
END
`); err != nil {
return err
}
_, err := s.db.Exec(`
INSERT INTO revision_search (
revision_uuid, object_uuid, filename, body
)
SELECT
r.uuid,
r.object_uuid,
COALESCE(r.filename, ''),
CASE
WHEN lower(COALESCE(r.content_type, '')) LIKE 'text/%'
OR lower(COALESCE(r.content_type, '')) LIKE 'application/json%'
OR lower(COALESCE(r.content_type, '')) LIKE 'application/xml%'
OR lower(COALESCE(r.content_type, '')) LIKE 'application/javascript%'
OR lower(COALESCE(r.content_type, '')) LIKE 'application/x-yaml%'
OR lower(COALESCE(r.content_type, '')) LIKE 'application/yaml%'
THEN COALESCE(r.body, '')
ELSE ''
END
FROM revisions r
WHERE NOT EXISTS (
SELECT 1 FROM revision_search_state state
WHERE state.revision_uuid = r.uuid
)
`)
if err != nil {
return err
}
_, err = s.db.Exec(`
INSERT OR IGNORE INTO revision_search_state (revision_uuid)
SELECT uuid FROM revisions
`)
return err
}
@@ -1,4 +1,4 @@
package store
package mestionora
import (
"context"
@@ -39,10 +39,7 @@ func TestSearchExcerptDeadlineIsOptionalButParentCancellationIsNot(t *testing.T)
t.Fatal(err)
}
defer database.Close()
if err := db.Migrate(database); err != nil {
t.Fatal(err)
}
New(database)
initSearchFixture(t, database)
tx, err := database.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true})
if err != nil {
t.Fatal(err)
@@ -72,10 +69,7 @@ func TestSearchFTSPlanScansMatchesOnce(t *testing.T) {
t.Fatal(err)
}
defer database.Close()
if err := db.Migrate(database); err != nil {
t.Fatal(err)
}
New(database)
initSearchFixture(t, database)
for _, field := range []string{"body", "filename"} {
plan := searchQueryPlan(t, database, searchFTSCandidatesSQL,
field+` : ("needle"*)`, "domain", "/", "/", "0", SearchCandidateLimit)
@@ -96,10 +90,7 @@ func TestSearchFilteredFTSPlanKeepsMatchFirstAndIndexedSeeks(t *testing.T) {
t.Fatal(err)
}
defer database.Close()
if err := db.Migrate(database); err != nil {
t.Fatal(err)
}
New(database)
initSearchFixture(t, database)
from, before := int64(1000), int64(2000)
for _, options := range []SearchOptions{
{FileType: "code"}, {Extension: "py"}, {FromMS: &from, BeforeMS: &before},
+102
View File
@@ -0,0 +1,102 @@
package mestionora
import (
"context"
"errors"
"testing"
)
func TestMissingPolicyDoesNotReadOrInitialize(t *testing.T) {
// A nil pool makes accidental construction, search or maintenance I/O fail.
engine := NewSQLite(nil)
ctx := context.Background()
if hits, err := engine.SearchPathsInDomainFiltered(ctx, "d", "/", "needle", 20, nil); err != nil || len(hits) != 0 {
t.Fatalf("missing page policy: %v %v", hits, err)
}
if hits, err := engine.SearchPathSuggestionsInDomain(ctx, "d", "/", "needle", 8, nil); err != nil || len(hits) != 0 {
t.Fatalf("missing suggestion policy: %v %v", hits, err)
}
if hits, err := engine.SearchCityInDomain(ctx, "d", "/", "needle", 30, nil, SearchOptions{}); err != nil || len(hits) != 0 {
t.Fatalf("missing city policy: %v %v", hits, err)
}
}
func TestDeniedPagesDoNotLoadSummaryOrHeading(t *testing.T) {
database := newSearchTestDB(t)
_, err := database.Exec(`
INSERT INTO objects(uuid,created_at) VALUES('o',1);
INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('d','/needle','o');
INSERT INTO revisions(uuid,object_uuid,filename,body,content_type,document_hash,created_at_ms,created_at)
VALUES('r','o','needle.html','<h1>PRIVATE_HEADING</h1><p>PRIVATE_SUMMARY</p>','text/html','hash',1000,1);
DROP TABLE revisions;
`)
if err != nil {
t.Fatal(err)
}
engine := NewSQLite(database)
deny := func(string) bool { return false }
ctx := context.Background()
// Candidates still exist in the independent index. Any attempt to hydrate
// denied summaries/headings now fails because their source table is absent.
if hits, err := engine.SearchPathsInDomainFiltered(ctx, "d", "/", "needle", 20, deny); err != nil || len(hits) != 0 {
t.Fatalf("denied summary loaded: %v %v", hits, err)
}
if hits, err := engine.SearchPathSuggestionsInDomain(ctx, "d", "/", "needle", 8, deny); err != nil || len(hits) != 0 {
t.Fatalf("denied heading loaded: %v %v", hits, err)
}
allow := func(string) bool { return true }
if _, err := engine.SearchPathsInDomainFiltered(ctx, "d", "/", "needle", 20, allow); err == nil {
t.Fatal("fixture did not exercise summary loading")
}
if _, err := engine.SearchPathSuggestionsInDomain(ctx, "d", "/", "needle", 8, allow); err == nil {
t.Fatal("fixture did not exercise heading loading")
}
}
func TestCancellationDuringPolicyNeverReturnsPartialSuccess(t *testing.T) {
database := newSearchTestDB(t)
if _, err := database.Exec(`INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('d','/needle','o')`); err != nil {
t.Fatal(err)
}
for _, excerpts := range []bool{false, true} {
ctx, cancel := context.WithCancel(context.Background())
allowed := func(string) bool { cancel(); return true }
// Filter-only search has no later FTS statement to notice cancellation.
hits, err := NewSQLite(database).searchPathsInDomainWithOptions(ctx, "d", "/", "", 20, allowed, SearchOptions{FileType: "other"}, excerpts)
cancel()
if !errors.Is(err, context.Canceled) || len(hits) != 0 {
t.Fatalf("partial success after cancellation: %v %v", hits, err)
}
}
}
func TestFTSExpressionTreatsOperatorsAsData(t *testing.T) {
for input, want := range map[string]string{
"": "", "alpha beta": `"alpha"* AND "beta"*`,
`a"b OR`: `"a""b"* AND "OR"*`,
`body:NEAR(x) -secret`: `"body:NEAR(x)"* AND "-secret"*`,
"日本語": `"日本語"*`,
} {
if got := ftsPrefixExpression(input); got != want {
t.Fatalf("%q: got %q want %q", input, got, want)
}
}
}
func TestSearchDoesNotMaintainIndexOnReadOnlyConnection(t *testing.T) {
database := newSearchTestDB(t)
database.SetMaxOpenConns(1)
if _, err := database.Exec(`INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('d','/needle','o'); PRAGMA query_only=ON`); err != nil {
t.Fatal(err)
}
engine := NewSQLite(database)
results, err := engine.SearchPathsInDomainFiltered(context.Background(), "d", "/", "needle", 20, func(string) bool { return true })
if err != nil || len(results) != 1 {
t.Fatalf("read-only search: %v %v", results, err)
}
// Schema maintenance is deliberately explicit, not performed by NewSQLite
// or a query. An accidental write in those paths would fail the test above.
if err := engine.EnsureSearchIndexSchema(); err == nil {
t.Fatal("read-only fixture unexpectedly accepted maintenance")
}
}
+34
View File
@@ -0,0 +1,34 @@
// Package mestionora owns bounded search, search projections and the SQLite
// search index. It consumes read-policy decisions; it never grants permissions.
// Search results and excerpts are untrusted text, not HTML-safe output.
package mestionora
import (
"database/sql"
"strings"
)
const Component = "mestionora"
// SQLite borrows a metadata connection pool. Construction performs no I/O or
// schema initialization. EnsureSearchIndexSchema is a separate startup step.
// The caller owns the pool lifetime and provisions the application schema.
type SQLite struct{ db *sql.DB }
func NewSQLite(db *sql.DB) *SQLite { return &SQLite{db: db} }
// Match the established Path scope rules, without filesystem path cleaning or
// URL decoding. Percent/underscore characters remain literal search data.
func normalizePath(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return "/"
}
if !strings.HasPrefix(value, "/") {
value = "/" + value
}
if value != "/" {
value = strings.TrimRight(value, "/")
}
return value
}
+40
View File
@@ -0,0 +1,40 @@
package ordoschnellic
import (
"go/parser"
"go/token"
"path/filepath"
"strconv"
"strings"
"testing"
)
func TestComponentBoundary(t *testing.T) {
if Component != "ordoschnellic" || PushEventType != "notification.push.deliver" {
t.Fatal("component/event identity changed")
}
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, file := range files {
if strings.HasSuffix(file, "_test.go") {
continue
}
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
if err != nil {
t.Fatal(err)
}
for _, spec := range ast.Imports {
path, err := strconv.Unquote(spec.Path.Value)
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"pageswiki/internal", "database", "os"} {
if path == forbidden || strings.HasPrefix(path, forbidden+"/") {
t.Errorf("%s imports %s: delivery must not own application policy, DB transactions or files", file, path)
}
}
}
}
}
+82
View File
@@ -0,0 +1,82 @@
package ordoschnellic
import (
"context"
"errors"
"time"
)
// Queue rechecks session/recipient eligibility in Claim. Empty Subscription
// means the claimed item is no longer deliverable and should be finalized.
// No outbound network operation runs inside a Queue transaction.
type Queue interface {
Cleanup(context.Context, time.Time) error
Claim(context.Context, time.Time) (*PushDelivery, error)
DeferFocused(context.Context, *PushDelivery, time.Time) error
Finish(context.Context, *PushDelivery, Outcome, time.Time) error
}
type Sender interface {
Send(context.Context, *PushDelivery) Outcome
}
type Stats struct{ Sent, Failed int }
type Dispatcher struct {
Queue Queue
Sender Sender
Now func() time.Time
}
func (d Dispatcher) Run(parent context.Context) (Stats, error) {
var stats Stats
if d.Queue == nil || d.Sender == nil {
return stats, errors.New("notification delivery dependencies are required")
}
if err := parent.Err(); err != nil {
return stats, err
}
now := d.Now
if now == nil {
now = time.Now
}
ctx, cancel := context.WithTimeout(parent, PushBatchTimeout)
defer cancel()
if err := d.Queue.Cleanup(ctx, now()); err != nil {
return stats, err
}
for i := 0; i < PushBatchLimit && ctx.Err() == nil; i++ {
item, err := d.Queue.Claim(ctx, now())
if err != nil {
return stats, err
}
if item == nil {
break
}
if item.Focused {
if err := d.Queue.DeferFocused(ctx, item, now()); err != nil {
return stats, err
}
continue
}
var outcome Outcome
if item.Subscription != "" {
outcome = d.Sender.Send(ctx, item)
if outcome.Attempted {
if outcome.Success {
stats.Sent++
} else {
stats.Failed++
}
}
}
// Preserve completion when the caller or outbound request is cancelled.
finishCtx, stop := context.WithTimeout(context.Background(), PushFinishTimeout)
err = d.Queue.Finish(finishCtx, item, outcome, now())
stop()
if err != nil {
return stats, err
}
}
return stats, parent.Err()
}
+150
View File
@@ -0,0 +1,150 @@
package ordoschnellic
import (
"context"
"errors"
"fmt"
"testing"
"time"
)
type queueFixture struct {
items []*PushDelivery
calls []string
errAt string
err error
finished []Outcome
finishContext func(context.Context)
at time.Time
t *testing.T
}
func (q *queueFixture) call(op string, at time.Time) error {
q.t.Helper()
if !at.Equal(q.at) {
q.t.Fatalf("clock for %s: %v", op, at)
}
q.calls = append(q.calls, op)
if op == q.errAt {
return q.err
}
return nil
}
func (q *queueFixture) Cleanup(_ context.Context, at time.Time) error { return q.call("cleanup", at) }
func (q *queueFixture) Claim(_ context.Context, at time.Time) (*PushDelivery, error) {
if err := q.call("claim", at); err != nil {
return nil, err
}
if len(q.items) == 0 {
return nil, nil
}
d := q.items[0]
q.items = q.items[1:]
return d, nil
}
func (q *queueFixture) DeferFocused(_ context.Context, _ *PushDelivery, at time.Time) error {
return q.call("defer", at)
}
func (q *queueFixture) Finish(ctx context.Context, _ *PushDelivery, o Outcome, at time.Time) error {
if q.finishContext != nil {
q.finishContext(ctx)
}
q.finished = append(q.finished, o)
return q.call("finish", at)
}
type senderFixture func(context.Context, *PushDelivery) Outcome
func (f senderFixture) Send(ctx context.Context, d *PushDelivery) Outcome { return f(ctx, d) }
func newQueueFixture(t *testing.T) *queueFixture {
return &queueFixture{t: t, at: time.Unix(1800000000, 0), err: errors.New("queue failure")}
}
func TestDispatcherBoundedBatchAndCounters(t *testing.T) {
q := newQueueFixture(t)
q.items = append(q.items, &PushDelivery{Focused: true}, &PushDelivery{})
for i := 0; i < 25; i++ {
q.items = append(q.items, &PushDelivery{Subscription: "valid"})
}
sends := 0
sender := senderFixture(func(context.Context, *PushDelivery) Outcome {
sends++
return Outcome{Attempted: true, Success: sends%2 == 0, Retry: sends%2 != 0}
})
stats, err := (Dispatcher{Queue: q, Sender: sender, Now: func() time.Time { return q.at }}).Run(context.Background())
if err != nil || stats != (Stats{Sent: 9, Failed: 9}) || sends != 18 || len(q.items) != 7 || len(q.finished) != 19 {
t.Fatalf("stats=%+v sends=%d pending=%d finished=%d err=%v", stats, sends, len(q.items), len(q.finished), err)
}
if q.finished[0] != (Outcome{}) {
t.Fatal("ineligible delivery was sent/retried")
}
if fmt.Sprint(q.calls[:5]) != "[cleanup claim defer claim finish]" {
t.Fatal(q.calls)
}
}
func TestDispatcherStopsOnQueueErrors(t *testing.T) {
for _, op := range []string{"cleanup", "claim", "defer", "finish"} {
t.Run(op, func(t *testing.T) {
q := newQueueFixture(t)
q.errAt = op
q.items = []*PushDelivery{{Subscription: "valid", Focused: op == "defer"}, {Subscription: "valid"}}
sends := 0
_, err := (Dispatcher{Queue: q, Sender: senderFixture(func(context.Context, *PushDelivery) Outcome { sends++; return Outcome{Attempted: true, Success: true} }), Now: func() time.Time { return q.at }}).Run(context.Background())
if !errors.Is(err, q.err) || q.calls[len(q.calls)-1] != op || sends > 1 {
t.Fatalf("calls=%v sends=%d err=%v", q.calls, sends, err)
}
})
}
}
func TestDispatcherCancellationFinalizesClaim(t *testing.T) {
q := newQueueFixture(t)
q.items = []*PushDelivery{{Subscription: "valid"}, {Subscription: "valid"}}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
q.finishContext = func(ctx context.Context) {
if ctx.Err() != nil {
t.Fatal("completion inherited cancellation")
}
deadline, ok := ctx.Deadline()
if !ok || time.Until(deadline) > PushFinishTimeout {
t.Fatal("completion is not bounded")
}
}
stats, err := (Dispatcher{Queue: q, Now: func() time.Time { return q.at }, Sender: senderFixture(func(ctx context.Context, _ *PushDelivery) Outcome {
if _, ok := ctx.Deadline(); !ok {
t.Fatal("send has no deadline")
}
cancel()
return Outcome{Attempted: true, Retry: true}
})}).Run(ctx)
if !errors.Is(err, context.Canceled) || stats.Failed != 1 || len(q.finished) != 1 || len(q.items) != 1 || !q.finished[0].Retry {
t.Fatalf("%+v %v %+v", stats, err, q.finished)
}
q.calls = nil
_, err = (Dispatcher{Queue: q, Sender: senderFixture(func(context.Context, *PushDelivery) Outcome { t.Fatal("cancelled batch sent"); return Outcome{} })}).Run(ctx)
if !errors.Is(err, context.Canceled) || len(q.calls) != 0 {
t.Fatal("cancelled batch accessed queue", err, q.calls)
}
}
func TestDeliveryPolicy(t *testing.T) {
at := time.Unix(1800000000, 0)
for _, attempt := range []int{-1, 0, 1, 2, 3, 4, 5, 1000000} {
next, ok := RetryAt(attempt, at)
if ok != (attempt >= 1 && attempt <= 4) {
t.Fatalf("attempt %d", attempt)
}
if ok && next.Sub(at) != time.Duration(1<<attempt)*time.Minute {
t.Fatal("backoff", attempt, next)
}
}
for _, code := range []int{200, 201, 204, 301, 400, 401, 403, 404, 410, 429, 500, 503} {
o := ClassifyPushStatus(code)
if !o.Attempted || o.Success != (code >= 200 && code < 300) || o.Retry != (code == 429 || code >= 500) || o.Expired != (code == 404 || code == 410) {
t.Fatal(code, o)
}
}
}
+26
View File
@@ -0,0 +1,26 @@
package ordoschnellic
import "time"
const (
PushBatchLimit = 20
PushBatchTimeout = 25 * time.Second
PushFinishTimeout = 3 * time.Second
PushLeaseDuration = time.Minute
PushFocusDelay = time.Minute
PushRetention = 24 * time.Hour
PushAttemptLimit = 5
)
// RetryAt preserves 2/4/8/16-minute backoff after attempts 1 through 4.
// Exhausted or invalid attempts are terminal and cannot overflow a shift.
func RetryAt(attempt int, now time.Time) (time.Time, bool) {
if attempt < 1 || attempt >= PushAttemptLimit {
return time.Time{}, false
}
return now.Add(time.Duration(1<<attempt) * time.Minute), true
}
func ClassifyPushStatus(status int) Outcome {
return Outcome{Success: status >= 200 && status < 300, Retry: status == 429 || status >= 500, Expired: status == 404 || status == 410, Attempted: true}
}
+38
View File
@@ -0,0 +1,38 @@
// Package ordoschnellic coordinates notification delivery. Business events,
// recipient authorization, Inbox transactions and durable claims belong to its
// callers. A transport acknowledgement is not proof of end-user receipt.
package ordoschnellic
import "errors"
const Component = "ordoschnellic"
const PushEventType = "notification.push.deliver"
type Notification struct {
UUID string `json:"uuid"`
DomainUUID string `json:"domain_uuid"`
RecipientUUID string `json:"recipient_uuid"`
EventUUID string `json:"event_uuid,omitempty"`
Kind string `json:"kind"`
Status string `json:"status"`
RequestUUID string `json:"request_uuid,omitempty"`
Href string `json:"href,omitempty"`
Message string `json:"message,omitempty"`
CreatedAtMS int64 `json:"created_at_ms"`
ReadAtMS int64 `json:"read_at_ms,omitempty"`
}
var ErrNotificationInput = errors.New("invalid notification")
type PushDelivery struct {
ID, Notification, Domain, Owner, Subscription string
Focused bool
Attempt int
}
type Outcome struct {
Success, Retry, Expired bool
// Attempted includes key/encryption setup failures, but excludes invalid
// stored subscriptions. This preserves the existing delivery counters.
Attempted bool
}
+86
View File
@@ -0,0 +1,86 @@
package ordoschnellic
import (
"context"
"crypto/ecdh"
"encoding/base64"
"encoding/json"
"net/url"
"strings"
webpush "github.com/SherClockHolmes/webpush-go"
)
type StoredBrowserPush struct {
webpush.Subscription
Origin string `json:"origin"`
}
// Only browser push vendors are destinations, never arbitrary user-provided URLs.
// DNS pinning/public-address checks at dial time provide a second SSRF boundary.
func ValidPushSubscription(sub webpush.Subscription) bool {
u, e := url.Parse(sub.Endpoint)
if e != nil || u.Scheme != "https" || u.User != nil || u.Fragment != "" || u.Port() != "" || len(sub.Endpoint) > 4096 {
return false
}
host := u.Hostname()
if host != "fcm.googleapis.com" && host != "updates.push.services.mozilla.com" && host != "web.push.apple.com" && !strings.HasSuffix(host, ".push.services.mozilla.com") {
return false
}
auth, e := base64.RawURLEncoding.DecodeString(sub.Keys.Auth)
if e != nil || len(auth) != 16 {
return false
}
key, e := base64.RawURLEncoding.DecodeString(sub.Keys.P256dh)
if e != nil {
return false
}
_, e = ecdh.P256().NewPublicKey(key)
return e == nil
}
type KeyProvider func(context.Context) (private, public string, err error)
// Client must enforce public-address DNS pinning, TLS, a bounded timeout and
// no redirects. There is deliberately no default HTTP client fallback.
type WebPush struct {
Keys KeyProvider
Client webpush.HTTPClient
}
// wakePayload intentionally has no access to Inbox titles, bodies or tokens.
func wakePayload(item *PushDelivery) []byte {
payload, _ := json.Marshal(map[string]string{"uuid": item.Notification, "owner": item.Owner, "domain": item.Domain})
return payload
}
func (w WebPush) Send(ctx context.Context, item *PushDelivery) Outcome {
if item == nil {
return Outcome{}
}
var sub StoredBrowserPush
if json.Unmarshal([]byte(item.Subscription), &sub) != nil || !ValidPushSubscription(sub.Subscription) {
return Outcome{Expired: true}
}
if w.Keys == nil || w.Client == nil || ctx.Err() != nil {
return Outcome{Retry: true, Attempted: true}
}
private, public, err := w.Keys(ctx)
if err != nil {
return Outcome{Retry: true, Attempted: true}
}
resp, err := webpush.SendNotificationWithContext(ctx, wakePayload(item), &sub.Subscription, &webpush.Options{HTTPClient: w.Client, Subscriber: sub.Origin, VAPIDPrivateKey: private, VAPIDPublicKey: public, TTL: 300, Urgency: webpush.UrgencyNormal, Topic: "pageswiki-inbox"})
if err != nil {
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
return Outcome{Retry: true, Attempted: true}
}
if resp == nil {
return Outcome{Retry: true, Attempted: true}
}
if resp.Body != nil {
defer resp.Body.Close()
}
return ClassifyPushStatus(resp.StatusCode)
}
+131
View File
@@ -0,0 +1,131 @@
package ordoschnellic
import (
"context"
"crypto/ecdh"
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"testing"
webpush "github.com/SherClockHolmes/webpush-go"
)
type httpFixture func(*http.Request) (*http.Response, error)
func (f httpFixture) Do(r *http.Request) (*http.Response, error) { return f(r) }
type trackedBody struct {
io.Reader
closed bool
}
func (b *trackedBody) Close() error { b.closed = true; return nil }
func subscriptionFixture(t *testing.T) StoredBrowserPush {
t.Helper()
key, err := ecdh.P256().GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
return StoredBrowserPush{Subscription: webpush.Subscription{Endpoint: "https://fcm.googleapis.com/fcm/send/synthetic", Keys: webpush.Keys{Auth: base64.RawURLEncoding.EncodeToString(make([]byte, 16)), P256dh: base64.RawURLEncoding.EncodeToString(key.PublicKey().Bytes())}}, Origin: "https://wiki.example"}
}
func deliveryFixture(t *testing.T, sub StoredBrowserPush) *PushDelivery {
t.Helper()
b, err := json.Marshal(sub)
if err != nil {
t.Fatal(err)
}
return &PushDelivery{Notification: "synthetic-notification", Owner: "synthetic-owner", Domain: "synthetic-domain", Subscription: string(b)}
}
func TestWebPushPrivacyAndResponseHandling(t *testing.T) {
private, public, err := webpush.GenerateVAPIDKeys()
if err != nil {
t.Fatal(err)
}
d := deliveryFixture(t, subscriptionFixture(t))
var payload map[string]string
if err := json.Unmarshal(wakePayload(d), &payload); err != nil || len(payload) != 3 || payload["uuid"] != d.Notification || payload["owner"] != d.Owner || payload["domain"] != d.Domain {
t.Fatal("payload contract", payload, err)
}
for _, code := range []int{201, 302, 400, 404, 410, 429, 503} {
body := &trackedBody{Reader: strings.NewReader("")}
w := WebPush{Keys: func(context.Context) (string, string, error) { return private, public, nil }, Client: httpFixture(func(r *http.Request) (*http.Response, error) {
data, err := io.ReadAll(r.Body)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(data), d.Notification) || strings.Contains(string(data), d.Owner) || r.Header.Get("Content-Encoding") != "aes128gcm" || r.Header.Get("TTL") != "300" || r.Header.Get("Topic") != "pageswiki-inbox" {
t.Fatal("unprotected/changed payload")
}
return &http.Response{StatusCode: code, Body: body}, nil
})}
if o := w.Send(context.Background(), d); o != ClassifyPushStatus(code) || !body.closed {
t.Fatal(code, o, body.closed)
}
}
}
func TestWebPushFailsClosedWithoutNetwork(t *testing.T) {
sub := subscriptionFixture(t)
d := deliveryFixture(t, sub)
for _, endpoint := range []string{"http://fcm.googleapis.com/x", "https://localhost/x", "https://127.0.0.1/x", "https://fcm.googleapis.com.evil.example/x", "https://fcm.googleapis.com:443/x", "https://user@fcm.googleapis.com/x", "https://web.push.apple.com/x#f"} {
sub.Endpoint = endpoint
w := WebPush{Keys: func(context.Context) (string, string, error) {
t.Fatal("unsafe target requested keys")
return "", "", nil
}, Client: httpFixture(func(*http.Request) (*http.Response, error) { t.Fatal("unsafe target contacted"); return nil, nil })}
if o := w.Send(context.Background(), deliveryFixture(t, sub)); !o.Expired || o.Attempted {
t.Fatal(endpoint, o)
}
}
for _, w := range []WebPush{{}, {Keys: func(context.Context) (string, string, error) {
t.Fatal("no HTTP client must fail closed")
return "", "", nil
}}, {Client: httpFixture(func(*http.Request) (*http.Response, error) { t.Fatal("no keys must fail closed"); return nil, nil })}} {
if o := w.Send(context.Background(), d); !o.Retry || !o.Attempted {
t.Fatal(o)
}
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
w := WebPush{Keys: func(context.Context) (string, string, error) {
t.Fatal("cancelled request loaded keys")
return "", "", nil
}, Client: httpFixture(func(*http.Request) (*http.Response, error) { t.Fatal("cancelled request sent"); return nil, nil })}
if o := w.Send(ctx, d); !o.Retry {
t.Fatal(o)
}
}
func TestWebPushTransportFailure(t *testing.T) {
private, public, err := webpush.GenerateVAPIDKeys()
if err != nil {
t.Fatal(err)
}
d := deliveryFixture(t, subscriptionFixture(t))
for _, mode := range []string{"keys", "network", "nil-response"} {
w := WebPush{Keys: func(context.Context) (string, string, error) {
if mode == "keys" {
return "", "", errors.New("secret must not escape")
}
return private, public, nil
}, Client: httpFixture(func(*http.Request) (*http.Response, error) {
if mode == "keys" {
t.Fatal("sent after key failure")
}
if mode == "network" {
return nil, errors.New("endpoint must not escape")
}
return nil, nil
})}
if o := w.Send(context.Background(), d); o != (Outcome{Retry: true, Attempted: true}) {
t.Fatal(mode, o)
}
}
}
+4 -10
View File
@@ -1,15 +1,9 @@
package render
import (
"github.com/gomarkdown/markdown"
)
import "pageswiki/internal/grammaratur"
// Markdown retains the renderer entry point while Grammaratur owns conversion.
// It deliberately does not enable authoring fences or sanitize display HTML.
func Markdown(input string) string {
out := markdown.ToHTML(
[]byte(input),
nil,
nil,
)
return string(out)
return grammaratur.MarkdownToHTML(input)
}
+114
View File
@@ -0,0 +1,114 @@
// Package schutzaria evaluates trusted, request-scoped authorization facts.
// It does not authenticate callers, load grants, resolve aliases, or commit
// publication. Callers must supply the correct Domain/principal snapshot.
package schutzaria
import "strings"
const Component = "schutzaria"
// AccessLayer is the union of matching subjects at one locality. Layers at
// the same Path and at ancestors intersect; Grant cannot override a denial.
type AccessLayer struct {
Path string
Read, Write, Grant bool
}
// LocalityAccess is immutable after construction and must not be cached
// across requests. Admin bypasses caps only; it does not create base grants.
type LocalityAccess struct {
admin bool
paths map[string][]AccessLayer
}
func NewLocalityAccess(admin bool, layers []AccessLayer) *LocalityAccess {
p := &LocalityAccess{admin: admin, paths: make(map[string][]AccessLayer)}
for _, layer := range layers {
p.paths[layer.Path] = append(p.paths[layer.Path], layer)
}
return p
}
// RequiresPathChecks permits the existing Object-alias fast path only when
// no local cap applies or the authenticated actor has recovery authority.
// Foreign-Domain restrictions must be checked separately before this shortcut.
func (p *LocalityAccess) RequiresPathChecks() bool {
return p != nil && !p.admin && len(p.paths) != 0
}
func (p *LocalityAccess) Decide(path, capability string, basis bool) bool {
if !strings.HasPrefix(path, "/") {
return false
}
if p == nil || p.admin {
return basis
}
grant := false
for at := path; ; {
for _, layer := range p.paths[at] {
allowed := layer.Write
if capability == "read" {
allowed = layer.Read
}
if !allowed {
return false
}
if layer.Grant && (capability == "read" || capability == "write" || capability == "create" || capability == "upload") {
grant = true
}
}
if at == "/" {
break
}
i := strings.LastIndex(at, "/")
if i <= 0 {
at = "/"
} else {
at = at[:i]
}
}
return basis || grant
}
// ReadPathPolicy contains only positive Path grants plus locality caps.
// Object grants must never expose all semantic aliases. A nil/zero policy denies.
type ReadPathPolicy struct {
access *LocalityAccess
all bool
exact, subtrees map[string]bool
}
// NewReadPathPolicy copies input slices; subsequent mutations cannot broaden
// an existing snapshot. Paths are already normalized by the application.
func NewReadPathPolicy(all bool, exact, subtrees []string, access *LocalityAccess) *ReadPathPolicy {
p := &ReadPathPolicy{all: all, access: access, exact: make(map[string]bool), subtrees: make(map[string]bool)}
for _, path := range exact {
p.exact[path] = true
}
for _, path := range subtrees {
p.subtrees[path] = true
}
return p
}
func (p *ReadPathPolicy) Allows(path string) bool {
if p == nil {
return false
}
return p.access.Decide(path, "read", p.allowsBase(path))
}
func (p *ReadPathPolicy) allowsBase(path string) bool {
if p == nil || !strings.HasPrefix(path, "/") {
return false
}
if p.all || p.exact[path] {
return true
}
for at := strings.LastIndex(path, "/"); at > 0; at = strings.LastIndex(path[:at], "/") {
if p.subtrees[path[:at]] {
return true
}
}
return false
}
+125
View File
@@ -0,0 +1,125 @@
package schutzaria
import (
"sync"
"testing"
)
func TestLocalityAccessIntersectionAndGrantLimits(t *testing.T) {
for _, parentRead := range []bool{false, true} {
for _, parentWrite := range []bool{false, true} {
for _, childRead := range []bool{false, true} {
for _, childWrite := range []bool{false, true} {
for _, grant := range []bool{false, true} {
for _, basis := range []bool{false, true} {
p := NewLocalityAccess(false, []AccessLayer{{Path: "/life", Read: parentRead, Write: parentWrite}, {Path: "/life/home", Read: childRead, Write: childWrite, Grant: grant}})
for _, cap := range []string{"read", "write", "create", "upload", "rename", "link", "delete", "manage"} {
allowed := parentWrite && childWrite
if cap == "read" {
allowed = parentRead && childRead
}
positive := grant && (cap == "read" || cap == "write" || cap == "create" || cap == "upload")
want := allowed && (basis || positive)
if got := p.Decide("/life/home/page", cap, basis); got != want {
t.Fatalf("parents=%v/%v children=%v/%v grant=%v base=%v cap=%s got=%v", parentRead, parentWrite, childRead, childWrite, grant, basis, cap, got)
}
}
if got := p.Decide("/lifes/other", "read", basis); got != basis {
t.Fatal("prefix boundary leaked")
}
}
}
}
}
}
}
// Multiple locality nodes may share a boundary; one denial still wins.
p := NewLocalityAccess(false, []AccessLayer{{Path: "/same", Read: true, Write: true, Grant: true}, {Path: "/same"}})
if p.Decide("/same/child", "read", true) {
t.Fatal("same-Path denial lost")
}
}
func TestLocalityAdminAndAbsentCapsNeverCreateGrants(t *testing.T) {
for _, p := range []*LocalityAccess{nil, NewLocalityAccess(false, nil), NewLocalityAccess(true, []AccessLayer{{Path: "/"}})} {
for _, basis := range []bool{false, true} {
if p.Decide("/a", "read", basis) != basis || p.Decide("relative", "read", basis) || p.Decide("", "read", basis) {
t.Fatal("admin/absent cap created a grant or allowed non-Path")
}
}
if p.RequiresPathChecks() {
t.Fatal("unexpected alias checks")
}
}
if !NewLocalityAccess(false, []AccessLayer{{Path: "/a"}}).RequiresPathChecks() {
t.Fatal("Object alias checks skipped")
}
}
func TestReadPolicyLiteralScopesAndImmutability(t *testing.T) {
exact := []string{"/exact", "/group_%"}
subtrees := []string{"/group_%"}
layers := []AccessLayer{{Path: "/group_%/private"}}
access := NewLocalityAccess(false, layers)
p := NewReadPathPolicy(false, exact, subtrees, access)
exact[0] = "/leak"
subtrees[0] = "/"
layers[0] = AccessLayer{Path: "/group_%/private", Read: true, Grant: true}
for path, want := range map[string]bool{"/exact": true, "/exact/child": false, "/group_%": true, "/group_%/a/b": true, "/group_%/private": false, "/group_%/private/child": false, "/groupXX/a": false, "/group_%2": false, "/leak": false, "/": false, "": false, "relative": false} {
if got := p.Allows(path); got != want {
t.Fatalf("path=%q got=%v want=%v", path, got, want)
}
}
for _, deny := range []*ReadPathPolicy{nil, {}} {
if deny.Allows("/anything") {
t.Fatal("empty policy failed open")
}
}
all := NewReadPathPolicy(true, nil, nil, access)
if !all.Allows("/other") || all.Allows("/group_%/private") || all.Allows("relative") {
t.Fatal("broad grant bypassed caps")
}
var wg sync.WaitGroup
for i := 0; i < 16; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 100; j++ {
if !p.Allows("/exact") || p.Allows("/leak") {
t.Error("snapshot changed")
}
}
}()
}
wg.Wait()
}
func TestGroupPermissionProfiles(t *testing.T) {
wants := map[string]map[string]bool{
"read": {"read": true},
"read_write_once": {"read": true, "create": true, "upload": true},
"read_write": {"read": true, "create": true, "write": true, "upload": true, "rename": true, "link": true},
"upload_only": {"upload": true},
"unknown": {},
}
for profile, want := range wants {
for _, cap := range []string{"read", "create", "write", "upload", "rename", "link", "delete", "manage", ""} {
for _, path := range []string{"/a_%", "/a_%/child"} {
if GroupPermissionAllows("/a_%", profile, cap, path) != want[cap] {
t.Fatal(profile, cap, path)
}
}
if GroupPermissionAllows("/a_%", profile, cap, "/aXX/child") {
t.Fatal("wildcard interpreted")
}
}
}
if p, err := NormalizeGroupPermission(" READ_WRITE_ONCE "); err != nil || p != "read_write_once" {
t.Fatal(p, err)
}
for _, p := range []string{"", "delete", "group_admin", "read/write"} {
if _, err := NormalizeGroupPermission(p); err == nil {
t.Fatal("unknown profile", p)
}
}
}
+22
View File
@@ -0,0 +1,22 @@
package schutzaria
import "errors"
var ErrApprovalTransition = errors.New("invalid approval request transition")
func ValidApprovalDecision(decision string) bool {
return decision == "approved" || decision == "rejected" || decision == "changes_requested"
}
// ApprovalNextState preserves the existing request state machine only.
// It does not verify reviewer authority, expiry, quorum, source revisions or
// audience. The caller owns those checks and the transactional CAS/write.
func ApprovalNextState(state string, currentGeneration, generation int64, decision string) (string, error) {
if generation < 1 || state != "pending" || currentGeneration != generation || !ValidApprovalDecision(decision) {
return "", ErrApprovalTransition
}
if decision == "changes_requested" {
return "rejected", nil
}
return decision, nil
}
+40
View File
@@ -0,0 +1,40 @@
package schutzaria
import (
"go/parser"
"go/token"
"path/filepath"
"strconv"
"strings"
"testing"
)
func TestComponentBoundary(t *testing.T) {
if Component != "schutzaria" {
t.Fatal("component identity changed")
}
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, file := range files {
if strings.HasSuffix(file, "_test.go") {
continue
}
ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly)
if err != nil {
t.Fatal(err)
}
for _, spec := range ast.Imports {
path, err := strconv.Unquote(spec.Path.Value)
if err != nil {
t.Fatal(err)
}
for _, prefix := range []string{"pageswiki/internal", "database", "net", "os", "time"} {
if path == prefix || strings.HasPrefix(path, prefix+"/") {
t.Errorf("%s imports %s: evaluation must not acquire state, perform I/O or read time", file, path)
}
}
}
}
}
+29
View File
@@ -0,0 +1,29 @@
package schutzaria
import (
"errors"
"strings"
)
var groupPermissions = map[string]map[string]bool{
"read": {"read": true},
"read_write_once": {"read": true, "create": true, "upload": true},
"read_write": {"read": true, "create": true, "write": true, "upload": true, "rename": true, "link": true},
"upload_only": {"upload": true},
}
func NormalizeGroupPermission(value string) (string, error) {
value = strings.ToLower(strings.TrimSpace(value))
if _, ok := groupPermissions[value]; !ok {
return "", errors.New("invalid group permission")
}
return value, nil
}
// GroupPermissionAllows evaluates a previously authenticated membership's
// profile. Creation-only existence checks and principal state belong to callers.
// This literal prefix rule deliberately preserves the existing stored scopes.
func GroupPermissionAllows(scope, permission, capability, targetPath string) bool {
inside := targetPath == scope || scope == "/" || strings.HasPrefix(targetPath, strings.TrimRight(scope, "/")+"/")
return inside && groupPermissions[permission][capability]
}
+52
View File
@@ -0,0 +1,52 @@
package schutzaria
import "strings"
type LocalityPolicy struct {
Kind string `json:"kind"`
UUID string `json:"uuid"`
Policy string `json:"policy"`
}
// EvaluateLocalityPublication combines a Domain-first chain's ceilings and
// intent. Neither result is an anonymous-read grant or a reviewer approval.
// The caller loads/validates topology and binds immutable revisions separately.
func EvaluateLocalityPublication(chain []LocalityPolicy) (allows, published bool) {
allows = true
for i, p := range chain {
if i == 0 && p.Policy == "inherit" {
return false, false
}
switch p.Policy {
case "authenticated_acl":
allows = false
case "public":
if i > 0 {
published = true
}
case "inherit":
default:
return false, false
}
}
return allows, allows && published
}
// PublicSemanticPath is a namespace eligibility check, never permission to
// publish content. User-space publication has its own authenticated policy path.
func PublicSemanticPath(path string) bool {
if !strings.HasPrefix(path, "/") || strings.ContainsAny(path, "\\\x00\r\n") || strings.Contains(path, "//") {
return false
}
for _, segment := range strings.Split(path, "/") {
if segment == "." || segment == ".." {
return false
}
}
first := strings.Split(strings.TrimPrefix(path, "/"), "/")[0]
switch first {
case "_", "-", ":", "o", "r", "v", "l", "e", "s", "u", "g", "static", "health", "new":
return false
}
return true
}
+89
View File
@@ -0,0 +1,89 @@
package schutzaria
import (
"errors"
"testing"
)
func TestPublicationCeilings(t *testing.T) {
values := []string{"public", "authenticated_acl", "inherit", "invalid"}
for _, domain := range values {
for _, quartier := range values {
for _, street := range values {
for _, spot := range values {
chain := []LocalityPolicy{{Kind: "domain", Policy: domain}, {Kind: "quartier", Policy: quartier}, {Kind: "street", Policy: street}, {Kind: "spot", Policy: spot}}
wantAllows := domain == "public" && quartier != "authenticated_acl" && street != "authenticated_acl" && spot != "authenticated_acl" && quartier != "invalid" && street != "invalid" && spot != "invalid"
wantIntent := wantAllows && (quartier == "public" || street == "public" || spot == "public")
a, p := EvaluateLocalityPublication(chain)
if a != wantAllows || p != wantIntent {
t.Fatalf("%+v got=%v/%v", chain, a, p)
}
}
}
}
}
// Preserve the legacy neutral empty-chain result; it is never a grant.
if a, p := EvaluateLocalityPublication(nil); !a || p {
t.Fatal("empty chain semantics changed")
}
if a, p := EvaluateLocalityPublication([]LocalityPolicy{{Kind: "domain", Policy: "public"}}); !a || p {
t.Fatal("Domain ceiling published content")
}
}
func TestPublicNamespaceEligibility(t *testing.T) {
for _, p := range []string{"/", "/life/home", "/blog/2026/11/01", "/日本語", "/user"} {
if !PublicSemanticPath(p) {
t.Fatal(p)
}
}
for _, p := range []string{"", "relative", "/a/../b", "/a/./b", "/a//b", "/a\\b", "/a\x00b", "/a\nb", "/a\rb"} {
if PublicSemanticPath(p) {
t.Fatal(p)
}
}
for _, root := range []string{"_", "-", ":", "o", "r", "v", "l", "e", "s", "u", "g", "static", "health", "new"} {
if PublicSemanticPath("/"+root) || PublicSemanticPath("/"+root+"/child") {
t.Fatal(root)
}
}
}
func TestApprovalStateTransitionNotPublicationAuthorization(t *testing.T) {
for _, state := range []string{"pending", "approved", "rejected", "withdrawn", "expired", "superseded", ""} {
for _, decision := range []string{"approved", "rejected", "changes_requested", "unknown", ""} {
for _, generation := range []int64{-1, 0, 1, 2} {
next, err := ApprovalNextState(state, 1, generation, decision)
valid := state == "pending" && generation == 1 && (decision == "approved" || decision == "rejected" || decision == "changes_requested")
if valid {
want := decision
if want == "changes_requested" {
want = "rejected"
}
if err != nil || next != want {
t.Fatal(state, decision, next, err)
}
} else if !errors.Is(err, ErrApprovalTransition) || next != "" {
t.Fatal(state, decision, generation, next, err)
}
}
}
}
}
func TestUserSpaceNarrowing(t *testing.T) {
p := UserSpacePolicy{Username: "alice", RootVisibility: "domain", DescendantVisibility: "public", Overrides: []UserSpacePathPolicy{{Path: "/u/alice/work", Visibility: "domain"}, {Path: "/u/alice/work/private", Visibility: "private"}, {Path: "/u/alice/work/private/reopen", Visibility: "public"}}}
for path, want := range map[string]string{"/u/alice": "domain", "/u/alice/blog": "public", "/u/alice/work": "domain", "/u/alice/work/page": "domain", "/u/alice/work/private": "private", "/u/alice/work/private/reopen/page": "private", "/u/bob/blog": "private", "/u/alice/_/edit": "private", "/u/alice/a/../blog": "private", "/u/alice/blog/": "private"} {
if got := EffectiveUserSpaceVisibility(p, path); got != want {
t.Fatal(path, got, want)
}
}
// Existing semantics: root and descendants have independent settings.
p.RootVisibility = "private"
if EffectiveUserSpaceVisibility(p, "/u/alice/blog") != "public" {
t.Fatal("root incorrectly became subtree cap")
}
if UserSpaceVisibilityRank("unknown") != -1 || UserSpaceDocumentPath("alice", "/u/alice/_", true) {
t.Fatal("invalid user-space policy/path")
}
}
+70
View File
@@ -0,0 +1,70 @@
package schutzaria
import (
"path"
"strings"
)
type UserSpacePathPolicy struct {
Path string `json:"path"`
Visibility string `json:"visibility"`
}
type UserSpacePolicy struct {
PrincipalUUID string `json:"principal_uuid"`
Username string `json:"username"`
ProfileVisibility string `json:"profile_visibility"`
AvatarVisibility string `json:"avatar_visibility"`
RootVisibility string `json:"root_visibility"`
DescendantVisibility string `json:"descendant_visibility"`
Version int64 `json:"version"`
DomainAllowsPublic bool `json:"domain_allows_public"`
Overrides []UserSpacePathPolicy `json:"overrides"`
}
func UserSpaceVisibilityRank(value string) int {
switch value {
case "private":
return 0
case "domain":
return 1
case "public":
return 2
default:
return -1
}
}
func UserSpaceDocumentPath(username, value string, descendant bool) bool {
root := "/u/" + username
if value != path.Clean(value) || strings.ContainsAny(value, "\\\x00\r\n") {
return false
}
if descendant {
if !strings.HasPrefix(value, root+"/") {
return false
}
rest := strings.TrimPrefix(value, root+"/")
return rest != "" && !strings.HasPrefix(rest, "_/") && rest != "_"
}
return value == root
}
func EffectiveUserSpaceVisibility(p UserSpacePolicy, target string) string {
root := "/u/" + p.Username
if target == root {
return p.RootVisibility
}
if !UserSpaceDocumentPath(p.Username, target, true) {
return "private"
}
visibility := p.DescendantVisibility
for _, item := range p.Overrides {
if target == item.Path || strings.HasPrefix(target, strings.TrimRight(item.Path, "/")+"/") {
if UserSpaceVisibilityRank(item.Visibility) < UserSpaceVisibilityRank(visibility) {
visibility = item.Visibility
}
}
}
return visibility
}
+39
View File
@@ -0,0 +1,39 @@
package seheweit
import "time"
const (
ResourceEventID = "builtin.resource.sample"
ResourceEventType = "resource.sample"
RetentionDays = 7
MaxHistoryPoints = 361
)
// HistoryWindow describes the existing inclusive, epoch-aligned sample query.
// Persistence chooses the most recently inserted sample in each bucket, not an
// average, a peak or necessarily the largest timestamp in out-of-order data.
type HistoryWindow struct {
FromMS int64
ToMS int64
BucketMS int64
}
func Window(hours int, now time.Time) HistoryWindow {
if hours < 1 {
hours = 1
}
if hours > RetentionDays*24 {
hours = RetentionDays * 24
}
span := int64(time.Duration(hours) * time.Hour / time.Millisecond)
bucket := span / (MaxHistoryPoints - 1)
if bucket < int64(time.Minute/time.Millisecond) {
bucket = int64(time.Minute / time.Millisecond)
}
return HistoryWindow{FromMS: now.UnixMilli() - span, ToMS: now.UnixMilli(), BucketMS: bucket}
}
// RetentionCutoff is inclusive: only samples strictly older are expired.
func RetentionCutoff(atMS int64) int64 {
return atMS - int64(RetentionDays*24*time.Hour/time.Millisecond)
}
+31
View File
@@ -0,0 +1,31 @@
package seheweit
import (
"testing"
"time"
)
func TestHistoryWindow(t *testing.T) {
now := time.Date(2026, 11, 1, 12, 34, 56, 0, time.UTC)
for _, tc := range []struct {
hours, wantHours int
bucket int64
}{
{-1, 1, 60000}, {0, 1, 60000}, {1, 1, 60000}, {24, 24, 240000}, {168, 168, 1680000}, {999, 168, 1680000},
} {
got := Window(tc.hours, now)
if got.ToMS != now.UnixMilli() || got.FromMS != now.Add(-time.Duration(tc.wantHours)*time.Hour).UnixMilli() || got.BucketMS != tc.bucket {
t.Fatalf("hours=%d %+v", tc.hours, got)
}
buckets := got.ToMS/got.BucketMS - got.FromMS/got.BucketMS + 1
if buckets > MaxHistoryPoints {
t.Fatalf("too many buckets: %d", buckets)
}
}
if RetentionCutoff(now.UnixMilli()) != now.Add(-7*24*time.Hour).UnixMilli() {
t.Fatal("retention changed")
}
if ResourceEventID != "builtin.resource.sample" || ResourceEventType != "resource.sample" {
t.Fatal("persisted identifiers changed")
}
}
+177
View File
@@ -0,0 +1,177 @@
// Package seheweit collects instance observations without controlling the
// observed workload. Authorization, queue execution and persistence belong to
// its callers. It does not inspect document contents or contact remote services.
package seheweit
import (
"errors"
"io"
"math"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"syscall"
"time"
"github.com/google/uuid"
)
// Identity survives multiple collectors, but not a process restart/PID reuse.
var processID = uuid.NewString()
type ProcessIO struct {
Scope string `json:"scope"`
Available bool `json:"available"`
ProcessID string `json:"process_id"`
ReadBytes uint64 `json:"read_bytes"`
WriteBytes uint64 `json:"write_bytes"`
}
type CPU struct {
LogicalCPUs int `json:"logical_cpus"`
LoadAverage string `json:"load_average"`
}
type Memory struct {
HeapBytes uint64 `json:"heap_bytes"`
GoSystemBytes uint64 `json:"go_system_bytes"`
GCCycles uint32 `json:"gc_cycles"`
Goroutines int `json:"goroutines"`
}
type Storage struct {
Path string `json:"path"`
TotalBytes uint64 `json:"total_bytes"`
UsedBytes uint64 `json:"used_bytes"`
AvailableBytes uint64 `json:"available_bytes"`
}
// Queue contains observations supplied by the application, not execution hooks.
type Queue struct {
Stats QueueStats `json:"stats"`
ActiveWorkers int64 `json:"active_workers"`
Paused bool `json:"paused"`
}
type QueueStats struct {
Queued int `json:"queued"`
Running int `json:"running"`
}
// Snapshot preserves the resource API and history JSON field names.
// Sources are sampled sequentially, not in an atomic cross-source transaction.
type Snapshot struct {
TimestampMS int64 `json:"timestamp_ms"`
IO ProcessIO `json:"io"`
CPU CPU `json:"cpu"`
Memory Memory `json:"memory"`
Storage Storage `json:"storage"`
Queue Queue `json:"queue"`
}
type sources struct {
now func() time.Time
readFile func(string) ([]byte, error)
statfs func(string, *syscall.Statfs_t) error
memory func(*runtime.MemStats)
cpus func() int
goroutines func() int
}
// Collect reads Go runtime counters, two bounded proc files and Statfs on the
// configured document root. No directory walk, shell or background loop.
func Collect(root string, queue Queue) Snapshot {
return collect(root, queue, sources{
now: time.Now, readFile: readProcFile, statfs: syscall.Statfs,
memory: runtime.ReadMemStats, cpus: runtime.NumCPU, goroutines: runtime.NumGoroutine,
})
}
func collect(root string, queue Queue, src sources) Snapshot {
var mem runtime.MemStats
src.memory(&mem)
if root == "" {
root = "."
}
if absolute, err := filepath.Abs(root); err == nil {
root = absolute
}
storage := Storage{Path: root}
var fs syscall.Statfs_t
if err := src.statfs(root, &fs); err == nil {
storage = storageCounters(root, fs.Blocks, fs.Bavail, fs.Bsize)
}
loadAverage := ""
if data, err := src.readFile("/proc/loadavg"); err == nil {
fields := strings.Fields(string(data))
if len(fields) >= 3 {
loadAverage = strings.Join(fields[:3], " ")
}
}
var ioData []byte
if data, err := src.readFile("/proc/self/io"); err == nil {
ioData = data
}
return Snapshot{
TimestampMS: src.now().UnixMilli(), IO: ParseProcessIO(string(ioData)),
CPU: CPU{LogicalCPUs: src.cpus(), LoadAverage: loadAverage},
Memory: Memory{HeapBytes: mem.HeapAlloc, GoSystemBytes: mem.Sys, GCCycles: mem.NumGC, Goroutines: src.goroutines()},
Storage: storage, Queue: queue,
}
}
func storageCounters(path string, blocks, available uint64, blockSize int64) Storage {
result := Storage{Path: path}
// Preserve the existing zero fallback rather than wrapping unsigned values.
if blockSize <= 0 || available > blocks || blocks > math.MaxUint64/uint64(blockSize) {
return result
}
result.TotalBytes = blocks * uint64(blockSize)
result.AvailableBytes = available * uint64(blockSize)
result.UsedBytes = result.TotalBytes - result.AvailableBytes
return result
}
const maxProcBytes = 16 << 10
func readProcFile(path string) ([]byte, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
data, err := io.ReadAll(io.LimitReader(f, maxProcBytes+1))
if err != nil {
return nil, err
}
if len(data) > maxProcBytes {
return nil, errors.New("resource counter file too large")
}
return data, nil
}
// ParseProcessIO uses actual storage bytes, not rchar/wchar (syscall traffic).
// Partial, missing or invalid counters are unavailable, not a zero I/O rate.
func ParseProcessIO(data string) ProcessIO {
result := ProcessIO{Scope: "process", ProcessID: processID}
read, write := false, false
for _, line := range strings.Split(data, "\n") {
fields := strings.Fields(line)
if len(fields) != 2 || (fields[0] != "read_bytes:" && fields[0] != "write_bytes:") {
continue
}
value, err := strconv.ParseUint(fields[1], 10, 64)
if err != nil {
return result
}
if fields[0] == "read_bytes:" {
result.ReadBytes, read = value, true
} else {
result.WriteBytes, write = value, true
}
}
result.Available = read && write
return result
}
+171
View File
@@ -0,0 +1,171 @@
package seheweit
import (
"encoding/json"
"errors"
"math"
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
"syscall"
"testing"
"time"
)
func fixtureSources(t *testing.T) sources {
t.Helper()
return sources{
now: func() time.Time { return time.UnixMilli(123456789) },
readFile: func(path string) ([]byte, error) {
switch path {
case "/proc/loadavg":
return []byte("0.01 0.20 3.00 1/99 1\n"), nil
case "/proc/self/io":
return []byte("rchar: 999\nread_bytes: 10\nwrite_bytes: 20\n"), nil
default:
t.Fatalf("unexpected file read: %s", path)
return nil, nil
}
},
statfs: func(path string, fs *syscall.Statfs_t) error {
if !filepath.IsAbs(path) {
t.Fatalf("relative root %q", path)
}
fs.Blocks, fs.Bavail, fs.Bsize = 100, 30, 4096
return nil
},
memory: func(m *runtime.MemStats) { m.HeapAlloc, m.Sys, m.NumGC = 1024, 4096, 9 },
cpus: func() int { return 8 }, goroutines: func() int { return 17 },
}
}
func TestSnapshotWireCompatibility(t *testing.T) {
queue := Queue{Stats: QueueStats{Queued: 2, Running: 1}, ActiveWorkers: 1, Paused: true}
snapshot := collect(t.TempDir(), queue, fixtureSources(t))
data, err := json.Marshal(snapshot)
if err != nil {
t.Fatal(err)
}
// Exact pre-extraction wire shape, including zeros/false (no omitempty).
want := map[string]any{
"timestamp_ms": int64(123456789),
"io": map[string]any{"scope": "process", "available": true, "process_id": processID, "read_bytes": 10, "write_bytes": 20},
"cpu": map[string]any{"logical_cpus": 8, "load_average": "0.01 0.20 3.00"},
"memory": map[string]any{"heap_bytes": 1024, "go_system_bytes": 4096, "gc_cycles": 9, "goroutines": 17},
"storage": map[string]any{"path": snapshot.Storage.Path, "total_bytes": 409600, "used_bytes": 286720, "available_bytes": 122880},
"queue": map[string]any{"stats": map[string]int{"queued": 2, "running": 1}, "active_workers": 1, "paused": true},
}
wantData, _ := json.Marshal(want)
var gotJSON, wantJSON any
if err := json.Unmarshal(data, &gotJSON); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(wantData, &wantJSON); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(gotJSON, wantJSON) {
t.Fatalf("wire changed: %s != %s", data, wantData)
}
zero, _ := json.Marshal(Snapshot{})
if strings.Contains(string(zero), "null") || !strings.Contains(string(zero), `"paused":false`) {
t.Fatalf("zero shape %s", zero)
}
}
func TestCollectionFailureAndIsolation(t *testing.T) {
src := fixtureSources(t)
src.readFile = func(string) ([]byte, error) {
return []byte("read_bytes: 5\nwrite_bytes: 6\n"), errors.New("unavailable")
}
src.statfs = func(_ string, fs *syscall.Statfs_t) error {
fs.Blocks, fs.Bavail, fs.Bsize = 100, 20, 4096
return errors.New("unavailable")
}
got := collect("", Queue{}, src)
if !filepath.IsAbs(got.Storage.Path) || got.Storage.TotalBytes != 0 || got.IO.Available || got.IO.ReadBytes != 0 || got.CPU.LoadAverage != "" {
t.Fatalf("failed source leaked partial values: %+v", got)
}
if got.Memory.HeapBytes != 1024 || got.CPU.LogicalCPUs != 8 || got.IO.ProcessID == "" {
t.Fatalf("independent sources lost: %+v", got)
}
// Each call has independent values; the process identity alone is shared.
for i := 0; i < 16; i++ {
t.Run("parallel", func(t *testing.T) {
t.Parallel()
other := collect("", Queue{}, fixtureSources(t))
if other.IO.ProcessID != got.IO.ProcessID || other.Storage.TotalBytes != 409600 {
t.Fatalf("identity/isolation %+v", other)
}
})
}
}
func TestStorageInvalidCounters(t *testing.T) {
for _, tc := range []struct {
blocks, available uint64
size int64
}{
{10, 11, 4096}, {10, 1, -1}, {10, 1, 0}, {math.MaxUint64, 1, 4096},
} {
if got := storageCounters("/root", tc.blocks, tc.available, tc.size); got != (Storage{Path: "/root"}) {
t.Fatalf("invalid storage wrapped: %+v", got)
}
}
if got := storageCounters("/root", math.MaxUint64, 0, 1); got.UsedBytes != math.MaxUint64 {
t.Fatalf("valid boundary %+v", got)
}
}
func TestProcessIOParser(t *testing.T) {
for _, tc := range []struct {
name, text string
available bool
read, write uint64
}{
{"valid", "read_bytes: 12\nwrite_bytes: 34\nrchar: 999\ncancelled_write_bytes: 99", true, 12, 34},
{"zero", "read_bytes: 0\nwrite_bytes: 0", true, 0, 0},
{"empty", "", false, 0, 0},
{"partial", "read_bytes: 12", false, 12, 0},
{"negative", "read_bytes: -1\nwrite_bytes: 34", false, 0, 0},
{"overflow", "read_bytes: 18446744073709551616\nwrite_bytes: 0", false, 0, 0},
{"invalid", "read_bytes: 12\nwrite_bytes: bad", false, 12, 0},
{"max", "read_bytes: 18446744073709551615\nwrite_bytes: 0", true, math.MaxUint64, 0},
} {
t.Run(tc.name, func(t *testing.T) {
got := ParseProcessIO(tc.text)
if got.Available != tc.available || got.ReadBytes != tc.read || got.WriteBytes != tc.write || got.Scope != "process" || got.ProcessID != processID {
t.Fatalf("%+v", got)
}
})
}
}
func TestProcFileBound(t *testing.T) {
path := filepath.Join(t.TempDir(), "counters")
for _, size := range []int{maxProcBytes, maxProcBytes + 1} {
if err := os.WriteFile(path, []byte(strings.Repeat("x", size)), 0600); err != nil {
t.Fatal(err)
}
data, err := readProcFile(path)
if size == maxProcBytes && (err != nil || len(data) != size) {
t.Fatalf("limit rejected: %v", err)
}
if size > maxProcBytes && (err == nil || data != nil) {
t.Fatal("oversize file accepted")
}
}
if _, err := readProcFile(path + "-missing"); err == nil {
t.Fatal("missing file accepted")
}
}
func TestCollectLocalRuntime(t *testing.T) {
root := t.TempDir()
before := time.Now().UnixMilli()
got := Collect(root, Queue{Paused: true})
if got.TimestampMS < before || got.TimestampMS > time.Now().UnixMilli() || got.Storage.Path != root || got.CPU.LogicalCPUs < 1 || got.Memory.Goroutines < 1 || !got.Queue.Paused || got.IO.ProcessID != processID {
t.Fatalf("runtime snapshot %+v", got)
}
}
+3
View File
@@ -12,6 +12,8 @@ import (
"regexp"
"sort"
"strings"
"pageswiki/internal/kunstzeal"
)
const adapterPluginsConfigKey = "projection_adapter_plugins"
@@ -50,6 +52,7 @@ type builtinAdapterInfo struct {
var adapterIDPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{1,63}$`)
func builtinAdapterManifest() []builtinAdapterInfo {
projectionAdapters := kunstzeal.Catalog()
out := make([]builtinAdapterInfo, 0, len(projectionAdapters))
for _, adapter := range projectionAdapters {
out = append(out, builtinAdapterInfo{
+2 -2
View File
@@ -7,7 +7,7 @@ import (
"path/filepath"
"strings"
"pageswiki/internal/compiler"
"pageswiki/internal/grammaratur"
"pageswiki/internal/render"
"pageswiki/internal/store"
)
@@ -63,7 +63,7 @@ func (s *Server) workspaceFile(session *store.AdapterSession, semanticPath strin
body := revision.Body
markdown := plugin.HTMLMarkdown && strings.HasPrefix(strings.ToLower(revision.ContentType), "text/html")
if markdown {
body, err = compiler.HTMLToMarkdown(body)
body, err = grammaratur.HTMLToMarkdown(body)
if err != nil {
return adapterWorkspaceFile{}, err
}
+2 -1
View File
@@ -91,7 +91,7 @@ func (s *Server) tryAnonymousProjection(w http.ResponseWriter, r *http.Request)
}
func (s *Server) serveAnonymousRevision(w http.ResponseWriter, r *http.Request, semantic string, raw bool, v *store.Revision) bool {
reader, err := s.store.OpenDocument(v.DocumentHash)
reader, err := s.store.OpenDocumentContext(r.Context(), v.DocumentHash)
if err != nil {
http.Error(w, "published document unavailable", 503)
return true
@@ -156,6 +156,7 @@ func (s *Server) serveAnonymousRevision(w http.ResponseWriter, r *http.Request,
return true
}
data["Text"] = string(body)
data["CodeLanguage"] = codeLanguage(contentType, filename, string(body))
}
data["SocialMetadata"] = s.socialMetadataForRequest(r, "anonymous_view.gohtml", data)
w.Header().Set("Content-Type", "text/html; charset=utf-8")
+40 -18
View File
@@ -59,17 +59,26 @@ func (s *Server) handleSaveRevisionAPI(
w http.ResponseWriter,
r *http.Request,
) {
var req struct {
ObjectUUID string `json:"object_uuid"`
Body string `json:"body"`
Message string `json:"message"`
Language string `json:"lang"`
ContentType string `json:"content_type"`
Filename string `json:"filename"`
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Cache-Control", "private, no-store")
if r.Method != http.MethodPost {
w.Header().Set("Allow", "POST")
api.WriteError(w, 405, "method_not_allowed", "POST required")
return
}
err := json.NewDecoder(r.Body).Decode(&req)
var req struct {
ObjectUUID string `json:"object_uuid"`
Body string `json:"body"`
Message string `json:"message"`
Language string `json:"lang"`
ContentType string `json:"content_type"`
Filename string `json:"filename"`
BaseRevision *string `json:"base_revision_uuid"`
EditorPrincipal *string `json:"editor_principal_uuid"`
}
err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4<<20)).Decode(&req)
if err != nil {
api.WriteError(
w,
@@ -80,6 +89,17 @@ func (s *Server) handleSaveRevisionAPI(
return
}
domainUUID := s.domainUUIDForRequest(r)
if req.EditorPrincipal != nil {
principal, e := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r))
id := ""
if e == nil && principal != nil {
id = principal.UUID
}
if id != *req.EditorPrincipal {
api.WriteError(w, 401, "editor_identity_changed", "editor identity changed; reopen with the original account")
return
}
}
if mode, _ := s.store.GetConfigInDomain(domainUUID, "authorization_mode"); mode == "enforcing" {
principal, authErr := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r))
if authErr != nil {
@@ -109,15 +129,16 @@ func (s *Server) handleSaveRevisionAPI(
}
}
revision, err :=
s.store.SaveRevisionContent(
req.ObjectUUID,
[]byte(req.Body),
req.Message,
req.Language,
req.ContentType,
req.Filename,
)
var revision *store.Revision
if req.BaseRevision != nil {
revision, err = s.store.SaveRevisionContentIfHead(req.ObjectUUID, []byte(req.Body), req.Message, req.Language, req.ContentType, req.Filename, *req.BaseRevision)
} else {
revision, err = s.store.SaveRevisionContent(req.ObjectUUID, []byte(req.Body), req.Message, req.Language, req.ContentType, req.Filename)
}
if errors.Is(err, store.ErrRevisionConflict) {
api.WriteError(w, http.StatusConflict, "revision_conflict", "document changed; reload the language head before saving")
return
}
if err != nil {
status := http.StatusInternalServerError
@@ -143,5 +164,6 @@ func (s *Server) handleSaveRevisionAPI(
"lang": revision.Language,
"content_type": revision.ContentType,
"filename": revision.Filename,
"body": revision.Body,
})
}
+27 -19
View File
@@ -2,7 +2,6 @@ package server
import (
"archive/zip"
"bytes"
"context"
"io"
"mime"
@@ -47,23 +46,20 @@ func (s *Server) handleArchiveEntry(w http.ResponseWriter, r *http.Request) {
return
}
if strings.EqualFold(filepath.Ext(revision.Filename), ".zip") {
hydrated, err := s.store.GetRevision(revision.UUID)
if err != nil {
http.NotFound(w, r)
return
}
s.serveZIPEntry(w, r, hydrated.Body, entry)
return
}
source, err := s.store.DocumentPath(revision.DocumentHash)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
s.serveZIPDocumentEntry(w, r, revision.DocumentHash, entry)
return
}
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
defer cancel()
lease, err := s.store.MaterializeDocument(ctx, revision.DocumentHash, 1<<30)
if err != nil {
http.Error(w, "archive source unavailable or exceeds materialization limit", http.StatusServiceUnavailable)
return
}
defer lease.Close()
command := exec.CommandContext(ctx, "nice", "-n", "10",
"7z", "x", "-so", "-y", "-spd", "-mmt1", "-p-", "--", source, entry)
"7z", "x", "-so", "-y", "-spd", "-mmt1", "-p-", "--", lease.Path, entry)
command.WaitDelay = time.Second
stdout, err := command.StdoutPipe()
if err != nil || command.Start() != nil {
http.Error(w, "cannot extract archive entry", http.StatusUnprocessableEntity)
@@ -85,16 +81,24 @@ func (s *Server) handleArchiveEntry(w http.ResponseWriter, r *http.Request) {
}
}
func (s *Server) serveZIPEntry(
func (s *Server) serveZIPDocumentEntry(
w http.ResponseWriter,
r *http.Request,
body string,
hash string,
entry string,
) {
reader, err := zip.NewReader(
bytes.NewReader([]byte(body)),
int64(len(body)),
)
input, err := s.store.OpenDocumentContext(r.Context(), hash)
if err != nil {
http.NotFound(w, r)
return
}
defer input.Close()
size, err := input.Seek(0, io.SeekEnd)
if err != nil {
http.Error(w, "cannot determine archive size", http.StatusUnprocessableEntity)
return
}
reader, err := zip.NewReader(input, size)
if err != nil {
http.Error(w, err.Error(), http.StatusUnprocessableEntity)
return
@@ -103,6 +107,10 @@ func (s *Server) serveZIPEntry(
if file.Name != entry || file.FileInfo().IsDir() {
continue
}
if file.UncompressedSize64 > 128<<20 {
http.Error(w, "archive entry exceeds 128 MiB limit", http.StatusRequestEntityTooLarge)
return
}
source, err := file.Open()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
+228
View File
@@ -0,0 +1,228 @@
package server
import (
"archive/zip"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"image"
"image/color"
"image/png"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"pageswiki/internal/db"
"pageswiki/internal/documentstore"
"pageswiki/internal/store"
)
func TestArchiveEntryReadsCanonicalDocumentWithoutRevisionHydration(t *testing.T) {
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
if err := db.Migrate(database); err != nil {
t.Fatal(err)
}
st := store.NewWithDocumentRoot(database, t.TempDir())
domain, _ := st.GetDefaultDomain()
object, err := st.CreatePageAtPathInDomain(domain.UUID, "/archive.zip")
if err != nil {
t.Fatal(err)
}
var payload bytes.Buffer
w := zip.NewWriter(&payload)
file, err := w.Create("hello.txt")
if err != nil {
t.Fatal(err)
}
file.Write([]byte("hello 日本語"))
if err := w.Close(); err != nil {
t.Fatal(err)
}
revision, err := st.SaveRevisionContent(object, payload.Bytes(), "test", "und", "application/zip", "archive.zip")
if err != nil {
t.Fatal(err)
}
// The canonical Document must work even after compatibility DB bodies
// have been removed. This never touches production records.
if _, err := database.Exec(`UPDATE revisions SET body='' WHERE uuid=?`, revision.UUID); err != nil {
t.Fatal(err)
}
s := &Server{store: st}
r := httptest.NewRequest(http.MethodGet, "/_/archive?path=/archive.zip&entry=hello.txt", nil)
response := httptest.NewRecorder()
s.serveZIPDocumentEntry(response, r, revision.DocumentHash, "hello.txt")
if response.Code != http.StatusOK || response.Body.String() != "hello 日本語" {
t.Fatalf("status=%d body=%q", response.Code, response.Body.String())
}
if response.Header().Get("Vary") != "Authorization, Cookie" {
t.Fatal("lost authorization cache partition")
}
missing := httptest.NewRecorder()
s.serveZIPDocumentEntry(missing, r, revision.DocumentHash, "missing")
if missing.Code != http.StatusNotFound {
t.Fatalf("missing status=%d", missing.Code)
}
ctx, cancel := context.WithCancel(r.Context())
cancel()
cancelled := httptest.NewRecorder()
s.serveZIPDocumentEntry(cancelled, r.WithContext(ctx), revision.DocumentHash, "hello.txt")
if cancelled.Code == http.StatusOK {
t.Fatal("cancelled request read Document")
}
}
// Measures the per-request cost of fetching a tiny entry from a large stored
// ZIP. Setup is outside the timer; no DB body hydration or scratch copy occurs.
func BenchmarkZIPDocumentEntry(b *testing.B) {
var payload bytes.Buffer
w := zip.NewWriter(&payload)
file, err := w.Create("hello.txt")
if err != nil {
b.Fatal(err)
}
file.Write([]byte("hello"))
padding, err := w.CreateHeader(&zip.FileHeader{Name: "padding.bin", Method: zip.Store})
if err != nil {
b.Fatal(err)
}
padding.Write(bytes.Repeat([]byte{0xaa}, 16<<20))
if err := w.Close(); err != nil {
b.Fatal(err)
}
root := b.TempDir()
backend, err := documentstore.NewLocal(root)
if err != nil {
b.Fatal(err)
}
sum := sha256.Sum256(payload.Bytes())
hash := hex.EncodeToString(sum[:])
if err := backend.PutIfAbsent(context.Background(), hash, int64(payload.Len()), bytes.NewReader(payload.Bytes())); err != nil {
b.Fatal(err)
}
s := &Server{store: &store.Store{DocumentRoot: root}}
r := httptest.NewRequest(http.MethodGet, "/archive", nil)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
response := httptest.NewRecorder()
s.serveZIPDocumentEntry(response, r, hash, "hello.txt")
if response.Code != http.StatusOK || response.Body.String() != "hello" {
b.Fatal("entry read failed")
}
}
}
func TestPreviewImageAndZIPWorkWithReaderOnlyLegacyStore(t *testing.T) {
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
if err := db.Migrate(database); err != nil {
t.Fatal(err)
}
st := store.New(database)
domain, _ := st.GetDefaultDomain()
s := &Server{store: st, previewRoot: t.TempDir()}
var picture bytes.Buffer
im := image.NewRGBA(image.Rect(0, 0, 10, 10))
im.Set(0, 0, color.White)
if err := png.Encode(&picture, im); err != nil {
t.Fatal(err)
}
var archive bytes.Buffer
w := zip.NewWriter(&archive)
for _, name := range []string{"1.png", "2.png", "3.png"} {
f, err := w.Create(name)
if err != nil {
t.Fatal(err)
}
f.Write(picture.Bytes())
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
kind, filename string
body []byte
}{
{"image", "picture.png", picture.Bytes()}, {"archive", "comic.zip", archive.Bytes()},
} {
object, err := st.CreatePageAtPathInDomain(domain.UUID, "/"+tc.filename)
if err != nil {
t.Fatal(err)
}
revision, err := st.SaveRevisionContent(object, tc.body, "test", "und", "application/octet-stream", tc.filename)
if err != nil {
t.Fatal(err)
}
if err := s.generatePreview(previewEventPayload{DocumentHash: revision.DocumentHash, Kind: tc.kind, Filename: tc.filename}); err != nil {
t.Fatal(err)
}
if info, err := os.Stat(s.previewCachePath(revision.DocumentHash, tc.kind)); err != nil || info.Size() == 0 {
t.Fatalf("preview: %v", err)
}
}
}
func TestProjectionMetadataMaterializesAndCleansSource(t *testing.T) {
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
if err != nil {
t.Fatal(err)
}
defer database.Close()
if err := db.Migrate(database); err != nil {
t.Fatal(err)
}
st := store.NewWithDocumentRoot(database, t.TempDir())
domain, _ := st.GetDefaultDomain()
object, err := st.CreatePageAtPathInDomain(domain.UUID, "/test.pdf")
if err != nil {
t.Fatal(err)
}
revision, err := st.SaveRevisionContent(object, []byte("synthetic PDF fixture"), "test", "und", "application/pdf", "test.pdf")
if err != nil {
t.Fatal(err)
}
// Synthetic tools validate only the storage handoff, not PDF rendering.
bin := t.TempDir()
for name, script := range map[string]string{
"nice": "#!/bin/sh\nshift\nshift\nexec \"$@\"\n",
"pdfinfo": "#!/bin/sh\ntest -r \"$3\" || exit 1\nprintf '%s' \"$3\" > \"$PAGESWIKI_TEST_SOURCE\"\nprintf 'Pages: 3\\n'\n",
} {
if err := os.WriteFile(filepath.Join(bin, name), []byte(script), 0700); err != nil {
t.Fatal(err)
}
}
logFile := filepath.Join(t.TempDir(), "source")
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
t.Setenv("PAGESWIKI_TEST_SOURCE", logFile)
s := &Server{store: st}
result, err := s.inspectProjectionMetadata(metadataProjectionPayload{DocumentHash: revision.DocumentHash, Kind: "pdf", Version: metadataProjectionVersion})
if err != nil || result.PDFPages != 3 {
t.Fatalf("result=%+v err=%v", result, err)
}
source, err := os.ReadFile(logFile)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(source), "pageswiki-document-") || strings.HasPrefix(string(source), st.DocumentRoot+string(os.PathSeparator)) {
t.Fatalf("tool received canonical path: %s", source)
}
if _, err := os.Stat(string(source)); !os.IsNotExist(err) {
t.Fatalf("scratch leaked: %v", err)
}
reader, err := st.OpenDocument(revision.DocumentHash)
if err != nil {
t.Fatal("canonical document removed", err)
}
reader.Close()
}
+12
View File
@@ -5,6 +5,7 @@ import (
"log"
"net/http"
"pageswiki/internal/store"
"strconv"
"time"
)
@@ -34,6 +35,17 @@ func (s *Server) handleAuthAdmin(w http.ResponseWriter, r *http.Request) {
}
panel := "auth"
switch r.FormValue("action") {
case "save_session_policy":
panel = "auth-sessions"
minutes, err := strconv.Atoi(r.FormValue("session_minutes"))
if err != nil || minutes < 5 || minutes > store.MaxAuthSessionMinutes {
http.Error(w, "session lifetime must be between 5 and 43200 minutes", http.StatusBadRequest)
return
}
if err := s.store.SetAuthSessionLifetimeMinutes(domainUUID, minutes); err != nil {
http.Error(w, "could not save session policy", http.StatusInternalServerError)
return
}
case "revoke_session":
panel = "auth-sessions"
if err := s.store.RevokeAuthSessionByID(domainUUID, r.FormValue("session_uuid")); err != nil {
+2 -2
View File
@@ -62,7 +62,7 @@ func requestIsHTTPS(r *http.Request) bool {
}
func (s *Server) setAuthCookie(w http.ResponseWriter, r *http.Request, token string) {
http.SetCookie(w, &http.Cookie{Name: authCookieName, Value: token, Path: "/", HttpOnly: true, Secure: requestIsHTTPS(r), SameSite: http.SameSiteLaxMode, MaxAge: 24 * 60 * 60})
http.SetCookie(w, &http.Cookie{Name: authCookieName, Value: token, Path: "/", HttpOnly: true, Secure: requestIsHTTPS(r), SameSite: http.SameSiteLaxMode, MaxAge: s.store.AuthSessionLifetimeMinutes(s.domainUUIDForRequest(r)) * 60})
}
func (s *Server) authTokenFromRequest(r *http.Request) string {
@@ -205,7 +205,7 @@ func (s *Server) handleAuth(w http.ResponseWriter, r *http.Request) {
}
if err == nil {
if strings.Contains(r.Header.Get("Accept"), "application/json") {
writeEventJSON(w, http.StatusOK, map[string]any{"access_token": token, "token_type": "Bearer", "expires_in": 86400})
writeEventJSON(w, http.StatusOK, map[string]any{"access_token": token, "token_type": "Bearer", "expires_in": s.store.AuthSessionLifetimeMinutes(domainUUID) * 60})
return
}
s.setAuthCookie(w, r, token)
+1 -1
View File
@@ -245,7 +245,7 @@ func (s *Server) handleLDAPLogin(w http.ResponseWriter, r *http.Request) {
return
}
if strings.Contains(r.Header.Get("Accept"), "application/json") {
writeEventJSON(w, http.StatusOK, map[string]any{"access_token": token, "token_type": "Bearer", "expires_in": 86400})
writeEventJSON(w, http.StatusOK, map[string]any{"access_token": token, "token_type": "Bearer", "expires_in": s.store.AuthSessionLifetimeMinutes(domainUUID) * 60})
return
}
s.setAuthCookie(w, r, token)
@@ -0,0 +1,50 @@
package server
import (
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
)
func TestSessionPolicyAdminUIAndCookie(t *testing.T) {
s, token, domain := configFixture(t)
form := url.Values{"action": {"save_session_policy"}, "session_minutes": {"60"}}
w := configRequest(s, token, "POST", "/-/sys/config/auth", form)
if w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/-/sys/config?panel=auth-sessions&saved=1" {
t.Fatalf("policy save %d %s", w.Code, w.Body.String())
}
if s.store.AuthSessionLifetimeMinutes(domain) != 60 {
t.Fatal("policy not saved")
}
w = configRequest(s, token, "GET", "/-/sys/config?panel=auth-sessions", nil)
if w.Code != 200 || !strings.Contains(w.Body.String(), `name="session_minutes" value="60"`) {
t.Fatal("policy control missing")
}
for _, bad := range []string{"", "abc", "0", "99999999999999999999999"} {
form.Set("session_minutes", bad)
if w = configRequest(s, token, "POST", "/-/sys/config/auth", form); w.Code != 400 {
t.Fatalf("bad lifetime %q accepted", bad)
}
}
user, err := s.store.CreatePasswordAccount(domain, "reader", "a long test password", "Reader", "")
if err != nil {
t.Fatal(err)
}
reader, err := s.store.CreateAuthSession(domain, user.UUID)
if err != nil {
t.Fatal(err)
}
form.Set("session_minutes", "120")
if w = configRequest(s, reader, "POST", "/-/sys/config/auth", form); w.Code != 403 {
t.Fatalf("non-admin policy change: %d", w.Code)
}
w = httptest.NewRecorder()
r := httptest.NewRequest("GET", "https://localhost/", nil)
s.setAuthCookie(w, r, "fixture-token")
cookies := w.Result().Cookies()
if len(cookies) != 1 || cookies[0].MaxAge != 3600 || !cookies[0].HttpOnly || !cookies[0].Secure {
t.Fatalf("cookie policy %#v", cookies)
}
}
+8
View File
@@ -58,6 +58,14 @@ func (s *Server) handleBrowser(
if !strings.HasPrefix(currentPath, "/") {
currentPath = "/" + currentPath
}
// Browser navigation is scoped to this semantic Path, not a Domain-wide
// read grant. Filter each child separately below as well.
if !s.requestCanReadPath(r, currentPath) {
http.Error(w, "forbidden", http.StatusForbidden)
return
}
w.Header().Set("Cache-Control", "private, no-store")
w.Header().Add("Vary", "Cookie, Authorization")
entries, err :=
s.store.ListBrowserEntriesInDomain(
+36
View File
@@ -0,0 +1,36 @@
package server
func init() {
for lang, entries := range map[string]map[string]string{
"en": {
"command.label": "Command · Tab: complete Path · Esc: close", "command.close": "Close command mode", "command.unavailable": "Path completion unavailable. Sign in or retry.", "command.empty": "No matching readable Paths.", "command.more": "More matches exist. Type a longer Path prefix.",
"auth.session_minutes": "Session lifetime (minutes)", "auth.session_policy_help": "5–43200 minutes (30 days); default 1440 (24 hours). Applies to new browser/API logins, including OIDC/LDAP. Existing sessions and CalDAV app passwords are unchanged. This is an absolute expiry, not an inactivity timeout.",
"browser.format": "Document format", "browser.wiki": "Wiki (default editor format)", "browser.plain": "PlainText (text/plain)",
"theme.quick": "Theme", "theme.default": "Domain default", "theme.local": "This browser only", "theme.failed": "Could not load the theme.",
"browser.address": "Path", "browser.go": "Go", "browser.up": "Parent folder",
"browser.command_help": "Enter /path or a relative path · cd .. · vi filename · view filename",
"browser.new": "New document", "browser.folder": "Open virtual folder", "browser.name": "Name or Path",
"browser.create_help": "Open the editor to create a document. If the Path already exists, edit that document.",
"browser.folder_help": "You can enter an empty Path without creating a folder, then create or upload documents there.",
"browser.invalid": "Enter a Wiki Path, not an external URL or a path containing control characters or backslashes.",
"browser.empty": "No visible items. Create a document, upload a file, or enter a different Path.",
},
"ja": {
"command.label": "コマンド · Tab: Path補完 · Esc: 閉じる", "command.close": "コマンドモードを閉じる", "command.unavailable": "Path補完を取得できません。ログイン状態を確認して再試行してください。", "command.empty": "閲覧可能な一致するPathがありません。", "command.more": "候補がほかにもあります。Pathをさらに入力してください。",
"auth.session_minutes": "セッションの有効期間(分)", "auth.session_policy_help": "5〜43200分(30日)。既定は1440分(24時間)。OIDC・LDAPを含む新規ブラウザ/APIログインに適用します。既存セッションとCalDAVのアプリパスワードは変更しません。無操作時間ではなくログインからの絶対期限です。",
"browser.format": "文書形式", "browser.wiki": "Wiki(既定の編集形式)", "browser.plain": "PlainText(text/plain)",
"theme.quick": "テーマ", "theme.default": "ドメインの既定", "theme.local": "このブラウザだけに適用", "theme.failed": "テーマを読み込めませんでした。",
"browser.address": "Path", "browser.go": "移動", "browser.up": "親フォルダ",
"browser.command_help": "/path または相対パス · cd .. · vi ファイル名 · view ファイル名",
"browser.new": "新規文書", "browser.folder": "仮想フォルダを開く", "browser.name": "名前またはPath",
"browser.create_help": "エディタを開いて文書を作成します。既存のPathでは、その文書を編集します。",
"browser.folder_help": "空のPathにもフォルダを作らず移動できます。移動先で文書の作成やアップロードができます。",
"browser.invalid": "WikiのPathを入力してください。外部URL・制御文字・バックスラッシュは使用できません。",
"browser.empty": "表示できる項目はありません。文書の作成、ファイルのアップロード、別のPathへの移動ができます。",
},
} {
for key, value := range entries {
translations[lang][key] = value
}
}
}
@@ -0,0 +1,82 @@
package server
import (
"encoding/json"
"strings"
"testing"
"time"
"pageswiki/internal/store"
)
func TestCitySearchContentDestinations(t *testing.T) {
s, token, domain := searchServerFixture(t)
street := seedSearchCity(t, s, token, domain)
createUserSpacePage(t, s, domain, "/city?#%", "<h1>Portal</h1>")
if _, err := s.store.CreateSpotAtPath(domain, "test", street, "nebula-way-undefined", "Undefined", "nebula", "/city?#%/street/_undefined"); err != nil {
t.Fatal(err)
}
principal, err := s.store.PrincipalForSession(domain, token)
if err != nil {
t.Fatal(err)
}
if _, err := s.store.DB.Exec(`INSERT INTO auth_capability_grants(uuid,role_uuid,capability,target_type,target_id,created_at_ms)
SELECT 'entrance-grant',b.role_uuid,'read','path','/city?#%/street/_undefined',1 FROM auth_role_bindings b JOIN domain_memberships m ON m.uuid=b.membership_uuid WHERE m.domain_uuid=? AND m.principal_uuid=? LIMIT 1`, domain, principal.UUID); err != nil {
t.Fatal(err)
}
s.mux.HandleFunc("/_/api/v1/search/suggest", s.handleSearchSuggestAPI)
s.mux.HandleFunc("/v/browser/", s.handleBrowser)
s.mux.HandleFunc("/", s.handleSemanticRoute)
w := configRequest(s, token, "GET", "/_/api/v1/search/suggest?q=nebula", nil)
var payload struct {
Items []searchSuggestionItem `json:"items"`
}
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &payload) != nil {
t.Fatalf("suggest: %d %s", w.Code, w.Body.String())
}
foundPortal, foundEntrance := false, false
for _, item := range payload.Items {
if strings.HasPrefix(item.Href, "/v/city/") || strings.Contains(item.Href, "_undefined") {
t.Fatalf("management/virtual link: %+v", item)
}
if item.Kind == "quartier" {
foundPortal = item.Href == "/city%3F%23%25"
}
if item.Kind == "spot" && item.Label == "Nebula Street" {
foundEntrance = item.Href == "/v/browser/city%3F%23%25/street"
}
}
if !foundPortal || !foundEntrance {
t.Fatalf("destinations missing: %+v", payload.Items)
}
for _, target := range []struct{ url, marker string }{
{"/city%3F%23%25", "<h1>Portal</h1>"},
{"/v/browser/city%3F%23%25/street", "browser-layout"},
} {
opened := configRequest(s, token, "GET", target.url, nil)
if opened.Code != 200 || !strings.Contains(opened.Body.String(), target.marker) || strings.Contains(opened.Body.String(), `class="city-node-detail"`) {
t.Fatalf("destination %s: %d", target.url, opened.Code)
}
}
if denied := configRequest(s, token, "GET", "/v/browser/private-city", nil); denied.Code != 403 {
t.Fatalf("unreadable folder: %d", denied.Code)
}
w = configRequest(s, token, "GET", "/v/search?q=nebula", nil)
if w.Code != 200 || !strings.Contains(w.Body.String(), `href="/city%3F%23%25"`) || strings.Contains(w.Body.String(), `href="/v/city/`) {
t.Fatalf("search: %d", w.Code)
}
}
func TestCitySearchViewAndSuggestionsAgree(t *testing.T) {
for _, hasPage := range []bool{false, true} {
result := store.CitySearchResult{Kind: "spot", Name: "notes-undefined", Title: "Undefined", PathPrefix: "/日本語 #?%/_undefined", NavigationPath: "/日本語 #?%", StreetTitle: "Notes", HasPage: hasPage}
view := citySearchResultView(result, "notes", time.UTC)
suggestion := citySuggestion(result, "notes")
if view.Href != suggestion.Href || view.Href != view.PathHref || suggestion.Label != "Notes" {
t.Fatalf("inconsistent: %+v %+v", view, suggestion)
}
if strings.Contains(view.Href, "#") || strings.Contains(view.Href, "?") || strings.Contains(view.Href, "_undefined") {
t.Fatal(view.Href)
}
}
}
+1 -1
View File
@@ -80,7 +80,7 @@ func (s *Server) handleCityMap(w http.ResponseWriter, r *http.Request) {
if detail.Street != nil {
streetHref = cityDetailHref("street", detail.Street.Name, detail.Quartier.Name, detail.Street.UUID)
}
s.renderWithContext(w, r, "city_node.gohtml", map[string]any{"Title": detail.Title, "Node": detail, "Pages": pageViews, "QuartierHref": quartierHref, "StreetHref": streetHref, "MapHref": "/v/city?" + kind + "=" + detail.UUID, "CanPlan": canPlan, "Locale": s.getLocale(domainUUID), "GovernanceEndpoint": s.governanceForUI(r, kind, detail.UUID)})
s.renderWithContext(w, r, "city_node.gohtml", map[string]any{"Title": detail.DisplayTitle(), "Node": detail, "Pages": pageViews, "QuartierHref": quartierHref, "StreetHref": streetHref, "MapHref": "/v/city?" + kind + "=" + detail.UUID, "CanPlan": canPlan, "Locale": s.getLocale(domainUUID), "GovernanceEndpoint": s.governanceForUI(r, kind, detail.UUID)})
return
}
s.renderWithContext(w, r, "city_map.gohtml", map[string]any{"Title": translate(s.getLocale(domainUUID), "city.map"), "CanPlan": canPlan, "Locale": s.getLocale(domainUUID), "GovernanceEndpoint": s.governanceForUI(r, "domain", domainUUID)})
+13
View File
@@ -34,6 +34,19 @@ func cityPageHref(path string) string {
return (&url.URL{Path: path}).EscapedPath()
}
// Search is a reading entry point, not a locality management shortcut. Both
// title and Path links use this projection; no additional per-hit DB reads.
func cityContentHref(result store.CitySearchResult) string {
path := result.ContentPath()
if !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") {
return "/v/browser"
}
if result.HasPage {
return cityPageHref(path)
}
return cityPageHref("/v/browser" + path)
}
type cityPageView struct {
store.CityPagePlacement
Href string
+6 -1
View File
@@ -29,6 +29,11 @@ func filterCityProjection(city store.CityMap, allowed func(string) bool) store.C
continue
}
spot.DetailHref = cityDetailHref("spot", spot.Name, quartier.Name, spot.UUID)
parentTitle := street.Title
if parentTitle == "" {
parentTitle = street.Name
}
spot.DisplayTitle = store.CitySpotDisplayTitle(spot.Name, spot.Title, spot.PathPrefix, parentTitle)
pages := []store.CityPagePlacement{}
unique := map[string]bool{}
spot.PortalPath, spot.PortalPageUUID = "", ""
@@ -113,7 +118,7 @@ func (s *Server) pageSpotAppearancesForRequest(r *http.Request, objectUUID strin
out := []store.CitySpotAppearance{}
allowed := s.pathReadFilter(r)
for _, spot := range spots {
if allowed(spot.PathPrefix) {
if allowed(spot.PathPrefix) && allowed(spot.StreetPath) && allowed(spot.QuartierPath) {
out = append(out, spot)
}
}
+13
View File
@@ -49,6 +49,19 @@ func TestCityReadPortalPriorityAfterFiltering(t *testing.T) {
}
}
func TestCityProjectionDisplayTitleDoesNotMutateCachedIdentity(t *testing.T) {
spot := store.CitySpot{UUID: "spot", Name: "home-undefined", Title: "Undefined", PathPrefix: "/life/home/_undefined"}
city := store.CityMap{Quartiers: []store.CityQuartier{{PathPrefix: "/life", Streets: []store.CityStreet{{Title: "Home", PathPrefix: "/life/home", Spots: []store.CitySpot{spot}}}}}}
projected := filterCityProjection(city, func(string) bool { return true })
got := projected.Quartiers[0].Streets[0].Spots[0]
if got.DisplayTitle != "Home" || got.Title != "Undefined" || got.Name != spot.Name || got.PathPrefix != spot.PathPrefix {
t.Fatalf("projection: %+v", got)
}
if city.Quartiers[0].Streets[0].Spots[0].DisplayTitle != "" {
t.Fatal("shared navigation cache mutated")
}
}
func TestCityReadProjectionHidesPagesLocalitiesPortalsAndRelations(t *testing.T) {
s, _, domain := configFixture(t)
s.mux.HandleFunc("/_/api/v1/city/", s.handleCityAPI)
+91
View File
@@ -0,0 +1,91 @@
package server
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"strings"
"testing"
"time"
)
func TestCommandCompletionReadACL(t *testing.T) {
s, token, _ := searchServerFixture(t)
s.mux.HandleFunc("/_/api/v1/path-picker", s.handlePathPickerAPI)
w := configRequest(s, token, "GET", "/_/api/v1/path-picker?path=/docs&mode=children", nil)
if w.Code != 200 || strings.Contains(w.Body.String(), "hidden") || !strings.Contains(w.Body.String(), "nebula guide?#%") {
t.Fatalf("ACL completion: %d %s", w.Code, w.Body.String())
}
}
func TestCommandPathCompletion(t *testing.T) {
s, token, domain := configFixture(t)
s.mux.HandleFunc("/_/api/v1/path-picker", s.handlePathPickerAPI)
createUserSpacePage(t, s, domain, "/manual", "<h1>Parent document and folder</h1>")
for i := 0; i < 260; i++ {
if _, err := s.store.CreatePageAtPathInDomain(domain, fmt.Sprintf("/manual/a%03d", i)); err != nil {
t.Fatal(err)
}
}
createUserSpacePage(t, s, domain, "/manual/日本語 #?", "plain")
w := configRequest(s, token, "GET", "/_/api/v1/path-picker?path=/manual&mode=children&prefix=日本語", nil)
var result struct {
Path string `json:"path"`
Entries []pathPickerEntry `json:"entries"`
}
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &result) != nil || result.Path != "/manual" || len(result.Entries) != 1 || result.Entries[0].Name != "日本語 #?" {
t.Fatalf("completion: %d %s", w.Code, w.Body.String())
}
w = configRequest(s, "", "GET", "/_/api/v1/path-picker?path=/manual&mode=children", nil)
if w.Code == 200 {
t.Fatal("anonymous API listed private paths")
}
}
func TestCommandAndHighlightChromium(t *testing.T) {
if os.Getenv("PAGESWIKI_BROWSER_TEST") != "1" {
t.Skip("opt-in Chromium")
}
s, token, domain, _ := editorFixture(t)
s.mux.HandleFunc("/v/browser/", s.handleBrowser)
s.mux.HandleFunc("/_/api/v1/path-picker", s.handlePathPickerAPI)
s.mux.HandleFunc("/notes", s.handleSemanticRoute)
s.mux.HandleFunc("/", s.handleHome)
createUserSpacePage(t, s, domain, "/notes", `<h1>Code sample</h1><pre><code class="language-python"># comment
def hello():
return "&lt;script&gt;literal&lt;/script&gt;"</code></pre>`)
createUserSpacePage(t, s, domain, "/notes/alpha", "<h1>Alpha</h1>")
createUserSpacePage(t, s, domain, "/notes/alpine", "<h1>Alpine</h1>")
createUserSpacePage(t, s, domain, "/日本語 #?/child", "<h1>Unicode</h1>")
script, err := os.ReadFile("internal/server/testdata/command-highlight.js")
if err != nil {
t.Fatal(err)
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r.Host = "localhost"
r.Header.Set("Authorization", "Bearer "+token)
if r.URL.Path == "/fixture" {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
_, _ = w.Write([]byte("<!doctype html><body style='margin:0'><script>" + string(script) + "</script></body>"))
return
}
s.withAuthorization(s.mux).ServeHTTP(w, r)
}))
defer server.Close()
for _, width := range []string{"1440", "375"} {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
shot := ""
if base := os.Getenv("PAGESWIKI_COMMAND_SCREENSHOT"); base != "" {
shot = base + "-" + width + ".png"
}
out, err := exec.CommandContext(ctx, "node", "internal/server/testdata/vim-browser-smoke.mjs", server.URL+"/fixture", t.TempDir(), shot, width, "950").CombinedOutput()
cancel()
if err != nil {
t.Fatalf("%s: %v\n%s", width, err, out)
}
}
}
+4 -2
View File
@@ -4,7 +4,7 @@ import (
"net/http"
"strings"
"pageswiki/internal/compiler"
"pageswiki/internal/grammaratur"
)
func (s *Server) compileMarkdownFromPath(
@@ -46,9 +46,11 @@ func (s *Server) transpileMarkdownFromPath(r *http.Request, path string, mode ma
html := latest.Body
if mode == markdownProjection {
html = s.expandMacrosForRequest(r, html, path, s.domainUUIDForRequest(r))
} else {
return grammaratur.HTMLToMarkdownAuthoring(html)
}
return compiler.HTMLToMarkdown(
return grammaratur.HTMLToMarkdown(
html,
)
}
+1
View File
@@ -1088,6 +1088,7 @@ func (s *Server) handleConfig(
"CampusTokens": campusTokens,
"AuthPrincipals": authPrincipals,
"AuthSessions": authSessions,
"AuthSessionMinutes": s.store.AuthSessionLifetimeMinutes(s.domainUUIDForRequest(r)),
"OIDCConfig": oidcConfig,
"SMTPConfig": smtpConfig,
"LDAPConfig": ldapConfig,
+24
View File
@@ -0,0 +1,24 @@
package server
func init() {
for lang, entries := range map[string]map[string]string{
"en": {
"contact.title": "Contact details", "contact.phone": "Phone number",
"contact.help": "Private by default, independently of your profile. A tap opens the phone app. Leave blank to remove. Use 3–15 digits with an optional leading +, spaces, hyphens or parentheses; extensions and service codes are not supported.",
"contact.visibility_help": "The profile audience remains the upper limit. Anonymous access also requires the domain to permit public profiles. Administrators do not automatically see private numbers.",
"contact.confirm": "I explicitly agree to make this phone number public.",
"contact.invalid_phone": "Enter a phone number with 3–15 digits and optional leading +; only spaces, hyphens and parentheses may separate digits.",
},
"ja": {
"contact.title": "連絡先", "contact.phone": "電話番号",
"contact.help": "プロフィールとは別に、既定で非公開です。タップすると電話アプリを開きます。空欄で削除。3〜15桁の数字と先頭の+、空白・ハイフン・括弧を使用できます。内線やサービスコードは未対応です。",
"contact.visibility_help": "プロフィールの公開範囲を上限とします。匿名公開にはドメイン側の許可も必要です。管理者でも非公開の番号は表示されません。",
"contact.confirm": "この電話番号をインターネットに公開することに同意します。",
"contact.invalid_phone": "電話番号は3〜15桁の数字と先頭の+で指定してください。区切りには空白・ハイフン・括弧のみ使用できます。",
},
} {
for key, value := range entries {
translations[lang][key] = value
}
}
}
+11 -216
View File
@@ -1,225 +1,20 @@
package server
import (
"path/filepath"
"strings"
)
import "pageswiki/internal/kunstzeal"
type projectionDelivery string
type projectionDelivery = kunstzeal.Delivery
type contentDescriptor = kunstzeal.Descriptor
type projectionAdapter = kunstzeal.Adapter
const (
deliveryHTML projectionDelivery = "html"
deliveryText projectionDelivery = "text"
deliveryRawMedia projectionDelivery = "raw-media"
deliveryRawFrame projectionDelivery = "raw-frame"
deliveryDownload projectionDelivery = "download"
deliveryHTML = kunstzeal.DeliveryHTML
deliveryText = kunstzeal.DeliveryText
deliveryRawMedia = kunstzeal.DeliveryRawMedia
deliveryRawFrame = kunstzeal.DeliveryRawFrame
deliveryDownload = kunstzeal.DeliveryDownload
inlineTextWindowThreshold = kunstzeal.InlineTextWindowThreshold
)
type contentDescriptor struct {
MediaType string
Filename string
SizeBytes int64
}
type projectionAdapter struct {
ID string
Kind string
Delivery projectionDelivery
NeedsText bool
WindowableText bool
Match func(contentDescriptor) bool
}
const inlineTextWindowThreshold = int64(2 << 20)
var projectionAdapters = []projectionAdapter{
{
ID: "html",
Kind: "html",
Delivery: deliveryHTML,
Match: mediaTypes("text/html"),
},
{
ID: "excalidraw",
Kind: "excalidraw",
Delivery: deliveryText,
NeedsText: true,
WindowableText: true,
Match: extensions(".excalidraw"),
},
{
ID: "csv",
Kind: "csv",
Delivery: deliveryText,
NeedsText: true,
WindowableText: true,
Match: matchAny(
mediaTypes("text/csv", "application/csv"),
extensions(".csv"),
),
},
{
ID: "archive",
Kind: "archive",
Delivery: deliveryDownload,
Match: matchAny(
mediaTypes(
"application/zip", "application/x-zip-compressed",
"application/x-rar-compressed", "application/vnd.rar",
"application/x-7z-compressed", "application/x-tar",
"application/gzip", "application/x-xz",
),
extensions(".zip", ".rar", ".7z", ".tar", ".gz", ".tgz", ".xz", ".txz"),
),
},
{
ID: "debian-package",
Kind: "metadata",
Delivery: deliveryDownload,
Match: matchAny(
mediaTypes(
"application/vnd.debian.binary-package",
"application/x-debian-package",
),
extensions(".deb"),
),
},
{
ID: "eml",
Kind: "email",
Delivery: deliveryText,
Match: matchAny(
mediaTypes("message/rfc822"),
extensions(".eml"),
),
},
{
ID: "pe",
Kind: "metadata",
Delivery: deliveryDownload,
Match: extensions(".exe", ".dll", ".sys"),
},
{
ID: "json",
Kind: "json",
Delivery: deliveryText,
WindowableText: true,
Match: matchAny(mediaTypes("application/json", "application/ld+json"), extensions(".json", ".jsonl")),
},
{
ID: "text",
Kind: "code",
Delivery: deliveryText,
NeedsText: true,
WindowableText: true,
Match: textMediaType,
},
{
ID: "image",
Kind: "image",
Delivery: deliveryRawMedia,
Match: matchAny(
mediaTypePrefix("image/"),
extensions(".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif", ".svg"),
),
},
{
ID: "audio",
Kind: "audio",
Delivery: deliveryRawMedia,
Match: matchAny(
mediaTypePrefix("audio/"),
extensions(".mp3", ".flac", ".wav", ".ogg", ".m4a", ".aac", ".opus"),
),
},
{
ID: "video",
Kind: "video",
Delivery: deliveryRawMedia,
Match: matchAny(
mediaTypePrefix("video/"),
extensions(".mp4", ".webm", ".mkv", ".mov", ".m4v", ".ogv"),
),
},
{
ID: "pdf",
Kind: "pdf",
Delivery: deliveryRawFrame,
Match: matchAny(
mediaTypes("application/pdf"),
extensions(".pdf"),
),
},
{
ID: "download",
Kind: "download",
Delivery: deliveryDownload,
Match: func(contentDescriptor) bool { return true },
},
}
func selectProjectionAdapter(descriptor contentDescriptor) projectionAdapter {
for _, adapter := range projectionAdapters {
if adapter.Match(descriptor) {
return adapter
}
}
return projectionAdapters[len(projectionAdapters)-1]
}
func mediaTypes(types ...string) func(contentDescriptor) bool {
return func(descriptor contentDescriptor) bool {
for _, candidate := range types {
if descriptor.MediaType == candidate {
return true
}
}
return false
}
}
func mediaTypePrefix(prefix string) func(contentDescriptor) bool {
return func(descriptor contentDescriptor) bool {
return strings.HasPrefix(descriptor.MediaType, prefix)
}
}
func extensions(values ...string) func(contentDescriptor) bool {
return func(descriptor contentDescriptor) bool {
extension := strings.ToLower(filepath.Ext(descriptor.Filename))
for _, candidate := range values {
if extension == candidate {
return true
}
}
return false
}
}
func matchAny(matchers ...func(contentDescriptor) bool) func(contentDescriptor) bool {
return func(descriptor contentDescriptor) bool {
for _, matcher := range matchers {
if matcher(descriptor) {
return true
}
}
return false
}
}
func textMediaType(descriptor contentDescriptor) bool {
switch descriptor.MediaType {
case "application/json",
"application/xml",
"application/javascript",
"application/x-javascript",
"application/x-sh",
"application/x-shellscript",
"application/x-yaml",
"application/yaml",
"application/toml":
return true
default:
return strings.HasPrefix(descriptor.MediaType, "text/")
}
return kunstzeal.SelectAdapter(descriptor)
}
+2 -2
View File
@@ -14,7 +14,7 @@ func (s *Server) windowedContentViewForRequest(r *http.Request, objectUUID, sema
if err != nil || metadata.UUID == "" {
return contentView{}, false, err
}
size, err := s.store.DocumentSize(metadata.DocumentHash)
size, err := s.store.DocumentSizeContext(r.Context(), metadata.DocumentHash)
if err != nil {
return contentView{}, false, nil
}
@@ -30,7 +30,7 @@ func (s *Server) windowedContentViewForRequest(r *http.Request, objectUUID, sema
if size <= inlineTextWindowThreshold || !adapter.WindowableText {
return contentView{}, false, nil
}
window, err := s.store.ReadDocumentWindow(metadata.DocumentHash, 0, 128<<10)
window, err := s.store.ReadDocumentWindowContext(r.Context(), metadata.DocumentHash, 0, 128<<10)
if err != nil {
return contentView{}, false, err
}
Loaded 100 of 302 files, more files were not shown because too many files have changed in this diff. Show more