diff --git a/.gitignore b/.gitignore index 8e07f45..d51957b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ data/* !data/pageswiki-cli !data/test_cli_assets.py !data/pageswiki_cli_test.py +!data/test_cli_dr.py +!data/pageswiki-remote /data2/ /.codex/ /.agents/ diff --git a/assets/editor/editor-core.test.mjs b/assets/editor/editor-core.test.mjs new file mode 100644 index 0000000..670eae8 --- /dev/null +++ b/assets/editor/editor-core.test.mjs @@ -0,0 +1,19 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import '../../static/js/editor-core.js'; +const {merge,validDraft}=globalThis.PagesEditorCore; +test('three-way merge keeps disjoint changes',()=>{ + assert.equal(merge('one two three','ONE two three','one two THREE'),'ONE two THREE'); + assert.equal(merge('abc','abc','Abc'),'Abc'); + assert.equal(merge('abc','ABC','ABC'),'ABC'); + assert.equal(merge('abc','Abc','aBc'),null); // adjacent ranges + assert.equal(merge('abc','axbc','aybc'),null); // competing insertions + assert.equal(merge('abc','axbc','abc!'),'axbc!'); + assert.equal(merge('one two three','one LOCAL three','one REMOTE three'),null); +}); +test('draft format, date, expiry and content validated',()=>{ + const now=Date.now(),draft={version:1,text:'source',baseBody:'base',baseRevision:'uuid',format:'markdown',savedAt:now}; + assert.ok(validDraft(draft,now)); + assert.ok(validDraft({...draft,format:'plain',text:''},now)); + for(const fields of [{version:2},{text:null},{baseRevision:1},{format:'projection'},{savedAt:now-8*86400000},{savedAt:now+120000}]) assert.ok(!validDraft({...draft,...fields},now)); +}); diff --git a/assets/editor/package-lock.json b/assets/editor/package-lock.json index cba5534..8e1b35e 100644 --- a/assets/editor/package-lock.json +++ b/assets/editor/package-lock.json @@ -27,6 +27,7 @@ "@codemirror/legacy-modes": "^6.5.3", "@codemirror/state": "^6.6.0", "@codemirror/view": "^6.43.0", + "@lezer/highlight": "^1.2.3", "@replit/codemirror-vim": "^6.3.0", "esbuild": "^0.28.0" } diff --git a/assets/editor/package.json b/assets/editor/package.json index f977dba..dff0550 100644 --- a/assets/editor/package.json +++ b/assets/editor/package.json @@ -4,12 +4,14 @@ "description": "", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "node --test editor-core.test.mjs", + "build": "node build.mjs" }, "keywords": [], "author": "", "license": "ISC", "dependencies": { + "@lezer/highlight": "^1.2.3", "@codemirror/autocomplete": "^6.20.3", "@codemirror/commands": "^6.10.3", "@codemirror/lang-cpp": "^6.0.3", diff --git a/assets/editor/src/editor.js b/assets/editor/src/editor.js index 3a206e2..c85b66a 100644 --- a/assets/editor/src/editor.js +++ b/assets/editor/src/editor.js @@ -1,9 +1,9 @@ -import { EditorView, keymap, lineNumbers } from "@codemirror/view"; +import { EditorView, keymap, lineNumbers, highlightActiveLine, highlightActiveLineGutter } from "@codemirror/view"; import { EditorState } from "@codemirror/state"; import { defaultKeymap, history, historyKeymap } from "@codemirror/commands"; import { bracketMatching, - defaultHighlightStyle, + HighlightStyle, StreamLanguage, syntaxHighlighting, } from "@codemirror/language"; @@ -28,32 +28,59 @@ import { java } from "@codemirror/lang-java"; import { cpp } from "@codemirror/lang-cpp"; import { shell } from "@codemirror/legacy-modes/mode/shell"; import { toml } from "@codemirror/legacy-modes/mode/toml"; -import { vim, Vim } from "@replit/codemirror-vim"; +import { vim, Vim, getCM } from "@replit/codemirror-vim"; +import { tags, highlightTree } from "@lezer/highlight"; + +// Stable semantic styles instead of selectors for generated Lezer class names. +// CSS variables follow the selected Wiki theme without rebuilding the document. +const pagesWikiHighlightStyle = HighlightStyle.define([ + {tag: tags.heading, class: "pw-syntax-heading"}, + {tag: [tags.link, tags.url], class: "pw-syntax-link"}, + {tag: tags.emphasis, class: "pw-syntax-emphasis"}, + {tag: tags.strong, class: "pw-syntax-strong"}, + {tag: [tags.keyword, tags.operatorKeyword], class: "pw-syntax-keyword"}, + {tag: [tags.tagName, tags.typeName, tags.monospace], class: "pw-syntax-type"}, + {tag: [tags.string, tags.special(tags.string)], class: "pw-syntax-string"}, + {tag: [tags.number, tags.bool, tags.null, tags.attributeName], class: "pw-syntax-number"}, + {tag: tags.comment, class: "pw-syntax-comment"}, + {tag: [tags.variableName, tags.propertyName], class: "pw-syntax-variable"}, + {tag: tags.function(tags.variableName), class: "pw-syntax-function"}, + {tag: tags.operator, class: "pw-syntax-operator"}, + {tag: tags.invalid, class: "pw-syntax-invalid"}, +]); const pagesWikiTheme = EditorView.theme({ ".cm-selectionBackground": { - backgroundColor: "rgba(255, 216, 102, 0.48) !important", + backgroundColor: "var(--editor-selection) !important", }, "&.cm-focused .cm-selectionBackground": { - backgroundColor: "rgba(255, 216, 102, 0.58) !important", + backgroundColor: "var(--editor-selection) !important", }, ".cm-selectionLayer .cm-selectionBackground": { - backgroundColor: "rgba(255, 216, 102, 0.58) !important", + backgroundColor: "var(--editor-selection) !important", }, ".cm-content ::selection": { - backgroundColor: "rgba(255, 216, 102, 0.48) !important", - color: "#ffffff !important", + backgroundColor: "var(--editor-selection) !important", + color: "var(--fg) !important", }, }); let vimMode = "INSERT"; +const languageAliases = {js:'javascript', ts:'typescript', py:'python', sh:'shell', bash:'shell', zsh:'shell', yml:'yaml', md:'markdown', c:'cpp', 'c++':'cpp', xml:'html', htm:'html'}; + function languageExtension(mode) { - switch (String(mode || "").toLowerCase()) { + mode = String(mode || "").toLowerCase(); + switch (languageAliases[mode] || mode) { case "html": return html(); case "markdown": - return markdown(); + return markdown({codeLanguages: info => { + const mode = info.trim().split(/\s+/)[0].toLowerCase(); + if (['md','markdown'].includes(mode)) return null; + const support = languageExtension(mode); + return support?.language || support; + }}); case "javascript": return javascript({ jsx: true }); case "typescript": @@ -129,6 +156,7 @@ function attachMinimap(view, minimap, text) { function setEditorMode(mode) { vimMode = mode; + document.body.dataset.editorMode = mode.toLowerCase(); const el = document.getElementById("editor-vim-mode"); @@ -152,6 +180,8 @@ function markdownCompletions(context) { return null; } + if (!context.explicit && !word.text.startsWith("{{")) return null; + return { from: word.from, options: [ @@ -163,10 +193,14 @@ function markdownCompletions(context) { { label: "```go", type: "snippet", apply: "```go\n\n```" }, { label: "```bash", type: "snippet", apply: "```bash\n\n```" }, { label: "[link](url)", type: "snippet", apply: "[text](url)" }, - { label: "{{children}}", type: "function", apply: "{{children}}" }, + { label: "{{children}}", type: "function", apply: "{{children}}", boost: 5 }, { label: "{{children:/}}", type: "function", apply: "{{children:/}}" }, { label: "{{children:/ exclude=/draft}}", type: "function", apply: "{{children:/ exclude=/draft}}" }, - ], + ].map(option => option.label.startsWith("{{") ? {...option, apply(view, completion, from, to) { + // Existing close-bracket text is part of this macro replacement. + const tail = view.state.sliceDoc(to, to + 2).match(/^\}{1,2}/)?.[0] || ""; + view.dispatch({changes:{from,to:to+tail.length,insert:option.label},selection:{anchor:from+option.label.length}}); + }} : option), }; } @@ -190,11 +224,9 @@ function installVimCommands() { window.PagesWikiCurrentPath || "/"; }; - const saveQuit = - () => { - save(); - setTimeout(quit, 120); - }; + const saveQuit = async () => { + if (await window.PagesEditor?.saveNow()) quit(); + }; // Register both canonical and short names. Vim.defineEx("write", "w", save); @@ -208,46 +240,13 @@ function installVimCommands() { } function attachVimModeTracking(view) { - view.dom.addEventListener( - "keydown", - (ev) => { - if (ev.key === "Escape" || (ev.ctrlKey && ev.key === "[")) { - setEditorMode("NORMAL"); - return; - } - - if (vimMode === "NORMAL") { - if (ev.key === "i" || ev.key === "a" || ev.key === "o" || ev.key === "O") { - setEditorMode("INSERT"); - return; - } - - if (ev.key === "R") { - setEditorMode("REPLACE"); - return; - } - - if (ev.key === "v" || ev.key === "V") { - setEditorMode("VISUAL"); - return; - } - } - - if (vimMode === "VISUAL" && ev.key === "Escape") { - setEditorMode("NORMAL"); - } - }, - true - ); - - view.dom.addEventListener( - "focusin", - () => { - if (!vimMode) { - setEditorMode("INSERT"); - } - } - ); + // Observe actual Vim transitions, including A/I/c/operator motions and + // command-line searches, rather than guessing the mode from individual keys. + getCM(view)?.on("vim-mode-change", event => { + const mode = String(event.mode || "normal").toUpperCase(); + setEditorMode(mode); + view.dom.classList.toggle("pageswiki-vim-visual", mode === "VISUAL"); + }); } function attachStatusBar(view) { @@ -281,10 +280,29 @@ function attachStatusBar(view) { } window.PagesWikiEditor = { + highlightCodeBlock(code, mode) { + mode = String(mode || '').toLowerCase(); + const language = languageExtension(mode); + const text = code.textContent; + // Projection only: never change canonical HTML, unlabelled/plain text, + // or large samples. Large code files retain the windowed viewer. + if (!language || text.length > 32768) return false; + const parser = (language.language || language).parser; + const fragment = document.createDocumentFragment(); + let offset = 0; + highlightTree(parser.parse(text), pagesWikiHighlightStyle, (from, to, classes) => { + if (from > offset) fragment.append(document.createTextNode(text.slice(offset, from))); + const span = document.createElement('span'); span.className = classes; + span.textContent = text.slice(from, to); fragment.append(span); offset = to; + }); + fragment.append(document.createTextNode(text.slice(offset))); + code.replaceChildren(fragment); + return true; + }, createViewer(target, text, mode, minimap) { const extensions = [ lineNumbers(), - syntaxHighlighting(defaultHighlightStyle), + syntaxHighlighting(pagesWikiHighlightStyle), pagesWikiTheme, EditorState.readOnly.of(true), EditorView.editable.of(false), @@ -305,11 +323,13 @@ window.PagesWikiEditor = { }, create(target, initialText, mode, options = {}) { - const keymapName = - String(options.keymap || "default").toLowerCase(); + const mobile = window.matchMedia("(pointer: coarse) and (hover: none)").matches; + const keymapName = mobile ? "default" : String(options.keymap || "default").toLowerCase(); const extensions = [ lineNumbers(), + highlightActiveLine(), + highlightActiveLineGutter(), history(), bracketMatching(), closeBrackets(), @@ -340,7 +360,7 @@ window.PagesWikiEditor = { ); if (options.syntax !== false) { - extensions.push(syntaxHighlighting(defaultHighlightStyle)); + extensions.push(syntaxHighlighting(pagesWikiHighlightStyle)); } if (options.autocomplete !== false) { @@ -358,6 +378,9 @@ window.PagesWikiEditor = { if (language) { extensions.push(language); } + if (mode === "html" || mode === "markdown") { + extensions.push(EditorState.languageData.of(() => [{closeBrackets:{brackets:["(", "[", "'", '"', "`"]}}])); + } const view = new EditorView({ @@ -372,116 +395,17 @@ window.PagesWikiEditor = { if (keymapName === "vim") { attachVimModeTracking(view); - attachPagesWikiExFallback(view); } return view; }, }; - -function attachPagesWikiExFallback(view) { - let active = false; - let buffer = ""; - - function save() { - if (window.PagesEditor && window.PagesEditor.saveNow) { - window.PagesEditor.saveNow(); - } - } - - function quit() { - location.href = - window.PagesWikiCurrentPath || "/"; - } - - view.dom.addEventListener( - "keydown", - (ev) => { - if (!active && vimMode === "NORMAL" && ev.key === ":") { - active = true; - buffer = ""; - return; - } - - if (!active) { - return; - } - - if (ev.key === "Escape" || (ev.ctrlKey && ev.key === "[")) { - active = false; - buffer = ""; - return; - } - - if (ev.key === "Enter") { - const cmd = - buffer.trim(); - - active = false; - buffer = ""; - - if (cmd === "w") { - ev.preventDefault(); - save(); - return; - } - - if (cmd === "q") { - ev.preventDefault(); - quit(); - return; - } - - if (cmd === "wq" || cmd === "x") { - ev.preventDefault(); - save(); - setTimeout(quit, 120); - return; - } - - return; - } - - if (ev.key.length === 1) { - buffer += ev.key; - } - - if (ev.key === "Backspace") { - buffer = - buffer.slice(0, -1); - } - }, - true - ); -} - - -let pagesWikiPreviewTimer = null; - function schedulePagesWikiPreview(view) { - const textarea = - document.getElementById("editor"); - + const textarea = document.getElementById("editor"); if (textarea) { textarea.value = view.state.doc.toString(); + textarea.dispatchEvent(new Event("input", {bubbles:true})); } - - if (pagesWikiPreviewTimer) { - clearTimeout(pagesWikiPreviewTimer); - } - - pagesWikiPreviewTimer = - setTimeout( - () => { - if ( - window.PagesEditor && - window.PagesEditor.renderPreview - ) { - window.PagesEditor.renderPreview(); - } - }, - 350 - ); } diff --git a/cmd/pageswiki/dr.go b/cmd/pageswiki/dr.go new file mode 100644 index 0000000..fa64a6a --- /dev/null +++ b/cmd/pageswiki/dr.go @@ -0,0 +1,127 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "time" + + "pageswiki/internal/dr" +) + +// All role changes are operator-only and offline. Never expose this as a +// domain-admin API: a data directory contains multiple campuses and domains. +func runDR(args []string, out io.Writer) error { + const usage = "pageswiki dr {init|status|snapshot|verify|restore|promote|resume-workers} [flags]\nUse COMMAND --help. Snapshot requires all writers stopped; promotion requires external fencing." + if len(args) == 0 || args[0] == "--help" { + _, err := fmt.Fprintln(out, usage) + return err + } + cmd := args[0] + switch cmd { + case "init", "status", "snapshot", "verify", "restore", "promote", "resume-workers": + default: + return errors.New(usage) + } + f := flag.NewFlagSet("dr "+cmd, flag.ContinueOnError) + f.SetOutput(out) + defaultDir := os.Getenv("PAGESWIKI_DATA_DIR") + if defaultDir == "" { + defaultDir = "./data" + } + dir := f.String("data-dir", defaultDir, "instance data directory") + var output, snapshot, hash, target, fenced, reference *string + var stopped, loss, reviewed *bool + switch cmd { + case "resume-workers": + reviewed = f.Bool("reviewed-restored-jobs", false, "attest restored queues and federation state have been reviewed for replay/duplicate effects") + case "snapshot": + output = f.String("output", "", "new snapshot directory outside data directory") + stopped = f.Bool("writers-stopped", false, "attest ALL servers, CLIs and external writers are stopped") + case "verify", "restore": + snapshot = f.String("snapshot", "", "snapshot directory") + hash = f.String("manifest-sha256", "", "manifest SHA-256 received over an authenticated channel") + if cmd == "restore" { + target = f.String("target", "", "new, nonexistent data directory (always standby)") + } + case "promote": + fenced = f.String("fenced-node", "", "exact former primary node UUID") + reference = f.String("fence-reference", "", "operator evidence/ticket for verified power/storage/network fencing") + loss = f.Bool("accept-data-loss", false, "accept loss of writes since the snapshot cutoff") + } + if err := f.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + return err + } + if f.NArg() != 0 { + return errors.New("unexpected positional arguments") + } + encode := func(v any) error { e := json.NewEncoder(out); e.SetIndent("", " "); return e.Encode(v) } + ctx := context.Background() + switch cmd { + case "resume-workers": + c, err := dr.ResumeWorkers(*dir, *reviewed) + if err != nil { + return err + } + return encode(c) + case "init": + c, err := dr.Init(*dir) + if err != nil { + return err + } + return encode(c) + case "status": + c, err := dr.Load(*dir) + if err != nil { + return err + } + return encode(struct { + Node *dr.Config `json:"node"` + StandbyReadable bool `json:"standby_readable"` + }{c, c != nil && c.Readable(time.Now())}) + case "snapshot": + if *output == "" || !*stopped { + return errors.New("snapshot requires --output and --writers-stopped; stop every writer first") + } + m, sum, err := dr.Snapshot(ctx, *dir, *output) + if err != nil { + return err + } + return encode(struct { + Manifest *dr.Manifest `json:"manifest"` + SHA256 string `json:"manifest_sha256"` + }{m, sum}) + case "verify": + if *snapshot == "" { + return errors.New("--snapshot is required") + } + m, err := dr.Verify(ctx, *snapshot, *hash) + if err != nil { + return err + } + return encode(m) + case "restore": + if *snapshot == "" || *target == "" { + return errors.New("--snapshot and --target are required") + } + c, err := dr.Restore(ctx, *snapshot, *hash, *target) + if err != nil { + return err + } + return encode(c) + case "promote": + c, err := dr.Promote(*dir, *fenced, *reference, *loss) + if err != nil { + return err + } + return encode(c) + } + return errors.New("unknown DR command") +} diff --git a/cmd/pageswiki/dr_test.go b/cmd/pageswiki/dr_test.go new file mode 100644 index 0000000..2ddf935 --- /dev/null +++ b/cmd/pageswiki/dr_test.go @@ -0,0 +1,21 @@ +package main + +import ( + "bytes" + "strings" + "testing" +) + +func TestDRCLIRequiresExplicitOfflineAttestation(t *testing.T) { + var out bytes.Buffer + err := runDR([]string{"snapshot", "--data-dir", t.TempDir(), "--output", "/not-used"}, &out) + if err == nil || !strings.Contains(err.Error(), "writers-stopped") { + t.Fatalf("unsafe invocation: %v", err) + } + if err = runDR([]string{"restore"}, &out); err == nil { + t.Fatal("missing restore target accepted") + } + if err = runDR([]string{"snapshot", "--help"}, &out); err != nil || !strings.Contains(out.String(), "writers-stopped") { + t.Fatal(err) + } +} diff --git a/cmd/pageswiki/main.go b/cmd/pageswiki/main.go index a2750da..3cf3b1b 100644 --- a/cmd/pageswiki/main.go +++ b/cmd/pageswiki/main.go @@ -1,16 +1,34 @@ package main import ( + "errors" "log" "os" "path/filepath" "pageswiki/internal/db" + "pageswiki/internal/dr" "pageswiki/internal/server" "pageswiki/internal/store" ) func main() { + if len(os.Args) > 1 && os.Args[1] == "schema" { + if err := runSchema(os.Args[2:], os.Stdout); err != nil { + log.Print(err) + if errors.Is(err, errSchemaPending) { + os.Exit(2) + } + os.Exit(1) + } + return + } + if len(os.Args) > 1 && os.Args[1] == "dr" { + if err := runDR(os.Args[2:], os.Stdout); err != nil { + log.Fatal(err) + } + return + } enrollment, err := parseExternalCampusEnrollment(os.Args[1:]) if err != nil { log.Fatal(err) @@ -34,6 +52,39 @@ func main() { log.Fatal(err) } + instanceLock, err := dr.Lock(dataDir, false) + if err != nil { + log.Fatal(err) + } + defer instanceLock.Close() + node, err := dr.Load(dataDir) + if err != nil { + log.Fatalf("invalid DR configuration (startup refused): %v", err) + } + if err = dr.RequireRole(node, os.Getenv("PAGESWIKI_DR_EXPECT_ROLE")); err != nil { + log.Fatal(err) + } + if node != nil && node.Role == "standby" { + if len(os.Args) > 1 { + log.Fatal("standby does not accept enrollment/startup mutation flags") + } + database, err := db.OpenReadOnly(filepath.Join(dataDir, "pages.db")) + if err != nil { + log.Fatal(err) + } + defer database.Close() + st, err := store.NewReadOnlyWithDocumentRoot(database, filepath.Join(dataDir, "documents")) + if err != nil { + log.Fatal(err) + } + st.StandbyReadUntil = node.ReadUntil() + log.Printf("DR standby: migrations, federation reconciliation and background workers disabled; snapshot=%s", node.SnapshotID) + if err := server.New(st).Run(":3000"); err != nil { + log.Fatal(err) + } + return + } + dbPath := filepath.Join( dataDir, @@ -87,7 +138,10 @@ func main() { if err := enrollExternalCampus(dataDir, st, enrollment); err != nil { log.Fatal(err) } - startManagedCampusReconciler(dataDir, st) + st.BackgroundJobsDisabled = node != nil && !node.WorkersEnabled + if !st.BackgroundJobsDisabled { + startManagedCampusReconciler(dataDir, st) + } srv := server.New(st) diff --git a/cmd/pageswiki/schema.go b/cmd/pageswiki/schema.go new file mode 100644 index 0000000..bf5c37d --- /dev/null +++ b/cmd/pageswiki/schema.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "pageswiki/internal/db" + "pageswiki/internal/store" +) + +var errSchemaPending = errors.New("schema migrations are pending; no changes were applied") +var errSchemaBlocked = errors.New("schema history or contract is incompatible; no changes were applied") + +func runSchema(args []string, out io.Writer) error { + const usage = "pageswiki schema {status|check} [--db FILE | --data-dir DIR] [--json] [--timeout 30s]\nRead-only inspection of managed components; legacy migrations and pending DDL are not rehearsed.\ncheck exits 2 for pending migrations, 1 for incompatible state, 0 when managed components are current." + if len(args) == 0 || args[0] == "--help" { + _, err := fmt.Fprintln(out, usage) + return err + } + if args[0] != "status" && args[0] != "check" { + return errors.New(usage) + } + f := flag.NewFlagSet("schema "+args[0], flag.ContinueOnError) + f.SetOutput(out) + defaultDir := os.Getenv("PAGESWIKI_DATA_DIR") + if defaultDir == "" { + defaultDir = "./data" + } + dir := f.String("data-dir", defaultDir, "instance data directory (not created)") + path := f.String("db", "", "explicit existing SQLite database path") + jsonOutput := f.Bool("json", false, "JSON report") + timeout := f.Duration("timeout", 30*time.Second, "inspection timeout") + if err := f.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + return err + } + if f.NArg() != 0 || *timeout <= 0 { + return errors.New("unexpected arguments or invalid timeout") + } + explicitDir := false + f.Visit(func(flag *flag.Flag) { + if flag.Name == "data-dir" { + explicitDir = true + } + }) + if *path != "" && explicitDir { + return errors.New("choose --db or --data-dir, not both") + } + if *path == "" { + *path = filepath.Join(*dir, "pages.db") + } + database, err := db.OpenReadOnly(*path) + if err != nil { + return err + } + defer database.Close() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + report, err := store.InspectSchema(ctx, database) + if err != nil { + return err + } + if *jsonOutput { + encoder := json.NewEncoder(out) + encoder.SetIndent("", " ") + err = encoder.Encode(report) + } else { + _, err = fmt.Fprintf(out, "Coverage: %s (not a full startup or data migration rehearsal)\n", report.Coverage) + for _, component := range report.Components { + if err != nil { + break + } + _, err = fmt.Fprintf(out, "%-24s %-8s %d -> %d\n", component.Component, component.State, component.AppliedVersion, component.LatestVersion) + for _, step := range component.Pending { + if err != nil { + break + } + _, err = fmt.Fprintf(out, " pending v%d %s (%d SQL statements, %d conditional columns)\n", step.Version, step.Name, step.Statements, len(step.AddColumns)) + } + for _, issue := range component.Issues { + if err != nil { + break + } + _, err = fmt.Fprintf(out, " %s\n", issue) + } + } + if err == nil { + _, err = fmt.Fprintf(out, "Legacy areas not covered: %v\n", report.LegacyAreas) + } + } + if err != nil || args[0] == "status" { + return err + } + pending := false + for _, component := range report.Components { + if component.State == "blocked" || component.State == "unknown" { + return errSchemaBlocked + } + pending = pending || component.State == "pending" + } + if pending { + return errSchemaPending + } + return nil +} diff --git a/cmd/pageswiki/schema_test.go b/cmd/pageswiki/schema_test.go new file mode 100644 index 0000000..18e7510 --- /dev/null +++ b/cmd/pageswiki/schema_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "pageswiki/internal/db" + "pageswiki/internal/store" +) + +func TestSchemaCommandMissingDatabaseDoesNotInitialize(t *testing.T) { + dir := filepath.Join(t.TempDir(), "missing") + var out bytes.Buffer + if err := runSchema([]string{"status", "--data-dir", dir}, &out); err == nil { + t.Fatal("missing DB accepted") + } + if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("directory created: %v", err) + } +} + +func TestSchemaCommandPendingAndBlocked(t *testing.T) { + dir := t.TempDir() + database, err := db.Open(filepath.Join(dir, "pages.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if err := database.Ping(); err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := runSchema([]string{"status", "--data-dir", dir, "--json"}, &out); err != nil { + t.Fatal(err) + } + var report store.SchemaInspection + if err := json.Unmarshal(out.Bytes(), &report); err != nil { + t.Fatal(err) + } + if report.Coverage != "managed-components-only" || len(report.Components) == 0 || len(report.LegacyAreas) == 0 { + t.Fatalf("report=%+v", report) + } + out.Reset() + if err := runSchema([]string{"check", "--db", filepath.Join(dir, "pages.db")}, &out); !errors.Is(err, errSchemaPending) { + t.Fatalf("pending: %v", err) + } + if _, err := database.Exec(`CREATE TABLE schema_migration_steps(component TEXT,version INTEGER,name TEXT,checksum TEXT,applied_at_ms INTEGER); INSERT INTO schema_migration_steps VALUES('future-component',1,'future','unknown',0)`); err != nil { + t.Fatal(err) + } + out.Reset() + if err := runSchema([]string{"check", "--data-dir", dir}, &out); !errors.Is(err, errSchemaBlocked) { + t.Fatalf("unknown component: %v", err) + } + var count int + if err := database.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='table'`).Scan(&count); err != nil || count != 1 { + t.Fatalf("command initialized DB: %d %v", count, err) + } +} + +func TestSchemaCommandArguments(t *testing.T) { + for _, args := range [][]string{{"apply"}, {"check", "--timeout", "0s"}, {"check", "extra"}, {"check", "--db", "a", "--data-dir", "b"}} { + if err := runSchema(args, &bytes.Buffer{}); err == nil { + t.Fatalf("accepted %v", args) + } + } + for _, args := range [][]string{nil, {"--help"}, {"check", "--help"}} { + if err := runSchema(args, &bytes.Buffer{}); err != nil { + t.Fatalf("help %v: %v", args, err) + } + } +} + +func TestSchemaCommandCurrentApplicationDatabase(t *testing.T) { + dir := t.TempDir() + database, err := db.Open(filepath.Join(dir, "pages.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if err := db.Init(database); err != nil { + t.Fatal(err) + } + if err := db.Migrate(database); err != nil { + t.Fatal(err) + } + // Initialize only this synthetic fixture. runSchema must not run the + // constructor itself, even for an already initialized application DB. + store.NewWithDocumentRoot(database, filepath.Join(dir, "documents")) + var out bytes.Buffer + if err := runSchema([]string{"check", "--data-dir", dir, "--json"}, &out); err != nil { + t.Fatalf("current database: %v\n%s", err, out.String()) + } + var report store.SchemaInspection + if err := json.Unmarshal(out.Bytes(), &report); err != nil { + t.Fatal(err) + } + for _, component := range report.Components { + if component.State != "current" { + t.Fatalf("component=%+v", component) + } + } +} diff --git a/data/pageswiki-cli b/data/pageswiki-cli index 2813459..16dcf24 100755 --- a/data/pageswiki-cli +++ b/data/pageswiki-cli @@ -9,6 +9,7 @@ Uses only the Python standard library. from __future__ import annotations import difflib +import fcntl import fnmatch import glob import hashlib @@ -336,6 +337,26 @@ class PagesWikiCLI: if not self.database_path.is_file(): raise CLIError(f"SQLite database not found: {self.database_path}") + # Coordinate with offline DR snapshots/promotions. Do this before any + # schema repair or content write. A standby never permits this CLI. + self._instance_lock = None + try: + lock_path = self.database_path.parent / ".pageswiki-instance.lock" + self._instance_lock = lock_path.open("rb") if lock_path.exists() else lock_path.open("a+b") + fcntl.flock(self._instance_lock.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + for root in {self.data_dir, self.database_path.parent, self.documents.parent}: + config_path = root / "dr-node.json" + if config_path.exists(): + config = json.loads(config_path.read_text()) + if not isinstance(config, dict) or config.get("version") != 1 or config.get("role") != "primary": + raise CLIError("DR standby or invalid configuration: data CLI is disabled") + if root != self.database_path.parent or self.documents != root / "documents": + raise CLIError("DR requires DB and documents in the same instance data directory") + except (OSError, ValueError, CLIError) as exc: + if self._instance_lock is not None: + self._instance_lock.close() + raise CLIError(f"DR maintenance guard: {exc}") from exc + self.db = sqlite3.connect( self.database_path, timeout=30, @@ -351,6 +372,8 @@ class PagesWikiCLI: def close(self) -> None: self.db.close() + if self._instance_lock is not None: + self._instance_lock.close() def _check_schema(self) -> None: required = {"objects", "paths", "revisions"} diff --git a/data/pageswiki-remote b/data/pageswiki-remote new file mode 100755 index 0000000..1f336f2 --- /dev/null +++ b/data/pageswiki-remote @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +"""Compatibility launcher for the unreleased prototype. Prefer pages-cli/pages.""" +from pathlib import Path +import runpy +import sys + +print('Renamed to pages (pages-cli). Use pages configure, then pages login. See pages-cli/README.md.', file=sys.stderr) +runpy.run_path(str(Path(__file__).resolve().parent.parent / 'pages-cli' / 'pages'), run_name='__main__') diff --git a/data/test_cli_dr.py b/data/test_cli_dr.py new file mode 100644 index 0000000..8793713 --- /dev/null +++ b/data/test_cli_dr.py @@ -0,0 +1,34 @@ +import fcntl +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from pageswiki_cli_test import pageswiki_cli + + +class DRGuardTest(unittest.TestCase): + def test_standby_rejected_before_database_open(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "pages.db").write_bytes(b"not a database; must not be opened") + (root / "dr-node.json").write_text(json.dumps({"version": 1, "role": "standby"})) + with patch.dict(os.environ, {"PAGESWIKI_DATA_DIR": directory, "PAGESWIKI_DB": str(root / "pages.db"), "PAGESWIKI_DOCUMENTS_DIR": str(root / "documents")}): + with self.assertRaisesRegex(pageswiki_cli.CLIError, "standby"): + pageswiki_cli.PagesWikiCLI() + + def test_offline_maintenance_lock_blocks_cli(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "pages.db").write_bytes(b"not a database") + with (root / ".pageswiki-instance.lock").open("wb") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + with patch.dict(os.environ, {"PAGESWIKI_DATA_DIR": directory, "PAGESWIKI_DB": str(root / "pages.db")}): + with self.assertRaisesRegex(pageswiki_cli.CLIError, "maintenance guard"): + pageswiki_cli.PagesWikiCLI() + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/design/README.md b/docs/design/README.md index dc7cf38..212c1c3 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -14,9 +14,39 @@ and future aligned translations, not as automatically current English editions. - Operations, i18n, ordered work, decisions and sources: chapters 18–22. - [Source-to-chapter inventory](ja/sources.md). - [Decision and inconsistency register](ja/decisions.md). +- [Component names and responsibilities](ja/component-names.md): adopted subsystem names, + all eight initial extractions, compatibility boundaries and the remaining component work. +- [Dregarnuhr scheduler contract](ja/dregarnuhr.md): application registration, queue persistence, + concurrency/retry tests and the remaining lease/HA/shutdown work. +- [Geduldh document storage contract](ja/geduldh.md): immutable local payloads, bounded scratch copies, + compatibility facades, Range/backfill/DR verification and remaining backend work. +- [Mestionora search and index contract](ja/mestionora.md): SQLite/City lookup, explicit read policies, + bounded enrichment, query-plan compatibility and remaining indexing/policy work. +- [Ordoschnellic notification delivery contract](ja/ordoschnellic.md): bounded Web Push dispatch, + transactional Inbox enqueue, privacy, retry/lease compatibility and remaining fencing/observability work. +- [Schutzaria authorization and publication contract](ja/schutzaria.md): pure policy evaluation, + immutable request snapshots, retained application checks and remaining consistency/reviewer work. +- [Kunstzeal projection and renderer contract](ja/kunstzeal.md): stable adapter selection, bounded + Kroki execution, source authorization, atomic derived cache and remaining provider/runtime work. +- [Seheweit observation contract](ja/seheweit.md): bounded local collection, compatible snapshots, + history windows/retention, cancellation and remaining availability/rollup/provider metrics work. +- [Editor resilience subproject](ja/editor-resilience.md): drafts, conditional saves, + conflict resolution, mobile authoring, language/history, round-trip preservation and collaboration gates. +- [Storage portability and operations](storage-portability-and-operations.md): backend contracts, + migration/backfill, backup/export/import and setup phases; + [read-only schema preflight and rollout](../operations/storage-schema-rollout.md). ## Document roles +DRの詳細設計・初版CLI・復旧手順は [単一writerのDR](ja/disaster-recovery.md) を参照。 +インスタンス単位の停止snapshotと手動昇格を先行し、自動切替・オンライン同期は未実装。 + +次段階のDR-D1は**詳細設計完了・実装未着手**: +[役割分離と再参加](ja/dr-authority-and-rejoin.md)、 +[protocol・同期・切替](ja/dr-protocol-and-sync.md)、 +[実施計画と受入試験](ja/dr-delivery-plan.md)。DB別件と独立して文書化し、 +保存側の能力契約が未提供の場合は自動切替を有効化しない。 + `arch.md`, `address-model.md`, `access-model.md`, `auth.md` and `i18n-delivery.md` include original conceptual drafts. Detailed topic documents describe contracts and implementation slices. `implementation-status.md` contains dated progress; diff --git a/docs/design/arch.md b/docs/design/arch.md index 33a0098..5929cf8 100644 --- a/docs/design/arch.md +++ b/docs/design/arch.md @@ -12,6 +12,25 @@ Pages Wiki is designed as **semantic locality driven knowledge system** rather t - Capability-centric authorization - Delegated access via macaroons xxx +## Implementation component names + +The adopted subsystem names and boundaries are defined in +[Component names and responsibilities](ja/component-names.md). +Grammaratur owns document-format conversion and authoring fidelity, Dregarnuhr +owns in-process scheduling, and Geduldh owns immutable payload storage and bounded +materialization. Mestionora owns SQLite search/index projections with caller-supplied +read policies. Ordoschnellic owns bounded Web Push delivery and transport policy, +with recipient checks and transactional Inbox enqueue retained in the application. +Schutzaria evaluates trusted Path/locality/group/publication policy facts without I/O; +authentication, fact loading and transactional authority remain application-owned. +Kunstzeal owns built-in projection selection and bounded diagram-provider execution; +source authorization and sandbox delivery remain in the application. +Seheweit owns local resource collection and history-window/retention policy; +authorization, queue control and SQL persistence remain in the application. +All eight initial extractions preserve compatibility; their remaining capabilities +are tracked separately. These names do not rename Knowledge +City Model entities, public URIs, stored identifiers or deployment services. + ## Knowledge City Model Pages Wiki don’t only comprises Machine friendly but also Human-Facing friendly with Knowledge City Model. @@ -362,6 +381,12 @@ PAGE --> REV2 PAGE --> REV3 ``` +The staged evolution to PostgreSQL/MySQL, Redis caches, S3 and network file +stores, together with backup/export/import and first-run setup, is defined in +[Storage Portability and Operations](storage-portability-and-operations.md). +The current supported runtime remains SQLite and local storage; the design +does not imply that remote backends are already selectable. + Metadata is stored in the database. Immutable Document payloads use a global content-addressed filesystem: diff --git a/docs/design/auth-backend.md b/docs/design/auth-backend.md index cab4c13..94aae09 100644 --- a/docs/design/auth-backend.md +++ b/docs/design/auth-backend.md @@ -67,6 +67,13 @@ to `enforcing` once the bootstrap flow is complete. ## HTTP flows +Implementation clarification (2026-11-01): clients requesting JSON from +`POST /_/auth/login` (or `/totp`) receive an opaque Bearer session rather than +extracting a browser cookie. The remote CLI follows this existing flow and the +same Domain membership/capability checks; see [remote CLI](ja/remote-cli.md). +It does not implement an OIDC device handoff yet. The cookie descriptions below +apply to browser sessions, not a requirement for all HTTP clients. + - `/_/auth/login`, `/_/auth/logout`, `/_/auth/callback/oidc/{provider}` - `/u/{username}` is the Domain-scoped profile and user-Wiki root. `/u/{username}/_/edit` is its owner-only profile editor and diff --git a/docs/design/data-cli.md b/docs/design/data-cli.md index 0689aac..afa15de 100644 --- a/docs/design/data-cli.md +++ b/docs/design/data-cli.md @@ -1,5 +1,10 @@ # Data CLI +Remote HTTP access is a separate subproject, `pages-cli`, with the `pages` +command (`pages-cli/pages`); see the +[remote CLI contract and usage](ja/remote-cli.md). This local tool retains its +trusted administrative database access and does not acquire Web credentials. + `data/pageswiki-cli` is a local administrative data surface. It provides rsync-like one-shot transfers and an interactive Semantic Path shell. diff --git a/docs/design/event-scheduler.md b/docs/design/event-scheduler.md index 69ca067..77db22f 100644 --- a/docs/design/event-scheduler.md +++ b/docs/design/event-scheduler.md @@ -25,6 +25,14 @@ notification delivery remains future work. External preview tools run through bounded commands with timeouts. +The in-process clock, loop, admission control and retry policy now live in +`internal/dregarnuhr`. Application handlers are registered in +`internal/server/event_handlers.go`; SQL ownership remains in `store` through +the server queue adapter. The existing API, JSON, IDs and schema are unchanged. +Admission is serialized within one runner, not across processes. Atomic claims +are not expiring leases, and startup recovery must not run alongside another +active writer. See [Dregarnuhr contracts and pending HA work](ja/dregarnuhr.md). + ## Event log retention and trends `event_logs` remains the recent, drill-down log. Each record is associated with diff --git a/docs/design/implementation-roadmap.md b/docs/design/implementation-roadmap.md index 83725aa..8236668 100644 --- a/docs/design/implementation-roadmap.md +++ b/docs/design/implementation-roadmap.md @@ -1,7 +1,75 @@ # Architecture Implementation Roadmap +## Current highest priority (2026-10-31) + +The user-prioritized [daily UX / browser / editor / CLI backlog](ja/priority-ux-20261031.md) +takes precedence over the feature backlog below. It covers type-to-search, virtual +Path navigation and browser commands, editor/plain-text and code styling, theme +switching, session timeouts, contacts, City labels and a session-aware terminal client. +Preserve authorization and storage safeguards. Track implementation, tests and +deployment separately; a source checkpoint includes the pre-existing dirty worktree. + ## Principles +### Named architecture components (2026-11-01) + +- [x] Adopt stable responsibility-based names for eight components; preserve UI terminology, + public APIs, persisted IDs, event consumer checkpoints and deployment boundaries. +- [x] Extract Grammaratur: ordinary and protected authoring conversions, compatibility + facades and application connections. Verification is recorded separately from deployment. +- [x] Extract Dregarnuhr clock/loop/admission/retry contracts and the queue adapter; + register the 14 existing business handlers outside the core. Preserve IDs/schema and atomic claims. +- [ ] Add Dregarnuhr expiring leases, fencing, multi-process scheduling and application shutdown integration; + these are not provided by the initial extraction. See [contract and remaining work](ja/dregarnuhr.md). +- [x] Extract Geduldh local payload storage and bounded materialization; preserve the Document Store + contract through aliases, connect application consumers, and test layout/Range/backfill/DR compatibility. + External backends and placement changes remain separate work; see [contract and remaining work](ja/geduldh.md). +- [x] Extract Mestionora SQLite search/index/City lookup with explicit read-policy callbacks, + legacy aliases, candidate/snippet budgets, plan and authorization tests; see [search contract](ja/mestionora.md). +- [ ] Preload lazy user-space read policies before Page search; retain snapshot consistency while + removing nested connection requirements. Keep current-only/Domain-partitioned and language-aware indexing as later migrations. +- [x] Extract Ordoschnellic bounded Web Push dispatch, retry policy and encrypted transport; + preserve transactional Inbox enqueue, authorization, event IDs and subscription checkpoints. + See [delivery contract and remaining work](ja/ordoschnellic.md). +- [ ] Add distinct Push claim generations and subscription-version fencing, Retry-After and + delivery metrics. Initial extraction retains existing retry/lease limitations, not exactly-once delivery. +- [x] Extract Schutzaria pure Path/locality/group/publication/user-space evaluation and approval state transitions; + retain authenticated fact loading, alias checks and transaction ownership in the application. + See [policy contract and remaining work](ja/schutzaria.md). +- [ ] Complete consistent authorization snapshots, lazy user-space preloading, reviewer inheritance/quorum, + and publication revalidation. State-machine extraction alone does not enable the reviewer workflow. +- [x] Extract Kunstzeal built-in adapter selection, bounded Kroki rendering and diagram hash/cache; + retain source authorization/CSP and add redirect refusal plus atomic cache replacement. + See [rendering contract and remaining work](ja/kunstzeal.md). +- [ ] Add provider-versioned cache keys, render concurrency/coalescing, unified disable/metrics and + additional diagram/slide providers. Initial extraction does not implement nwdiag or slides. +- [x] Extract Seheweit resource collection, typed snapshots and history window/retention policy; + retain APIs/schema/event IDs, connect foreground and background paths, bound proc reads, + guard storage arithmetic, propagate write cancellation and reject read-only sample writes. + See [observation contract and remaining work](ja/seheweit.md). +- [ ] Add source availability, bounded/coalesced collection, rollups/retention budgets, + component/provider metrics and instance-scoped HA/SNMP collectors. These are not part of the initial extraction. + +See [component names, boundaries and acceptance criteria](ja/component-names.md). +All eight initial component extractions are implemented; unchecked items describe remaining capabilities. + +### Storage portability and operations (2026-09-27) + +Implemented and fixture-tested in the working tree; **not deployed**. + +- [x] Immutable local Document Store contract, bounded materialization and streaming archive access. +- [x] Resumable document backfill with store identity, UUID cursor and batch checkpoints. +- [x] SQLite migration ledger with version/checksum, atomic adoption and current-schema assertions. +- [x] Managed projection, resource, broadcast, navigation, adapter-session, import and event-queue schemas. +- [x] Read-only `pageswiki schema status|check` with JSON output, pending steps and explicit legacy exclusions. +- [ ] Remaining legacy schema/repository contracts and context propagation; configurable materialization limits/metrics. +- [ ] PostgreSQL/MySQL, S3/SFTP and mounted-store contracts, optional Redis cache. +- [ ] Coordinated backup/restore, portable export/import, Web operations and initial setup wizard. + +See the [Japanese design and acceptance criteria](storage-portability-and-operations.md) and +[preflight / deployment procedure](../operations/storage-schema-rollout.md). +Passing managed-schema checks does not replace an isolated full-startup rehearsal. + ### Task management first slice (2026-09-24) 2026-09-25 task CalDAV (`0.8.0`, implemented and fixture-tested, **deployed to app/app2 on 2026-09-26 at 00:24 JST**; [deployment record](../operations/deployments/2026-09-26-task-caldav.md)): @@ -443,6 +511,12 @@ without changing user-visible Page identity. ## Phase 6 — Activity, notification, and collaboration +- [x] Editor resilience E1–E5 initial delivery: local drafts, conditional language-head saves, + conservative merge/conflict review, revision restore, language forks, mobile input, macro completion, + and protected HTML authoring blocks. See [Japanese implementation contract](ja/editor-resilience.md). +- [ ] Editor E6: authorized cross-instance editing presence and feature-gated CRDT collaboration. + Follow-up: block-level HTML protection, hosted-editor/CLI migration, and extensible insertion palette. + - [ ] Formalize a Domain-sequenced event outbox for Page, Revision, Path, topology, policy, share, comment, reaction, quality, and federation events. - [ ] Add Page/Spot/Street/Quartier subscription and change-notification rules. @@ -479,6 +553,43 @@ without changing user-visible Page identity. ## Phase 8 — Controller disaster recovery and high availability +First slice: [Japanese DR design and operator runbook](ja/disaster-recovery.md). +This is **instance-scoped offline recovery**, not automatic controller HA. + +DR-D1 follow-up is **design complete, implementation pending** (2026-09-27): +[authority/rejoin](ja/dr-authority-and-rejoin.md), +[protocol/synchronization](ja/dr-protocol-and-sync.md), +[delivery plan and acceptance matrix](ja/dr-delivery-plan.md). +No DB/schema/code changes were made for this design-only follow-up. + +- [x] Specify separate controller/data authority, shared-writer coupling, + authenticated rejoin/probe, Learning state, checkpoint verification, + fenced handover/preempt, API contracts, rollout gates and acceptance tests. +- [ ] DR01–DR03: implement validated models, pure state machine and + observation-only authenticated probe; discovery must not grant write access. +- [ ] DR04–DR05: connect restart write gates and authority providers only after + the separately managed storage interfaces and all writer paths are agreed. +- [ ] DR06–DR10: connect consistent checkpoints, bounded resumable transfer, + epoch-aware control manifests, verified fencing and manual group transitions. +- [ ] DR11–DR13: certify preempt policy, scoped operator UI and failure drills; + automatic lossless failover requires an independently verified durability boundary. + +- [x] Node-local primary/standby configuration; read-only SQLite startup without + migrations, key generation, scheduler, GC or managed-campus reconciliation. +- [x] Offline snapshot/verify/restore CLI: committed WAL via SQLite Backup API, + canonical files and keys, payload/reference verification, atomic new generations. +- [x] Offline operator-attested promotion, old-primary identity/data-loss checks, + promotion audit and explicit restored-job resumption. External fencing itself + is not implemented or independently verified by these commands. +- [x] Reject standby writes and stale reads; separate read/write readiness; + coordinate server and data CLI with an instance maintenance lock. +- [ ] Rebuild-free read-only organizer/CalDAV paths, standby UI/role indicators, + DR settings panel and instance-operator authorization. +- [ ] Online cross-store snapshot barrier, GC pins, resumable bounded delta + transfer, encrypted/signed generations, retention and measured restore drills. +- [ ] Restore controller sequence safely across accepted newer manifests; local + audit epoch alone does not solve managed-instance replay protection. + - [ ] Add controller manifest journal replication to a designated DR instance, including signing-key rotation/escrow, encrypted backup, sequence verification, last-known-good snapshots, RPO/RTO reporting, and restore drills. @@ -519,6 +630,8 @@ without changing user-visible Page identity. freezing and configured hosted-editor actions. - [ ] Add Kroki and PlantUML diagram adapters with bounded input, pinned renderer version, cached immutable output, timeout, and network policy. + Existing bounded Kroki execution is now in Kunstzeal; renderer-version keys and complete + provider network/resource policy remain pending rather than marking this whole item complete. - [ ] Add nwdiag as a bounded network-diagram projection and editor adapter; support static SVG fallback, pinned renderer versions, and explicit source import/save without executing authored code. diff --git a/docs/design/ja/active-notifications.md b/docs/design/ja/active-notifications.md index 7c936c9..f03bbab 100644 --- a/docs/design/ja/active-notifications.md +++ b/docs/design/ja/active-notifications.md @@ -2,6 +2,10 @@ 更新:2026-09-25。実装・隔離fixture検証済み。同日05:59 JSTに本番2インスタンスへ反映済み。[反映記録](../../operations/deployments/2026-09-25-active-notifications-calendar-mobile.md)。実機Push受信確認は未実施。 +2026-11-01追記:配送処理を [Ordoschnellic](ordoschnellic.md) へ初期分離。 +以下のUI・API・DB・購読を維持するソース変更であり、この分離自体は未デプロイ。 +claim世代と購読更新競合を含む配送保証・残件は同契約に記録する。 + ## UIと設定 通知ベルを開いたInboxの「⚙ 通知設定」に集約する。アカウント・ドメイン・ブラウザごとにlocalStorageへ保存し、他ユーザの設定を継承しない。通知本文はlocalStorageに保存しない。 @@ -29,7 +33,7 @@ - POST: PushSubscription、focused、seen(最大30通知ID)。購読更新/閲覧中のack。 - DELETE: 現在の利用者が所有する購読を削除。 -購読はDomain・Principal・**元の認証セッションのハッシュ**に束縛する。Bearer tokenは保存しない。配信時にセッション期限/失効、Principal、membership、Domain配置を再確認する。ログアウト後や無効化後は送信しない。最大8購読/利用者・Domain、30日lease(閲覧時に更新)。 +購読はDomain・Principal・**元の認証セッションのハッシュ**に束縛する。Bearer tokenは保存しない。claim時にセッション期限/失効、Principal、membership、Domain配置を再確認する。この時点でログアウト済み・無効化済みなら送信しない。claim後の失効と送信の競合は残る。最大8購読/利用者・Domain、30日lease(閲覧時に更新)。 VAPID秘密鍵は`notification_push_keys`に保存するため、DBバックアップを秘密情報として扱う。鍵を失った復旧では再購読が必要。端末の購読URL・鍵・通知本文をログへ出さない。 diff --git a/docs/design/ja/auth-permissions.md b/docs/design/ja/auth-permissions.md index dcc680e..082c4e9 100644 --- a/docs/design/ja/auth-permissions.md +++ b/docs/design/ja/auth-permissions.md @@ -2,6 +2,10 @@ 状態:パスワード・TOTP・セッション、OIDC、LDAP、SMTPリンク、グループと主要な認可経路は実装済み。全経路の監査完了を意味しません。 +2026-11-01、純粋な権限・公開範囲の評価を [Schutzaria](schutzaria.md) へ初期分離。 +認証・権限情報の読込・更新transactionは既存側に残し、API・ACL設定・公開範囲は変更しません。 +承認の継承・reviewer定足数を実装済みとするものではありません。 + ## 認証から認可へ 認証器はPrincipal identityを証明します。その後、Domain membership、グループ・Role、対象操作のcapability、上位Localityの制限を評価します。OIDCのclaimやメールアドレスだけで管理権限を付与しません。 diff --git a/docs/design/ja/campus-federation.md b/docs/design/ja/campus-federation.md index c1d6f6a..a69efb7 100644 --- a/docs/design/ja/campus-federation.md +++ b/docs/design/ja/campus-federation.md @@ -33,6 +33,12 @@ Domain間PSKによるidentity・group継承は設計段階です。署名付き ## TODOと合格条件 +DRの次段階は [管理主系とデータ主系の分離](dr-authority-and-rejoin.md) と +[管理manifestの世代・受理契約](dr-protocol-and-sync.md) を参照する(詳細設計のみ)。 +Campusを管理するprimaryと、Domainデータに書き込むprimaryは別authority。 +ただし同じDBを正本として共有する役割は物理的なwriter切替を連動させる。 +保存済みroleやprobe応答だけで復帰ノードをprimaryにしない。 + - 反映ACK、設定管理の解除、削除ドメインの保持・GC。 - 同名・異originのグループ衝突、PSK失効、sequence欠落、鍵交換の試験。 - 秘密の外部管理と明示的なローテーション手順。 diff --git a/docs/design/ja/component-names.md b/docs/design/ja/component-names.md new file mode 100644 index 0000000..7477bf3 --- /dev/null +++ b/docs/design/ja/component-names.md @@ -0,0 +1,212 @@ +# PagesWiki のコンポーネント命名と責務 + +2026-11-01 採用。神名を実装アーキテクチャの部品名として使用する。 +Grammaratur、Dregarnuhr、Geduldh、Mestionora、Ordoschnellic、Schutzaria、Kunstzeal、Seheweit の +8部品の初期分離を実装した。初期分離と各部品の後続機能・本番展開の完了は区別する。 + +## 由来と適用範囲 + +プロジェクト名 pages は、作者の意図として Little Glee Monster の楽曲 pages に由来する。 +Knowledge City Model は『とある科学の超電磁砲』の学舎の園に着想を得た知識の配置モデル。 +このモデルの Campus、Domain、Quartier、Street、Spot、Page は変更しない。 + +内部の機能分担には『本好きの下剋上』の神々の名前を用いる。 +作品の役割とソフトウェアの役割を重ねる命名の参考は +[TOMOYO Linux 作者の説明](https://i-love.sakura.ne.jp/tomoyo/)。 +名前の参照表は [プロジェクト作者の Gist](https://gist.github.com/anya-tokugawa/d3939b16298937f8b7e39a6ec3595464)。 +以下の英字表記は PagesWiki 内の採用表記であり、作品の公式翻訳表記を定義するものではない。 + +## 採用した部品名 + +| 部品名 | 内部識別名 | 担当範囲 | 担当しないもの | +| --- | --- | --- | --- | +| Mestionora メスティオノーラ | `mestionora` | 索引、検索、知識間の関連性の解決 | 正本保存、認可の独自判断、編集UI | +| Grammaratur グラマラトゥーア | `grammaratur` | 文書形式変換、変換時の情報保全、変換診断 | 認可、画面用sanitizer、macro展開、Revision保存 | +| Geduldh ゲドゥルリーヒ | `geduldh` | 内容アドレス付きDocumentの保存・取得・完全性検証 | 認証やタスク等の全DB、参照なしを独断判定するGC | +| Dregarnuhr ドレッファングーア | `dregarnuhr` | 実行時刻、ジョブ投入・実行制御、リース、再試行 | カレンダーやタスクの業務ルール、通知本文・宛先 | +| Ordoschnellic オルドシュネーリ | `ordoschnellic` | イベントに応じた通知、宛先別配送、配送状態 | 業務変更の生成、正本トランザクションの所有 | +| Schutzaria シュツェーリア | `schutzaria` | 権限評価、公開範囲、共有・承認ポリシーの評価 | 個々のOIDC/LDAPプロバイダー実装、表示用HTML変換 | +| Kunstzeal キュントズィール | `kunstzeal` | 図・スライド等の描画プロバイダー実行基盤 | 独自描画エンジンの強制、文書正本の変更 | +| Seheweit ゼーエヴァイト | `seheweit` | リソース・稼働状態の収集、集約、観測 | ジョブ制御や観測対象の業務処理 | + +名前はリリースごとに変えず、責務と共に維持する。バージョンは従来の SemVer 2.0.0 を継続する。 +機能を増やすためだけに神名を追加せず、境界と受入条件が成立してから部品を作る。 +神々の系譜や関係を依存グラフ、権限の上下関係、配置単位へ強制的に写さない。 + +## 実装と運用の命名規則 + +- パッケージ名は小文字の内部識別名、設計上の部品名は先頭大文字の表記とする。 + `DocumentStore` 等の契約・インターフェースと `PutIfAbsent` 等の操作名は技術用語を使う。 +- 1部品は1コンテナを意味しない。プロセス内のパッケージとして分離でき、独立配置は別判断。 +- UIは検索、通知、権限設定などの通常名を維持する。神名の理解を利用条件にしない。 + 管理者向けの診断や設計文書には、神名と一般的な役割名を併記できる。 +- URI、既存のprovider/extension ID、設定キー、DB名、保存形式、イベント型、consumer ID、 + レジストリ上のイメージ名を命名のためだけに変更しない。特にconsumer IDの改名で + 既存checkpointを失い、通知や処理を再実行することを防ぐ。 +- 新たなログ・トレースで識別が必要なら固定の `component=grammaratur` 等を用いる。 + 文書本文や認証情報は記録しない。無条件に変換ごとのログを追加しない。 +- 既存の `server` / `store` 一式を神名へ置換しない。呼び出し契約・テストを確立してから + 部品を抽出し、移行中は一方向の互換ラッパーを残す。実装を二重に保守しない。 + +## Grammaratur の実装契約 + +初期実装は `internal/grammaratur`。HTTP、DB、ファイル、ネットワークへ直接アクセスせず、 +文字列の変換だけを行う。認可と入力サイズ上限は呼び出し元が検証する。 + +| 関数 | 契約 | +| --- | --- | +| `HTMLToMarkdown` | 従来の通常変換。保護フェンスを理解しない外部エディタ等にも渡せる。変換損失の可能性は従来どおり | +| `MarkdownToHTML` | 従来のMarkdown方言とオプションを維持した通常変換 | +| `HTMLToMarkdownAuthoring` | 往復変換で構造・属性・文字が変わる場合は元HTMLを保護ブロックに保持 | +| `MarkdownAuthoringHTML` | 明示的にauthoringを選んだ呼び出しだけで `{=html}` ブロックをHTMLへ戻す | + +出力は表示を許可されたHTMLではない。ブラウザー表示時は従来どおり認可・macro展開・ +sanitizer・CSPを適用する。原文の保全とスクリプトの実行許可を混同しない。 +通常変換を保護ブロック対応へ暗黙に切り替えず、hosted editor の保存形式も変更しない。 + +`internal/compiler` のHTML/Markdown変換関数と `render.Markdown` は Grammaratur へ委譲する。 +組み込みエディタ、authoring API、Markdown出力、hosted editor のHTML入力変換は +Grammaratur を直接呼び出す。既存の通常Markdown呼び出しは `render.Markdown` 経由でも +同じ実装を使う。図のhash等、文書形式変換以外の `compiler` の処理は今回移さない。 + +依存方向は呼び出し元 → Grammaratur → 変換ライブラリ。 +Grammaratur から server/store/db/render/compiler へ戻る依存は作らない。 +新たな形式や詳細診断を追加する場合も、この境界を維持する。 + +## 移行順序と受入条件 + +1. **Grammaratur 初期分離**:通常変換と保護変換を集約。既存の関数・API・返却形式を維持。 + 通常変換の旧ライブラリ出力との一致、互換ラッパー、保護ブロック、エディタ、 + hosted editor、表示安全性、ドキュメントexportを検証する。 +2. **Dregarnuhr 初期分離**:現行schedulerから汎用の時刻・retry・dispatch契約を抽出。 + preview、City、calendar、taskの実行ハンドラーは登録型の利用者として残す。 + fake clockとworker競合試験で優先度・実行上限・既存claimを検証する。 + 期限付きlease・fencingは既存実装にもなく、初期分離には含めない。 + [実装契約と後続TODO](dregarnuhr.md) を参照。 +3. **Geduldh 初期分離**:既存 `documentstore.Store` 契約をaliasで維持し、 + ローカル保存と検証付き一時展開の実装を `internal/geduldh` に分離。 + filesystem layout、hash、Range、streaming、移行・DR手順を変えない。 + 大きな `store` パッケージ全体の改名はしない。[文書保存契約と残件](geduldh.md) を参照。 +4. **Mestionora 初期分離**:SQLite検索・索引・City検索を分離し、 + 呼出し元の権限判定と既存の候補・抜粋予算を維持。[検索と索引契約](mestionora.md) を参照。 +5. **Ordoschnellic 初期分離**:Web Pushの有界配送・再試行・暗号化を抽出し、 + Inboxの原子的登録と購読checkpointを維持。再送が起こり得る既存の保証範囲を変更しない。 + [通知配送契約と残件](ordoschnellic.md) を参照。 +6. **Schutzaria 初期分離**:DB非依存の読み取り・Locality・グループ・公開範囲・承認状態評価を分離。 + 認証・DB読込・公開確定transactionは既存側に残す。[権限評価契約と残件](schutzaria.md) を参照。 +7. **Kunstzeal 初期分離**:ビルトイン表示方式の選択、Krokiの有界描画、図hash/cacheを分離。 + 認可・ソース由来・CSPを維持し、redirect拒否とcache原子的置換を追加。[描画契約と残件](kunstzeal.md) を参照。 +8. **Seheweit 初期分離**:リソース収集と履歴保持・bucket規則を分離。現在値APIと背景監視を接続し、 + 収集読込上限・容量計算・取消/read-only保存を補強。認可とSQLは既存側に残す。[観測契約と残件](seheweit.md) を参照。 + +この移行は配置変更、DB migration、機能追加を必須にしない。各段階でソース変更、 +テスト、commit、CI、デプロイを別々に記録する。本番反映は検証後の別操作とする。 + +## 今回の変更範囲 + +- 採用済み:8部品の名称と責務、互換性と依存方向の規則。 +- 実装済み:8部品の初期分離、旧入口の委譲、利用側の接続、境界・互換性テスト。 +- 後続:Dregarnuhr のリース・fencing・終了処理、 + Grammaratur の形式追加・詳細診断・ブロック単位の保護、Geduldh の外部backend・配置分離、 + Mestionora のポリシー事前読込・現行専用索引・言語別検索、Ordoschnellic のclaim世代・購読更新競合・配送観測、 + Schutzaria の権限読込の一貫性・承認継承・権限総覧、Kunstzeal のprovider版・実行予算・描画形式の拡充、 + Seheweit の情報源別availability・収集予算・rollup・component別観測・HA/SNMP。 +- 対象外:既存DBと本番データの変更、UIの改名、画像やロゴの利用、コンテナ分割、本番展開。 + +## 検証記録 + +2026-11-01、合成データと一時DBで以下を確認した。本番データへの操作は行っていない。 + +- `go test ./...` 成功。保存層・サーバー・CLI・設計書exportを含む。 +- `go test -race ./internal/grammaratur -count=1` 成功。 +- `go vet ./internal/grammaratur ./internal/compiler ./internal/render` 成功。 +- `PAGESWIKI_BROWSER_TEST=1` の `TestEditorResilienceChromium` と + `TestHTMLProjectionChromium` 成功。編集・保存・競合・言語分岐・HTML保全と、 + 9つの表示/編集経路の安全性を確認。 +- `git diff --check` 成功。 + +同日、Dregarnuhr の初期分離後にも以下を確認した。 + +- `go test ./...` 成功。store・serverを含め全Goパッケージの回帰テストを実行。 +- `go test -race ./internal/dregarnuhr -count=1` 成功。 +- store/serverの `TestDregarnuhr*`、`TestClaimDueEvent*` とコアの境界・実行制御試験を + race detector付きで実行し成功。外部カレンダーは実行せず、登録の確認だけを行った。 +- `go vet ./internal/dregarnuhr ./internal/store ./internal/server` 成功。 +- `git diff --check` 成功。今回ブラウザ試験の再実行はしていない。 + +同日、Geduldh の初期分離後にも以下を確認した。 + +- `go test ./...` 成功。保存・サーバー・CLI・DR・設計書exportを含む。 +- `go test -race ./internal/geduldh ./internal/documentstore -count=5` 成功。 + 新旧入口の型・エラー・保存キー互換、並行保存、一時コピーの共有と回収を確認。 +- store/server/DRの文書保存、バックフィル、Range配信、認可付き不変オブジェクト配信、 + ZIPの部分取得、Materialize、snapshot/restore試験をrace detector付きで実行し成功。 +- 保存失敗時のRevision非確定、DB競合時のpayload保全、Geduldhから保存したバイト列のDR復元を確認。 +- `go vet ./internal/geduldh ./internal/documentstore ./internal/store ./internal/dr` 成功。 +- `git diff --check` 成功。今回ブラウザ試験は再実行していない。 + +同日、Mestionora の初期分離後にも以下を確認した。 + +- `go test ./...` 成功。既存DBを読む任意診断は無効にし、一時DB・合成データのみで実行。 +- Mestionoraの境界・認可・FTS実行計画をrace detector付きで3回反復し成功。 +- store/serverを含む検索、City、型・エラー互換、Domain/alias分離のテストを + race detector付きで実行し成功。 +- `go vet ./internal/mestionora ./internal/store ./internal/server` 成功。 +- `TestSearchResultsChromium` 成功。1440pxと390pxでCity先頭表示、3表示モード、 + フィルター、リンク、危険なHTMLの文字列表示と横方向のはみ出し防止を検証。画像も目視確認。 +- 索引初期化・更新SQLはpackage/receiver/DBフィールドの移動以外が同一であることを確認。 +- `git diff --check` 成功。ポリシーの遅延DB参照に伴う接続制約は後続TODOとして記録。 + +同日、Ordoschnellic の初期分離後にも以下を確認した。 + +- `go test ./...` 成功。既存DBを読む任意診断は無効にし、一時DB・合成データのみで実行。 +- `go test -race ./internal/ordoschnellic -count=5` 成功。 + バッチ上限、キャンセル後の終了保存、暗号化payload、HTTP分類、未設定clientでの送信禁止、依存境界を確認。 +- store/serverの通知・Activity Subscription・Task Activity・失効済みfocus通知と新規配送試験を + race detector付きで3回反復し成功。並行claim、lease境界、旧attempt完了の拒否、 + 認証/Domain/受取人の不一致、focused延期、backoff、Inbox保全、checkpoint継続を確認。 +- 追加した有界清掃・read-only拒否試験もrace detector付きで3回反復し成功。 +- `go vet ./internal/ordoschnellic ./internal/store ./internal/server` と `git diff --check` 成功。 +- 外部通知サービスへの送信・実端末Push受信・ブラウザ試験は今回実施していない。 + UIとService Workerは変更せず、配送の障害競合は後続TODOとして記録。 + +同日、Schutzaria の初期分離後にも以下を確認した。 + +- `go test ./...` 成功。本番DBを読む任意診断を無効化し、一時DB・合成データで実行。 +- `go test -race ./internal/schutzaria -count=5` 成功。親子制限の交差、肯定grantの範囲、 + snapshotのコピー・並行参照、グループプロファイル、公開上限、予約Path、ユーザー領域、承認状態を検証。 +- store/serverのPath・Locality・匿名公開・公開ナビゲーション失効・グループ継承失効・ + governance・承認・endpoint判定をrace detector付きで2回反復し成功。 +- 新規のstore型/JSON/エラー互換、4,096件上限、DB失敗時の不完全policy拒否試験を + race detector付きで3回反復し成功。 +- `TestCityGovernanceChromium` 成功。隔離fixtureで所有者委譲・公開上限・アクセス設定を操作し、保存結果を確認。 +- 抽出対象8ファイルのSQLリテラル(スキーマ・triggerを含む)が変更前と同一であることを確認。 +- `go vet ./internal/schutzaria ./internal/store ./internal/server` と `git diff --check` 成功。 + +同日、Kunstzeal の初期分離後にも以下を確認した。 + +- `go test ./...` 成功。実Krokiではなくlocalhostの偽provider、一時DB・合成ソースを使用。 +- `go test -race ./internal/kunstzeal -count=5` 成功。表示順・catalogコピー・kind/hash互換、 + 入出力上限、取消、エラー非開示、cacheの並行読取と原子的置換を検証。 +- serverのKunstzeal・既存diagram認可分離・表示形式・hosted adapter設定・HTML projection等を + race detector付きで3回反復し成功。301/302/307/308の転送拒否、cache保存失敗時の表示継続も確認。 +- 追加の由来情報なし/期限切れのディスクcache拒否試験もrace detector付きで3回反復し成功。 +- `TestHTMLProjectionChromium` 成功。既存の9表示/編集経路で正本と表示の信頼境界を確認。 +- registry定義は型名の公開化以外が同一であることを確認。nwdiagは実装済みcatalogに存在しないため、設計書の状態を訂正。 +- `go vet ./internal/kunstzeal ./internal/compiler ./internal/server` と `git diff --check` 成功。 +- 実Kroki・本番ネットワーク・取り込み済み外部カレンダーへのアクセスは行っていない。 + +同日、Seheweit の初期分離後にも以下を確認した。 + +- `go test ./...` 成功。本番DBの任意診断は無効化し、一時DB・合成データで実行。 +- `go test -race ./internal/seheweit -count=5` 成功。JSON互換、情報源の失敗、proc読込上限、 + 容量のoverflow/underflow防止、I/Oの取得可否とprocess identity、並行収集、履歴windowを検証。 +- store/serverのSeheweit・既存リソース監視・履歴・実行状態API試験をrace detector付きで3回反復し成功。 + 同一jobの重複防止、保持境界、bucket代表の互換、取消、read-only保存拒否、清掃失敗時rollback、 + APIの期間fallbackと空配列、GETが履歴保存しないこと、認可・設定保持も確認。 +- `TestExecutionMonitorChromium` 成功。隔離fixtureで実行状態・リソース画面と設定ダイアログを操作し、 + 監視間隔120秒の保存まで確認。2画面のスクリーンショットも確認。 +- `go vet ./internal/seheweit ./internal/store ./internal/server` と `git diff --check` 成功。 +- 本番DB・本番ネットワーク・取り込み済み外部カレンダーへのアクセスは行っていない。 + +8部品の初期分離はソース変更と検証まで完了。commit・push・本番展開は本作業に含めていない。 diff --git a/docs/design/ja/disaster-recovery.md b/docs/design/ja/disaster-recovery.md new file mode 100644 index 0000000..ef939ad --- /dev/null +++ b/docs/design/ja/disaster-recovery.md @@ -0,0 +1,194 @@ +# DR:単一書き込みノードと検証済み復旧世代 + +状態:2026-09-27、設計と第1段階の実装。自動フェイルオーバー、オンライン同期、設定画面は未実装。本番への適用・実データでの復旧訓練は別の承認を得て行う。 + +**次段階の詳細設計DR-D1が完了(文書のみ)**: +[管理主系・データ主系と再参加](dr-authority-and-rejoin.md)、 +[通信・同期・切替protocol](dr-protocol-and-sync.md)、 +[実施計画・受入試験](dr-delivery-plan.md)。 +本書は既存offline実装の記録として保持する。次段階の判断・実装順はDR-D1を優先する。 +DBは別件の変更対象のため、本詳細設計作業ではDB・schema・実装コードを変更していない。 + +## 1. 基本方針 + +Campus / Domain の意味、UUID、FQDN、所属関係を変えず、**そのデータを動かすインスタンスの復旧構成**を追加する。通常時は primary だけが書き込み、standby は検証済みの過去世代を読み取り専用で扱う。ネットワーク疎通失敗だけでは昇格しない。 + +「相手が落ちた」と「こちらから相手へ接続できない」は区別できない。2台だけで相互監視して両方が書き込みを開始すると、SQLite のトランザクションが各ノードで正常でも、全体では異なる正本が生まれる。旧 primary の電源・ストレージ・通信経路を外部で確実に隔離する **fencing** を昇格の前提とする。初版のコマンドは運用者の確認記録を残すもので、fencing 自体を実行・保証しない。 + +### 復旧の単位 + +現状の `pages.db` は複数 Campus / Domain の情報を共有する。したがって第1段階は **データディレクトリ全体=1インスタンス**を複製・昇格する。Campus の設定欄だけで1 Campusを昇格させ、同じDB内の別Campusを読み取り専用に残す操作は提供しない。 + +コントローラと配下の管理インスタンスはそれぞれ別の復旧単位。コントローラのスナップショットにはリモート配置 Domain の設定は含まれるが、別インスタンスにある記事の実体までは含まれない。それぞれ保全する。 + +## 2. 論理モデル + +| 項目 | 意味・保全先 | +| --- | --- | +| Campus / Domain UUID・FQDN | 既存の論理 identity。復旧でも変更しない | +| `cluster_id` | 同じ復旧単位の primary / standby をまとめるUUID | +| `node_id` | DRノード固有のUUID。復元時に新規採番する | +| `primary_node_id` | スナップショットの作成元、または昇格後の自ノード | +| `role` | `primary` / `standby`。プロセス起動時に読み、稼働中には変更しない | +| `epoch` | 手動昇格の監査世代。**分散ロック・署名manifestのfencing tokenではない** | +| `snapshot_id`, `captured_at` | 復元世代と保全開始時刻。再起動で鮮度をリセットしない | +| `max_read_age_seconds` | 待機系の閲覧期限。初期値3,600秒、設定範囲1〜86,400秒 | +| `workers_enabled` | キュー、GC、管理Campus reconcilerの再開許可。復元・昇格直後はfalse | +| `promotion_history` | 時刻、旧primary UUID、隔離の証跡、復元世代、epoch | + +設定は `data/dr-node.json`。これは**ノード固有**なのでprimaryのファイルをstandbyへ丸ごとコピーしない。`dr restore` がstandby用に生成する。`dr-node.json` を削除して昇格してはいけない。設定のない既存インスタンスは後方互換のため従来の書き込み可能起動となる。 + +DRホストの起動環境には `PAGESWIKI_DR_EXPECT_ROLE=standby` を指定する。設定が欠けた空volumeや、異なるroleのvolumeを誤mountした場合はDBを作らず起動を拒否する。昇格後に再起動するときだけ、配置設定も `primary` へ変更する。これは役割を変更する設定ではなく、期待した役割と実際が一致することを確認する安全装置。 + +既存の `instance-ed25519.key` と認証鍵は復旧する論理サービスの鍵であり、DRの物理 `node_id` とは別。鍵の複製を持つ旧ノードは確実に隔離する。 + +## 3. 初版の状態遷移とAPI + +1. 既存 primary を停止し、`dr init` でノード設定を作成する。 +2. 全writer停止中に `dr snapshot` で世代を作成する。primaryは完了後に再開できる。 +3. 別ホストへ転送し、`dr verify` → `dr restore`。復元結果は常にstandby。 +4. standby を別のローカルデータディレクトリから起動する。データvolumeは **read-only mount** とする。primaryとSQLiteファイルを共有しない。 +5. 障害時は旧primaryを永続的に隔離し、standbyも停止して `dr promote`。 +6. 復元後の権限・鍵・設定・キューを確認し、必要な修正をした上で、書き込み可能mountで再起動する。LB/DNSの切替は外部運用。 +7. キューの重複実行・外部送信を検討して、停止中に `dr resume-workers`。再起動でジョブとreconcilerが動く。 + +`dr promote` と `dr resume-workers` はホスト運用者向けCLIに限定する。`campus_admin` や `domain_admin` が他Campusを巻き込んでインスタンス全体の役割を変えるHTTP APIは設けない。 + +### 待機系の動作 + +- SQLiteを `mode=ro` + `query_only` で開く。マイグレーション、Ensure系の初期化、認証鍵生成を行わない。 +- 書き込みHTTPメソッドを503で拒否する。認証発行、エディタ起動、外部エディタ起動、外部図表レンダリング、設定・連携・インポートも拒否する。POSTを成功扱いで捨てない。 +- GC、イベントスケジューラ、preview生成、外部カレンダー同期、Campus reconcilerを起動しない。 +- 普通のWiki文書・RAWの読み取りは従来の認可を通す。既存sessionもスナップショット時点のものに限る。新規ログインは不可。 +- **初版のorganizer/CalDAVと拡張機能APIはstandbyでは503**。既存の読み取り経路にDB索引の修復・資格情報の更新があるため。昇格後は通常の機能を使える。全機能のread replicaを実装済みとはしない。 +- レスポンスは `X-PagesWiki-DR-Role: standby`、`Cache-Control: private, no-store`。RAWの長期cache指定よりも優先する。 +- 古くなった復元世代からの閲覧は503で閉じる。`/health` は生存確認だけで、書き込み可能の判定に使用しない。 +- `GET /health/read` はDB接続と閲覧期限を検査する。`GET /health/write` はstandbyで常に503。primary側の200も**他ノードが停止している証明ではない**。 + +公開範囲縮小、アカウント失効、セッション失効は保全時点より後の変更が反映されない。閲覧期限はその時間を制限するだけで即時失効を保証しない。第1段階ではstandbyは運用者の隔離された検証環境とし、通常の読み取りLBには組み込まない。実際に外部公開へ切り替える前に、復旧時点以降の権限縮小・失効を再適用する。 + +## 4. バックアップの内容と整合性 + +| 対象 | 取扱い | +| --- | --- | +| `pages.db` | SQLite Backup APIで複製。コミット済みWALを含め、rowidを保存する | +| `documents/**` | 記事・asset・taskのCASと、カレンダーの月別ICS・metadataを含む | +| `themes/**` | ローカルテーマ | +| `auth.key`, `instance-ed25519.key` | 存在する場合に保全。極めて機密性が高い | +| `federation/**` | 管理Campus参加token・endpointなどの復旧情報 | +| `dr-node.json` | **除外**。復元先でstandby用に新規生成 | +| preview・一時import・編集中adapter workspace | 除外。未保存編集は復旧対象外 | +| Docker image、Compose、環境変数、TLS鍵、外部volume | このCLIの対象外。別の保全対象台帳で管理 | + +`PAGESWIKI_AUTH_KEY` を環境変数で与えている場合、それも別の暗号化されたsecret保全が必要。復旧先に**同じ値**を供給しないと既存認証・暗号化情報を復旧できない。CLIは環境変数の秘密値をmanifestに書かない。復旧に使用するアプリのimage digestと設定・secretの世代を運用記録に残し、まず同じバージョンで復元する。 + +### なぜ初版は停止バックアップか + +SQLite Backup APIはDB内の整合性を確保するが、同時に更新中の `calendar.json` や別ファイルまで同じ論理時点へ固定しない。DBとファイル正本の境界にglobal snapshot barrierがない現状では、稼働中の `cp` / `rsync data/` を整合バックアップとはみなさない。 + +全てのWebサーバ、CLI、旧バイナリ、SQLite直接編集、外部writerを止める。新しいサーバ・データCLIは `.pageswiki-instance.lock` の共有ロックを保持し、保全・昇格は排他ロックで稼働中を拒否する。ただしこれは協調ロックであり、旧版や外部ツールを強制停止する機能ではない。`--writers-stopped` は実際の停止を運用者が確認したことの明示。 + +### 世代の作成・転送・復元 + +- 同じfilesystem内の一時ディレクトリへ作成し、ファイルfsync、manifest検証後にno-replaceのatomic renameで公開する。既存の保存先へ上書きしない。Linuxの `renameat2(RENAME_NOREPLACE)` を必要とする。 +- manifestはsnapshot/cluster/source-node/epoch、保全開始・完了時刻、相対path・size・SHA-256の一覧を持つ。初版は最大100,000ファイル、manifest最大64MiB。 +- 全payloadのSHA-256、CASファイル名、DBの `quick_check`、Revision/asset/task正本参照の存在、カレンダーmetadataから参照する月別世代のhashを確認する。カレンダーイベント本文の意味的再パースや外部同期はしない。 +- symlink、絶対path、`..`、重複path、未知の保全root、manifestにないpayloadを拒否する。 +- 転送はSSH等の認証された経路で行う。検証に渡す**期待manifest SHA-256を同じ未認証転送から自己申告させない**。現在のmanifestは署名付きではなく、期待hashの真正性は運用者が保証する。 +- バックアップは暗号化済み保管先、暗号化transport、復旧担当者だけのアクセス権で扱う。CLI自身の暗号化・KMS連携は未実装。ファイル0600、ディレクトリ0700で生成する。 +- 復元は新規ディレクトリへコピーし、コピー後もhash検証する。稼働中のデータへoverlayしない。失敗世代は公開せず、旧世代を勝手に削除しない。 +- 世代更新は新規復元→standby停止→mount切替→再起動。初版の復元では物理node UUIDが毎回変わるため、その都度配置台帳を更新する。 + +世代保持の初期案は「直近3世代+日次7世代+週次4世代」。容量・法的保持・鍵ローテーションの要件を確認して決定する。自動削除はまだ実装しない。破損・誤削除が複製されても戻せるよう、同期先とは独立した世代バックアップを最低1系統持つ。 + +## 5. CLIの運用例 + +以下はLinux上のバイナリ例。実環境の絶対pathへ置換する。全コマンドは対象ディレクトリを明示し、保全先はデータディレクトリ外にする。コンテナでは一時CLI用コンテナへ同じdataとbackup保存先をmountし、entrypointに `/pageswiki` を指定する(image内の実際の配置を確認する)。 + +```sh +# 先にprimaryのサーバ・全writerを停止する。 +pageswiki dr init --data-dir /srv/pageswiki/data +pageswiki dr status --data-dir /srv/pageswiki/data +pageswiki dr snapshot --data-dir /srv/pageswiki/data \ + --output /srv/backup/pageswiki/generation-001 --writers-stopped + +# 出力された manifest_sha256 を保管し、snapshotとともにDR側へ安全に転送する。 +# この時点でprimaryは再開できる。 + +# DR側。期待hashは安全に受け取った値。 は置換する。 +pageswiki dr verify --snapshot /srv/backup/pageswiki/generation-001 \ + --manifest-sha256 '' +pageswiki dr restore --snapshot /srv/backup/pageswiki/generation-001 \ + --manifest-sha256 '' --target /srv/pageswiki/standby-001 +pageswiki dr status --data-dir /srv/pageswiki/standby-001 +# standby-001を /data:ro として起動。元の論理FQDN/Hostと環境secretを維持する。 +# PAGESWIKI_DR_EXPECT_ROLE=standby も起動環境に設定する。 +``` + +昇格例: + +```sh +# 旧primaryを停止・隔離し、再起動しても復帰しない外部fenceを確認する。 +# standbyも停止し、失われる範囲(captured_at以降)を確認する。 +pageswiki dr promote --data-dir /srv/pageswiki/standby-001 \ + --fenced-node '' \ + --fence-reference 'incident-123: power fencing independently verified' \ + --accept-data-loss + +# 隔離環境でrw mount、PAGESWIKI_DR_EXPECT_ROLE=primaryへ切替。 +# 権限失効を再適用し、起動・読み書き確認後に経路切替。 +# ジョブはまだ停止。内容を確認・調整し、再びプロセスを停止してから実行する。 +pageswiki dr resume-workers --data-dir /srv/pageswiki/standby-001 \ + --reviewed-restored-jobs +# 再起動でスケジューラ/GC/Campus reconcilerが再開する。 +``` + +旧primaryはそのまま再参加させない。旧データは隔離して差分調査・必要な手動救出用に保全し、新primaryの最新スナップショットから新しいstandbyを作る。古いDBで新primaryを上書きする「切り戻し」は禁止。自動の双方向マージは提供しない。 + +## 6. Campusコントローラに固有の注意 + +Campus manifestの署名、発行sequence、管理ノードの受理sequenceと参加tokenを一体で保全する。記事バックアップとmanifest配信は同義ではない。 + +初版では既存Campus FQDNをDR側へ向け直す方式を採り、管理ノードへ勝手に別Campusを登録しない。ホスト名・証明書・reverse proxyの扱いは復旧手順に含める。`PAGESWIKI_INSTANCE_FQDN` も復旧する論理サービスに合わせる。 + +**重大な未解決事項:** 保全後にprimaryが新しいmanifestを発行し、管理ノードが既に受理していた場合、復元したcontrollerのsequenceが後退して再送を拒否され得る。DRのローカル `epoch` を増やすだけでは既存署名プロトコルのsequence後退を解消しない。reconcilerを既定停止する理由の一つである。強引なDB sequence書換えで回避せず、次段階で「外部fenceで承認されたcontroller epoch+epoch内sequence」のプロトコルと管理側の受理規則を実装する。それまではcontroller配信の完全自動復旧を保証しない。 + +## 7. 次段階:オンライン同期と自動昇格の詳細設計 + +以下は初版時点の構想。詳細化後の実施順は [DR01〜DR13](dr-delivery-plan.md) を正とし、 +probe/再参加gateと役割・writer境界の分離を先行する。実装済み項目ではない。 + +1. **オンライン整合checkpoint**:全正本writer(Web、CLI、calendar metadata、GC)で共有する短時間barrierを設ける。DB backup cutoffとcalendar metadata世代一覧を固定し、その世代が参照するCAS/monthly blobのGC pinを持つ。大きいblob転送はbarrier解除後に低優先度で行う。SQLite単独online backupをこの代替としない。 +2. **転送と受理**:instance-boundの別用途資格情報、mTLSまたは短命署名、cluster/node/epoch/sequence固定。Campus参加tokenを丸用しない。CAS差分転送→hash確認→DBとmetadata世代の受理→active pointer切替の順。途中世代はservableにしない。retryは同じsnapshot IDで冪等、帯域・IO・CPU上限、foreground優先、同時転送数を設定可能にする。 +3. **監視とUI**:設定画面の「運用 / DR」にノード、role、最終検証世代、保全時刻、受信時刻、遅延、容量、最終訓練結果を表示。Campus画面から復旧単位への関連リンクを出す。緑は「HTTP疎通」ではなく「検証済み世代・鮮度・役割整合」、黄は遅延、赤は破損/期限切れ。Domain設定へDRのhostnameを重複入力させない。 +4. **operator権限分離**:インスタンス運用者がreplica/転送鍵/役割を管理し、campus_adminは所属Campusの復旧状態を参照する。共有DBを跨ぐ昇格・secret操作はcampus_adminだけでは許可しない。初版のCLIに後からAPIを接続する際も同じ境界とする。 +5. **キュー・認証の復旧**:session/policy revocation watermarkを同期し、古いACLからのreadをfail closedにする。jobにepoch leaseとidempotency keyを導入し、外部副作用は送信先も含む重複制御または確認フローを設ける。lease期限だけで旧executorを安全とはみなさない。 +6. **controller epoch**:署名対象にleader identityとepochを含め、管理インスタンスは承認された単調増加epochとsequenceを永続化する。epoch移行はout-of-bandで信頼したfencing承認だけが行えるようにする。古い鍵/epochの再送と復元後のrollbackを試験する。 +7. **自動昇格**:第三のwitnessまたはconsensus storeと外部fencing backendを選択。lease失効時は旧primaryがcommitを拒否し、すべてのDB/ファイルwriterとジョブがepochを照合する。fence失敗・quorumなしでは書き込み停止を選び、二重writerを選ばない。これができるまで自動昇格ボタンや「疎通断だけでwrite ok」は実装しない。 +8. **LB参照構成**:write readiness、read freshness、drain、長時間upload、WebSocket、Host検証、接続中断時の冪等再試行を統合する。read/write振り分けはHTTPメソッドだけで決めない。RO nodeをwrite routeへ入れない。 + +API案は `/_/api/v1/instance/dr/...` を予約候補とするが現時点で未登録。転送はユーザーのPath URIと分離し、`POST snapshots`、snapshot ID配下のmanifest/payload、`POST acknowledge` をinstance-to-instance限定で設ける。公開 `/_/api/v1/federation/enroll` にbackupの読み出し能力を追加しない。 + +## 8. RPO / RTOと受入基準 + +初版のRPOは「障害発生時刻 − 最後に**DR側で検証完了した**snapshotのcaptured_at」。保全予定間隔や転送開始時刻ではない。RTOには障害判断、外部fence、復元/検証、権限失効の再適用、サービス検査、経路切替を含む。目標値は実データ量と許容停止時間から別途決定する。RPO=0・無停止とは表明しない。 + +自動試験は合成データを使い、次を確認する。 + +- 未checkpointのコミット済みWALを取り込み、rowidを維持する。 +- DB参照先の欠落・payload改変・manifest改変・path traversal・symlinkを拒否する。 +- 同じsnapshotを繰り返しverifyしても書き換えない。復元先既存ディレクトリを上書きしない。 +- サーバ/CLIの共有ロック中は保全・昇格を拒否する。 +- 復元後はノードUUIDが別になり、role=standby、worker停止となる。 +- standbyの起動とWiki読取でDBを変更せず、書込API/Store、外部起動、期限切れ閲覧を拒否する。認可をバイパスしない。 +- 旧primary identity・隔離の証跡・データ損失受容のいずれかがない昇格を拒否する。 +- 昇格後もjobを勝手に再送せず、明示的再開が必要。 + +実機の電源断、ネットワーク分断、鍵失効、容量不足、転送中断、DNS切替、controller sequence後退、復旧元と受信先のバージョン差異は別の復旧訓練が必要。訓練未完了を成功と扱わない。 + +## 9. 根拠資料と関連設計 + +- [SQLite Online Backup API](https://www.sqlite.org/backup.html):DB単位の整合したコピー。 +- [SQLite WAL](https://www.sqlite.org/wal.html):WALとDBファイルの扱い、共有filesystemの制約。 +- [SQLite VACUUM](https://www.sqlite.org/lang_vacuum.html):rowidが変わり得るため初版のコピー方式には採用しない。 +- [Campus / Domain](../campus-domain-model.md)、[署名manifest](../federation-manifest-integrity.md)、[運用と保全](operations.md)、[Phase 8](../implementation-roadmap.md#phase-8--controller-disaster-recovery-and-high-availability)。 diff --git a/docs/design/ja/dr-authority-and-rejoin.md b/docs/design/ja/dr-authority-and-rejoin.md new file mode 100644 index 0000000..7ba1885 --- /dev/null +++ b/docs/design/ja/dr-authority-and-rejoin.md @@ -0,0 +1,285 @@ +# DR詳細設計:管理主系・データ主系・再参加 + +設計版:DR-D1、2026-09-27。**設計のみ。記載したAPI・状態機械・設定は未実装。** +今回の作業は文書のみを変更し、DB、schema、migration、実行コード、配置設定、本番データを変更しない。 +既存の停止snapshot/手動昇格は [初版の実装・運用記録](disaster-recovery.md) に残す。 + +## 1. 文書体系・決定の適用範囲 + +- 本書:責務、identity、再参加状態、書き込み条件、障害時判断、設定・UI。 +- [通信・同期・主導権移譲契約](dr-protocol-and-sync.md):メッセージ、URI、検証、切替と保全。 +- [実施計画・受入試験](dr-delivery-plan.md):作業単位、変更予定箇所、依存、導入・撤退手順。 + +DR-D1は初版文書の「次段階」の設計を具体化・更新する。初版の `role` や `epoch` の意味を +黙って変更せず、旧形式の移行は別の明示的操作とする。以下のMUST相当の条件は、 +「できれば実装する」事項でなく、機能有効化の前提とする。 + +DB別件の完了を待たず、wire型・純粋な状態機械・認証probe・表示の実装準備はできる。 +一方、writerを開く接続、online snapshot、manifest受理checkpointの永続化、 +独立した主系切替は、保存層の契約確認と別途の実装承認まで無効とする。 + +## 2. 決定一覧 + +| ID | 決定 | +| --- | --- | +| D01 | Campusの管理primaryとData replica groupの書き込みprimaryを分離する | +| D02 | Instanceに全体共通のprimaryフラグを持たせず、役割はgroupごとに表現する | +| D03 | 同じ物理writer境界を共有する役割は連動切替する。論理分離だけで独立writerにしない | +| D04 | DR対象プロセスは再起動時に必ずrecoveringから入る。保存済みroleではwriterを開かない | +| D05 | discovery/probeは情報収集。参加許可、昇格承認、正本の決定とは別 | +| D06 | 発見したpeerの数や、最大の自己申告epochだけでは主系を選ばない | +| D07 | 主系選定の既定はmanual、preempt既定はoff。自動モードは別の能力認定が必要 | +| D08 | 自動裁定の最初の実装先は外部etcd v3 APIとする。PagesWiki独自Raftは作らない | +| D09 | lease切れ・HTTP不通だけでは旧writer停止の証拠にならない。新grant前にfencingを要する | +| D10 | 元primaryの未同期更新は隔離保存し、現primaryへ自動マージしない | +| D11 | 共有DB環境での最初の同期単位はデータディレクトリ全体。Domain単位のSQL差分複製はしない | +| D12 | バックアップ世代、データ書込世代、管理manifest世代を混同しない | +| D13 | 状態収集/authority監視は通常のイベントキューに依存させない。大容量転送は低優先度 | +| D14 | 主系復帰は同期→drain→最終checkpoint→権限移譲の協調的切替とする | + +etcd採用は自動モードの設計上のprovider選択であり、この変更でサービスを導入しない。 +運用でetcdを用意しない場合もmanualの保全・復元を使える。実際のprovider配置、電源/保存先の +fence手段、容量と目標RPO/RTOは導入時の入力であり、推測して本番設定に書かない。 + +## 3. 三つの境界 + +### 3.1 論理サービス + +| 対象 | 不変のidentity | 主系の責務 | 待機系に複製するもの | +| --- | --- | --- | --- | +| Controller group(CG) | group UUID、Campus UUID | Campus設定、Domain配置、管理側継承情報、設定manifest発行 | 管理正本、発行履歴、承認済み構成、復旧用鍵参照 | +| Data replica group(DG) | group UUID、dataset UUID | Domainのページ/asset/カレンダー/タスク/ローカル権限等の更新 | metadata、CAS、可変正本、失効情報、outbox/checkpoint | +| Instance | node UUID、ノード固有鍵 | 上記役割を実行する場所 | Instance自体を業務正本のownerとしない | + +1 Campusにつき管理authorityは1 CG。1 DGは複数Domainを持てる。1 Domainは同一時点では +1 DGへ配置する。複数Campusのデータが1 DGに共存することも、既存DB構成の間は許容する。 +CGのprimaryはDGのprimaryである必要はない。ただし次節の物理境界制約に従う。 + +Campus/DomainのUUID、FQDN、Path、ユーザー領域のURIは維持する。 +Domain配置の将来形式は `replica_group_id` を正本にし、現在primaryのFQDNは解決結果とする。 +物理ノード移動をDomainの改名として記録しない。旧 `placement_instance_fqdn` は移行中の +互換表示に限り、二つの値を別々のauthorityが更新する方式は採らない。 + +### 3.2 保存領域のwriter境界 + +`WriterResource` はDB、可変正本ストア等の実際の排他単位を表す。ローカルpath文字列だけで +なく保存層が提供する安定したresource identityで識別する。複製コピー同士も同じ論理正本の +writer resourceに属する。NFS上のSQLiteを別resourceと見せるような登録を拒否する。 + +各役割は更新するresourceの集合を宣言し、共有resourceを持つ役割の推移的なまとまりを +`coupling_bundle` とする。同じbundleのwriter許可は一括で一つのnodeにだけ与える。 +例:CG1とDG1がpages.dbを共有、CG2も同じDBを更新するなら三役割すべてを連動させる。 +テーブル名が異なっていても、DB全体を物理snapshotで複製する間は分離しない。 + +独立切替を許可する条件は、保存担当が次の能力を明示すること。 + +- 対象の正本、派生projection、署名発行状態に独立した所有者とwriter境界がある。 +- 役割間の更新はversion付きのprotocolとして受理でき、別役割のDBへ直接書かない。 +- 対象resourceへの旧writerのアクセスを実際に停止できる。 +- role/resource対応の構成世代が全参加者で一致する。 + +能力が未申告なら `shared-instance` とし、独立切替要求は `WRITER_BOUNDARY_COUPLED` で拒否する。 +不足能力を理由にDB分割・DDLを自動実行しない。 + +### 3.3 障害・隔離の影響範囲 + +role単位で保存領域が分かれても、fenceがホスト電源断なら同居する全groupへ影響する。 +切替planは「writer連動範囲」と「fenceの影響範囲」を別々に計算する。 +他groupのprimaryまで停止するplanは、自動preemptしない。運用者の明示承認と、影響groupの +サービス継続計画を必要とする。group単位のfence能力があると偽装して回避しない。 + +## 4. identity、保存位置、世代 + +以下は論理レコードの契約であり、RDBのテーブル定義ではない。 + +| レコード/値 | 保存主体 | 更新ルール | +| --- | --- | --- | +| Membership | AuthorityStore | group種別、member/learner、鍵、resource対応。expected revisionでCAS | +| NodeIdentity | ノードローカル保護領域 | node UUIDと専用鍵。業務snapshotから他ノードへ複製しない | +| `boot_id` | 各プロセス | 起動ごとに乱数UUID。過去のgrant/probe応答を再利用しない | +| `trust_generation` | AuthorityStore+運用保全 | authority自身の災害復旧時に明示更新するtrust世代 | +| `control_epoch` | CG authority | CGの主導権交代ごとに単調増加 | +| `data_epoch` | DG authority | DGの書込権交代ごとに単調増加 | +| `resource_epoch` | writer resource authority | 共有保存境界の許可更新。bundle全体で矛盾なく確定 | +| `manifest_sequence` | CGの管理正本 | control_epoch内の管理配信順序。data_epochとは比較しない | +| Checkpoint | 保存層+受信側 | dataset、epoch、cut ID、親checkpoint、内容hash、検証ACK | +| Operation | AuthorityStore | 永続したintent、phase、expected世代、証跡、結果。timeoutで削除しない | +| Node runtime state | メモリ+ノードローカル記録 | 観測状態。書込権の正本にはしない | + +初版 `dr-node.json` の `epoch` は監査値なので、control/data epochへコピーしてauthorityと +みなさない。旧 `cluster_id` は移行mapで既存の復旧単位へ対応させる。 +DR-D1では物理ノードを維持した世代更新でnode UUIDを変えず、`dataset_generation`を更新する。 +新しい物理ノードは新規identityを登録する。初版のrestoreによる採番との差は移行計画に明記する。 + +単に最大のepochを採用しない。有効なのは、信頼済みauthorityが登録したtrust世代・group・ +membershipに結び付いた証明だけ。timestampは監視用で、履歴の新旧や正本選定に使わない。 + +## 5. 設定契約 + +設定案の中心は次のGroupPolicy。ノード一覧に秘密値を埋め込まず `secret_ref` を用いる。 + +| 設定 | 既定/検証 | 意味 | +| --- | --- | --- | +| `kind` | 必須:control/data | 二つの役割を混同しない | +| `coordination_mode` | manual | manual / coordinated。Campusの既存authority_modeとは別項目。自動モードは必要能力の全検査を要求 | +| `members` / `learners` | 明示登録、重複不可 | 発見しただけのpeerをmemberへ追加しない | +| `preferred_node_id` | 未指定 | 期待する主系。実際のprimaryを上書きする値ではない | +| `preempt` | off | off / manual / automatic。controlとdataで別設定 | +| `stable_for_seconds` | 300、範囲60〜86400 | preferred nodeが継続して正常・同期済みである必要時間 | +| `switch_cooldown_seconds` | 1800、stable以上 | 交代直後の再切替を抑制。障害対応は別の明示操作 | +| `preempt_window` | 空=時間制限なし | 任意の許可時間帯。タイムゾーン必須、DSTは計画時にUTCへ解決 | +| `probe_interval_seconds` | 5、範囲2〜60 | ±20% jitter。leaseの更新周期とは別 | +| `probe_timeout_seconds` | 2、範囲1〜10 | 接続・応答の合計deadline | +| `suspect_after_failures` | 3、範囲2〜10 | 表示をsuspectにするだけ。これで昇格しない | +| `read_policy` | disabled | disabled / fresh-authorized。失効watermark未対応なら後者は選べない | +| `max_read_staleness_seconds` | 300、範囲1〜3600 | read replica能力実装後に適用。初版3600秒設定とは別version | +| `rpo_target_seconds` | 必須・導入時入力 | 目標。同期の成功や無損失を保証するフラグではない | +| `rto_target_seconds` | 必須・導入時入力 | fence・起動・routingを含む実測比較の目標 | +| `data_loss_policy` | require-approval | 自動failoverで不明/未複製の損失を黙認しない | +| `fence_provider_ref` | coordinatedでは必須 | 対象identity・resource・影響範囲を検証できるprovider | + +上記は試験開始用の既定値。実環境のRTTやI/Oに応じて変更できるが、同期完了、現主系の隔離、 +相互排他、未知世代拒否といった安全条件は設定で無効化できない。 +preempt=automaticはcoordinated、全target能力、writer境界、適合fence、失効同期を必須とする。 +manualでは自動選挙も自動preemptも行わず、復帰しただけではwriterを再開しない。 + +## 6. 再参加の状態機械 + +設定上の優先順位、現在の役割、実行状態を別フィールドにする。 +状態遷移の入力は検証済みイベントのみ。単なるHTTP 200やpeerの文字列申告では遷移しない。 + +| 状態 | 入る条件 | 許可するもの | 次状態の条件 | +| --- | --- | --- | --- | +| `recovering` | 全DR対象プロセスの起動、許可失効、重大な構成変更 | ローカル検証、認証済みprobe | identity/構成が有効ならlearning | +| `learning` | 書込gateを閉じてauthority/peerを照合中 | probe、checkpoint一覧、同期plan | authoritative state一致→syncing。矛盾→conflict | +| `syncing` | 正しいsourceとbase世代が確定 | 不変世代の受信・検証 | 内容・schema・ACL等が一致→standby | +| `standby` | 検証済み世代、最新membership | 同期追従、能力認定済みreadのみ | 書込grant取得→activating | +| `activating` | 新grantとfence証跡が確定 | root切替、互換性検証、gate接続 | 二重確認+readiness成立→primary | +| `primary` | group/resource/bootすべて一致した有効grant | 許可範囲のread/write | 計画移譲→draining。許可失効→recovering | +| `draining` | operationが停止cutを要求 | 既存処理の完了待ち、checkpoint | writer停止証明後standby/停止。失敗ならoperation停止 | +| `conflict` | 履歴分岐、同一世代で異hash、trust不一致 | 証拠保全、管理状態参照 | 運用者のreseed/解決planのみ | +| `blocked` | 同期先不在、容量不足、fence不明、互換性不足 | 状態・監視 | 原因解消後、learningから再検査 | + +learning/syncing中は一般のreadも公開しない。復元されたprivate文書と古いACLをうっかり公開しない。 +standbyでreadを許す場合も、内容の鮮度とは別に必要な失効watermarkを確認する。 +複数groupを持つInstanceは各groupの状態を表示する。単一の緑ランプで全役割をprimaryと見せない。 + +### 起動順序 + +1. ノードの専用identity、deployment期待値、承認済みmembershipを読む。 +2. 通常HTTP handler、migrations、worker、CLI接続より先にすべてのwriter gateを閉じる。 +3. read-only互換性検査と管理用control listenerだけを起動する。 +4. 新boot_idで認証probeし、authorityの現在値と照合する。誰にも到達できなければblocked。 +5. 必要なら旧ローカルデータを隔離保全し、正しいsourceから新generationへ同期する。 +6. 同期済みでもまずstandby。新bootに対するgrant取得を別処理で行う。 + +正常終了時にprimaryだった場合もこの順序を省略しない。単一ノードの非DR運用は従来起動を +維持するが、DR設定の欠落を非DRへ自動フォールバックする経路はdeployment guardで拒否する。 + +## 7. 書き込み許可の条件 + +次の論理式をすべて満たす場合だけwriterを開く。 + +```text +CanWrite(role, resource, boot) = + approved_membership AND validated_trust_generation + AND active_group_grant(holder=node, boot=boot, epoch=current) + AND active_resource_grant(holder=node, bundle=current) + AND verified_dataset AND schema_compatible AND revocations_applied + AND local_gate_open AND NOT draining AND NOT expired +``` + +Web/APIだけでなくCalDAV、CLI、managed設定受理、migrations、GC、asset処理、calendar保存、 +外部同期、outbox配信もwriter一覧に含める。HTTP GETでも修復/発行をする経路はwriteである。 +RO standbyへのmanifest適用は、そのreplicaのauthoritative DBへ直接行わず、現DG primaryへ +適用するか同期対象のprojectionとして受け取る。ローカル状態報告の保存先と業務正本を混ぜない。 + +許可喪失時は新規処理を拒否し、実行中処理をキャンセル・停止する。しかしプロセス停止や +ネットワーク遅延によりキャンセルが間に合わない可能性があるため、新primaryのgrant発行は +外部fence確認または認定済み保存側fencingの完了まで待つ。local mutex/lease確認だけでは代替不可。 + +## 8. 裁定・信頼モデル + +発見peer、複製peer、authority参加者は別。既存Campus参加トークンにbackup読出しや昇格権を +追加しない。ノード間通信は専用identityとgroup限定の許可を使う。 + +coordinatedの初段providerは外部etcdの通常整合性readと比較更新transactionを使う。 +watchは再確認のきっかけで、watch結果だけから新grantを発行しない。epoch、membership、 +operation intentは期限切れで消さず、leaseは生存・短期許可にのみ用いる。 +etcdのKV transactionとwatchの保証は異なるためである。[etcd API guarantees](https://etcd.io/docs/v3.5/learning/api_guarantees/) + +authorityは業務snapshotと別の保全・失効境界に置く。旧authority backupを戻してepochを +巻き戻すことは禁止。authority全損時は全writerをfenceし、新trust_generationを +out-of-bandで承認・再配布するbreak-glass手順を要求する。単にepochを大きくする操作ではない。 +authority quorumが失われた場合は新grantを発行しない。既存writerも許可更新不能なら閉じる。 + +2台のPagesWikiを相互監視するだけではcoordinated modeを認定しない。 +etcdは別途適切な障害領域に配置する。配置数やfence backendの接続情報は導入計画で確定する。 +manual modeの運用者証跡は自動検証済みfenceと区別し、UIと監査に表示する。 + +## 9. コントローラとデータ側の障害マトリクス + +| 事象 | 管理側 | データ側 | 判定 | +| --- | --- | --- | --- | +| CG停止、DG正常 | 配置変更・新manifestを停止 | 有効なdata grantと許可済み設定で継続 | CGへの疎通だけをDGの書込条件にしない | +| DG停止、CG正常 | 管理操作は継続可 | 対象DGのfailover判定 | CGが配置文字列を書き換えるだけでは昇格不可 | +| 旧CG復帰 | learning→管理checkpoint同期 | DGの切替は不要 | CG preempt設定のみ評価 | +| 旧DG復帰 | 配置group identityを維持 | learning→現DG primaryから同期 | 古いデータを優先しない | +| 同じDBでCG/DGが別nodeを要求 | 連動planが必要 | 連動planが必要 | 独立要求を拒否 | +| primaryとreplicaが分断 | authoritative grantとfenceを照合 | 同左 | 応答者だけの多数決はしない | +| authorityのみ停止 | 更新許可の有効期間内だけ処理 | 同左 | 新grantなし、期限到達でwriter停止 | +| 最新受理manifestより復元内容が古い | 自動配信しない | 古い権限を再適用しない | 管理復旧の明示承認が必要 | +| role fenceが同居別groupも停止する | impact planを提示 | 対象groupも判定 | 自動preemptを拒否 | + +## 10. UI・権限・監視 + +設定の「運用 / DR」を親にし、「概要」「グループ」「同期・保全」「切替履歴」を子パネルにする。 +概要は1group 1行、kind / Campus・Domain範囲 / 実primary / preferred / state / 同期点 / +遅延 / 書込許可 / 原因を表示。ノード追加・ポリシー編集はアイコンからダイアログに収める。 +フォルダ名やpathではなくnode・groupを選択させ、秘密は参照名だけ表示する。 + +キャンパス設定にはCGの役割・状態とDR画面へのリンクを追加する。Domain設定では配置先を +DG単位で選び、現在のnodeを補助表示する。旧配置方式は移行前であることを示す。 + +| 権限 | 許可 | +| --- | --- | +| instance operator(新しいホスト運用者権限) | node登録、trust/fence、backup秘密、bundle全体の切替・復元 | +| campus_admin | 自Campusの管理状態参照、ポリシー変更/切替の申請 | +| domain_admin | 自Domainの配置・同期状態参照。ホスト/鍵/他Domain情報は非表示 | +| replica service identity | 明示groupのprobe、許可された世代の同期。operator API不可 | + +instance operatorはcampus_adminへの暗黙追加ではない。初段はホストCLIだけでこの権限を +実現し、既存認証DBに新roleを勝手に加えない。Webでのapplyは認可契約が完成するまで非表示。 +切替UIはplan→影響範囲・損失・fence表示→承認→進行状態。操作中も平常画面をoverlayしない。 +通知Inboxへ失敗/blocked/完了を集約し、probe失敗を毎回broadcastしない。 + +メトリクス:role state、grant残時間、最後のprobe成功、同期cut時刻、検証ACK時刻、 +pending bytes、転送速度/制限、hash失敗数、fence結果、operation phase、未複製の更新有無。 +UUID別の無制限時系列は避け、ノード/group別集計と履歴画面を分ける。秘密・文書本文をログに出さない。 +緑=primary許可あり/検証済みstandby、黄=learning/syncing/suspect、赤=conflict/blocked。 +色だけで判断させずラベルと原因コードを併記し、日本語・英語キーを用意する。 + +## 11. 保存担当との接続条件 + +DB変更の担当へ渡す成果物は、テーブル要求ではなく以下の能力・整合性契約。 + +- `DescribeResources`:各正本とwriter排他境界、共有するrole、schema対応範囲。 +- `InspectCheckpoint`:正本を変更せず、適用世代・内容digest・schema fingerprintを取得。 +- `FreezeWriters/ReleaseBarrier`:全writerの停止、未完了処理、最終cutの確定。 +- `CaptureCheckpoint/Pin/Unpin`:DBと可変正本を同じcutへ固定し、必要blobをGCから保護。 +- `StageGeneration/Verify/ActivateGeneration`:非稼働rootへ復元し、検証後に切替。 +- `ApplyControlManifest`:管理内容と受理checkpointを同じ永続化境界で確定。 +- `CheckWritePermit`:対応できる場合のみ保存側fencingとして能力宣言。 + +詳細な入出力と失敗時条件は [protocol](dr-protocol-and-sync.md) に定義する。 +契約を満たす実装がなければoffline capture/shared-instance/manualのままとし、DB別件を +妨げるDDL、直接SQL、独自の並行正本ファイルを追加しない。 + +## 12. 参考と非目標 + +Learnerという考え方は、同期前のノードを選挙・サービス提供から外すための参考である。 +PagesWikiの状態名をetcdの状態と同一視しない。[etcd learner design](https://etcd.io/docs/v3.5/learning/design-learner/) + +非目標:active-active更新、複製DBのSQL行単位マージ、任意LAN参加、gossipによる独自投票、 +DNSだけによるfencing、DB製品選択の変更、既存Campus/FQDNの改名、未同期更新の黙った破棄。 +同期とバックアップは別であり、replicaの存在を履歴世代の保全の代替としない。 diff --git a/docs/design/ja/dr-delivery-plan.md b/docs/design/ja/dr-delivery-plan.md new file mode 100644 index 0000000..1b90212 --- /dev/null +++ b/docs/design/ja/dr-delivery-plan.md @@ -0,0 +1,231 @@ +# DR詳細設計:実施計画・受入試験・導入判定 + +設計版:DR-D1、2026-09-27。**詳細設計完了、実装作業は未着手。** +ここでいう実装作業はDR-D1の再参加・役割分離protocol。既存のoffline DR CLIは別の初版。 +今回の成果物は文書のみであり、DB・migration・本番配置を変更する作業指示ではない。 + +## 1. 設計成果物と完了条件 + +| 成果物 | 本版で定義した範囲 | +| --- | --- | +| [役割・再参加](dr-authority-and-rejoin.md) | CG/DG/Instance、writer連動境界、identity、状態機械、設定・UI・権限 | +| [protocol・同期](dr-protocol-and-sync.md) | 認証probe、API、authority契約、checkpoint、manifest v3案、切替・失敗時動作 | +| 本書 | 実施順、予定変更箇所、DB別件への接続条件、試験、導入/撤退手順 | +| [初版記録](disaster-recovery.md) | 既存機能と制約を保持し、DR-D1と区別 | + +- [x] primaryを管理/データに分け、物理保存境界による連動条件を記述。 +- [x] 起動・probe・Learning・同期・昇格・preempt・失敗からの再参加を定義。 +- [x] 身元、世代、許可、同期内容、承認、未知損失の扱いを定義。 +- [x] API入出力、サイズ上限、再試行、冪等性、切替phase、認可主体を定義。 +- [x] DBを変更せずに準備できる範囲と、保存担当との合意待ち範囲を分離。 +- [x] 実装タスク、依存、合格条件、障害注入試験、導入/撤退条件を定義。 +- [ ] 実装・実機でのfencing認定・復旧訓練・本番反映。 + +「詳細設計完了」は実装方針と判断規則を定義した状態であり、未知の本番設備の能力を保証した +意味ではない。RPO/RTO値、ノード数/FQDN、fence機器、容量、鍵保管先は導入時の必須入力として残す。 +これらを未設定のまま自動運転へ移行することはできない。 + +## 2. DB別件との作業境界 + +本タスクでは以下を行わない。 + +- アプリDBへのSELECTを含むアクセス、DBのコピー・バックアップ・復元、schema inspectの実行。 +- SQL/DDL/migration/repositoryの編集、コード生成、サーバ起動、CLIによるデータ操作。 +- compose、認証鍵、federation token、node設定、稼働コンテナ、外部カレンダーへのアクセス。 +- 文書を本番Wikiへアップロードする操作(アップロードもDBへの書き込みになるため)。 + +読み取る対象は設計書と必要最小限のソースコードだけ。ドキュメントの静的整合性確認を実施する。 +別件の変更を前提にDBの物理分割方法やテーブル名を確定させない。 + +保存担当へ渡すinterface要求は次の通り。応答はSQLの実装ではなく、能力の宣言とテスト結果で受け取る。 + +| 契約 | 保存担当に確認する事項 | 未提供時の挙動 | +| --- | --- | --- | +| Resource map | metadata/可変正本/署名発行状態が共有するwriter境界 | shared-instance連動のみ | +| Schema inspection | 非破壊・read-onlyでの互換性/世代取得 | 復旧世代をservableにしない | +| Consistent cut | DBとcalendar等を同じ論理時点で固定できる範囲 | 全writer停止snapshotのみ | +| Durable pin | snapshot/GC/転送/再起動を跨ぐ参照保持 | online同期不可 | +| Restore activation | 新generationの検証、root切替、途中停止からの復旧 | 自動復元不可、既存手動手順のみ | +| Manifest apply | 管理内容と受理位置のatomic commit | manifest v3配信不可 | +| Write fencing | 旧許可を実際の書込み先で拒否できるか | 外部fence必須 | +| Durability receipt | client成功応答済み更新がreplica上で永続化された境界 | 自動無損失failover不可 | +| Identity/secret inventory | logical service keyとphysical node keyの区別 | nodeを自動参加させない | + +合意前に作業できるのは仕様型、pure state machine、mock provider、観測専用probe、 +静的UI/認可契約の設計まで。runtimeへのwrite gate接続も、別件の保存経路を網羅できることを +確認してから行う。この順序は「DBが共有のままでもroleのflagだけ先に動かす」ことを禁止する。 + +## 3. 実装タスクと依存順 + +以下は将来の予定。行が存在しても対象ソースを今回変更しない。 + +| ID | 実施内容 | 主な変更予定箇所 | 依存 | 完了判定 | +| --- | --- | --- | --- | --- | +| DR01 | version付き型、ID/epoch/構成検証、resource coupling planner | `internal/dr`の新model/config/planner | 設計D1 | 不正構成・独立不可・未知版を拒否。DB依存なし | +| DR02 | pure state machineとoperation phase reducer、fake clock | `internal/dr`のstate/operation | DR01 | 不変条件N1〜N10の遷移試験。失敗時はwriterを開かない | +| DR03 | 専用node identity、mTLS/署名/nonce、観測専用probe | `internal/dr`のtransport、server管理route | DR01 | 未認証/別group/replayを拒否。probeでroleが変わらない | +| DR04 | runtime startup gateの接続、非DR互換、旧CLI禁止 | `cmd/pageswiki/main.go`、server middleware、store/CLI全writer入口 | DR02/03+保存経路合意 | 再起動で必ずrecovering。起動修復/workerも許可前にwriteしない | +| DR05 | AuthorityStore interface、manual provider、etcd providerと限定service | `internal/dr`のauthority、専用運用cmd | DR01/02 | CAS競合/timeout/再起動でgrantが二重にならない | +| DR06 | checkpoint adapter、世代/pin台帳、schema/read-only検証 | 既存`internal/dr/snapshot.go`と保存担当adapter | DR01+保存契約合意 | offline fixtureから再開可能。online未対応を明示拒否 | +| DR07 | 認証済みmanifest/Range転送、recipient ACK、負荷制御 | DR transport、documentstore adapter、scheduler表示 | DR03/06 | 中断再開/破損/別groupアクセス/容量不足を検出 | +| DR08 | v1/v2 anchor移行、管理manifest v3発行/受理/ACK | federation manifest、enroll/reconcile、受理repository | DR03/05/06+受理永続化合意 | 同内容再送・旧epoch・欠落履歴・rollbackを正しく処理 | +| DR09 | fence provider契約、impact planner、証跡照合 | DR fence/operation、運用connector | DR05 | 隔離未確認/誤node/別boot/別group影響で昇格拒否 | +| DR10 | 手動switchover/failover/reseedをoperationへ統合 | DR CLI/API、root activation、routing adapter | DR04〜09 | 完全fixtureでroleごとの切替と共有DB連動が成立 | +| DR11 | preempt planner、自動coordinated gate、cooldown | DR policy executor | DR10+設備認定 | flapping抑制、0損失の計画復帰、不可条件時の提案のみ | +| DR12 | DR管理UI、i18n、health、Inbox、監査集約 | config panels、status APIs、metrics | DR03/10+operator認可 | group別表示、権限境界、秘密非表示、plan/applyが一致 | +| DR13 | v0移行/本番runbook、復旧訓練、負荷/障害試験 | docs/operations、隔離検証環境 | DR10/12 | 受入matrix全通過、RPO/RTO実測、明示承認後のみ有効化 | + +DR04以降のコード接続は、DB別件の差分が落ち着き接続点が合意された後。 +DR08のschema実装は保存担当の変更計画に載せ、DRタスクが独自migrationを追加しない。 +DR09の機器connectorは任意shell入力ではなく、許可された設備・resource IDへのtyped操作とする。 + +### 段階リリース + +| 段階 | 提供するもの | 明示的に提供しないもの | +| --- | --- | --- | +| L0:現行初版 | offline snapshot、検証、手動復元・昇格 | 再参加判定、オンライン同期、自動fencing | +| L1:観測 | DR01〜03、状態の参照UI | 自動昇格、既存writerのruntime変更 | +| L2:再参加gate | DR04/05、operator承認下の起動、shared bundle判定 | 未接続の保存経路があるprofileの有効化 | +| L3:同期・手動切替 | DR06〜10、認証転送、group別の計画操作 | 未認定providerの自動切替 | +| L4:自動主系復帰 | 認定済みDR11/12、coordinated preempt | snapshotのみを根拠にした無人・無損失failover | +| L5:自動障害復旧 | replica durability receipt等が揃うprofileのみ | 未複製更新や未知損失の黙認 | + +各段階で有効化フラグは既定off。capabilityの揃わない段階へUI操作で飛び越せない。 +L1の観測機能が動作しても「Learningから自動で昇格できる」と表示しない。 + +## 4. CLI・API・UIの実施内容 + +将来CLI案(すべて未実装、DB直接更新用コマンドではない): + +```text +pageswiki dr config validate --file +pageswiki dr groups status --group +pageswiki dr rejoin plan --group --node +pageswiki dr switchover plan --group --to +pageswiki dr failover plan --group --to +pageswiki dr operations apply --plan --plan-sha256 +pageswiki dr operations show --id +pageswiki dr operations cancel --id +pageswiki dr reseed plan --group --node --checkpoint +``` + +現行の `dr promote` はL0専用とする。D1管理されたdatasetでは旧epochの手動書換えに +ならないよう拒否し、operationへ案内する。旧binary/旧CLIもwriterとして動けない配置制約を +設ける。新設定ファイルの存在を古いbinaryが無視する可能性は、file markerだけでは解決しない。 + +online CLI、Web、policy executorは同じplan/operation serviceを使う。 +dry-run/planはfence・permission・同期実体を変更しない。operatorがCLIから秘密付きbackupを +扱えることと、campus_adminがWebから全DBをダウンロードできることを同一にしない。 + +UIは最初に観測だけを実装し、operationが完成した段階で切替ボタンを追加する。 +コンパクトなgroup一覧、control/dataの別ラベル、preempt状態、同期点、ブロック理由を優先する。 +確認dialogには「どのCampus/Domain/同居groupが止まるか」「失う可能性のある期間」 +「隔離手段」「未達条件」を列挙する。UNIX時刻入力や、色だけの安全判定は使わない。 + +## 5. 受入試験マトリクス + +テストはfake clock/fake authority/fake fence/一時保存領域から開始し、 +実カレンダー・個人文書・本番ノードには接続しない。各テストは最終状態だけでなく、 +途中のwriter許可集合を記録し、同一resourceで2 holderになっていないことを検査する。 + +| ID | 入力・障害注入 | 期待結果 | +| --- | --- | --- | +| T01 | primary記録を持つnodeが再起動、peer応答なし | recovering→blocked、migration/worker含めwrite 0 | +| T02 | 非DR instance起動 | 従来の起動を維持。DR期待環境で設定欠落なら起動拒否 | +| T03 | 別group、未知鍵、同FQDNで別fingerprint、失効nodeからprobe | 認証/認可拒否、WARNING。membership変化なし | +| T04 | 古いboot/nonce、応答遅延、重複JSON key、未知protocol、大きすぎるbody | 状態遷移なし。入力種別に応じたerror | +| T05 | peerが大きいepoch/primaryを自己申告、authorityは異なる | 再確認・conflict、安全側停止。自己申告で昇格なし | +| T06 | 発見peerの半数以上が同じ内容、登録authorityに到達不能 | 観測の多数決でgrantを出さない | +| T07 | 同一DBのCG/DGが別holderを要求、alias resourceも混在 | WRITER_BOUNDARY_COUPLED、連動planを提示 | +| T08 | 独立storeのCG停止、DGは有効grantと設定を保持 | CGだけ停止。DGは権限鮮度条件の範囲で継続 | +| T09 | CGのpreempt on、DGのpreempt off | 独立profileならCGのみ戻る。sharedなら連動影響を承認前に提示 | +| T10 | primaryの停止・再開、時計逆行、VM pause、permit応答喪失 | stale permitでwrite再開なし。新grantにはfenceを要求 | +| T11 | 2 executor/2 candidateが同時に同group操作、response消失、manual agent二重起動 | 一つのoperation/grantだけ確定。同じIDで結果照合、manual agent排他 | +| T12 | fence要求送信済みだが確認不明、別node/古いbootの証跡 | FENCE_UNCONFIRMED、grantなし | +| T13 | ホストfenceが別DG primaryにも影響 | 自動preempt禁止、impactに全groupを列挙 | +| T14 | capture中にDB更新・calendar世代更新・GC・schema変更 | barrier/capabilityに従い排他。保証不能ならcapture中止 | +| T15 | 転送中断/Range再開、chunk改変、symlink/path traversal、未登録file ID | 検証失敗、他データを読ませずactive root不変 | +| T16 | blobの一部欠落、calendar参照世代欠落、schema不適合、secret欠落 | verified/applicable ACKなし、standby公開なし | +| T17 | pin後にsource再起動、transfer期限切れ、GC/保持処理 | 必要世代を消さない。残存pinを照合して回収 | +| T18 | 同期中に新しいwrite、表示上lag=0だが最終cut未確定 | switchover不可。final checkpointの一致を必須とする | +| T19 | metadataのみ最新、ACL/session失効は古い | read/write readinessとも不成立 | +| T20 | 旧primaryに現系列にない更新 | 隔離保全、conflict。自動マージ/削除なし | +| T21 | manifest旧epoch、同sequence異hash、欠落親、receiptなし | 適用拒否。v3導入後のv1/v2 downgradeも拒否 | +| T22 | 内容適用と受理checkpointの間で失敗、publication receipt応答喪失 | 一体rollback/同内容照合。別payloadで再発行しない | +| T23 | controller復元点が管理ノード受理位置より古い | blocked、epochを増やして古い権限を再配信しない | +| T24 | 各operation phaseの前後でprocess kill | 文書化したphaseから再開。REVOKING以降は旧epoch復活なし | +| T25 | GRANTED後root切替失敗、RO filesystem、disk full、fsync失敗 | 新primaryを公開せず、次の復旧operationへ | +| T26 | routing更新失敗、DNS cacheに旧宛先、既存keep-alive | 新primaryは一つ。旧宛先はwrite拒否、routing修復 | +| T27 | 同期済みpreferred nodeが繰り返しflap、時間帯/DST境界 | 安定時間リセット、cooldown遵守、不要な再切替なし | +| T28 | snapshotの鮮度はRPO内だがclient成功応答の反映が不明 | 自動障害昇格不可。損失確認をoperatorへ要求 | +| T29 | grant lease失効/authority quorum喪失/watch欠落 | 新grantなし。local gate期限で停止、外部fence前は他holder不可 | +| T30 | AuthorityStoreを古いbackupへ復元、manual台帳の破損/消失 | 全writerを隔離、新trust世代の手動復旧。epochの黙認・自動初期化なし | +| T31 | high IO/CPU/foreground要求増、巨大snapshot | stream/帯域/hash上限を維持。制御通信が転送に埋もれない | +| T32 | campus_admin/domain_adminが他scope切替/backup秘密を要求 | 拒否。read UIも他Campus・secretを漏らさない | +| T33 | restart後workerが過去jobを拾う、外部通知結果が曖昧 | 許可と再送条件を確認。曖昧な副作用は自動で成功/再送しない | +| T34 | operatorがcancel/disable/旧binaryへのrollbackを要求 | 安全点だけ取消。active grantを消して旧系を復活させない | + +T24はPLANNED〜COMPLETEDの全境界で行う。ネットワークを切るタイミングも +「要求送信前」「相手がcommit後、応答前」「応答後、ローカル永続化前」に分ける。 +fixtureの全writer経路を一度ずつ実行し、HTTP guardだけで満たしたと判定しない。 + +## 6. 性能・運用の合格条件 + +固定のRTO保証値を現時点で作らず、次の測定結果を導入記録に残す。 + +- 通常foreground p50/p95/p99と、同時同期時の差。目安の許容悪化率は10%以内、超過なら転送予算を縮小。 +- DB/可変正本のbarrier時間、checkpoint作成時間、初回/増分転送時間、検証時間、必要空き容量。 +- suspect検知、fence確認、grant、起動、routing完了の各所要時間と合計RTO。 +- 障害時点と最後のDR側検証cutの差、client成功応答境界の確認可否、未複製損失の有無。 +- スナップショット10万/100万file相当のmanifest paging、RSS/heap、descriptor数の上限。 + 初版v1の10万file上限を、v2の性能試験なしに解除しない。 +- 一日継続のprobe/転送/再起動試験で、timer・goroutine・scratch・pin・operationが無制限に増えない。 + +10%は候補性能予算であり、実機測定前に達成と表明しない。別件の保存層の性能評価と同じ +fixture条件を使用する。外部サービス/実データにアクセスする実機試験は別途承認を得る。 + +## 7. 導入時に入力・承認する事項 + +設計として未決のアルゴリズムではなく、設備・運用に依存する導入パラメータを列挙する。 + +| 入力 | 決定する主体 | 未入力時 | +| --- | --- | --- | +| node FQDN/管理origin、node identity、対象Campus/DG、resource対応 | 運用者+保存担当 | 登録・同期を開始しない | +| RPO/RTO、retention、容量/帯域予算 | 運用者 | 本番enable不可 | +| authority配置、CA、専用service key保管/更新、quorum障害領域 | 運用者 | manualのみ | +| fence機器/credential参照、再起動抑止、影響group | 運用者 | coordinated enable不可 | +| backup/secret暗号化と復号担当、別障害領域の保全先 | 運用者 | 秘密付き転送不可 | +| CG/DGごとのpreferred、preemptと許可時間帯 | 運用者+管理者 | preempt off | +| 保存profile能力と対応app/image/schema | 保存担当 | offline/shared-instanceのみ | + +## 8. 本番導入手順 + +1. 運用者が対象と入力を確定し、保存担当がresource map・能力契約を承認する。 +2. 本番とは通信しない隔離fixtureで対応段階のT01〜T34と負荷試験を実施する。 +3. 既存環境の保全・image digest・schema互換性・復旧手順をレビューする。実データ保全は別途承認。 +4. L1観測のみを配置し、比較表示を確認する。旧roleに基づくサービス挙動はこの段階では変えない。 +5. 全writerと旧CLIを停止する保守枠を取り、L2への移行planを確認する。 +6. 旧cluster/node/roleを新group/resourceに対応させ、fresh authorityをbootstrapする。 + 旧primaryの確認記録を付け、元のepochを新authorityの根拠にしない。 +7. 新startup gate下で起動し、他nodeはlearnerとして参加。sourceと同期内容を検証する。 +8. L3の手動switchoverと復帰を1往復実施し、RPO/RTO・監査・権限失効・queueを検証する。 +9. 保存分離済みprofileならCG/DGの独立切替、共有profileなら連動拒否/連動切替を確認する。 +10. 能力が揃ったgroupに限り、明示承認を得てL4を有効化する。L5は別の認定を要求する。 + +ローリング移行中、旧ノードをwriter候補に残さない。mixed protocolで書込み可能nodeを +増やす移行はしない。旧nodeからのprobeは対応版を表示し、新grant候補から除外する。 +管理manifest v3は全対象が移行anchorをACKした後に切替える。 + +## 9. 撤退・失敗時 + +DRの「無効化」は設定ファイル削除ではない。新operation受付を止め、active operationを照合し、 +一つのwriterを運用者が確定・隔離確認してから非DR運用への移行planを実行する。 +authorityを停止しただけで旧nodeに書き込みを許可しない。 + +L1だけなら観測プロセスを停止して戻せる。L2以降はrole/許可/DB互換性の記録を保持する。 +新epochでwriteが発生した後は、旧snapshotを起動することでrollbackしない。 +旧binaryに戻す場合は、最新データとschema互換性を検証し、DRのtrust/gateを無視するbinaryが +同時に動かないことを外部で保証する。保証できなければ停止を継続する。 + +実施結果には、適用していない段階、満たしていない能力、最後に検証した世代、 +次に必要な承認を必ず残す。設計完了・実装完了・復旧訓練完了・本番稼働を同じチェックにしない。 diff --git a/docs/design/ja/dr-protocol-and-sync.md b/docs/design/ja/dr-protocol-and-sync.md new file mode 100644 index 0000000..5e4fa18 --- /dev/null +++ b/docs/design/ja/dr-protocol-and-sync.md @@ -0,0 +1,397 @@ +# DR詳細設計:通信・同期・主導権移譲契約 + +設計版:DR-D1、2026-09-27。**すべて次段階の設計。新endpoint・設定・永続化は未実装。** +責務と用語は [管理主系・データ主系](dr-authority-and-rejoin.md)、実装の順序は +[実施計画](dr-delivery-plan.md) を正とする。DB変更は本作業に含めない。 + +## 1. 構成と永続化の契約 + +### 1.1 Authority Service + +manual providerは単一のoperator管理agentと専用のdurable操作台帳を使用する。 +台帳は業務DB/snapshot外のローカル保護領域に置き、プロセス排他、revision比較、 +operationの追記・fsync後の応答を必須にする。grant発行・holder変更にはoperator承認が必要。 +同じ許可の短期renewは自動でもよいが、別boot/holderへの変更にはならない。 +台帳破損/消失/別agent起動で自動再初期化せずblockedにする。manual provider自体のHAは +提供せず、復旧時は旧agentと全writerを隔離してtrust再承認する。これは独自分散合意ではない。 +この運用台帳をアプリmetadataや管理manifest受理checkpointの代わりには使わない。 + +独立した運用サービスとしてgroup membership、grant、operation、fence証跡の参照を扱う。 +coordinated providerはetcd v3の線形化readと比較更新transaction。業務nodeにはetcdの +任意Put権を与えず、認証されたAuthority Serviceの限定操作を通す。APIプロセスは複数起動できるが、 +決定の正本は一つのetcd keyspaceであり、プロセスのメモリや通常event queueではない。 + +論理keyは `groups/{kind}/{uuid}/membership`、`.../active`、`resources/{uuid}/active`、 +`operations/{operation_uuid}`、`trust/generation`。これはetcdの論理配置案で、アプリDBのDDLではない。 +group/resource activeとoperation phaseの変更は同じCAS transactionで確定する。 +共有bundleは関連全group/resourceを同じtransactionに含める。provider上限を超えるbundleは +登録時に拒否し、一部だけ切替えない。初期上限は1 bundleあたりgroup+resource合計32。 + +`active` は `{membership_revision, trust_generation, epoch, holder_node_id, holder_boot_id, +state, operation_id, resource_epochs, checkpoint_id, revision}` を持つ。 +`state` は ACTIVE / REVOKING / FENCED / PREPARED / NONE。epochは10進文字列uint64、 +上限でエラー。membership revisionとprovider revisionもJSON数値精度に依存しない文字列にする。 +epochと操作intentはleaseに付けず保存し続ける。短期permitだけが期限付き。 + +### 1.2 契約インターフェース + +| 契約 | 入力→結果 | 必須の性質 | +| --- | --- | --- | +| `ObserveAuthority` | group/bundle→membership、active、revision | 線形化read。watch/probeのcacheで代替不可 | +| `PlanTransition` | expected revision、target、reason→immutable plan/hash | 不足能力・影響・損失・必要承認を全列挙。副作用なし | +| `BeginTransition` | plan hash、承認、idempotency key→operation | CASで一つだけ開始、競合は409 | +| `RecordFence` | operation、provider evidence→FENCED | exact node/boot/resource/operationへの新鮮な証跡を検証 | +| `CommitGrant` | 検証ACK、fence、expected各revision→PREPARED | group/resourceを原子的に新holderへ。旧grantは復活不可 | +| `RenewPermit` | active grant、boot、nonce→短期permit | ACTIVE holder、またはPREPARED holderの限定初期化scopeだけ。期限更新は権限移譲ではない | +| `CompleteActivation` | writer gate/root検証ACK→ACTIVE | checkpoint/root、schema、grantの一致。未完なら配信不可 | +| `DescribeResources` | role→resource集合・能力 | aliasを含む共通正本を検出。未申告は独立と見なさない | +| `FreezeWriters` | bundle、operation、deadline→cut候補 | 新規受付停止、実行中writerをdrain。未知writerがいれば失敗 | +| `CaptureCheckpoint` | freeze token→checkpoint、pins | metadata/可変正本の同一cut、checksum、復旧に必要な参照集合 | +| `Stage/VerifyGeneration` | checkpoint、payload→verified ACK | 非稼働rootのみ。権限/互換性も検証し、writerを開かない | +| `ActivateGeneration` | verified ACK、grant→root ACK | root pointerをdurableに切替。曖昧な状態はrecovering | +| `ApplyControlManifest` | envelope、expected受理checkpoint→ACK | 内容・受理checkpointを同じ永続化境界でcommit | + +PREPAREDではmigration/正本修復に必要な限定writerを新holderだけに許可できる。 +通常リクエストとworkerはACTIVEになるまで閉じる。移行が長引けばpermitを更新する必要がある。 +初段は同じschema/imageで復元し、DR切替にschema migrationを混ぜない。 + +`FreezeWriters` の成立はアプリDBの外から書くCLI等も止められる場合に限る。 +未知writerが存在する保存profileは、サーバ/CLI全停止と外部保守lockによるoffline modeだけを許可する。 +保存層で実現できない契約を、別ファイルへの二重書きで「同じtransaction」と称しない。 + +## 2. 信頼・ネットワーク + +接続はHTTPS、TLS 1.2以上、mTLSによる専用node identityとgroup許可を必須にする。 +検証無効化、HTTPフォールバック、通常Campus参加tokenの転用はしない。 +既存の開発用 `--add-external-campus-http` はこのprotocolを許可しない。 +初回信頼はoperatorがCA/鍵fingerprintとnode UUIDを別経路で承認する。probeによるTOFUはしない。 + +TLS終端proxyを用いる場合、外部からのidentity headerを除去し、信頼済みproxy接続だけで +検証済みidentityを伝える。加えて下記のnode署名で要求を束縛する。任意の +`X-Client-Cert` やFQDNだけで認証しない。FQDNの再利用・鍵変更はWARNING+quarantine。 + +peer endpointはoperatorが登録したHTTPS originと固定prefixのみ。ユーザーのURL入力や +probe応答の未知URLへ自動接続しない。redirectは禁止。私設networkの許可はgroupごとの +allowlistで定義し、DNS解決後の宛先IPも検証する。TLSのserver nameと登録originは維持する。 + +### 署名envelope + +共通JSON envelopeは `protocol_version="dr/1"`, `key_id`, `payload_b64`, `payload_sha256`, +`signature_b64`。payloadはUTF-8 JSONの**送信した正確なbyte列**を署名し、再serializeしない。 +署名対象は用途別固定prefix(例 `pageswiki.dr.probe.request.v1\n`)+payload bytes。 +Ed25519を用い、種類の異なるenvelope間で署名を使い回さない。JSON重複key、未知version、 +不正UTF-8、深さ過大、型違い、未知の必須capabilityを拒否する。canonical JSONの独自実装は不要。 + +通信payloadは `request_id`, `sender_node_id`, `sender_boot_id`, `group_id`, `group_kind`, +`membership_revision`, `trust_generation`, `nonce`, `issued_at`, `expires_at` を共通に持つ。 +nonceは32 random bytesのbase64url、request_idはUUID。応答は要求ID/nonce/宛先node/bootと +要求payload hashをechoし、観測したauthority revisionを含める。nonce未要求・別boot・期限切れ +応答は状態遷移に使わない。UTC時刻の許容ずれは5秒、response有効時間は10秒以内を初期値とする。 +時刻だけに依存せず、boot/nonce/authority revisionも検証する。 + +node用秘密鍵はnode-local。Campus manifest署名鍵とは別。信頼世代・鍵ローテーションは +旧新keyの承認期間と撤回revisionを記録し、同一nodeを二つのbootが同時に有効化しない。 + +要求payloadには `http_method`, `request_target`, `audience_id` も含め、実際のmethod/管理URI/ +受信nodeまたはserviceに一致させる。署名検証前に別URIへrewriteしない。未知query、dot-segment、 +encoded slashなどの曖昧なpathを拒否する。path groupとpayload groupの不一致も拒否する。 +管理manifestの署名鍵をbackupから持ち出せても、Authority Serviceが確定したpublication receipt +なしに新manifestを受理させない。署名鍵を保持することと、現在の発行権限を持つことを分離する。 + +POSTの通信envelopeはJSON body、GETは8KiB以内の `PagesWiki-DR-Request` headerへ格納し、 +GET bodyを要求しない。Range要求はその値も署名payloadへ含める。通常JSONの復号payload上限は +256KiB/深さ16、manifest pageだけは1MiBまで。binary転送の応答検証は署名済みmanifestの +chunk/全体hashで行い、全ファイルをbase64 JSONに包まない。 +保管するcheckpoint/manifest署名は通信envelopeと用途prefixを分け、request nonceや10秒の +応答期限を付けない。不変artifactの由来/完全性検証と、今このnodeがアクセス・書込みできるかの +短期許可検証を分離する。古いartifact署名が有効でも現行のwrite grantにはならない。 + +## 3. probeと再参加 + +`POST /_/api/v1/instance/dr/probes` を認証済みgroup member/learnerだけに提供する。 +無認証でpeer一覧、group一覧、snapshot一覧を返さない。 + +要求の追加項目: + +```json +{ + "group_kind": "data", + "group_id": "", + "sender_node_id": "", + "sender_boot_id": "", + "known_authority_revision": "123", + "known_epoch": "7", + "applied_checkpoint_id": "", + "supported_protocols": ["dr/1"], + "capabilities": ["probe-v1", "offline-checkpoint-v1"], + "nonce": "" +} +``` + +例は共通項目の一部を省略した型説明で、稼働設定として投入しない。 +応答はprimary観測値、membership revision、state、checkpoint id/hash/親、 +内容cut時刻、verified/applicableの区別、schema互換性、roleに応じたcontrol受理位置を返す。 +資格情報・文書本文・ユーザー一覧・内部path・任意SQLは返さない。 + +probe結果は `observed`、AuthorityStore read結果は `authoritative` と区別する。 +処理順:node identity→group所属→nonce等→protocol→authority照合→履歴照合→同期plan。 +新しいepochを発見したら必ずauthorityへ再確認し、peerだけを根拠に旧許可を新primaryへ移さない。 +矛盾の疑いがあれば自ノードのwriterは安全側へ閉じるが、他ノードの昇格は承認しない。 + +既知peerへ並列fan-outする。初期同時数4、groupごと最大32 endpoint、1応答256KiB、 +要求/応答rateは1 nodeあたり10req/sまで。初回は即時、その後5秒±20%jitter。 +失敗時は最大60秒までbackoff、安定した成功で戻す。タイマー終了はsuspect表示だけを変える。 +membership変更はoperatorによるexpected revision付き操作。join要求はlearner候補に留め、 +既存memberやauthority quorumを自動的に増減させない。 + +## 4. API境界とエラー + +全URIは予定。通常ユーザーのPath/公開Domainからは独立した管理経路に置く。 + +| method / prefix以下のURI | 呼出主体 | 契約 | +| --- | --- | --- | +| `GET capabilities` | 登録node/operator | protocol、schema/profile能力。対応申告であり同期完了証明ではない | +| `POST probes` | group member/learner | 情報収集のみ。grantを発行しない | +| `GET groups/{kind}/{id}/status` | node/operator、範囲制限した管理者 | observed/authoritative/as_ofを明示 | +| `POST groups/{kind}/{id}/plans` | operator | rejoin / switchover / failover / reseedの副作用なしplan | +| `POST operations` | operator/認定済みpolicy executor | plan hashと承認を照合して開始 | +| `GET operations/{id}` | 許可された操作主体 | phase、未達条件、証跡、結果 | +| `POST operations/{id}/cancel` | operator | cancel可能点だけ。grant移譲後を勝手に巻き戻さない | +| `GET groups/{kind}/{id}/checkpoints/{id}` | 承認replica/operator | 署名metadata、transfer manifest参照 | +| `POST transfers` | 承認replica/operator | group/checkpoint/recipientを固定した転送session | +| `GET transfers/{id}/manifest` | session recipient | paged immutable manifest、snapshot ETag | +| `GET transfers/{id}/files/{file_id}` | session recipient | 許可されたopaque file IDのRange取得 | +| `POST transfers/{id}/acks` | session recipient | bytes受信/検証/適用ACKを別stageで記録 | +| `POST groups/{kind}/{id}/permits/renew` | active holderのみ | node/boot/epoch/resourceを束縛した期限更新 | + +prefixは `/_/api/v1/instance/dr/`。node-local probe/transferはnode agentが処理し、 +plans/operations/grantはAuthority Serviceが処理する。配置proxyは経路を区別し、 +node agentがauthority不通時に代わりのgrantを作らない。 + +state変更要求は `Idempotency-Key: UUID` とplan/expected revision必須。同じkey・同じbodyは +同じoperationを返し、同じkeyで異bodyは409。応答喪失後は元operationをGETして照合する。 +planの有効期間は60秒。membership、epoch、checkpoint、fence影響範囲が変われば期限内でも失効。 +grant commit前にはもう一度最新条件を再検査する。 + +Web操作はoperator認可gatewayを介し、human actor・scope・承認内容・plan hashを短命の +delegationに束縛してAuthority Serviceへ渡す。nodeの秘密鍵やetcd資格情報をブラウザへ渡さない。 +gatewayが認可しただけでimpact planの承認を省略しない。instance operatorの認可信頼源が +接続されるまではWeb applyを無効にし、ホスト運用者のCLIだけを提供する。 + +| HTTP | code例 | 呼出側の動作 | +| --- | --- | --- | +| 400 | INVALID_ENVELOPE / UNSUPPORTED_VERSION | 修正までretryしない | +| 401/403 | IDENTITY_REQUIRED / GROUP_FORBIDDEN | 権限修正。未知groupの内容は隠す | +| 404 | UNKNOWN_RESOURCE | 他groupの存在を推測できる詳細を返さない | +| 409 | STALE_EPOCH / OPERATION_CONFLICT / HISTORY_DIVERGED / WRITER_BOUNDARY_COUPLED | 再probeまたは新plan。自動上書きしない | +| 412 | MANIFEST_CHANGED / STALE_PLAN | 同一転送・旧planを継続しない | +| 413/429 | LIMIT_EXCEEDED / RATE_LIMITED | 上限内へ縮小、Retry-Afterを遵守 | +| 501 | CAPABILITY_UNAVAILABLE | offline/manualへ明示的に再計画。暗黙downgradeしない | +| 503 | AUTHORITY_UNAVAILABLE / NOT_READY / FENCE_UNCONFIRMED | 書き込みを閉じて待機 | + +JSON errorは `code, message_key, request_id, retryable, operation_id`。本文・秘密・SQLは含めない。 +管理レスポンスはno-store。状態取得はETag可、ただしcached statusを昇格承認には使わない。 + +### 配信とreadiness + +管理listenerは一般DomainのHost選択/ログインsessionから独立して認証し、learning中もprobeを +受けられるようにする。復旧した業務DBの認証設定だけに、再参加の身元検証を依存させない。 +通常routeはHost→Domain→DG→resource/grantの順に解決し、各要求で対象groupを判定する。 +コントローラ用routeはCampus→CGを判定する。HTTP methodだけでどちらのprimaryか決めない。 + +node全体の `/health` は生存確認。複数groupで混在する場合、従来の `/health/write` は +対象listenerの全write groupがreadyなときだけ200とする保守的な集約値で、個別routingには使わない。 +認証済みgroup statusの `read_ready`, `write_ready`, `state`, `reason`, `as_of` を +routing adapterへ渡し、切替時はfresh authorityと実際のgate状態も照合する。 +PREPARED/ACTIVATINGは通常write readyにしない。既存接続も切替後の各要求で再検査し、 +古い接続が旧primaryへの書込権を保持することはない。URIやHostをDRノードFQDNへ書き換えない。 + +## 5. Checkpointとバックアップ契約 + +### 5.1 内容 + +checkpoint v2(初版snapshot v1とは別format)は次を持つ。 + +- format/protocol version、group kind/id、dataset ID、checkpoint UUID、親checkpoint hash。 +- source node/boot、trust generation、control/data/resource epoch、membership revision。 +- snapshot cut ID、cut時刻、capture完了時刻、内容集合digest。 +- app SemVer、image digest、保存profile、schema component/version/checksumの集合。 +- DB snapshot descriptor、可変正本manifest一覧、CAS参照集合、総byte数、file数。 +- 管理manifest受理位置、ACL/session失効watermark、outbox/job再送判定位置。 +- 暗号化方式・key参照・別保全のsecret一覧。secret値は一覧に含めない。 +- 操作状態 `capturing/sealed/transferring/verified/activated`。sealed前はservableにしない。 + +「送信完了」「受信した」「hash検証済み」「配信可能」「昇格可能」は別。 +checkpoint sequenceはその系列の順序であり、clientへ成功応答した全変更の境界とは限らない。 +初段のsnapshot同期は物理snapshot+CAS不足分転送。SQL差分適用やWALファイルの独自転送はしない。 +オンラインのtransaction log方式は保存層が将来別profileとして保証するまで使わない。 + +### 5.2 一貫したcapture + +1. capture対象のbundle/resourceを確定し、schema変更と同時に走らないことを確認する。 +2. 新規writer受付を閉じ、DB・可変正本の処理中更新をdrainする。GCも対象世代を消せない状態にする。 +3. 保存層に同一cutのDB snapshotと可変正本manifestを固定させ、参照集合を確定する。 +4. 参照CAS/月別世代へのdurable pinを確立する。pinの永続化とcut固定を確認する。 +5. sealedしたDB像・metadataから後続コピーできることを確認してbarrierを解除する。 +6. 大きな不変payloadのhash確認/転送はforegroundより低優先度で継続する。 + +SQLite Backup APIはDB単位の整合コピーであり、別ファイルの同一cutまで自動保証しない。 +DB像の固定にコピー完了まで必要なprofileでは、その間barrierを維持する。 +「短時間online」と無条件に表明せず、barrier予算超過ならpublish前にcaptureを中止し、 +primaryを再検査して通常処理へ戻す。最初はoffline profileだけでもよい。 +[SQLite Backup API](https://www.sqlite.org/backup.html) + +pinの保持はtransferの期限と独立。transfer sessionが切れても、保全世代/進行中operationから +参照されるblobを消さない。完了/中止後もretentionに従って解放する。ノード再起動でpinが +失われるprofileはオンラインcapture不可。保持台帳の実装は保存担当の契約に従う。 + +### 5.3 受信・再開・切替 + +受信は非稼働generationに対して行う。file IDはmanifest内でscope付きに割り当て、 +任意pathや他groupのhashを直接読めるAPIにしない。manifestは1page最大1,000件/1MiB、 +sealed manifestのroot digestとpage hashで全体を検証する。各payloadは全体hash/sizeと +16MiB単位のchunk hashを持ち、Range再開時にもchunkと全体の両方を検証する。 + +転送sessionはrecipient node/boot、group、checkpoint、期限に結び付く。初期期限15分、 +再認証後に同じsealed checkpointへ再開可能。期限はreaderの権限であり正本の寿命ではない。 +宛先の空き容量は必要byte+20%と運用予約分を確認し、既存稼働世代を自動削除して空きを作らない。 + +受信後:全payload→DB/正本参照整合→schema互換→失効/鍵の可用性→内容digestの順に検証する。 +DB inspectionはread-onlyで行い、初期化/migrationによって検証対象を変化させない。 +検証ACKを受けても一般routingは開かず、standbyのread条件または新grantを別途確認する。 +root切替は新generationをfsyncした後、atomic pointer更新で行う。旧世代を保持し、 +再起動時にpointerとoperation recordが食い違えばrecoveringで照合する。 + +転送負荷の初期値:nodeごとpayload stream最大2、合計10MiB/s、hash worker 1、 +foreground高負荷時は新chunk受付を停止。制御用probe/permitは別connection・別予算。 +リソース監視値が取れない場合は固定上限を維持し、無制限へ拡張しない。 +pause/backoff/cancelの最大応答遅延は1 chunkのI/O期限内。候補値30秒、実測して認定する。 +通常event schedulerにはcapture/verify/転送等を表示するが、grant更新は専用loopとする。 + +### 5.4 保全・秘密・分岐 + +複製は保全世代の代替ではない。retention初期案は直近3+日次7+週次4、導入時に容量確認する。 +世代依存とpinsを検査してから削除し、root trust/復号鍵の保全はsnapshotとは別の障害領域に置く。 +業務復旧に必要な暗号化鍵は許可済みgroup nodeへ保護して配布するが、物理nodeの専用秘密鍵は +他ノードへ配布しない。初版が保全するlegacy instance keyとは移行時に区別する。 + +復帰ノードに現primaryの系列にない変更があれば、旧DB/正本を隔離したgenerationとして保全する。 +件数・由来・時点を示し、手動export/importや内容比較へ渡す。データの新旧をmtimeだけで比較せず、 +自動マージ、自動削除、旧DBの現primaryへの逆コピーを禁止する。 + +## 6. 管理manifestの世代設計 + +既存schema_version 1/2、署名と受理checkpointを維持し、新契約は **schema_version 3案** とする。 +未対応受信先が一つでも対象に含まれる間はv3配信/CG自動移譲を有効化しない。 +capability応答だけでなく、移行済み受理状態のACKを必要とする。 + +v3 payloadは既存管理情報に `control_group_id`, `trust_generation`, `control_epoch`, +`manifest_sequence`, `configuration_checkpoint_id`, `previous_manifest_hash`, `payload_hash`, +`authority_commit_receipt` を追加する。署名は確定した管理内容を一度だけserializeしたbyte列に行う。 + +発行手順:同じ管理snapshotから内容を取得→候補payloadを永続化→有効control grant下で +authorityへ `expected previous hash/sequence` によるpublication commitを要求→返却receiptと +署名を保存→配信。応答喪失なら同じpayload hashでcommit結果を照合する。新規内容に同じsequenceを +割り当てない。receiptのない「大きなepochのmanifest」は配信・受理しない。 +snapshot内容の取得とcheckpoint固定は保存担当の契約が必要で、既存の連番SQLだけで代替しない。 + +受信手順: + +1. Campus/CG/署名鍵/trust世代/対象Domain ownershipを検証する。 +2. authority receiptと、現在許可されたcontrol epochを確認する。epoch切替時はfresh authority確認必須。 +3. 同epochの小さいsequence、同sequenceで異hash、親hash不一致は拒否する。 +4. 同epoch・同sequence・同hashは冪等ACKとする。 +5. 新epochは署名済み移譲recordとbase管理checkpointを検証する。単なる数値の大小では許可しない。 +6. 管理内容と受理位置を一つのtransaction境界で保存し、そのcommit後に適用ACKを返す。 + +管理ノードprobeから回収した最大受理位置は「これより前へ戻してはいけない」下限の証拠。 +完全な管理正本の代わりではない。checkpointが不足すればCGはblockedとし、新epochを使って +古い権限を再発行しない。復旧snapshotがauthorityの最後のpublicationより古い場合は、 +正しい履歴の取得または運用者による明示的な再構成・失効再適用を要求する。 + +v1/v2からの移行は最後のrevision/hashをmigration anchorとして固定し、最初のv3がそれを参照する。 +trust pinを削除してやり直す方法は採らない。v3受理後のv1/v2自動fallbackも拒否する。 +この永続化の追加は将来のDB担当との合意対象であり、本設計作業では実装しない。 + +## 7. 計画切替とpreempt + +preemptはpreferred nodeへ向けたswitchover operationをpolicy executorが作るだけであり、 +特別な高速昇格経路を持たない。offなら現primaryを維持、manualなら提案のみ通知する。 +automaticでも安定時間/時間帯/cooldown/能力/影響範囲を満たさない限りplanをapplyしない。 + +| operation phase | 確定させること | crash/timeout時 | +| --- | --- | --- | +| PLANNED | target、expected構成、影響、必要能力、見込時間 | 期限切れで再plan。副作用なし | +| VALIDATING | authority最新値、target同期/鍵/schema/ACL、fence能力 | 旧primaryの権利は変更せず停止 | +| DRAINING | 旧primaryの新規write/job停止、実行中処理の結果 | 旧grantがまだ有効とauthorityで確認できる場合だけdrain解除可能 | +| SEALED | final cutと内容集合、targetの最終検証ACK | 不足があれば旧権利の扱いを照合。勝手に切替えない | +| REVOKING | 旧permit再発行禁止を永続化 | ここから旧epochの自動復活は禁止 | +| FENCING | 旧writerへの外部隔離と、その確認証跡 | 不明ならblocked。新grantなし | +| GRANTED | 新epochのgroup/resource許可をPREPAREDで確定 | holderをGETで照合し再開。別targetへretryしない | +| ACTIVATING | target root/gate/schema/readinessを検証 | targetを隔離、次のoperationを計画。旧系に単純rollback不可 | +| ROUTING | TLS/Host/接続先を更新しread/write確認 | 新primaryは一つのままrouting修復 | +| COMPLETED | sourceの降格、監査、Inbox、保持世代を記録 | 重複完了は冪等 | + +planned switchoverの損失許容は常に0。最終cutのDBだけでなくCAS・可変正本・失効・outboxも +targetに適用済みでなければSEALEDを超えない。「遅延がほぼ0」「同期率100%表示」だけでは不可。 +grantを移す瞬間はwriterが一時的に0になってよいが、2になってはいけない。 + +fence証拠はkind、provider、対象node/boot、対象resource、操作ID、確認時刻、結果、 +再起動/再接続を阻止する方法を含める。接続不能やSIGTERM送信済みだけではconfirmedにならない。 +初段の自動認定は全旧writerを確実に隔離できる外部providerだけ。保存側fencing契約が後で +認定されれば、影響範囲を小さくするprofileとして追加できる。 + +**fenceで認定する範囲:** HTTP/LB経路を閉じただけではローカルjob/CLIの書込を止められない。 +プロセス停止なら全writerと再起動経路、電源断なら同居group、保存資格情報の失効なら対象となる +すべての正本backendを対象にする。remote backendへ既に送った可変正本更新が遅れて完了する +可能性も評価する。認定された保存側世代拒否、drain確認、または明確な処理終了証明がなく、 +旧更新が後から確定し得るprofileでは新grantを出さない。単にtimeoutを待ったことは終了証明ではない。 +メール等の既送信副作用はfenceで取り消せないため、outboxの曖昧結果は別途照合・承認対象にする。 + +## 8. 障害時昇格と許可喪失 + +障害時はprobe失敗でsuspect表示→authority照合→fence確認→最後の検証済み世代と損失評価→ +新grant、という順。旧primaryと最終cutを交渉できないため、計画切替と同じ0損失を仮定しない。 +RPO内というだけでは、成功応答済みの更新が失われない証拠にならない。 + +DR-D1の既定は未知損失/未複製変更があればoperator承認必須。自動failoverは保存profileが +`acknowledged_writes_durable_on_replica` を提供し、authorityがその確定境界を照合できる場合に限る。 +現在の非同期snapshot profileはこの能力を持たないので、自動preemptは将来可能でも、 +無人・無損失failoverまで実装済みとはしない。許容損失による自動昇格profileは本版では提供しない。 + +短期permitの初期候補はTTL30秒、更新5秒、local safety margin10秒、更新RPC期限2秒。 +有効期限を受信時刻だけから延長せず、要求開始のmonotonic時刻と署名された有効期間から +保守的なlocal deadlineを計算する。遅れた応答、clock異常、別boot、別epochは破棄する。 +許可喪失でwriter gateを閉じるが、プロセス停止/遅延commitの安全性はそれだけに依存せず、 +次holderにgrantする前の外部fenceで保証する。TTLが経過しただけの自動grantは禁止。 + +旧nodeが復帰しても、既存許可をdiskから復活させない。nonce付きprobeと新bootに束縛した +fresh permitが必要。authority自身が復元され旧recordしかない場合は全体をblockedにする。 + +## 9. cancellation、監査、rollback + +operationは長時間処理なので202を返し、通常event/queue画面には参照を出す。 +authorityのphase更新は通常キューの「再試行」ボタンで巻き戻さない。 +同じ操作を複数executorが拾ってもCAS phase+idempotency key+fence providerの操作IDで重複を抑える。 +外部副作用のexactly-onceを無条件に約束せず、曖昧な送信結果は照合またはoperator確認にする。 + +REVOKINGより前は、現grantが旧holderに有効と確認できる場合にだけcancelして復帰できる。 +REVOKING以降の失敗は新しい復旧operationと新epochで処理する。以前の設定ファイルに戻すだけの +rollback、epoch減少、旧writerの再起動、受理checkpoint削除は禁止。 + +監査はactor、group/bundle、plan hash、old/new node/boot、各epoch、checkpoint、損失承認、 +fence証跡参照、phase、結果、request IDを残す。書き換え可能な業務snapshot内だけを監査の +保全先にしない。probe全件は短期集約し、状態変化と操作の証跡は長期保存する。 + +## 10. 不変条件 + +- N1:同じwriter resourceに二つの有効holderを同時に作らない。 +- N2:probe、priority、preempt設定、watch通知は単独でwriterを開けない。 +- N3:新epochで古い管理内容/権限を無条件に正当化しない。 +- N4:未検証・不完全なgenerationへactive pointerを向けない。 +- N5:旧primaryの未同期変更を保全せずに削除・上書きしない。 +- N6:通常DBのrestorationでnode trust/authorityの単調性を巻き戻さない。 +- N7:すべてのwrite経路と背景処理がgate/隔離の範囲内にある。 +- N8:control切替とdata切替を区別し、共有resourceの場合だけ明示的に連動する。 +- N9:grant移譲後の失敗を旧epoch復活で回避しない。 +- N10:不足能力・到達不能・曖昧な副作用はfail closedにし、成功と表示しない。 diff --git a/docs/design/ja/dregarnuhr.md b/docs/design/ja/dregarnuhr.md new file mode 100644 index 0000000..e48ce99 --- /dev/null +++ b/docs/design/ja/dregarnuhr.md @@ -0,0 +1,108 @@ +# Dregarnuhr:イベント実行基盤 + +2026-11-01。初期分離を実装。公開API・永続データ形式は変更しない。 +名称と全体の移行順序は [コンポーネント命名と責務](component-names.md)、 +イベント仕様は [Event Scheduler](../event-scheduler.md) を参照。 + +## 目的と今回の境界 + +Dregarnuhr は「いつ、何件を実行し、失敗後にいつ再試行するか」を担当する。 +カレンダー同期、通知配送、City再構成、プレビュー生成の内容は担当しない。 +神名の導入に合わせ、従来の `server` にあった実行ループと制御を +`internal/dregarnuhr` に分離した。別サービスや別コンテナにはしない。 + +| 層 | 責務 | 主な実装 | +| --- | --- | --- | +| 汎用実行基盤 | 時刻、定期フック、実行枠、ハンドラー呼出し、完了判定、再試行計算 | `internal/dregarnuhr` | +| アプリケーション | 14種類の既存イベント登録、業務payload、監視・設定との接続 | `internal/server/event_handlers.go`、`dregarnuhr_queue.go` | +| 永続化 | 優先順位付きatomic claim、実行状態、イベント定義、ログ・保持設定 | `internal/store/event_jobs.go` | + +依存方向は server/store → dregarnuhr。Dregarnuhr は server、store、DB、 +ネットワーク、ファイルI/Oを直接参照しない。パッケージ境界をテストで固定する。 +SQLのスケジュール定義評価や投入時の重複排除は、今回の分離ではstoreに残す。 + +## 契約 + +- `Clock` / `Timer`:現在時刻とタイマーを注入する。テストは仮想時間を進め、実時間の待機に依存しない。 +- `Loop`:定期dispatch、初回のみの処理、繰り返し処理、外部wakeをまとめる。 + 遅延したtickはまとめて処理し、取り戻すために何千回も連続実行しない。 +- `Registry`:イベント型から `Handler(context.Context, *Job)` を引く。 + 空名、nilハンドラー、重複登録は拒否。未登録型は既存同様エラーになる。 +- `Queue`:設定読込、atomic claim、キャンセル状態確認、開始ログ、完了保存のアダプター契約。 + `ErrNoJob` は正常な空キューを表す。既存storeの `sql.ErrNoRows` はアダプターで変換する。 +- `Runner`:実行枠の確認からclaimまでを同一runner内で直列化し、ハンドラーをgoroutineで実行する。 + 1回のdispatchは最大32件。完了で枠を解放し、次のdispatchをwakeする。 +- `Complete` / `RetryDelay`:DBを触らず完了状態と再試行時刻を算出する。 + `store.EventJob` / `QueueSettings` / `ErrEventSkipped` は互換aliasを維持する。 + +既存のHTTPエンドポイント、JSONフィールド、イベント型・ID・キー、consumerのcheckpoint、 +DBスキーマ、UI表示名は変更しない。実行ログには `component=dregarnuhr` を付ける。 +payloadを新たにログへ出力しない。業務エラー本文の安全性は引き続き各ハンドラーで担保する。 + +## 維持する実行規則 + +優先順位は `priority DESC, run_at_ms, created_at_ms, uuid`。 +SQLiteの単一 `UPDATE ... RETURNING` で選択とclaimを行い、同じジョブの二重claimを防ぐ。 +同一プロセスの実行上限はキュー設定に従う。foregroundリクエストがある間は、 +既存ジョブを強制終了せず、実行中が0件の場合に限り1件を開始できる。 +これにより長時間の編集接続があってもバックグラウンドを完全停止させない。 + +| フック | 間隔 | +| --- | --- | +| 定義評価・dispatch | 1秒、またはwake時 | +| 初回プレビュー走査 | 起動後5秒 | +| 定期プレビュー走査 | 15分 | +| 履歴整理 | 1時間 | +| 承認期限・City変更の走査 | 10秒、またはwake時 | + +遅延後の定期フックは実行時刻から次の間隔を数える。厳密な壁時計への位相固定や +取り逃した回数分のcatch-upは行わない。業務イベント定義のスケジュール方式は変更しない。 + +成功は `succeeded`、適用対象外は `skipped`、キャンセル要求を伴う失敗は `cancelled`。 +その他の失敗は上限前なら `queued`、上限到達なら `failed`。 +再試行は基準秒数 × 2^(試行回数−1)、上限24時間。 +例えば基準30秒なら最初の失敗後30秒、次は60秒となる。 + +## 同時に是正した点 + +1. 複数呼出しが同時に空き枠を確認して上限を超える競合を、runnerのadmissionロックで防ぐ。 +2. 再試行秒数・試行回数が極端に大きくても、durationのオーバーフロー前に24時間へ飽和させる。 + 従来の通常設定での待ち時間は同じだが、初回から24時間を超える基準値も上限対象となる。 +3. 業務ハンドラーのpanicを失敗として記録・再試行対象にする。panicの内容自体は記録しない。 +4. キャンセル状態のDB読込失敗を「キャンセルなし」と扱わない。業務処理を止め、失敗経路へ送る。 +5. worker contextが取り消されても、独立した5秒contextで完了保存を試みる。 + 保存失敗はログへ出す。旧式の設定・ログ用DB呼出しにはcontext未対応部分が残るため、 + 保存処理全体の厳密な5秒上限を保証するものではない。 + +## 未実装:リース・HA・終了処理 + +**atomic claimは期限付きリースではない。現時点は単一writerプロセスの運用を前提とする。** +別プロセス同士の同時実行数は共有されず、定期定義の投入もHA対応ではない。 +完了UPDATEはUUIDで対象を指定しており、旧workerの遅延完了を防ぐfencingはない。 +既存の起動時復旧は `running` を再投入するため、他のwriterが生きている状態では実行してはならない。 +exactly-onceの配送・実行は保証しない。外部副作用の冪等性は業務ハンドラー側の責務となる。 + +- [ ] owner、attempt/claim token、lease expiry、heartbeatを永続化する。 +- [ ] 完了・キャンセル更新で所有権とattemptを比較するfencingを導入する。 +- [ ] 期限切れ回収と定期投入を複数プロセスから安全に実行し、全体の実行上限を定義する。 +- [ ] 成功後・完了保存前の障害、lease更新停止、旧workerの遅延完了を障害注入試験する。 +- [ ] アプリケーションの停止contextをschedulerに接続し、受付停止・実行中処理のdrain・期限付き終了を実装する。 +- [ ] 各業務ハンドラーへcontextを伝播する。現状は独自のBackground/timeoutを使う処理も残る。 +- [ ] 古い設定・ログDB呼出しをcontext対応にする。完了保存失敗の回収方針も定義する。 + +コアのLoop/Runnerはキャンセルを受け付けるが、現行アプリケーションは従来どおり +Background contextで起動する。これをgraceful shutdown対応済みとは扱わない。 +これらはDB migrationとDR/authority設計を伴う後続作業であり、命名・初期分離に混在させない。 + +## 検証方針 + +本番DB、外部カレンダーの内容、実際の通知配送先には触れず、一時DB・合成ハンドラーで検証する。 + +- fake clock:初回・定期・wake、長時間遅延、閉じたwake channel、キャンセルとtimer解放。 +- race detector:並行dispatch、foreground制限、registryの排他、実行カウンター。 +- 一時DB:優先順・独立DB接続でのclaim、時刻境界、試行回数、30秒/60秒retry、各終了状態。 +- server接続:14ハンドラーの登録、実行間隔、永続状態と監視counter、read-only/ジョブ無効化の抑止。 +- 全Go回帰テスト・静的検査。既存API・業務イベントのテストも併用する。 + +実行結果は [命名文書の検証記録](component-names.md#検証記録) に追記する。 +ソース変更・テスト・commit・CI・本番展開は別段階として記録する。 diff --git a/docs/design/ja/editor-resilience.md b/docs/design/ja/editor-resilience.md new file mode 100644 index 0000000..788eeb7 --- /dev/null +++ b/docs/design/ja/editor-resilience.md @@ -0,0 +1,117 @@ +# エディタ改善サブプロジェクト + +状態:E1–E5 の初期実装と回帰検証完了。E6 は未実装・後続ゲート。 +既存 HTML 正本・不変 Revision・Page UUID・言語別 head を維持する。 +本書は `html-editor.md` の挿入パーツ設計を含むエディタ横断の実行計画。 + +2026-11-01:文書変換と情報保全の実装を `internal/grammaratur` へ分離。 +通常変換・保護変換の契約と既存APIは維持する。 +[コンポーネント命名と責務](component-names.md) に移行境界を定義する。 + +## 優先順位と完了条件 + +1. **E1 編集内容の保全**:端末内下書き、認証切れ/通信断/権限失効の区別、 + 再認証後の明示的な保存再開、保存完了を待つ `:wq`、二重保存防止。 +2. **E2 保存競合**:言語別 head を DB トランザクション内で比較する CAS。 + 同一箇所を変更していないと確定できる範囲だけ自動マージし、その他は + 基点/自分/最新の内容を表示して手動調整。相手の編集を無条件に上書きしない。 +3. **E3 操作改善**:モバイルで Vim を使わない、macro 補完時の終端重複を防ぐ、 + HTML/Markdown の切替で未保存データを捨てない、少数の挿入コンポーネント。 +4. **E4 履歴・言語**:旧 Revision を作業バッファに読み込み、確認保存で新 Revision。 + BCP 47 の明示言語は暗黙に `und` へフォールバックして上書きしない。 + 翻訳は同じ Page UUID の別言語として分岐し、URL Path を別 Page にしない。 +5. **E5 変換保全**:HTML→Markdown→HTML の構造比較。不一致なら HTML を + `{=html}` 保護フェンス内に保持する。projection 展開結果を正本や下書きへ保存しない。 +6. **E6 共同編集(後続)**:サーバーをまたぐ presence、選択範囲、CRDT/OT、 + 編集セッションと永続化境界。単なる同時保存やポーリングを共同編集と呼ばない。 + +## 保存と下書きの契約 + +- 正本の保存単位は Page UUID × canonical language。開始 head(空も有効)を + `base_revision_uuid` で渡す。比較と Revision 追加は同一 write transaction。 +- 旧クライアントの base 未指定は移行互換として残す。UI は必ず base を指定する。 + 全入口の必須化・hosted editor/CLI 追従は後続ゲート。 +- 401 は再ログイン、403 は権限の再確認。変換 API の失敗時には Markdown を + HTML として保存しない。リダイレクト/非 JSON のログイン応答も成功にしない。 +- 保存要求中の編集は別の dirty 状態として保持する。保存成功したスナップショット + と現在内容が一致するときだけ下書きを削除する。応答喪失時は head 比較で回復する。 +- 下書きは origin・Domain・Principal・Page・language 単位。別ユーザーの下書きを + 自動表示しない。サーバー同期はしない。端末を共有する場合の平文保存リスクを明記。 +- 端末ストレージは容量制限・拒否・プライベートモード・ブラウザー終了で失われ得る。 + 保存失敗を明示し、テキストのダウンロード導線を用意する。下書きには保存時刻、 + 形式、基点 Revision と基点内容を保持し、期限とサイズの上限を設ける。 +- 復旧候補がある場合は無条件に server autosave しない。復元/破棄をユーザーが選ぶ。 + 再接続だけで公開 Revision を自動更新しない。 + +## 初期実装の API・UI と制限 + +- `POST /_/api/v1/revisions/save`:任意の `base_revision_uuid` を追加。 + このエディタは常に指定し、409 を競合として処理する。`editor_principal_uuid` も照合し、 + 編集中に別アカウントへ切り替わった状態で保存しない。旧クライアントの無条件保存は残る。 +- `GET /_/api/v1/editor/state?uuid=…&lang=…`:対象 Domain・Page の読取権限を検証。 + `metadata=1` は本文ファイルを読み込まず head を取得。`revision=…` は対象 Page の + 過去本文を取得する。非表示タブでのポーリングは停止し、表示時は30秒間隔。 + **これは保存済みの別更新検出であり、未保存の同時編集者 presence ではない。** +- `POST /_/api/v1/editor/convert`:作業中 HTML を安全な authoring Markdown に変換。 + `POST /_/api/v1/render/markdown` の `authoring:true` で保護フェンスを元の HTML に戻す。 + `projection:true` と併用したプレビューでは従来の sanitizer・macro 展開を適用する。 +- 変換できない HTML は文書全体を保護する保守的な実装。フェンスは + `` ```{=html} `` 形式(本文にバッククォートがある場合はそれより長い区切り)。 + 任意の属性、コメント、未知要素、表の結合、画像キャプション、コード内空白を保持。 + フェンス内を編集でき、外側に Markdown を追記できる。通常のコードフェンス内にある + `{=html}` サンプルは HTML として実行しない。保全は実行許可を与えるものではない。 +- 自動マージは各版を1つの変更範囲として比較する保守的方式。重複・隣接・同一挿入点は + 手動確認へ回す。自動再保存は3回まで再照合し、それ以上は確認画面へ移行。 + ファイル形式が変わった場合は自動マージせず退避と再読込を案内する。 +- 下書きは `localStorage`。UTF-8 JSON 全体で2 MiBまで、復元対象は7日以内。 + 起動時に期限切れを削除するため、ブラウザーを開かない期間の物理削除は保証しない。 + Domain/Principal/Page/language/tab ごとに分離し、他タブの下書きを一括削除しない。 + IndexedDB・サーバー同期・暗号化は未実装。保存失敗時はダウンロードを案内する。 +- `+` は挿入・下書き、`文` は言語の編集・分岐、`↶` は旧リビジョンを作業バッファへ + 読み込む。復元は明示保存で新 Revision を追加し、旧 Revision を書き換えない。 + 履歴は50件ずつ追加取得し、現在言語で絞り込む。分岐は翻訳生成ではなく本文のコピー。 +- 挿入は子ページ・表・画像キャプション・補足・コード・引用の6種の初期パレット。 + 画像 URL 等のプレースホルダーは編集が必要。ファイルピッカー、図形・外部 widget の + パラメータ編集、プラグインによる挿入パーツ登録は既存の HTML editor 設計に残る。 +- touch/coarse pointer + hover 不可の端末ではデスクトップの Vim 設定を上書きせず、 + 当該編集セッションだけ通常入力にする。実機 iOS/Android の IME は別途受入試験。 + +## 次段階の実装順序 + +1. **E6a presence**:編集中リースの共有ストアと認可付き API、別端末を含む表示。 + 管理者にだけ見える監視ではなく、同じ Page/language を編集できる参加者に限定する。 +2. **E6b realtime**:下記 CRDT 詳細設計・採用ライブラリ・同時変換ルールを確定した後に + feature flag 付きで実装。現時点の CAS を最終コミット時の保護として残す。 +3. **E5b / E3b**:HTML ブロック単位の保護、hosted editor/CLI への保護フェンス対応、 + アセットピッカーと拡張挿入パーツ。通常の Markdown だけを扱う外部エディタへ + このフェンスを無断で渡さない。既存の外部エディタ変換経路は本変更の対象外。 +4. 保存 API の旧クライアント移行と base 必須化、下書き大容量化・明示破棄 UX、 + 共有端末ポリシー、モバイル実機テスト。 + +## 共同編集の詳細設計 TODO + +- presence キーは Domain × Page UUID × language × Principal × tab session。 + TTL 付きリース、退出、バックグラウンド抑制、権限失効時の購読停止を定義。 +- API は `/_/api/v1/editor/` 配下に置く。WebSocket/SSE は origin・認証・対象 Page の + read/write を接続時と継続時に検証。本文や利用者名を未認可者へ漏らさない。 +- 本文 CRDT は authoring source(HTML/Markdown の混在規則と基点を含む)を同期し、 + projection HTML を同期しない。言語・形式の不一致を同一 room に混在させない。 +- SQLite に毎キー入力を記録しない。スナップショット間隔・増分上限・圧縮・GC・ + HA 配信基盤と Revision commit 境界を定義。offline 分岐復帰・undo と IME を検証。 +- Yjs/Automerge 等は権限、再接続、肥大化、ライセンス、bundle サイズを比較する。 + この選択とサーバー共有状態を未設計のまま本番へ有効化しない。 + +## 検証・展開ゲート + +テスト用 DB と合成文書だけを用いる。実データ・外部カレンダーにはアクセスしない。 +保存 CAS、空 head、言語分離、旧 Revision 復元、認証失効、別 Principal、通信断、 +変換失敗、保存中の追記、再試行、複数タブ、モバイル、IME、macro 終端、未知 HTML の +round-trip を検証する。CRDT と複数インスタンス presence は別途負荷・安全性試験。 +本番反映・リリース採番は実装検証の後に行い、既存リリースタグは変更しない。 + +初期回帰試験:`TestEditorRevisionCAS`、`TestEditorResilienceAPI`、 +`TestAuthoringRoundTrip` / `TestAuthoringProtectedHTML`、`assets/editor/editor-core.test.mjs`。 +実ブラウザーは `PAGESWIKI_BROWSER_TEST=1` で `TestEditorResilienceChromium` と +`TestHTMLProjectionChromium`。通信断、401、応答喪失、競合 UI、自動マージ、保存中の追記、 +変換失敗、下書き復元、旧版復元、言語分岐、マクロ補完、モバイル判定、テーマ、 +authoring/projection の分離を合成データで検証する。 diff --git a/docs/design/ja/geduldh.md b/docs/design/ja/geduldh.md new file mode 100644 index 0000000..ba12905 --- /dev/null +++ b/docs/design/ja/geduldh.md @@ -0,0 +1,96 @@ +# Geduldh の文書保存契約 + +2026-11-01、ローカルDocument Storeと一時展開の初期分離を実装。 +GeduldhはSHA-256で識別する不変の文書バイト列を保存・取得・検証する。 +従来の `internal/documentstore` の実装を移し、保存形式、DBスキーマ、公開URLは変えない。 +名称の位置付けは [コンポーネント命名と責務](component-names.md)、 +外部ストレージと運用の計画は [保存基盤の多様化](../storage-portability-and-operations.md) を参照。 + +## 実装と責務 + +| 層 | 所有するもの | 所有しないもの | +| --- | --- | --- | +| `internal/geduldh` | Store契約、ローカルpayload、保存キー、サイズ・hash検証、容量制限付き一時展開 | SQL、認証・認可、Page/Revision、GCの削除判断 | +| `internal/documentstore` | 旧公開型のalias、旧コンストラクター・Keyの委譲 | 独自の保存実装・検証ロジック | +| `internal/store` | Revisionと参照のtransaction、DB-only互換、バックフィルのidentity・checkpoint、read-only運用 | 別実装のローカルpayload writer | +| server / DR / CLI | 認可後の配信、外部ツール、保全・復元の手順 | hashを知っていることによる認可の省略 | + +アプリケーションと旧入口からGeduldhへの一方向依存とする。Geduldhからstore、DB、 +server、旧documentstoreへ戻る参照は作らない。HTTPレスポンスや権限判定も持たせない。 +`Component = "geduldh"` は内部の固定識別名であり、保存ディレクトリやサービス名ではない。 + +## 保存と取得の契約 + +`geduldh.Store` は既存の `documentstore.Store` と同じ型として扱える。 + +| 操作 | 契約 | +| --- | --- | +| `Stat(ctx, hash)` | 正規化したhashとバイト数を返す。内容全体の検証ではない | +| `Open(ctx, hash)` | `Read`、`ReadAt`、`Seek`、`Close` に対応するReaderを返す。全体を先読みしない | +| `PutIfAbsent(ctx, hash, size, reader)` | サイズとSHA-256を検証し、完全なpayloadだけを上書きせず公開する | +| `Verify(ctx, hash)` | ストリーム全体を読み、SHA-256の一致を検証する | + +通常の `Open` / `Stat` は全体をhash検証しない。Range配信やZIPの部分読み取りで +ファイル全体の走査を強制しないためである。検証が必要な経路は `Verify` または検証付き +Materializerを選ぶ。既存のRevision本文取得時のhash検査はstore側に残る。 + +保存キーは `sha256//<64桁hash>`、ルートは既存の `documents`。 +hashの前後空白の除去・小文字化、既存エラーの同一性を維持する。Path、ファイル名、 +Domain、言語、Revision UUIDは保存キーに混ぜない。配置の変更で公開URLやETagを変えない。 + +ローカル保存はルート内アクセス、64 KiB単位の検証コピー、一時ファイル、ファイルfsync、 +hardlinkによるno-replace公開、ディレクトリfsyncを維持する。 +既存payloadは再検証し、破損していても入力で上書き修復しない。 +`NewLocal` は保存先を作成しない。ルートの準備とバックフィルは起動・運用側の責務とする。 + +書込みではpayloadの永続化を先に完了し、その参照を持つRevisionのDB transactionを +後で確定する。DB側の競合・rollbackで未参照payloadが残ることは許容する。 +Geduldhは参照数を判断せず、`Store` にDeleteを設けない。GCは保持・参照・backup pinを +判断する別の経路とし、今回の分離では変更しない。 + +## 外部ツール用の一時コピー + +`Materializer.Acquire` は同じhashの取得を共有し、検証済みのノードローカルコピーを +`Lease` で返す。ここでのleaseは一時ファイルの利用権であり、分散ジョブの期限付きリースではない。 +最後の利用者がCloseするとコピーを削除する。正本のhardlinkを外部ツールへ渡さない。 +一人の待機中断で他の利用者を巻き込まず、全員が中断した場合は取得を取り消す。 + +既存アプリケーションの上限を維持する:同時取得2件、待機・利用中を含め64件、 +予約容量合計2 GiB、共有取得期限2分。呼出し側のファイル単位上限とcontextも適用する。 +一時ディレクトリは0700、コピーは0400。削除失敗時には予約容量を解放済みと扱わない。 +これらはアクセス元アプリケーションの設定であり、外部ストレージ対応の宣言ではない。 + +## 互換性と今回の変更範囲 + +- 旧documentstoreの型・エラーはalias、関数はGeduldhへの委譲とし、実装を二重に持たない。 +- storeの保存、読み取り、サイズ・部分取得、一時展開はGeduldhへ接続する。 +- `.document-store-id`、バックフィルのUUID cursor、DB-only互換、起動時移行は変更しない。 +- DR manifest、スナップショットの構造、復元手順、保存済みデータは変更しない。 +- UI、URI、設定キー、イメージ名、イベント型、consumer checkpointは変更しない。 + +これは既存の保存実装の分離であり、新たなファイル移行・DB migrationを必要としない。 +任意のmount先での永続性、外部backend、マルチノードHAを追加するものではない。 + +## 残件 + +- [ ] S3 / SFTPなどのbackendを、条件付き作成・部分取得・破損・切断の共通契約試験を経て追加する。 +- [ ] NFS / SMB / SSHFSはno-replace公開・fsync・mount喪失の実環境試験を行う。ローカルパスであるだけで対応済みとしない。 +- [ ] 旧Revision保存・本文取得を含む全経路へcontextを伝播する。 +- [ ] payload、可変正本、秘密情報、scratchの配置設定を分離する。 +- [ ] 一時展開の上限を設定化し、削除失敗・クラッシュ残骸の安全な回収とメトリクスを追加する。 +- [ ] Documentのサイズをメタデータ化し、外部backend導入時の一覧のStat/HEAD多発を防ぐ。 +- [ ] GCの参照snapshot・保持・backup pinと、外部ストレージの削除契約を整備する。 + +## 検証 + +合成payloadと一時DB・ディレクトリで、次を検証する。 + +- 移動前レイアウトの読み取り、新旧入口の型・エラー・キー互換、既存inodeの保持。 +- 並行書込み、不完全入力・既存破損の拒否、キャンセル、ルート外へのsymlink逸脱防止。 +- Range / ReaderAtと逐次読み取りの独立性、ZIP配信、認可を伴う不変オブジェクト配信。 +- 一時コピーの共有、容量・件数・時間制限、最終Closeでの削除、正本からの分離。 +- 永続化失敗時のRevision非確定、metadata競合時の正本保全、バックフィルの再開。 +- Geduldhで保存 → 既存DR snapshot → verify → restore → Geduldhで検証・読取り。 + +実行結果は [コンポーネントの検証記録](component-names.md#検証記録) に記載する。 +本番データの操作、commit・push・本番展開はこの初期分離には含めない。 diff --git a/docs/design/ja/html-editor.md b/docs/design/ja/html-editor.md index 8b88c9f..2e01da9 100644 --- a/docs/design/ja/html-editor.md +++ b/docs/design/ja/html-editor.md @@ -2,6 +2,11 @@ 状態:設計案。全面WYSIWYG採用は未決定。既存エディタを置換しない段階的改善を第一案とします。 +2026-09-27 追記:[エディタ改善サブプロジェクト](editor-resilience.md) で +端末下書き・CAS 保存・競合確認・言語分岐・旧版復元・HTML 保護ブロックと +6種の初期挿入パレットを実装。以下の型付きパーツ schema、画像ピッカー、 +限定 block 視覚編集、リアルタイム共同編集は引き続き後続段階です。 + ## 解決したい問題 表、画像とcaption、引用、注意書き、コード、リンク、macro、埋め込みを毎回手書きする負荷を減らします。単にHTMLを隠すことより、既存文書を壊さず効率よく編集できることを重視します。 diff --git a/docs/design/ja/kunstzeal.md b/docs/design/ja/kunstzeal.md new file mode 100644 index 0000000..d1a1d8b --- /dev/null +++ b/docs/design/ja/kunstzeal.md @@ -0,0 +1,125 @@ +# Kunstzeal の表示方式と描画プロバイダー契約 + +2026-11-01、既存の表示方式の選択とKroki図描画を `internal/kunstzeal` に初期分離した。 +Kunstzealは描画エンジンを自作する部品ではなく、表示方式と外部描画の呼出し契約をまとめる。 +Page/Revisionの正本、認可、HTMLの信頼境界は引き続きアプリケーションが所有する。 +スライド、nwdiag、新しい描画サービスの追加は今回含めない。 +[コンポーネント命名](component-names.md) と [表示アダプタ設計](../projection-adapters.md) を併読する。 + +## 責務と依存方向 + +| 部品 | 責務 | +| --- | --- | +| `kunstzeal.SelectAdapter` | 有効MIME・ファイル名から、既存の優先順で表示方式を選択 | +| `kunstzeal.Catalog` | ビルトイン14種類の定義のコピーを返す。呼出し元から共有registryを変更させない | +| `DiagramProvider` / `Kroki` | 型・入出力サイズ・時間を制限したSVG描画要求。エラーを一般化 | +| 図のhash・cache | 既存hashの計算、有界読込、一時ファイルからの原子的なcache置換 | +| server | 対象解決・認可・Domain/Path由来情報・HTTP応答・CSP・Kroki接続設定・hosted plugin管理 | +| Grammaratur | HTML/Markdown等の文書形式変換。描画された図を正本へ書き込まない | +| Dregarnuhr / 既存preview処理 | PDF・動画等の背景サムネイル生成。今回は移動しない | + +server/compiler → kunstzeal の一方向依存とする。 +Kunstzealはstore/server/DBへ依存せず、認証CookieやBearer tokenを受け取らない。 +図cacheのファイルI/Oと設定済みKrokiへのHTTP以外に、新たな実行環境を持たない。 +任意の実行ファイルやshellを起動する機構ではない。 + +## 表示方式の選択と互換性 + +順序は `html`、`excalidraw`、`csv`、`archive`、`debian-package`、`eml`、`pe`、 +`json`、`text`、`image`、`audio`、`video`、`pdf`、`download` を維持する。 +特殊形式を一般JSON/textより先に選択し、未知の形式はdownloadへ落とす。 +MIME検出、文字コード変換、各形式のparserとViewer実装自体は今回移動しない。 +MIMEと拡張子の優先関係も変更しない。 + +`Descriptor`・`Adapter`・`Delivery` はserverの旧型をaliasで接続する。 +ID、Kind、Delivery、NeedsText、WindowableText、2MiBのテキストwindow閾値は同一。 +`/_/api/v1/adapter` のビルトイン一覧も同じcatalogを参照する。 +Catalogは複製したsliceを返すため、一覧編集が別要求の選択結果を変えない。 + +hosted viewer/editorの設定キー・有効無効・URL検証・起動ticket・保存経路は変更しない。 +ビルトイン選択registryを任意コードをロードするプラグイン機構へ変更するものではない。 +Excalidrawの別コンテナ/ブラウザ内描画、PDF、media、archive等の実行環境は従来のまま。 + +## Krokiの実行契約 + +`DiagramProvider.Render(ctx, kind, source)` はSVGバイト列またはエラーを返す。 +既存Kroki呼出しはmermaid・plantuml・graphviz・dot・d2のみ許可し、dotはgraphvizへ正規化する。 +任意のURL・出力形式・エンジン名は文書から指定できない。 + +| 項目 | 契約 | +| --- | --- | +| 図ソースPOST | 最大1MiB。超過時は送信せず413 | +| HTML内の図ブロック | 32KiBまで。超過時は元のコードブロックを残す | +| 描画要求 | POST、`text/plain; charset=utf-8`、最大15秒。親contextの取消も反映 | +| 結果 | 2xxのみ成功、最大4MiB。非2xxの応答本文は利用者へ返さない | +| endpoint | 管理者の `KROKI_URL`、空なら既存の `http://kroki:8000`。API入力から選択しない | +| client | 呼出し元のHTTP clientが必須。未設定時に既定clientで送信しない | +| redirect | serverのclientで全redirectを拒否。301/302/307/308も描画失敗として扱う | +| エラー | 設定不備500、描画失敗502。接続先URL・provider本文・元ソースをエラー文へ含めない | + +今回の是正として、endpointにuserinfo・query・fragmentを含む設定は拒否する。 +内部ComposeサービスへのHTTP接続は既存仕様なので許可する。公開IP限定の一般URL取得とは異なる。 +既存のHTTP clientによるproxy環境設定の扱いは変えない。接続先は信頼された運用設定として扱う。 +DNS/IP固定、providerのTLS認証・egress制限、描画サービス自体のファイル/ネットワークアクセス制限は +この抽出だけでは提供しない。配置先・ネットワーク隔離と合わせた後続契約とする。 + +clientはcontextを尊重する必要がある。コアの時間制限は任意の悪意ある実装を強制停止するsandboxではない。 +HTTP clientとcoreの両方に時間予算を設けるが、描画サービス側のCPU/RAM使用量は別管理である。 + +## Cacheと認可 + +hashは従来の `SHA256(kind + NUL + format + NUL + source)` を維持し、 +`compiler.DiagramHash` とserverの旧入口は共通実装へ委譲する。 +HTML由来ではsourceの前に既存のHTML projection version・Domain・Pathを組み込み、 +同じ図でも異なるPathの権限参照を共有しない。POSTの入力そのものをhash化する方式も維持する。 + +保存先は既存の `//.svg`。DB migrationやcache移行は不要。 +読込は最大4MiB。書込は同じディレクトリの一時ファイルを完成させてからrenameする。 +並行読込に途中までのSVGを渡さず、失敗時は一時ファイルを削除する。 +cacheの保存に失敗しても、正常に描画できた応答は返す。これは再生成可能なcacheであり、 +fsyncを伴う正本文書の耐障害commitや、同一キーの二重生成抑止ではない。 + +GETの認可はcache読込より先に行う。既存のソース由来情報はメモリ上で128件、30分以内。 +kind・Domain・Pathの現在の読み取り権限を検査し、期限切れ・由来不明・別Domain・権限なしは404。 +ディスクにSVGが残っていること自体はアクセス権にならない。 +POSTは既存のdomain read endpoint policyを通り、利用者が送ったソースを描画する別経路である。 +これを新たな匿名描画APIへ変更しない。 + +## SVGの信頼境界 + +providerが返すSVGをアプリケーションのHTMLへ直接挿入しない。 +cache hit、新規描画、直接アクセスでも、従来どおり次を付与する。 + +```text +Content-Type: image/svg+xml; charset=utf-8 +Content-Security-Policy: sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data: +X-Content-Type-Options: nosniff +Cache-Control: private, no-store +``` + +バイト数を制限することとSVGの内容を安全と認定することは異なる。 +今回SVG sanitizerや完全な構文検証は追加しない。Kunstzealの出力は信頼済みHTMLではなく、 +上記のsandbox応答で扱う必要がある。HTML本文のsanitizer・macro展開・CSPはserverが引き続き適用する。 +図の正本ソース、Page/Revision、既存の表示URIは変更しない。 + +## 後続TODO + +- [ ] provider ID/version・テーマ・オプションを含む新しいcache keyと、旧cacheからの移行/失効を設計する。 +- [ ] 同時描画上限、同一キーの実行集約、優先度、キャンセル、Dregarnuhrへの背景生成、過負荷時の安全な再試行表示を整備する。 +- [ ] provider単位の停止・切戻し・health・CPU/RAM/時間/件数/失敗の観測を統一する。今回hosted pluginの既存設定は維持したが、この統一機構は追加していない。 +- [ ] ソース由来cacheをプロセスglobalからinstance単位へ整理し、再起動・複数nodeでも認可と失効を保てる取得方式にする。 +- [ ] ディスクcacheの容量・TTL・清掃、renderer更新時の不整合、障害後の一時ファイル残留を管理する。 +- [ ] provider宛通信と描画サービス内の外部参照を分け、送信許可・接続先・ネットワーク隔離・バージョン固定を契約化する。 +- [ ] nwdiagの有界parser・安全なSVG出力・編集preview、Draw.io等のproviderを追加する。現在のregistryにnwdiagは実装されていない。 +- [ ] Reveal互換・HedgeDoc・Slidev・CSS-onlyスライドのimport/save/printを独立providerとして設計する。 +- [ ] Excalidrawのhosted/browser実行とPDF/動画サムネイル・metadata subprocessを、各々の安全境界を維持したまま段階的に接続する。 + +## 検証と反映 + +合成ソース・一時ディレクトリ・偽HTTP client・localhostの偽Krokiだけで検証する。 +入力/出力上限、取消、全redirect拒否、エラー情報の非開示、cacheの並行読込/置換、 +kind/hashとcatalog互換、Domain/Pathの認可、由来情報の期限、CSPを確認する。 +実Krokiの版ごとの描画品質や本番通信は今回の検証対象ではない。 + +実行記録は [コンポーネント検証記録](component-names.md#検証記録) に記載する。 +本番DB・取り込み済み外部カレンダーへアクセスせず、commit・push・デプロイは別段階とする。 diff --git a/docs/design/ja/mestionora.md b/docs/design/ja/mestionora.md new file mode 100644 index 0000000..58551a7 --- /dev/null +++ b/docs/design/ja/mestionora.md @@ -0,0 +1,110 @@ +# Mestionora の検索と索引契約 + +2026-11-01、既存のSQLite検索・検索索引・City検索を `internal/mestionora` へ初期分離した。 +検索の意味、順位、予算、HTTPレスポンス、索引の形式は維持する。 +権限ポリシーそのものはアプリケーションが所有し、Mestionoraは渡された読み取り判定を適用する。 +[コンポーネント命名](component-names.md) と [検索結果の仕様](../search-results.md) を併読する。 + +## 所有する処理 + +| 部品 | 責務 | +| --- | --- | +| `mestionora.SQLite` | Pageとファイルの候補抽出、絞り込み、順位付け、認可後の抜粋、H1サジェスト | +| City検索 | Domain内のQuartier・Street・Spotのメタデータ照合、祖先を含む認可、表示名・ポータルPathの射影 | +| 検索索引の初期化 | FTS5、現行Revisionの投影、既存のINSERT/DELETE trigger、従来のbackfill | +| store互換入口 | 従来の関数を委譲し、結果型・検索条件・エラーをaliasで維持 | +| server | Domainとscopeの解決、リクエストの権限ポリシー、日付のタイムゾーン変換、HTMLの安全な表示、URL生成 | + +依存方向は server/store → mestionora → database/sql とHTML tokenizer。 +現段階はSQLite実装であり、ストレージから独立した検索サービスではない。 +DB接続の作成・所有、認証、認可ルール、Documentの正本保存、外部検索への送信は担当しない。 +`Component = "mestionora"` は内部識別名であり、公開URLやDB名を変更しない。 + +## 呼出し契約と互換性 + +`NewSQLite(db)` は接続プールを借用するだけで、I/O・DDL・索引再構成を行わない。 +`EnsureSearchIndexSchema` は従来の起動経路から明示的に呼ぶ。 +検索リクエストから初期化・migrationを起動しないため、読み取り専用接続でも検索できる。 + +- `SearchPathsInDomainFiltered`:追加のファイル・日付条件を使わない、認可付きキーワード検索。 +- `SearchPathsInDomainWithOptions`:ファイル種別、拡張子、更新日/作成日を組み合わせたPage検索。 +- `SearchPathSuggestionsInDomain`:本文抜粋を生成しない候補検索。許可されたHTML結果だけにH1を補う。 +- `SearchCityInDomain`:Cityのメタデータ検索。全CityMapや配下Pageの展開はしない。 + +新入口はすべて `allowed func(path string) bool` を要求する。nilなら結果を返さない。 +旧 `Store.SearchPathsInDomain` の全許可入口は信頼された既存呼出しとの互換のためにだけ残し、 +Mestionoraには移さない。HTTPの検索・サジェストはMestionoraへ直接、要求ごとの判定を渡す。 + +旧 `SearchResult`、`PathSuggestion`、`SearchOptions`、`CitySearchResult` の型とメソッド、 +`ErrInvalidSearchOptions` の同一性を保持する。H1とCity表示名の共通投影は一つの実装へ委譲する。 +UI・URI・GETパラメーター・JSON、表示順位、Domain/Path/Revisionの識別子は変更しない。 + +## 認可と読み取りの境界 + +Page検索は現在のDomain・scope・フィルターに合う候補をSQLで絞り、Path単位の認可を行う。 +同じ文書に公開と非公開のaliasがあっても、非公開Pathを別名として返さない。 +本文抜粋・要約・H1は許可された表示対象にだけ補う。FTS自体は非公開データも含む索引を +走査するので、「非公開文書の索引を一切読まない」という保証ではない。 +未許可候補の件数、Path、ファイル名、タイトル、抜粋は返さず、全件拒否は通常の空結果とする。 + +City結果は境界PathとQuartier・Street祖先のすべてを確認する。 +未分類Spotのポータルを親Streetへ向ける既存の投影と、親側の判定を維持する。 +Mestionoraはユーザー間・Domain間で結果や許可判定を共有キャッシュしない。 + +Pageの候補と抜粋は同じ読み取りtransactionで扱う。認可callbackもその中で呼ぶため、 +事前読込済みの判定が望ましい。現行アプリケーションでは `/u/` のユーザー領域ポリシーに +遅延DB参照が残る。このcallbackを使う場合は別接続の余力が必要であり、プールを1本へ制限すると +待ち合わせ・timeoutの原因になる。Cityは候補transactionを終了してから認可する。 +認可とPage検索を別snapshotへ分割する変更は、Path/Revisionの再検証と合わせた後続設計とする。 + +検索データはHTMLとして信頼しない。抜粋の文字列化・script/style除去・強調表示と +リンクのエンコードは既存のserver/template経路で行う。H1抽出はsanitizerではない。 + +## 保持する検索予算と意味 + +| 項目 | 上限・動作 | +| --- | --- | +| Page/Cityの入力 | 512 bytes・16語まで | +| Page候補 | Path/filename部分一致、本文FTS、filename FTSの各streamで1,000件まで | +| Page結果 | 既定100件、最大200件。認可後に順位付け・表示上限を適用 | +| City候補・結果 | 種類ごとに1,000候補、許可された結果は最大30件 | +| 検索時間 | 各検索2秒。HTTPはCityとPageで2秒を共有 | +| 本文snippet | 1文書64 KiB、重複Revisionを除いた合計1 MiB、追加処理の時間予算400 ms | +| サジェストHTTP | 700 ms、Page最大8件・City最大4件。Cityは任意の補足 | + +Domain・scope・ファイル/日付条件を候補LIMITより先に適用する。 +FTS-firstのCROSS JOIN、現行RevisionとPathへの索引参照、rowidで限定したsnippet取得を維持する。 +通常検索でDocumentファイルをstatしたり、全件数を数えたりしない。 +大きい本文やsnippet予算超過は抜粋だけ省略し、許可されたヒットは残す。 + +FTSは各語を引用したprefix条件をANDで結合する。入力中の演算子や引用符をSQL/FTSの命令として +扱わない。Path scopeは既存の正規化と区間比較を使い、`%`・`_` をワイルドカードにしない。 +MIMEと拡張子の優先順位、日付の半開区間、Cityにファイル種別を適用しない規則も変更しない。 + +今回の是正点として、Pageの認可中にcontextが取り消された場合は返却直前にも検査し、 +候補が残っていても部分的な成功として返さない。DBエラーのHTTP出力は従来どおり一般化する。 + +## 索引と残件 + +`revision_search`、`revision_search_state`、`latest_revision_index`、`search_schema_state`、 +既存triggerとbackfill SQLは名称・内容を維持した。分離による新しいmigrationや索引再作成は不要。 +起動時の従来の整備処理がなくなったわけではない。 + +- [ ] ユーザー領域を含む読み取りポリシーの事前読込と予算を整理し、接続1本でも認可中に待ち合わない構造にする。 +- [ ] 現行Revisionのみ・Domain別の索引、更新watermark、継続取得を設計する。現候補上限は網羅的な検索を保証しない。 +- [ ] 日本語の中間一致、HTMLの可視テキスト化、言語別順位と対象Revisionの選択をmigration付きで整備する。 +- [ ] 現状の最新RevisionはObject単位であり、言語別検索ではない。履歴検索・言語別検索の意味を別途決める。 +- [ ] 既存FTS backfillとtrigger再作成の起動コスト、再開可能性、readinessを整理する。 +- [ ] H1取得を索引メタデータへ移す。現在は認可後にDB本文の先頭65,536文字を取得する互換処理。 +- [ ] 外部検索・関連性検索は別provider契約とし、認可済みの範囲・送信先・送信内容を明示する。 +- [ ] 結果cacheを追加する場合はDomain、principal、policy version、索引versionを含め、失効を先に実装する。 + +## 検証と反映 + +合成データと一時DBで新旧入口の一致、nil/拒否ポリシー、alias・Domain・リクエスト分離、 +認可後の抜粋とH1、キャンセル、読み取り専用動作、FTS/Cityの実行計画を検証する。 +既存の600 Page / 1,200 Revision試験、City祖先認可、MIME・日付条件、XSS回帰試験を維持する。 +Chromiumではデスクトップ・モバイルの3表示モード、City先頭表示、絞り込み、リンクを確認する。 + +実行結果は [コンポーネント検証記録](component-names.md#検証記録) に記載する。 +本番DBの診断や外部カレンダー内容へのアクセスは行わない。commit・push・本番反映は別操作とする。 diff --git a/docs/design/ja/operations.md b/docs/design/ja/operations.md index f38690e..a1c3856 100644 --- a/docs/design/ja/operations.md +++ b/docs/design/ja/operations.md @@ -35,6 +35,13 @@ MWDB/Karton等の接続はpending/clean/suspicious/malicious/error/expiredの状 ## DRとHAの順序 +次段階の詳細設計DR-D1は [役割・再参加](dr-authority-and-rejoin.md)、 +[通信・同期・切替](dr-protocol-and-sync.md)、[実施計画](dr-delivery-plan.md) に分離した。 +設計のみの完了であり、DB別件に介入していない。管理primaryとデータprimaryを独立させるが、 +保存resourceを共有する場合は連動切替とし、probeを昇格の証明としない。 + +2026-09-27追記:[DR詳細設計と運用手順](disaster-recovery.md)に、インスタンス単位の停止snapshot、検証付き復元、読み取り専用起動、外部fenceを前提にした手動昇格の初版を定義した。オンライン同期・自動昇格・管理GUIは次段階。初版では待機系のorganizer/CalDAVは閉じ、昇格後に再開する。 + コントローラDRは署名manifest journal、鍵の保全、sequence、昇格epoch、旧leaderのfencingから設計します。記事DBとDocumentの複製は別の責務です。Campus manifestは記事バックアップではありません。 LB/HAにはsession失効共有、job lease、cache invalidation、uploadのnode非依存、WebSocket drain、proxy origin検証を要します。SQLiteをネットワーク共有して複数writerで動かす構成は採りません。fenced single-writer+replicaか、適切なmetadata backendを先に選びます。 diff --git a/docs/design/ja/ordoschnellic.md b/docs/design/ja/ordoschnellic.md new file mode 100644 index 0000000..69e790c --- /dev/null +++ b/docs/design/ja/ordoschnellic.md @@ -0,0 +1,107 @@ +# Ordoschnellic の通知配送契約 + +2026-11-01、既存のWeb Push配送を `internal/ordoschnellic` に初期分離した。 +通知の作成・Inboxへの登録と、登録済み通知の配送を区別する。 +今回の対象は配送ループ、再試行方針、購読データ検証、暗号化したWeb Pushの送信。 +メール等の新チャネル追加、通知UI変更、独立サービス化は含めない。 +[コンポーネント命名](component-names.md) と [アクティブ通知](active-notifications.md) を併読する。 + +## 責務と依存方向 + +| 部品 | 所有する処理 | +| --- | --- | +| `ordoschnellic.Dispatcher` | 有界バッチ、閲覧中の延期、配送結果の集計、終了処理、キャンセルの伝播 | +| `ordoschnellic.WebPush` | 保存済み購読の検証、通知識別子の暗号化、HTTP結果の分類 | +| `ordoschnellic.Queue` | 清掃・取得・延期・完了を要求する契約。永続化の実装は所有しない | +| store | Inboxとキューの原子的登録、購読とセッションの束縛、claim時の適格性確認、lease、結果保存 | +| server | 認証済み購読API、VAPID鍵の取得、接続先制限付きHTTP client、キューadapter、イベント登録・集計ログ | +| Dregarnuhr | 既存の `notification.push.deliver` を定期実行。通知本文や受取人は決めない | + +依存方向は server/store → ordoschnellic → Web Pushライブラリ。 +Ordoschnellicからserver/store/DB/ファイルへ戻る依存を持たない。 +接続先の公開IP検証・DNS固定・TLS・redirect禁止はserverが提供するHTTP clientで行う。 +鍵providerやHTTP clientがない場合、既定clientで送信せず再試行扱いにする。 + +`Notification` と `PushDelivery` は共通型として分離し、既存store型はaliasで維持する。 +通知APIのJSON、エラー同一性、UUID、DBテーブルとtrigger、購読ID・session hash、イベント型、 +`builtin.notification.push` の定義IDは変えない。新規migrationや再購読は不要。 + +## 通知の生成と購読位置 + +業務処理が `CreateNotificationTx` を呼び、Inbox INSERT triggerが該当購読のキューを作る。 +業務側のrollbackはInboxと配送キューの両方を取り消す。ネットワーク送信をこのtransactionへ入れない。 +Inboxの重複キー `(domain_uuid, recipient_uuid, event_uuid, kind)` と、 +キューの `(subscription_id, notification_uuid)` を維持し、同じ通知を新UUIDで作り直さない。 + +Activity Subscriptionのconsumer ID・watermark、Task Activityのjournal→outbox橋渡しは +別契約としてそのまま残す。Ordoschnellicという名前への置換やcheckpointの初期化は行わない。 +新規Push購読への過去Inboxの一斉再送もしない。 + +## 認可と送信情報 + +claimのDB transactionで、Domain・受取人・Inbox・購読・元セッションの対応を確認する。 +セッション期限/失効、Principalのactive/readonly、membershipのactive、Domainの有効性と +local配置、未読状態、24時間以内の通知であることを検査する。 +条件を満たさないclaimは購読本文を渡さず、送信せずにキューを終了する。 +これはclaim時点の検証であり、検証直後からHTTP送信までの失効との競合を完全には排除しない。 + +送信先は従来のFCM・Mozilla・Appleの許可ホストのみ。 +HTTPS、userinfo/fragment/明示portの禁止、P-256鍵とauth鍵の形式検証を維持する。 +任意URLへの通知チャネルではない。HTTP clientは公開IPへのDNS固定とredirect禁止を維持し、 +サーバーの既存接続防御を迂回しない。 + +暗号化前のwake payloadは `uuid`・`owner`・`domain` の3項目のみ。 +件名・本文・Bearer tokenをPushへ入れない。OS側は一般的な新着表示、詳細は認可済みInboxで読む。 +VAPID秘密鍵は既存DB内にあり、今回の分離で格納場所・rotation方針は変えない。 +ログは `component=ordoschnellic` とsent/failedの件数のみ。購読URL・鍵・本文は記録しない。 +sentはPushサービスの2xx受付であり、端末への到達や利用者の閲覧を意味しない。 + +## 配送予算と終了規則 + +| 項目 | 維持する動作 | +| --- | --- | +| 実行間隔 | 既定30秒、priority -3。既存管理者設定を上書きしない | +| バッチ | 最大20 claim・25秒。focusedや失効済みも件数予算に含む | +| HTTP | 1要求5秒、redirectを追わない | +| claim lease | 60秒。ネットワーク処理はclaim transactionの外で実行 | +| 前面表示 | foreground lease 90秒。延期は1分、配送試行数を消費しない | +| 成功 | 2xxで現在のキュー項目を完了 | +| 再試行 | 通信/鍵取得失敗・429・5xx。試行1~4の後に2/4/8/16分、5試行目で終了 | +| 購読失効 | 404・410、または不正な保存購読データで購読を削除 | +| その他HTTP結果 | 現行仕様どおり当該通知の配送を終了。401/403も自動再試行しない | +| 保持 | キュー24時間、Push TTL 300秒。清掃は購読100件・キュー500件/run、Inboxは消さない | +| 終了保存 | 送信側contextが取り消されても独立した最大3秒で結果保存を試す | + +`Queue`操作には時刻を渡し、テストでlease境界とbackoffを待ち時間なしで検証できる。 +旧store入口は現在時刻を渡して委譲する。タイムアウトには実時間contextを使う。 +新たにDregarnuhrから配送へcontextを伝播し、開始前キャンセルではDBにも接続しない。 +不正な試行数でshiftを行わず、終端扱いにする防御も追加した。 + +DB失敗は呼出し元へ返し、後続claimへ進まない。バッチ時間を使い切った残件は次回実行へ回す。 +親contextのキャンセルはエラーとして返す。Dregarnuhrの汎用ジョブに期限付きleaseが未実装でも、 +Pushキューには既存の60秒leaseがある。この二つを混同しない。 + +## 配送保証と残件 + +初期分離はexactly-onceを導入しない。HTTP成功直後にプロセスが停止したり結果保存に失敗すると、 +lease後に再送され得る。ブラウザ側のtagによる集約は端末表示の対策であり、配送の一回性ではない。 +通常の完了更新はattempt一致を検査するが、これを完全なfencingと呼ばない。 + +- [ ] claim世代をattemptと分離する。focused延期によるattempt減算では世代が再利用されるため、古いworkerの結果との競合が残る。 +- [ ] 404/410による購読削除を購読version/claim世代で条件付けする。現在はIDだけで削除するため、送信中に更新された同一購読を削除し得る。 +- [ ] claim後の失効と送信の競合を整理する。送信直前の再検証も完全な原子性を保証しないことを明記する。 +- [ ] Retry-After、失敗理由別のメトリクス、キュー深さ・最古待機時間、枯渇/失効/廃棄件数を追加する。本文・endpointは観測属性にしない。 +- [ ] lease超過や多重workerの障害試験、HTTP成功後のDB失敗を含む復旧手順を整備する。 +- [ ] quiet hours、digest、チャネル別設定、メール等の配送はrecipient/channel/versionの冪等キーと受入条件を決めてから追加する。 +- [ ] 実機のPush受信・再訪・ログアウト、VAPID rotation、端末一覧UIは既存の残件として継続する。 + +## 検証と反映 + +合成データ・一時DB・偽HTTP clientのみで検証する。実通知サービスへの送信、 +本番DBや外部から取り込んだカレンダー内容へのアクセスは行わない。 +境界・型/JSON/イベントID互換、バッチ上限、キャンセル後の終了保存、HTTP分類、暗号化、 +認可失効、並行claim、lease回収、旧attempt完了の拒否、focus延期、Inbox保全を対象とする。 +既存のtransaction rollback・重複抑止・Activity Subscriptionのcheckpoint試験も維持する。 + +検証結果は [コンポーネント検証記録](component-names.md#検証記録) に記載する。 +ソース変更とテスト、本番反映は別段階。commit・push・デプロイは今回含めない。 diff --git a/docs/design/ja/priority-ux-20261031.md b/docs/design/ja/priority-ux-20261031.md new file mode 100644 index 0000000..ea2a8b2 --- /dev/null +++ b/docs/design/ja/priority-ux-20261031.md @@ -0,0 +1,134 @@ +# 最優先:日常操作・ブラウザ・エディタ・CLI(2026-10-31) + +ユーザ指定の次期開発最優先。従来の機能追加より先に進める。ただし認可、正本の保全、競合検出を省略しない。本番反映とローカル検証済みは区別する。 + +## 実装順と受け入れ条件 + +### 1. ホームとファイルブラウザ + +- [x] ホームで他の入力欄にフォーカスしていないとき、文字入力でグローバル検索へ移る。最初の文字を失わず、IME開始・修飾キー・モーダル・選択中テキストを妨げない。中央検索、検索+ブックマーク、従来トップバーを対象とする。実機IMEでの入力確定は下記受け入れ残件。 +- [x] ブラウザに `/` 始まりのアドレス欄を設け、絶対/相対Path、`cd`、`vi`、`view` を扱う。URLのquery/hashや外部URLと混同せず、同一Domain内のPathとして解決する。シェル実行はしない。 +- [x] ブラウザから名前を指定して新規文書エディタへ進む。既存Pathでは既存文書を編集する。 +- [ ] 後続:エディタの新規作成をGET時ではなく明示的な保存時へ遅延する。既存エディタはGET時にPage/Pathを作成するため、今回のナビゲーション追加ではその契約を維持する。UUIDを必要とする下書き/競合/言語/保存APIを合わせて変更する。 +- [x] 未作成Pathに仮想フォルダとして潜れる。空フォルダの実体は作らず、そこから作成/アップロードへ進む。現在Pathを貼り付け先など既存操作と共有する。 +- [x] キーボード移動、空フォルダ、特殊文字、日本語、ルート境界、読み取り専用アカウントからの作成拒否、既存の複数選択/ドラッグ操作をfixture検証する。 + +### 2. エディタと表示 + +- [x] カーソル/選択範囲/現在行をテーマに合わせる。Vimの実際のmode-changeで通常/挿入/置換/選択の状態とカーソル色を更新。`A` 等の移行もfixture確認済み。 +- [x] ライトモードで背景・文字・構文色・ガター・補完UIをテーマ化。生成されたLezerクラス名への依存を除去し、意味タグによる構文色へ移行。既存のエディタ復旧試験も通過。 +- [x] PlainTextの初期実装:ブラウザの新規文書ダイアログで指定、または `/e/-/{path}?edittype=plain`。既存の `text/plain` は自動選択。保存・プレビューは文字列のまま、HTML/マクロとして実行しない。下書き・競合処理にplainを追加し、異なる形式の下書きを自動復元しない。 +- [ ] PlainText後続:既存Wiki HTMLとの形式変換は損失確認・保存MIME・過去版復元の意味を確定してから。現在は409で拒否し、既存HTMLの原本と履歴を変更しない。Domain既定形式と外部アダプタのPlainText設定も後続。 +- [x] codeブロックのテーマ対応装飾、ブロック内の横スクロール、インラインcodeの区別、印刷時の折り返しCSS。300px幅での局所overflowをChromium検証。実機モバイルと印刷の目視確認は残件。 + +### 3. テーマ・アカウント・連絡先 + +- [x] 共通ナビの◐からプリセットを選択、リロードせず切替。host限定Cookieに保持し、Domain既定に戻せる。登録済みテーマのみ許可。Domain設定/アカウントの全端末設定ではなくブラウザ上書き。失敗時は現在テーマを保持する。 +- [x] 絶対セッション期限:設定→セッション、5〜43200分、既定1440分。Domain単位で新規ブラウザ/APIログイン(OIDC/LDAPを含む)に適用。発行時にDBの期限を固定し、Cookie/APIのexpires_inにも反映。既存セッションの期限とCalDAVアプリパスワードは変更しない。管理者認可とサーバ側失効をfixture検証。 +- [ ] 無操作タイムアウトとアカウント別の短縮上書き:background pollingを人間の操作とみなさないactivity契約、再認証、既存セッションへの即時適用を別途設計する。 +- [x] 電話連絡先の初期実装(2026-11-01ローカル検証済み、本番未反映):本人編集、独立した公開範囲、表示用文字列と正規化した `tel:` リンク。公開は明示確認とプロフィール/ドメイン上限に従う。クリックなしで発信しない。 + +### 4. Cityの表示名 + +- [x] 検索結果/ホーム・ビューア検索サジェストのCityリンクは設定詳細ではなく閲覧導線にする(2026-11-01本番反映済み)。対応するPathにPageがあればそのページ、なければそのPathのディレクトリ一覧へ。仮想未整理Spotは実在しない `/_undefined` ではなく親Streetを入口とする。タイトルリンクとPathリンクを統一し、認可、Domain分離、特殊文字のエンコードを維持する。管理詳細への明示的なリンクは残す。 +- [x] 初期対応:生成済みSpotの既定プレースホルダー名 `Undefined` を閲覧時の表示から外し、親Streetのタイトル(未設定なら名前)を使用する。独自タイトルは保持し、UUID/識別名/Path境界は書き換えない(2026-11-01本番反映済み)。 +- [ ] 「子Spotがあるならカルチエ名」という案の階層対応は別途確定する。現行はQuartier > Street > SpotでSpotを入れ子にできないため、今回の表示変更では昇格/再配置/権限境界の変更をしない。 + +### 5. セッション付きCLI/TUI + +- [x] HTTPクライアント初期版 `pages-cli`/コマンド `pages`(2026-11-01ホストのユーザーPATHへ配置済み。実ユーザログインは未実施):configureによる接続先・既定プロファイル・言語管理、パスワード/TOTP、Bearer保持・失効、ls/cd/cat/view/get、既存Page edit/put、Path補完、メモリ内履歴、検索候補、ブラウザ引き継ぎ。詳細と原設計整合は [遠隔CLI](remote-cli.md)。既存DB直アクセスCLIは変更しない。 +- [ ] 既存 `data/pageswiki-cli` を拡張する専用起動コマンドとセッション保持。履歴や資格情報に0600相当の保護と失効/ログアウトを設ける。 +- [ ] `Ctrl+P` のfuzzy検索と、ローカル `$EDITOR` 編集。現在版/ETagを保持してアップロード時の競合を検出し、上書き・差分・キャンセルを選べるようにする。 +- [ ] パスワード認証に加え、ターミナルにURLを表示するブラウザ認証(OIDC)。既存Web認証と同じ本人性/MFA制約を適用し、短命・単回の認可コード+PKCEやdevice-flow相当の仕様を検討する。URLや履歴に長期トークンを含めない。 +- [ ] ターミナル表示が難しい内容はブラウザに引き継ぐ。short linkは認証を迂回しない内部リンクとし、公開share link発行とは分離する。自動公開しない。 + +## チェックポイントと進捗 + +### 2026-11-01:コマンドバーと共通シンタックス色(本番反映済み) + +- ブラウザのPathアドレス欄をtopbarへ移設。1100px以下では非表示。左下に独立した `cd` / `vi` / `view` バーを常設し、入力欄などにフォーカスがない状態の文字入力を最初の1文字から引き継ぐ。 +- 共通ナビのあるホーム以外の画面では `g` でバーを表示、Escで閉じる。ホームのtype-to-searchは維持。フォーム、モーダル、テキスト選択、修飾キー、編集中のCodeMirror/Vimの `g` は奪わない。ブラウザのEscはバーを消さず入力をクリア。 +- Tab/Shift+TabでPathを補完・候補巡回。候補が1件のフォルダは再度Tabで配下を補完。相対Path、日本語、空白、`#` / `?` / `%` をそのまま扱い、Enter時にsegment単位でURLエンコード。シェル実行はしない。入力欄末尾以外/コマンド未入力時のTabは通常のフォーカス移動。 +- `path-picker?mode=children&path=...&prefix=...` を追加。親Path自身にPageが存在しても親へ戻らず配下を取得。名前専用SQLでRevision本文やプレビューを読み込まない。Domainと既存Path認可を維持。DB処理2秒、候補走査1000件、認可後最大250件、UI最大100件。上限時は絞り込みを促し、入力変更・Esc時はfetchを中止して古い結果を捨てる。prefixはASCII英字を大小区別せず照合し、それ以外はSQLiteの文字比較に従う。 +- モバイルのブラウザは中央一覧を優先。サイドバー/プレビューはボタンでオーバーレイ表示し、デスクトップの表示設定は変更しない。 +- CodeMirrorエディタ、コードファイルの読み取りビューア、文書中のコードブロックで `pw-syntax-*` とテーマの `--editor-*` 色を共有。Markdown内の対応言語のfenceも構文解析。全プリセットの切替、読み取りビューアとの色一致を確認。 +- Wiki本文・編集Preview・ブラウザPreview・公開ページ/公開ブログの言語指定付きコードブロックは、見えてきたものから1ブロックずつ遅延処理。32,768文字超と未対応言語は原文のまま表示し、正本・HTML保存データは変更しない。大きいコードファイルは既存のwindowed viewerを維持。新規CDN・外部ライブラリは追加しない。 +- Chromium 1440px/375pxでコマンド入力/補完/キャンセル/ナビゲーション/レスポンシブ/Vim保護/コード原文保持/動的Preview追加を確認。ホーム3種・エディタ復旧の既存回帰も通過。実機IMEの確定とiPhone Safariのソフトウェアキーボードは未確認。 +- 開始時ソース:`.deployment-backups/dev-command-20261101/source-before.tar.gz`。2026-11-01 07:14 JST頃、fix2をベースに対象ファイルだけを差し替えた `pageswiki-app:command-20261101` をapp/app2へ反映。電話連絡先/SQLite初期接続の後続変更は含めない。[反映記録](../../operations/deployments/2026-11-01-command-ux.md)を参照。 + +2026-10-31:上記チェック済み項目はローカル実装・fixture検証済み。本番未反映。新たなDBテーブル/migrationはない。セッション期限は既存Domain configに保存する。 + +2026-11-01:上記UXを本番app/app2へ反映済み。既存の保存基盤変更も含むため、隔離コピーでmigration/backfillを確認し、旧イベント索引の採用不具合を修正して反映した。Docker実行基盤の版ずれによる一時停止と復旧、切替直前の整合バックアップ、イメージ識別子・検証結果は [本番反映記録](../../operations/deployments/2026-11-01-priority-ux.md) を参照。 + +- 開始時のHEAD:`9dd425f7658b2eecb1152ec8d457d6224142f9fc`。既存の未コミット変更を含むソースを退避済み。 +- バックアップ:`.deployment-backups/dev-priority-20261031/source-before.tar.gz`。 +- 検証済みの区切り:同ディレクトリの `source-after-ux.tar.gz`。ハッシュと復元時の注意は同ディレクトリの `README.md` に記録する。 +- SHA-256:`8fa6864eeeaa89b3019c06f9c1975a13f380731efa033df810870fb04c4f94e2`。 +- 対象はGit管理対象と非ignoredの新規ソース。`.git`、秘密設定、本番DB・Document実体、コンテナイメージは対象外。本番の復元用バックアップではない。 +- 復旧時は別ディレクトリに展開して対象ファイルを比較・復元する。新規追加ファイルは個別に退避する。既存のユーザ変更を失う `git reset --hard` は使わない。 +- 本番へのデプロイ、DB変更、外部サービスへの接続はこの段階では行わない。 + +### 検証 + +- `go test ./...`:通過(opt-in browserを除く全パッケージ)。 +- Chromium:ホーム3レイアウトの入力検索、他入力/モーダル/修飾キー、IME開始イベント、仮想Path移動、新規Wiki/PlainText作成、既存ドラッグ選択、テーマ変更、Vimモード、PlainText保存とリテラルプレビュー、既存の下書き復旧・競合処理。 +- Node:パス正規化/URLエンコード/外部URL拒否、編集coreのmerge/draft検証。 +- 編集bundle再ビルド、`git diff --check`。Index Whiteのエディタをスクリーンショットでも確認。 +- 本番・実機Safari/iPhone・日本語IMEの確定動作・印刷・カスタムテーマ全種類は未確認。 + +## 後続実装の境界と状態 + +### 連絡先 + +まず `/u/{username}/_/edit` の本人連絡先を対象とし、オーガナイザーの汎用連絡先帳/CardDAVとは区別する。電話番号はDomain+Principalで保存し、既定private。プロフィール全体をpublicにしただけで電話番号を公開しない。独立した公開確認と上位公開制限を適用する。 + +表示文字列と正規化済みの電話番号を分離する。`tel:` の値はサーバが組み立て、入力された任意URIをそのまま採用しない。サービスコード/制御文字/改行/スキーム混入は初期版では受理しない。スマホでタップしたときだけOSへ委譲し、自動発信や外部の番号検証サービスへの送信はしない。 + +保存先はmanaged schema moduleとして追加し、本人更新と状態確認をtransaction内で行う。公開profile投影、API、検索、exportで非公開値を漏らさないfixtureを先に用意する。 + +2026-11-01実装:`profile-contacts` component v1、Domain+Principalを主キーとする。Principalや認証API/継承データには電話番号を追加しない。プロフィール画面のみ専用の認可済み取得を使い、非公開の番号はdomain_adminにも出さない。管理者の閲覧権限を本人の同意と扱わない。 + +- `/u/{username}/_/edit#contact` の独立フォームで本人のみ更新。CAS versionで古いフォームは409。activeの本人とactiveのDomain membershipをtransaction内で検査し、readonly/suspended/他Domainからの更新を拒否。 +- 既定private、domain/publicはプロフィール公開範囲でさらに制限。publicの新規設定・番号の差し替えは確認チェックが必要。ドメイン公開の撤回/プロフィール非公開への変更を次の読み取りから反映する。 +- 電話番号は3〜15桁、先頭の+、表示用の空白・ハイフン・括弧に限定。内線、URIパラメータ、サービスコード、制御文字、任意スキームを拒否。空欄保存は番号を消去してprivateに戻す。 +- JA/ENの説明・フォーム・保存後アンカーを追加。デスクトップ1440px/375pxで実ブラウザの入力・保存・telリンク・フォーム幅を検証。iPhoneの電話アプリ起動そのものは未検証。 +- store/HTTPで本人・匿名・他メンバー・管理者・越境・公開確認・公開撤回・競合・不正番号を検証。個人情報の本文を監査ログへ追加しない。CardDAV、住所録、複数番号、内線は後続。 +- **本番反映済みfix2にはこの機能を含めない。** 次回リリースでは追加componentの適用と、旧版への切り戻し条件を別途検証する。 + +### 新規DBの初期接続競合(後続修正、本番未反映) + +全体回帰で `TestSQLMigrationLedgerConcurrencyAndChecksum` がまれにSQLITE_BUSYとなることを確認。診断メッセージをmigration connection取得/BEGINに分け、2,000回の反復で接続取得段階の失敗を再現した。`sql.Open` がlazyのため、最初の並行接続でWALへの切替が競合していた。 + +`db.Open` 内で5秒上限のPingを完了してからプールを公開し、初期接続失敗時はCloseして返す。migrationのBEGIN IMMEDIATE/checksum/transaction境界は変更しない。修正後の同テスト2,000回が成功。既にWALで稼働している本番DBのジョブ競合修正とは別件として扱う。 + +### Cityの名称規則 + +現行の未整理Spotは `name` の `-undefined` や `/_undefined` によって識別され、Street直下Pageの収容にも使われる。`city-estates.js` のラベルだけを書き換えても、API・パンくず・詳細・検索に別名が残る。 + +2026-11-01実装・本番反映済み:内部ID/境界と `display_title` を分離。生成Spotで既定名(空、識別名自身、Undefined等)の場合のみ親Street表示名へ射影し、独自タイトルは維持する。生成由来かどうかの永続フラグはないため、既存と同じ識別名suffix/仮想Path規則を使用する。通常Spotに意図的に付けた `Undefined` は変更しない。[反映記録・切り戻し](../../operations/deployments/2026-11-01-city-pages-cli.md)。 + +- 地図ラベル、パンくず、ディレクトリ、整理モーダル、詳細見出し、検索・サジェスト、ページ上の所属リンク、Cityマクロで同じ規則を使用。管理用識別名/Pathはそのまま残す。 +- 匿名地図は公開用に縮退させた名前から再計算し、認証済み投影のカスタムタイトルを再利用しない。所属リンクはSpotに加えて親Street/Quartierの閲覧権限も確認する。共有ナビゲーションキャッシュを表示名処理で変更しない。 +- 検索導線は `NavigationPath` と `HasPage` により本文/ブラウザ一覧へ分岐。検索結果のタイトルとPath、サジェストのURLを統一。`/_undefined` は親Streetへ案内し、無関係な先頭Pageをポータルにしない。 +- 遷移先の `/v/browser/{path}` はDomain全体のreadではなく対象Pathのreadで認可し、各子要素も個別に検査する。対象Pathを読めない利用者は403。匿名公開の一覧は従来の公開ナビゲーション処理を維持する。検索から本文/一覧を実際に開くHTTP試験を追加。 +- Pageの存在判定は既存Path索引のDomain+完全一致で検索SQL内に含める。本文、Revision、City全体の展開、結果ごとの追加HTTP/DB呼び出しは不要。候補上限/タイムアウト/親階層の認可を維持。EXPLAIN QUERY PLANでPath全走査がないことをfixture検査。 +- 検索画面は700px以下で検索欄をナビアイコンの下段に置き、プロフィールアイコンによる横はみ出しを解消。390pxと1440pxのブラウザ試験・モバイルスクリーンショットで確認。 +- 全Goテスト、Cityラベル(通常/popup)・整理操作・名前付き詳細遷移・検索表示(desktop/mobile)のChromium試験が成功。公開キャッシュ撤回、別Domain、親ACL、特殊文字、生成名と手動タイトル保持を検査。本番データ/外部カレンダーは使わない。 +- 復旧点:`.deployment-backups/dev-city-20261101/source-before.tar.gz`(SHA-256 `2675aff20450d24eb0fe1707ea5f3a38ee26c9bdf20dec1a29926d52ef5af4bf`)。DB/設定変更なし。別ディレクトリへ展開して差分復元する。前からある未コミット実装をまとめて取り消さない。 +- 検索CSSの変更前は同ディレクトリの `search-css-before.tar.gz` に別途保全(このファイルは着手時にHEADと同一)。変更後は `source-after.tar.gz`。範囲とハッシュは同ディレクトリのREADMEを参照。 + +ユーザの「子Spotがあるならカルチエ名」という案は、現行のSpotをネストしない制約と区別する必要がある。今回の対応では昇格/再配置はせず、空の仮想要素を生成し直さない。 + +### リモートCLI + +現行 `data/pageswiki-cli` はSQLite/Document Storeへ直接アクセスする運用CLI。shell、補完、履歴、`$PAGER`/`$EDITOR` は既にあるが、Web認証を受けるリモートクライアントではない。Webユーザの認可をSQLite直アクセスへ継ぎ足さず、HTTP transportを持つ別コマンド(仮称 `pageswiki remote`)として段階導入する。 + +2026-11-01:試作名 `data/pageswiki-remote` から、ユーザ指定によりサブプロジェクト `pages-cli`/実行名 `pages` へ整理。`pages configure` と既定値により通常操作のprofile/lang引数を不要とした。以下は全体計画であり、Ctrl+P、OIDCハンドオフ、新規作成、強制上書きの対話選択、short linkは未実装。[初期版の範囲・制限・検証](remote-cli.md)を参照。 + +1. パスワード+TOTPの既存JSONログイン、失効/logout、接続先・Principal・Domain別セッション保存。秘密情報はコマンド引数/履歴/ログへ出さず、保存先0700・資格情報0600相当、TLS検証必須。 +2. `ls/cd/view/get/edit` と現在Pathの保持。検索は認可付き・件数制限・キャンセル可能なAPIを使用し、`Ctrl+P` のfuzzy選択へ接続。端末の制御シーケンスは本文/タイトルから除去する。 +3. `$EDITOR` は引数を分離して起動し、shellへ未検証Pathを連結しない。取得時のRevisionを保存し、CAS書込・409時の差分確認・一時ファイルの保持/削除を実装する。Wiki HTMLと編集Markdownは既存authoring変換APIを使い分ける。 +4. OIDCはブラウザ側で既存ログイン/MFAを完了し、短命・単回・認証先/クライアント紐付きのハンドオフでCLIへ交換する。長期BearerをURLへ載せない。確認画面、CSRF/PKCE相当、リプレイ拒否、発行上限、poll backoffを含めてから実装する。 +5. 表示できないオブジェクトは認証を維持した短い内部URLでブラウザへ引き継ぐ。公開share linkは自動発行しない。別originへのredirectにBearerを転送しない。 + +初期の遠隔アクセス試験は偽サーバと一時データのみ。本番資格情報や外部取り込み済みカレンダーの内容をテストに使わない。 diff --git a/docs/design/ja/remote-cli.md b/docs/design/ja/remote-cli.md new file mode 100644 index 0000000..0bbf88b --- /dev/null +++ b/docs/design/ja/remote-cli.md @@ -0,0 +1,95 @@ +# pages-cli:`pages` HTTPクライアント + +2026-11-01。合成データで検証済み。同日、本番サーバのCity更新と合わせてクライアントをホストの `/home/anya/.local/bin/pages` に配置した。実ユーザの資格情報取得・ログイン・本番Page編集は未実施。接続設定とログインは利用者が行う。[反映記録](../../operations/deployments/2026-11-01-city-pages-cli.md)。 + +## 原設計との整合 + +原設計の [Address Model](../address-model.md)、[URI](../uri.md)、[認証バックエンド](../auth-backend.md)、[エディタの保存契約](editor-resilience.md)を使用する。 + +- Pathは到達経路、UUIDはPageの識別子、Revisionは履歴。取得は `/v/raw/path:/...`、`uuid:`、`rev:` で行い、`/raw/...` を増設しない。 +- 編集開始時にObject UUIDを確定し、指定言語のheadを取得。保存はUUID+言語+`base_revision_uuid`+`editor_principal_uuid`。Path変更に追従するため保存先を再解決して別Pageに書き込まない。 +- Wiki正本はHTML。編集用Markdownはauthoring射影であり、children等の表示展開結果を保存しない。保護HTMLフェンスを既存authoring APIで往復する。 +- Domainは接続originで決まり、HTTPで既存の認証・権限制御を受ける。Campus連携用トークン/PSKをユーザー認証に流用しない。 +- 旧認証設計にはCookie中心の記載が残るが、実装済みのJSONログインはBearerセッションを発行する。CLIはこのフローを使用し、Web Cookieを抽出しない。 +- `data/pageswiki-cli` は信頼されたローカル運用ツールとして維持。遠隔版はサブプロジェクト `pages-cli`、実行名 `pages`(`pages-cli/pages`)とし、DB・Document Storeを直接開かない。 + +サーバAPI/DBスキーマの追加・変更はない。初期実装はPython 3.9以降の標準ライブラリで動作するUnix系クライアント(fcntl、readline使用)。Windowsネイティブ版は未対応。 + +## 利用方法 + +```sh +# 初回設定。パスワードはlogin時の非表示プロンプト、必要ならTOTPを続けて入力。 +pages configure --origin https://pages.example.test --username anya +pages login +pages status +pages + +# ワンショット。--lang はコマンドより前に指定する。 +pages ls /manual +pages edit /manual/start +pages get /manual/start ./start.html +pages put ./start.html /manual/start --base REVISION_UUID +pages logout +``` + +配置方法と詳しい設定例は [pages-cli README](../../../pages-cli/README.md)。既定プロファイル名は `default`、編集言語は `und`。`pages configure --lang ja` で変更し、`configure set --profile work --origin ... --username ... --lang en` で接続先を追加する。`configure use work` で既定を切り替え、通常操作ではprofile/langの引数を省略する。一時的な `--profile`/`--lang` 上書きも残す。 + +設定は `config.json` の version/default_profile/profiles と、各接続のorigin/username/langに限定。パスワード・Bearerは設定表示へ含めない。configureはネットワーク通信をせず、ログイン中のorigin/username変更を拒否する。mainは設定ロック→セッションロックの順で取得し、起動時の接続先すり替えを防ぐ。手編集時も保存したセッションの接続先・アカウントとの一致を確認してから使用する。logoutは設定を手編集した場合でも、保存済みの元接続先に対して失効する。 + +対話シェルは `ls`、`cd`、`pwd`、`status`、`cat`、`view`、`get`、`put`、`edit`、`search`、`open`、`logout`、`exit` を扱う。`--help` で一覧、`edit --help` 等で引数を確認する。`cd` は仮想Pathにも移動でき、存在や閲覧可能な子の存在を保証しない。 + +TabはPath候補を取得し、上下キーは当該プロセス内の履歴をたどる。Pathに空白等があればシェルと同様に引用する。履歴はディスクへ保存しない。現在Pathは接続プロファイルへ保存する。同じプロファイルの同時実行はロックで拒否し、別プロファイルは独立して使える。 + +`search QUERY` は権限付きサジェストを表示する初期コマンド。Ctrl+Pのインクリメンタルfuzzy選択は未実装であり、通常のreadlineキー割当のまま。 + +### 閲覧・編集 + +- `get ADDRESS LOCAL` は最大64 MiBの正本を取得し、SHA-256を照合する。ローカル出力は0600で新規作成し、既存ファイルを上書きしない。Revision UUIDを表示する。 +- `cat` はpipe時に正本バイト列を出力。TTY時には制御文字を除去する。`view` はテキストを `$PAGER`(既定less)に渡し、HTMLを端末上で実行しない。 +- `edit` は `$EDITOR`(既定vi)をshellを介さず起動。HTMLはauthoring Markdownが既定、その他の対応テキストは原文。`--format raw` でHTMLを直接編集できる。`--format markdown` はHTML文書専用。 +- ローカル作業ファイルは0700の一時ディレクトリ内に0600で保存。成功・無変更時は削除、失敗/409/通信断時は原稿とrecovery.jsonを保持して場所を表示する。recovery.jsonにはorigin、UUID、基点Revision、言語、形式を記録し、Bearerは含めない。 +- 409では最新内容とローカル原稿の差分を表示し、保存は停止する。自動マージ、強制上書き、自動再送はしない。応答喪失時は取得してサーバの状態を確認してから再試行する。 +- `put` は既存Pageの既存言語のみ。`--base` が必須で、現在headと不一致なら送信しない。サーバ保存も同じbaseでCASを行う。編集を保持したい場合は最新を取得して差分を確認し、調整済み原稿とその最新Revisionを明示して保存する。 +- `put --format markdown` はauthoring変換を使用。rawは正本形式を保持する。ローカル拡張子で保存MIMEを勝手に変更しない。テキスト編集は1 MiB、JSON送信は2 MiBまで。 +- `rev:` は取得専用。新規Page、未作成の言語head、バイナリ編集、Asset更新は拒否する。新規作成のためにエディタGETの副作用を利用しない。 +- `open` は同一originのPath/UUID/Revisionビューアをブラウザで開く。BearerはURLに含めず、ブラウザ側のログインが必要。共有リンクは自動発行しない。 + +## HTTP契約 + +| 操作 | 既存エンドポイント | +| --- | --- | +| ログイン/MFA | POST `/_/auth/login`、`/_/auth/totp`、フォーム本文+Accept application/json | +| 失効 | POST `/_/auth/logout`、Bearer | +| 一覧・補完 | GET `/_/api/v1/path-picker?mode=children&path=...&prefix=...` | +| 検索候補 | GET `/_/api/v1/search/suggest?q=...` | +| 正本・識別子 | GET/HEAD `/v/raw/{explicit-address}?lang=...` | +| 言語別編集状態 | GET `/_/api/v1/editor/state?uuid=...&lang=...` | +| HTMLへの復帰 | POST `/_/api/v1/render/markdown`、authoring=true、projection=false | +| CAS保存 | POST `/_/api/v1/revisions/save` | + +一覧は既存の件数制限を維持し、truncatedを表示する。巨大な全階層を再帰取得しない。読み取りのRaw言語選択は既存サーバの候補選択に従うが、編集は指定言語のheadを必須とし、存在しない場合は拒否する。 + +## セキュリティと運用上の制限 + +- 遠隔はHTTPS必須、証明書検証を無効化するオプションなし。HTTPはlocalhost/127.0.0.1/::1の開発専用。 +- 接続先はoriginのみ。userinfo、Path、query、fragmentを含むURLは拒否。リダイレクトは同一originも含め追従しない。認証画面HTMLや302/303を成功と扱わない。 +- 設定は `$XDG_CONFIG_HOME/pages-cli/config.json`、セッションは同じルートの `sessions/PROFILE.json`(既定 `~/.config/pages-cli/`)。ディレクトリ0700/ファイル0600、所有者を検査し、シンボリックリンクは拒否。atomic replaceとプロファイルロックを使用。旧試作の `~/.config/pageswiki-remote/` は自動読込・移行・削除しない。再configure/loginし、不要な旧セッションはサーバ側で失効する。 +- 資格情報は平文のため、同一OSユーザーやrootからは保護できない。OS keyring/暗号化保存は後続。共有端末ではlogoutし、残した作業原稿は利用者が整理する。 +- パスワード/TOTP/Bearerを引数や履歴へ入力させない。保存したexpiryを超えたセッションでは操作を開始せず、実際の失効・権限はサーバで毎回判定する。logout失敗時は失効済みと報告せず資格情報を残す。 +- ネットワーク要求には20秒のsocket timeoutと応答サイズ上限。総転送時間の厳密deadline/ストリーミング巨大ファイル対応は後続。 + +## 検証と次段階 + +Python unittest:origin制限、端末制御文字、別origin redirect遮断、非JSON/403/サイズ上限、TOTP、資格情報権限・symlink・並行実行拒否、authoring/CAS payload、ダウンロード上書き拒否、409時の原稿保全。 + +設定追加後は12件:既定値/複数接続切替/言語保存、一時上書き、設定表示からのトークン除外、セッション保持中の再接続先変更拒否、手編集での接続先不一致拒否も検査。HTTP統合試験では公開エントリーポイントのconfigure→login→引数省略のls/status/edit→logoutを通す。 + +Goの `TestRemoteCLIHTTPIntegration`:一時DBと本物のHTTPハンドラでログイン、一覧、編集状態、authoring変換、保存、古いRevision拒否、Rawハッシュ、immutable拒否、存在しない言語拒否、put、ログアウト後の拒否を検査。Pythonがないビルド環境ではこの統合試験をskipする。外部カレンダーや本番資格情報は使わない。 + +次段階の順序: + +1. Ctrl+Pによる検索選択、補完の引用文字/途中カーソル対応、永続履歴の明示opt-in。 +2. 新規Page+Path+初回Revisionの原子的な作成API、create/upload-only権限、並行作成の衝突、言語分岐。 +3. 競合の対話選択・復旧コマンド、巨大転送のstreamingとキャンセル、mget/mput(globのlocal/remoteを明示)。 +4. OIDCブラウザ認証の短命・単回・PKCE相当ハンドオフ。既存device approvalをそのまま転用せず、本人確認・Domain・CLIリクエストを束縛する。 +5. OS keyring、Windows対応、非公開の短縮リンク引き継ぎ。自動的な公開shareは行わない。 diff --git a/docs/design/ja/roadmap.md b/docs/design/ja/roadmap.md index ec6c600..dcbc3c9 100644 --- a/docs/design/ja/roadmap.md +++ b/docs/design/ja/roadmap.md @@ -4,6 +4,8 @@ ## 優先順 +2026-10-31:ユーザ指定により[日常操作・ブラウザ・エディタ・CLI](priority-ux-20261031.md)を次期開発の最優先とする。下記の従来機能追加より優先するが、認可・データ保全の安全条件は省略しない。TODO、受け入れ条件、復元用チェックポイントを同文書に集約する。 + 2026-09-27追記:`organizer.tasks 0.11.1`で[既定タスクリスト名の変更](task-dav-default-list-name.md)を実装し、12:51 JSTに[本番両インスタンスへ反映](../../operations/deployments/2026-09-27-default-task-list-name.md)。PROPPATCH 405を解消し、利用者・domain別の表示名を保存、全探索結果へ反映する。実機再試行は確認待ち。色・アイコン・通常プロジェクトのPROPPATCHは別件。 2026-09-27追記:`organizer.tasks 0.11.0`の[タスク所属統一と上書き同期](task-dav-project-unification.md)を実装し、12:18 JSTに[本番両インスタンスへ反映](../../operations/deployments/2026-09-27-task-unification.md)。全Webプロジェクトを個別CalDAVリストとして公開し、既定リストはプロジェクトなしに限定。無条件PUT/DELETEはトークン別設定なしで受理し、版履歴を保持する。実機再同期は確認待ち。次の優先タスクは競合候補の保存、Inbox通知、上書き前/受信版の比較・採否選択(同文書のTODO)。反映時に観測したスケジューラclaimの一時的DBロックとディスク容量整理も運用記録へ残した。 diff --git a/docs/design/ja/schutzaria.md b/docs/design/ja/schutzaria.md new file mode 100644 index 0000000..c0b5439 --- /dev/null +++ b/docs/design/ja/schutzaria.md @@ -0,0 +1,130 @@ +# Schutzaria の権限と公開範囲の評価契約 + +2026-11-01、既存の権限・公開範囲の純粋な評価処理を `internal/schutzaria` に初期分離した。 +認証済みPrincipalと対象Domainに対して読み込んだ事実を評価する部品であり、 +認証器、DBアクセス、権限変更、匿名公開の確定を一括で置き換えるものではない。 +既存の許可・拒否、公開の明示的な承認、URI・JSON・DB・監査記録の形式を維持する。 +[コンポーネント命名](component-names.md)、[認証と権限](auth-permissions.md)、 +[Localityの権限・公開](../locality-access-publication.md) を併読する。 + +## 責務と境界 + +| 所有者 | 担当する処理 | +| --- | --- | +| Schutzaria | Path読み取りgrant、Locality制限の交差、グループ権限プロファイル、公開上限と意図、公開対象Pathの適格性、ユーザー領域のaudience、既存承認要求の状態遷移 | +| store | Principal/membership/role/groupの有効性を含むDB読込、Objectの全alias検査、所有者権限、epoch/topology、更新transaction、trigger、監査、承認対象とmanifest | +| server | セッション・HTTP認証、endpointと対象の解決、legacy/enforcingの選択、ユーザー領域の追加制限、匿名と認証済み経路の分離、表示・エラー応答 | +| Mestionora等 | 呼出し元が渡す読み取り判定の適用。検索自体で権限を作らない | + +依存は server/store → schutzaria → 標準ライブラリの文字列等の処理のみ。 +SchutzariaからDB、HTTP、ファイル、時計、server/storeへ依存しない。 +認証情報やネットワーク接続を受け取らず、ログに対象本文・ユーザー情報を追加しない。 +OIDC/LDAP、Cookie/Bearer、共有用credentialの検証方式は変更しない。 + +`store.ReadPathPolicy`、`LocalityPolicy`、`UserSpacePolicy`、`UserSpacePathPolicy` は型aliasで維持。 +従来の公開評価・ユーザー領域評価関数は新実装へ委譲し、承認遷移エラーの同一性も保つ。 +新しいmigration、既存ACLの再登録、ポリシー設定の初期化は不要。 + +## PathとLocalityの評価 + +`ReadPathPolicy` は読み取り専用・要求単位のスナップショット。 +構築時に入力sliceを内部mapへコピーし、外部の変更で既存要求の権限が広がらないようにする。 +nilとゼロ値は拒否する。新たなglobal cacheやユーザー間の共有cacheは作らない。 + +- RoleのPath grantは完全一致、Domain grantまたは既存Path wildcardはDomain内の全Path。 +- グループscopeはそのPathと配下。`/a` と `/ab` を区別し、`%`・`_` はワイルドカードにしない。 +- Object grantはこのスナップショットへ含めない。Objectが読めても全aliasの存在を公開しない。 +- Locality内の該当ユーザー/グループは和集合、祖先と同一Pathの複数Locality制限は積集合。 +- 一つでもread/writeが拒否なら、下位の肯定grantで回避できない。 +- Localityの肯定grantで追加できるのはread/write/create/uploadのみ。rename/link/delete/manageは増やさない。 +- 管理者の回復用制限回避は基礎grantを維持するだけで、新しい権限は作らない。 + +`LocalityAccess.Decide` は制限と基礎grantを合成する低水準操作である。 +nilのLocality制限は「制限なし」としてbasisを返すが、nilのReadPathPolicyは常に拒否する。 +読込失敗と「制限なし」を混同しない。storeは全読込が成功してからスナップショットを返す。 +今回、Locality読込エラー時に部分的なReadPathPolicyを返さずnilを返すよう明確化した。 + +Path正規化・Domain/Principalの選択・アカウント状態の評価は呼出し元の責務。 +Schutzariaのコンストラクターに利用者入力のadmin/allフラグを直接渡さない。 +既存のscope文字列の意味を保つため、この分離では追加のURL decodeやPath正規化を行わない。 + +Object取得は引き続きstoreで全ローカルaliasの制限を交差させる。 +外国Domain側の制限検査はローカル管理者のshortcutより先に行い、独立したObject grantで +元Domainの制限を迂回させない。正本やPathの更新triggerも既存のまま残す。 + +## 権限プロファイル + +| プロファイル | 許可するcapability | +| --- | --- | +| read | read | +| read_write_once | read、create、upload | +| read_write | read、create、write、upload、rename、link | +| upload_only | upload | + +どのプロファイルもdeleteや管理権限を暗黙に追加しない。 +read_write_onceの「未存在対象のみ」は、更新するtransactionでの存在確認と組み合わせて成立する。 +プロファイル判定単独で、既存文書を上書きできないことを保証するわけではない。 +Principalのreadonly・停止やmembership/groupの失効は、従来のDB読込で先に適用する。 + +## 公開設定と実際の匿名公開 + +`EvaluateLocalityPublication` はDomainから下位への、検証済みの順序付きchainを受け取る。 +Domainのpublicは公開を許す上限であり、それだけではPageを公開しない。 +祖先のauthenticated_aclを下位のpublicで解除できず、不明なpolicyは拒否する。 +既存互換として空chainは `(allows=true, published=false)` の中立値を返す。 +呼出し側は実在するDomainと階層を解決し、allowsだけを匿名アクセス許可に使わない。 + +`PublicSemanticPath` は予約名前空間・不正なPathの除外だけを担当する。 +これがtrueでも、内容の読取権限、alias、現行Revision、hash、公開manifest、 +governance/topology versionの検証は省略できない。 +匿名要求は既存の公開経路を通し、無効な認証情報を匿名へ降格させない。 + +既存の[自動公開](../automatic-publication.md)は明示的なStreet/Spot設定に限る。 +上位required、設定時のpublisherの現在の権限、固定したPath境界、公開範囲の確認は +storeのtransactionに残す。今回の分離で自動公開の既定値を変えたり、旧記事を新たに公開しない。 + +ユーザー領域はrootと配下の設定が独立しており、rootのprivateを全配下のprivateへ読み替えない。 +配下overrideはさらに狭くするだけで、親のprivateを子のpublicで再公開できない。 +プロフィール・avatar・管理Path、所有者とDomain管理者の扱いは既存の呼出し元で維持する。 +不明なvisibilityの入力検証とDomainの公開上限は保存側で行う。 + +## 承認と共有の範囲 + +`ApprovalNextState` は現在のpending状態、generation一致、既知のdecisionだけを評価する。 +changes_requestedは従来どおりrejectedへ遷移する。ここでapprovedを返しても、 +公開manifestを作成したり、reviewer権限を付与したりしない。 +要求UUID・Domainの限定、認可、更新transactionとCAS、監査・Inboxは既存側の責務とする。 + +[承認ワークフロー](../publication-approval-inbox.md)の上位requiredの継承、reviewerの委譲制約、 +定足数、自己承認の禁止、期限と現行sourceの再検証、承認と公開確定の接続には未完了部分がある。 +状態遷移の抽出を、それらの実装完了や本番利用の許可とは扱わない。 +共有リンクのpurpose・期限・失効・対象の検証も今回新たに有効化・省略しない。 + +## 予算と一貫性の残件 + +既存の読み取りgrant最大4,096件、Locality制限最大4,096件、Object alias最大10,000件、 +ユーザー領域override最大256件を維持する。上限超過やDBエラーは拒否し、途中までの許可一覧を使わない。 +評価コアはDBの件数を増やさず、Path評価も保存されたancestor境界に対して行う。 + +不変の要求内スナップショットは、DB上の全権限情報を単一時点で読み込んだ保証ではない。 +`LoadReadPathPolicy`の通常DB経路はgrantとLocalityを別queryで読む。 +更新処理や匿名公開側にあるtransactionを、この分離で取り除かない。 + +- [ ] 権限・アカウント状態・対象更新を同一transaction/versionで固定する経路を拡充する。 +- [ ] ユーザー領域の遅延DB読込を有界の事前読込へ整理する。Mestionoraの認可callbackで別接続を必要とする制約は残る。 +- [ ] ReadPathPolicy全体の一貫した読込snapshotとcontext予算、group権限列挙の上限を設計する。 +- [ ] 自動公開の上位requiredを含む評価、所有者の委譲範囲、承認要求の期限・定足数・現行sourceの再検証を次の分離単位として整理する。 +- [ ] 説明可能な権限総覧を追加する。付与元・拒否元は閲覧者の権限内でのみ開示する。 +- [ ] 保護されたPathの粗い更新禁止triggerを、権限影響確認付きの原子的移動・mergeへ置き換える。 +- [ ] 共有リンクとCampus間ポリシー継承を、purpose・version・失効・対象束縛を持つ別契約として完成させる。 + +## 検証と反映 + +コアでは親子のread/write・基礎grant・肯定grantの組み合わせ、同一Pathの複数制限、 +文字列境界、snapshotのコピーと並行参照、公開上限、予約Path、ユーザー領域の絞り込み、 +承認状態・generationを検証する。型alias・JSON・エラー同一性と読込失敗の拒否も確認する。 +既存の匿名ナビゲーション、公開失効、グループ継承失効、governance、Object/Path認可の試験を維持する。 + +結果は [コンポーネント検証記録](component-names.md#検証記録) に記録する。 +一時DBと合成データで検証し、本番ACLの変更や外部カレンダーへのアクセスは行わない。 +ソース変更・テストと、commit・push・本番反映は別段階とする。 diff --git a/docs/design/ja/seheweit.md b/docs/design/ja/seheweit.md new file mode 100644 index 0000000..6b2d483 --- /dev/null +++ b/docs/design/ja/seheweit.md @@ -0,0 +1,104 @@ +# Seheweit のリソース収集と履歴契約 + +2026-11-01、リソース収集と履歴の期間・保持ルールを `internal/seheweit` に初期分離した。 +現在値APIとバックグラウンド監視は同じ収集実装を使う。Seheweitは観測を担当し、 +ジョブの実行制御・認可・DBトランザクション・表示は呼出し元に残す。 +これで採用した8部品の初期分離が揃うが、各部品の後続機能が完了したことは意味しない。 +[コンポーネント命名](component-names.md) と [システム拡張](../system-extensions.md) を併読する。 + +## 責務と接続 + +| 部品 | 責務 | +| --- | --- | +| `seheweit.Collect` | Go runtime、load average、プロセスI/O、Document Rootの容量を型付きSnapshotへ収集 | +| `seheweit.ParseProcessIO` | 実ストレージ読み書きの累積値と、プロセス単位の識別子・取得可否 | +| `seheweit.Window` / `RetentionCutoff` | 期間・bucket幅・保持境界。SQLやDB接続は持たない | +| server | 管理権限の検査、キュー状態の読込、実行中worker数、HTTP応答、監視handlerの登録 | +| store | 既存SQLiteへの保存・期限切れ削除・履歴query。schemaの所有は移さない | +| Dregarnuhr | 監視イベントの実行時刻・claim・worker制御。観測結果から仕事を勝手に停止しない | +| 設定画面 | ライブ値・履歴グラフ・実行中ジョブの表示、I/O差分速度の計算 | + +server/store → seheweit の一方向依存とする。Seheweitからserver/store/DBは参照しない。 +文書内容・認証情報・ジョブpayload・外部カレンダーを読まない。 +ネットワーク接続、shell、ディレクトリ再帰走査、独自の定期loopは追加しない。 +Queueは呼出し元から渡す観測値だけであり、停止・再開・再試行のコールバックではない。 + +## 収集値と互換性 + +`Snapshot` のJSONは既存の `timestamp_ms`、`io`、`cpu`、`memory`、`storage`、`queue` を維持する。 +従来のmapを型付きstructへ置換し、背景保存時のtimestamp型assertionをなくす。 +旧 `processIOCounters` はalias、parserは委譲とし、取得ロジックを二重に持たない。 + +| 観測値 | 意味と制限 | +| --- | --- | +| CPU | Go runtimeの論理CPU数と `/proc/loadavg` の先頭3値。CPU使用率やcgroup上限ではない | +| メモリ | Go heap、Go system bytes、GC回数、goroutine数。RSSやホスト全体の使用量ではない | +| 容量 | Document Rootに対するStatfs。使用量は従来のtotal−Bavailで、予約領域も含む | +| I/O | `/proc/self/io` のread_bytes/write_bytes。rchar/wcharやデバイスIOPSとは別 | +| キュー | queued/running件数、プロセス内active worker数、paused。履歴ジョブ全件は取得しない | + +procファイルの読込は各16KiBまで。超過・読込失敗時は部分データを採用しない。 +Statfsの失敗、非正のblock size、BavailがBlocksを上回る値、乗算overflowは容量0へ落とす。 +符号なし減算で巨大な使用量を作らない。絶対Pathへの解決失敗時も空Pathへ上書きしない。 + +I/O取得失敗は `available=false`。プロセスIDはプロセス起動ごとのランダムUUIDを共有し、 +collectorの呼出しごとには変えない。PID 1の再利用と、同一プロセス内の複数呼出しを区別する。 +既存のブラウザ計算は、再起動・カウンタ減少・時刻逆転・欠測・JS安全整数超過を速度0にせず欠測にする。 +並行収集では可変の共有Snapshotを使わない。 + +容量失敗時の0、load取得失敗時の空文字、キュー読込失敗時のゼロ値は既存互換のfallbackである。 +I/O以外の取得失敗を正常値と区別するavailability契約は後続とする。 +各情報源は順番に採取し、DB・runtime・OSを跨ぐ同一瞬間のsnapshotではない。 + +## 背景保存と履歴 + +イベントID `builtin.resource.sample`、型 `resource.sample`、既定60秒、許可範囲30〜86400秒、 +priority 100とmax attempts 1を維持する。設定済みのenabledとintervalは起動時に上書きしない。 +画面のライブ更新既定1秒と、背景の永続化既定60秒は別の周期である。 + +保存は `resource_samples` の既存schemaを使う。job_uuidの一意性で同一jobの重複登録を防ぎ、 +挿入と7日より古い行の削除を同一transactionに置く。既存行のpayloadをretryで上書きしない。 +保持境界は今回の採取timestampから7日前。境界と同じtimestampは残す。 +retryで挿入を無視しても、その採取時刻で清掃する既存動作は維持する。 + +`SaveResourceSampleContext` は取消contextをBeginTx・各SQLへ渡す。 +監視handlerは取消済みなら収集を始めず、保存時もcontextを引き継ぐ。 +read-only Storeでは保存を拒否する。旧 `SaveResourceSample` はBackground contextで委譲する。 +清掃失敗時には挿入もrollbackし、半端な成功を返さない。 +contextはDB待機を中断できるが、実行中のStatfs等を強制停止する仕組みではない。 + +履歴APIは `GET /_/api/v1/queue/resources/history?hours=1|24|168` のまま。 +UIの期間既定1時間と選択の記憶、APIの未指定・不正値時の24時間fallbackは変えない。 +内部Windowは1〜168時間へ制限し、幅は期間/360か最低1分、epoch基準でbucketを切る。 +期間両端を含み、最大361点を返す。空履歴は `[]`、retention_daysは7を維持する。 + +bucket代表は既存SQLの `MAX(id)`、つまり最後に挿入された行である。 +時刻の順序が逆転した入力では最大timestampの行とは限らず、平均・最大値・ピークでもない。 +期間はtimestamp索引で絞るが、361という上限は返却点数であり走査行数の固定上限ではない。 +現在値GETは収集だけ、履歴GETは読込だけで、どちらも保存や補間を行わない。 +ブラウザを閉じても背景イベントは止まらず、監視無効化・キュー停止では新規採取を止める。 + +## 権限と配置 + +既存queue APIの管理権限を維持し、新しい公開endpointやDomain間共有を追加しない。 +応答は引き続き `Cache-Control: no-store`。リソースはインスタンス全体の情報なので、 +個々の文書の公開権限とは分けて扱う。URI・設定キー・schema・保存済みJSONの移行は不要。 +追加コンテナや監視サービス、外部送信先の導入も行わない。 + +## 後続TODO + +- [ ] 各情報源のavailability・診断コード・取得時刻を追加し、旧履歴とUIを互換移行する。DB失敗を空キューに見せない。 +- [ ] foreground多重要求の短時間共有と収集予算を設ける。runtime読込や遅いマウントのStatfsを計測し、画面要求を待たせない。 +- [ ] キュー状態読込へのcontext伝播と一貫性、観測API専用のinstance_operator権限を整理する。 +- [ ] 大量履歴の保持清掃を分割し、粒度別rollup・平均/最大/欠測数・容量予算を設計する。返却点数の制限だけで走査負荷は保証しない。 +- [ ] component/provider別の件数・失敗・待機時間・実行時間を低cardinalityで追加する。共有GoプロセスのCPU/RAMを根拠なく按分しない。 +- [ ] cgroup、デバイスI/O、外部adapterプロセスを独立scopeで観測する。現在のprocess I/Oと混合しない。 +- [ ] HAのinstance UUID・boot ID・node別収集・集約を定義する。現行process_idは永続instance identityではない。 +- [ ] SNMPv3 collectorを任意機能として追加する。接続先・OID許可、秘密管理、timeout、再試行、counter resetを別途検証する。 + +## 検証と反映 + +合成counter・注入clock・一時DB・localhostのブラウザfixtureで、wire互換、収集失敗、容量の異常値、 +I/O識別、並行収集、保持境界、idempotence、rollback、取消、read-only拒否、認可、設定保持を検証する。 +実行記録は [コンポーネント検証記録](component-names.md#検証記録) に記載する。 +本番DBや取り込み済み外部カレンダーにアクセスせず、commit・push・本番展開は別段階とする。 diff --git a/docs/design/platform-extension-architecture.md b/docs/design/platform-extension-architecture.md index ffa6df1..6c7888d 100644 --- a/docs/design/platform-extension-architecture.md +++ b/docs/design/platform-extension-architecture.md @@ -7,6 +7,11 @@ collaboration, federation, high-availability, content, security, and storage features. It defines extension boundaries, not permission to bypass existing identity or authorization rules. +Responsibility-based subsystem names are adopted in +[Component names and responsibilities](ja/component-names.md). Grammaratur, +Dregarnuhr, Geduldh, Mestionora, Ordoschnellic, Schutzaria, Kunstzeal and Seheweit have initial extractions. Naming does not change the contracts below, +persisted extension IDs, authorization boundaries or deployment topology. + ## Non-negotiable identity boundaries Future features attach to stable identities: @@ -74,17 +79,20 @@ origin allowlist, timeout, and privacy policy. ### Document Store contract -The canonical interface is content-addressed and supports: +The implemented canonical interface is `geduldh.Store`, with a compatible +`documentstore.Store` alias. It is content-addressed and supports: ```text -Stat(sha256) -Open(sha256) / ReadAt(sha256, offset, length) -PutIfAbsent(expected_sha256, stream) -Verify(sha256) -DeleteUnreferenced(sha256, retention proof) +Stat(ctx, sha256) +Open(ctx, sha256) -> Reader / ReaderAt / Seeker / Closer +PutIfAbsent(ctx, expected_sha256, size, stream) +Verify(ctx, sha256) ``` -Filesystem and S3 implementations obey the same contract. Bucket/key URLs are +The local implementation and future S3 implementations must obey this contract. +`DeleteUnreferenced(sha256, retention proof)` is a separate future GC contract, +not a method available to ordinary payload consumers. See [Geduldh](ja/geduldh.md). +Bucket/key URLs are placement data and never appear in Revision identity. Metadata databases, credentials, signing keys, sessions, and comments do not belong in this store. diff --git a/docs/design/projection-adapters.md b/docs/design/projection-adapters.md index d83a780..8c3655e 100644 --- a/docs/design/projection-adapters.md +++ b/docs/design/projection-adapters.md @@ -20,7 +20,8 @@ Revision UUID, and raw payload delivery. ## Current Adapter Contract -The server registry in `internal/server/content_adapter.go` describes: +The registry in `internal/kunstzeal/adapters.go`, accessed through compatible +server types and selection wrappers, describes: | Field | Meaning | | --- | --- | @@ -42,7 +43,7 @@ The initial registry reserves these stable IDs: | `text` | CodeMirror text | windowed text | | `csv` | bounded table with sticky header | paginated virtualized table | | `excalidraw` | validated interactive canvas + source fallback | canvas + editor action | -| `nwdiag` | bounded network diagram projection | static SVG preview + hosted/editor action | +| `json` | bounded JSON structure/text projection | indexed structural retrieval | | `archive` | ZIP/RAR/7z/TAR entry table + comic mode | archive directory | | `eml` | decoded headers and body | multipart mail viewer | | `debian-package` | ar/control metadata | package metadata | @@ -57,6 +58,12 @@ These adapters are built in and versioned with Pages Wiki. They are always listed by `GET /_/api/v1/adapter`; they are not configuration records and cannot be deleted by an administrator. +2026-11-01: adapter selection and existing Kroki execution were initially +extracted into [Kunstzeal](ja/kunstzeal.md). Stable IDs/order and source authorization +remain unchanged. nwdiag and slide providers are still TODO, not entries in the +implemented registry. Kroki redirects are now refused; derived SVG cache writes +use atomic replacement. Provider versioning and unified resource controls remain pending. + ## Integration Adapter Plugins Additional hosted viewers and editors are Domain-scoped integration adapter @@ -401,6 +408,8 @@ CSV detection uses MIME type or `.csv`. The derived adapter must: ## nwdiag network diagrams +Status: design/TODO. No nwdiag parser or built-in adapter is currently registered. + `nwdiag` is a source/document format, not an instruction to execute arbitrary code. The adapter parses a bounded, version-pinned grammar and produces a static SVG (or PNG) projection. It must reject oversized graphs, recursive diff --git a/docs/design/search-results.md b/docs/design/search-results.md index 498a294..84ea277 100644 --- a/docs/design/search-results.md +++ b/docs/design/search-results.md @@ -2,6 +2,11 @@ Implemented: 2026-09-14. +2026-11-01: SQLite search/index maintenance and City metadata lookup are extracted +into `internal/mestionora`. Store APIs/types remain compatibility facades; HTTP +search and suggestions call Mestionora with the existing request read policy. +No new index schema or migration is introduced. See [Mestionora's contract](ja/mestionora.md). + ## City-first results and search options Search now places matching Quartiers, Streets and Spots above Page/file hits. @@ -114,7 +119,7 @@ Global counts, denied Paths, denied filenames, and denied excerpts are never included in the rendered data. An all-denied candidate set uses the same empty state as an ordinary no-match query. -All search statements use one read-only SQLite transaction, preserving a +Page candidate and excerpt statements use one read-only SQLite transaction, preserving a consistent current Path / latest Revision snapshot. The three candidate streams are: @@ -142,7 +147,10 @@ snapshot. `octet_length(body)` counts bytes, not Unicode characters; see Eligible body excerpts are fetched together with one FTS snippet query restricted by those rowids, not the unindexed UUID payload column. Name-only textual summaries use one additional bounded Revision-prefix query. There are no -per-result authorization queries or Document-file stat calls. Existing preview +Document-file stat calls. The normal Path policy is preloaded; lazy `/u/` policy +lookups can still borrow another DB connection during authorization. Do not +combine that callback with a single-connection pool until the policy preloading +follow-up is implemented. Existing preview cache checks run only for displayed authorized results, without scheduling foreground generation. @@ -240,7 +248,7 @@ are local observations, not latency guarantees or authenticated HTTP timings. This feature adds no schema migration or index rebuild. ```sh -go test -race ./internal/server ./internal/store -run '^TestSearch' -count=1 +go test -race ./internal/mestionora ./internal/server ./internal/store -run '^TestSearch' -count=1 PAGESWIKI_BROWSER_TEST=1 go test ./internal/server -run '^TestSearchResultsChromium$' -count=1 # Optional: opens an existing DB with mode=ro and query_only, with no migrations. PAGESWIKI_SEARCH_DIAGNOSTIC_DB=/absolute/path/pages.db go test ./internal/store -run '^TestSearchReadOnlyDatabase$' -count=1 -v diff --git a/docs/design/storage-portability-and-operations.md b/docs/design/storage-portability-and-operations.md new file mode 100644 index 0000000..b4c93f3 --- /dev/null +++ b/docs/design/storage-portability-and-operations.md @@ -0,0 +1,410 @@ +# 保存基盤の多様化と運用機能の統合 + +状態:詳細設計・段階導入。2026-09-27。 +本書は [基本設計](arch.md) と [拡張契約](platform-extension-architecture.md) を補完する。 +各 TODO は実装済みを意味しない。本番 DB・保存先の移行は別途承認する。 + +## 1. 優先順位と対象 + +先に SQLite / PostgreSQL / MySQL、Redis 等の KV、ローカル / S3 / SSH / NFS / SMB の +役割・契約を整える。その上にバックアップ、リリース、CLI、export/import、Web バックアップ、 +初回「ようこそ」画面を載せる。Web と CLI で別の保存実装を作らない。 + +実装順は **保存契約 → 正本の分離 → DB 移植 → 外部 Document → 運用ジョブ → UI**。 +Redis を RDB の代わりに選ぶ UI や、S3 URL をローカルディレクトリ欄へ指定する方式は採用しない。 +初期設定が SQLite + ローカルだけでも、単一ノード運用が追加サービスなしで成立すること。 + +## 2. 現行実装の監査 + +| 箇所 | 現状 | 移植前に必要な変更 | +| --- | --- | --- | +| `internal/db/db.go` | SQLite 固定、WAL、busy_timeout、NORMAL | 接続プロファイル、durability 設定、driver ごとの検証 | +| `internal/db/migrate.go` / `Ensure*Schema` | DDL とデータ移行が分散。core bootstrap の ALTER エラー無視は是正、他の Ensure は継続監査 | 新 migration runner への段階集約と driver 別 DDL | +| `internal/store` | `*sql.DB` への直接 SQL。SQLite 固有式が多い | repository ごとの SQL 分離・契約テスト | +| `internal/store/document_store.go` | SHA-256 正本と DB body の移行互換 | payload backend の分離、復旧時以外の暗黙再作成の縮小 | +| `internal/calendar/repository.go` | 月別 ICS + calendar.json、flock、rename、DB はキャッシュ | mutable canonical repository と条件付き世代更新 | +| `DocumentRoot` の利用箇所 | calendar、auth.key の親、preview、import、payload を兼用 | payload / canonical state / secrets / scratch の分離 | +| `DocumentPath` の利用箇所 | 外部コマンドへ直接ローカルパス | 期限付き Materialize、容量制限・後処理 | +| `data/pageswiki-cli` | Python sqlite3 とローカル Document を直接操作 | online API モードを主経路に、offline repair を明示分離 | + +調査時点で SQLite 固有語(PRAGMA、sqlite_master、rowid、INSERT OR、strftime 等)が +テストを除く約 50 ファイルに存在する。接続ドライバだけ追加しても動作しない。 +「接続できた」と「PagesWiki の全機能を利用できる」をサポート状態として区別する。 + +## 3. 正本・派生物・機密情報を分離する + +| 層 | 主なデータ | 保存先 | 消失時の扱い | +| --- | --- | --- | --- | +| MetadataRepository | UUID、Path、Revision、ACL、ユーザー/グループ、タスク、設定、outbox、確定ジョブ | SQLite / PostgreSQL / MySQL | バックアップから復元が必要 | +| DocumentStore | SHA-256 で識別する不変バイト列 | local / S3 / 検証済み mounted filesystem / SFTP | バックアップから復元が必要 | +| CanonicalStateRepository | カレンダーのメタ情報、月ごとの ICS 世代、ユーザー設定 | 世代付きファイル正本(local から段階拡張) | バックアップから復元が必要 | +| CacheStore | 検索・表示・サジェスト・リソース集計の再生成可能データ | memory / SQL cache / Redis | 正本から再生成 | +| SecretStore | auth.key、署名鍵、暗号化鍵、接続資格情報 | secret file / 環境 / 外部 Secret manager | 別権限・暗号化して保全 | +| Scratch/Materialization | import 展開、外部ビューア用一時ファイル、生成中 preview | ノードローカルの専用領域 | 破棄して再実行 | + +セッション失効、承認、ジョブ完了、タスクの変更を Redis のみの情報にしない。 +「Redis の AOF を有効にしたから正本のバックアップを省略できる」とはしない。 + +## 4. 共通インターフェース + +### 4.1 不変 Document + +Go 契約は `internal/geduldh.Store`。2026-11-01に既存ローカル実装を +[Geduldh](ja/geduldh.md) へ分離し、`internal/documentstore.Store` は同じ型の互換aliasとした。 + +```text +Stat(ctx, sha256) -> { hash, size } +Open(ctx, sha256) -> Reader + ReaderAt + Seeker + Closer +PutIfAbsent(ctx, sha256, size, stream) +Verify(ctx, sha256) +``` + +- 保存キーは既存互換の `sha256/<先頭2文字>/<64桁hash>`。Path やファイル名ではない。 +- サイズとハッシュの検証前に、完全な Document として読める状態にしない。 +- 不変 payload を先に永続化し、その後に参照する DB transaction を commit する。 +- DB rollback で未参照 blob が残るのは許容するが、存在しない blob の参照確定はしない。 +- 読み取りの Open/Seek/ReadAt は大容量全体の先読みをしない。remote range は上限・期限付き。 +- ETag は HTTP の条件付き取得に使い、SHA-256 とは別フィールドとする。 +- timeout、not found、破損、認証失敗、容量不足を区別する。エラー時に local へ勝手に書かない。 +- 通常の Put/Read インターフェースに Delete を含めない。GC は参照 snapshot、保持、backup pin、 + 進行中 upload の保護を持つ専用経路から実施する。 +- local path を返すことを全 backend の必須機能にしない。 + +接続済みの範囲はローカルの保存・通常読み取り・サイズ取得・部分読み取りと、 +外部ツール用 Materialize。HTTP の主要な本文配信・部分取得には Request.Context を接続した。 +remote backend、従来の Revision 本文取得/保存 API 全体への context 伝搬は未接続。 +既存の DB body fallback は互換のため維持する。起動時 backfill は保存先ごとの進捗を記録し、 +完了済みの保存先では revisions の走査を省略する(後述)。 + +### 4.2 可変正本 + +カレンダーはファイル正本を維持し、月ごとの不変 ICS 世代を作成してから +calendar manifest を `CompareAndSwap(expectedVersion, nextManifest)` で更新する。 +DB のカレンダー cache は manifest version に従って再生成する。 +S3 で POSIX rename/flock を模倣せず、条件付き manifest 更新を使う。 +カレンダーの競合は明示的に返し、古いキャッシュで正本を上書きしない。 +世代の削除はバックアップ pin と参照中 version を考慮する。 + +### 4.3 外部ツールへの受け渡し + +`Materialize(ctx, hash, quota) -> localPath, cleanup` を別サービスにする。 +archive/PDF/動画/PE/DEB 等のローカルパス必須ツールはこれを利用する。 +SHA-256 検証、容量上限、同一 blob の取得集約、参照中の退避禁止、期限、キャンセルを持つ。 +remote Object URL や presigned URL を無条件にツールへ渡さない。 +ZIP の範囲参照で足りる処理は Materialize せず ReaderAt を使う。 + +実装は `internal/geduldh.Materializer`(旧 `internal/documentstore.Materializer` は互換alias)と +`Store.MaterializeDocument(ctx, hash, maxBytes)`。同じ Store 内で同じ hash の取得をまとめ、 +利用者ごとの lease で参照中のコピーを保持する。待機者の一人がキャンセルしても他の待機者の +取得は中断しない。全員がキャンセルしたら共有取得を中断し、最後の lease の Close で削除する。 +取得・検証に成功するまで外部コマンドへパスを渡さない。コピーは 0700 の専用一時ディレクトリに +0400 で作り、正本への hardlink は渡さない(外部ツールが誤って正本を変更するのを防ぐ)。 + +現時点の固定上限(将来 deployment profile に移す): + +- Store ごとに同時取得 2 件、取得待ち/利用中を含め 64 件、予約容量合計 2 GiB。 +- 共有取得の期限は 2 分。呼び出し元の request/preview/metadata の期限でも待機を中断する。 +- metadata / PDF は 512 MiB、動画 / 非 ZIP 展開は 1 GiB、Excalidraw は 16 MiB。 +- 保存先は `TMPDIR`(未設定時は OS の一時領域)。DocumentRoot やネットワークマウントを使わず、 + ノードローカルの十分な空き容量を持つ領域を指定する。tmpfs の場合はメモリ消費になる。 +- 削除失敗時には予約容量を解放済みと見なさない。プロセス強制終了で残った一時ファイルの + 安全な回収とリソース監視へのメトリクス接続は残件。 + +画像・ZIP サムネイル、ZIP 内の個別ファイル配信は Reader/ReaderAt を直接利用する。 +ZIP の個別配信は互換 DB body の全体 hydration をしない。非 ZIP / PDF / 動画などは、 +ローカル正本でも検証付き一時コピーを作るため追加 I/O が発生する。永続キャッシュや +reflink 最適化は容量/失効/破損検出の契約を保った上で別途導入する。 + +## 5. RDB の対応設計 + +### 接続と migration + +- 接続プロファイルはインスタンス管理で指定。Domain 管理者に DSN を公開しない。 +- driver ごとに host/port/database/TLS/secret reference/pool/timeout を持つ。 +- 未対応 driver は起動前に明示エラー。SQLite にフォールバックしない。 +- SQLite を既存互換で維持し、まず PostgreSQL、次いで MySQL 8.4 LTS / InnoDB を対象にする。 + PostgreSQL の最低対応版は CI matrix の確定時に明示する。MariaDB を MySQL と同一扱いしない。 +- 起動時 Ping、schema version、driver と schema の一致を検証する。 +- migration は driver/version/checksum/applied_at の履歴と排他的ロックを持つ。 + 起動ごとの分散 Ensure* を段階的に集約し、未知の DDL エラーを無視しない。 +- MySQL の DDL は暗黙 commit を前提に、再開可能な移行段階を記録する。 + migration rollback を DB transaction だけで保証しない。 + +#### 初段の migration runner(実装済み範囲) + +`internal/db.ApplySQLiteMigrations` は component ごとに連続した version、name、SQL の +SHA-256 checksum、適用時刻を `schema_migration_steps` に記録する。 +専用 DB connection の `BEGIN IMMEDIATE` で他プロセスとの適用を直列化し、 +DDL と履歴を一つの transaction で commit する。失敗時は rollback し、transaction の +状態が確定できなかった connection を pool へ戻さない。 +適用済み SQL の変更、version の欠落、バイナリより新しい移行履歴は起動時にエラーにする。 +適用済みの手順は編集せず、新しい version を追加する。 + +移行済みの利用先は Document backfill の進捗テーブルと、下表の 6 領域。 +既存の `schema_migrations` / `revision_metadata_v1` は変更せず維持する。 +core bootstrap の列追加は PRAGMA で存在確認し、未知の ALTER エラーを無視しない。 +古い revision の hash 計算は 128 件ずつ UUID を選択し、本文は一度に一件だけ読み込む。 +既存のすべての Ensure*Schema が新 runner へ移行したわけではなく、 +PostgreSQL/MySQL 対応や schema drift の完全な検査を意味しない。 + +| component | 対象 | 旧版から不足時に追加する列 | +| --- | --- | --- | +| `projection-metadata` | `projection_metadata` | なし | +| `resource-history` | `resource_samples` | なし | +| `broadcasts` | `broadcasts` | なし | +| `browser-navigation` | `browser_navigation_links` | `is_directory` | +| `adapter-sessions` | `adapter_sessions` | `root_path`, `principal_uuid` | +| `imports` | import session/item/journal/City 対応の 5 テーブル | `import_journal.completed` | +| `event-jobs` | jobs/definitions/logs/log policies/log buckets/queue settings | `event_uuid`, `priority`, `max_attempts`, `started_at_ms`, `finished_at_ms`, `cancel_requested` | +| `document-store` | 本文移行の進捗 `document_backfills` | なし(v2 で契約を検証) | + +定義は `internal/store/schema_modules.go` / `schema_jobs.go` と +`document_backfill.go` に置き、診断時には同じ定義を参照する。既存 `Ensure*Schema` メソッドは +互換ラッパーとして残し、テーブルの再作成・行の削除はしない。 +条件付き列追加は存在確認と ALTER を同じ migration transaction 内で実施する。 +追加列の宣言と schema assertion も checksum 対象とする。旧式定義の checksum は、 +新しい空フィールドを JSON へ出さないことで維持する。 + +component の最新 version に、現在のスキーマ全体に必要な assertions を記載する。 +適用済みでも毎回検査する。必要な列、一意性の対象列、インデックスの所属と列順、 +追加列の型/NOT NULL/default、adapter session の部分 UNIQUE 条件を確認する。 +すべての列の型・CHECK・照合順序・外部キーまでを網羅する汎用 diff ではない。 +後続 version が意図的に列を削除する場合に備え、過去 version の assertions は最新の契約で +置き換える(過去の定義と checksum は改変しない)。 + +初回採用時は既存の行を保持したまま不足する既知の列・索引を作成する。 +採用後に schema が欠落/変更した場合は自動再作成せず、起動を止めて調査対象とする。 +event_jobs 用の検索索引は projection の初期化から Event Jobs の初期化へ移し、 +派生キャッシュからジョブ正本のスキーマへ向かう不要な依存を取り除いた。 +イベントキューは、既知の不足列を追加する v1 と、列を使う索引・周辺テーブルを採用する v2 に分ける。 +実行中ジョブの再キューイングは通常起動の状態復旧処理として分離し、migration runner や診断から呼ばない。 + +`pageswiki schema status|check` は `mode=ro` / `query_only` 接続で、管理対象の履歴/checksum と +最新 schema assertions を検査する。読み取り transaction の snapshot から判断し、 +未作成 DB の初期化・鍵生成・本文移行・ジョブ復旧を実行しない。 +`check` の終了コードは current=0 / pending=2 / blocked・unknown・実行エラー=1。 +未適用 DDL は実行せず計画だけ表示するため、pending の成功を保証する dry-run ではない。 +未移行の従来 schema 処理は診断範囲外として出力し、起動リハーサルを別途要求する。 +反映前の確認項目は [保存基盤のスキーマ変更・反映手順](../operations/storage-schema-rollout.md) を参照。 + +#### 再開可能な Document backfill(実装済み範囲) + +- `Store.BackfillDocuments(ctx, batchSize)` が一回最大 128 件の UUID/hash を取得する。 + 本文は不足 blob の一件分だけ読む。単一の巨大な legacy 本文のメモリ上限は別の課題である。 +- 保存先の `.document-store-id` と DB の `document_backfills` を対応させ、 + UUID cursor・処理数・完了状態・更新時刻を記録する。UUID cursor は VACUUM で変わらない。 +- payload の検証・永続化後にバッチの checkpoint を更新する。失敗時は未完了バッチを再実行する。 + 保存先単位のファイルロックで二重実行を防ぎ、ロック待ちも context のキャンセルを受け付ける。 +- 正本が既にある場合はその SHA-256 を検証し、古い/空の DB body は使わない。 + 破損した正本は互換キャッシュで上書きせず、明示エラーにする。 +- 保存先を空のディレクトリへ差し替えると identity が変わり、以前の完了状態を流用しない。 + 既存正本が一部欠落した場合の検出は verify/repair の役割であり、通常起動で全件監査しない。 +- DB-only の旧 writer/importer を止めた保守状態で実施する。通常の Page 保存は + Document 永続化が先行するため、移行 cursor より前の UUID が増えても不足しない。 +- 初回移行は引き続き起動前に完了させる。完了済みの通常起動では revisions を問い合わせない。 + 非同期化する際は未移行 payload の配信・readiness・保守モードの契約を先に整備する。 +- backup/restore は `.document-store-id` も含めて保存先を保全する。これは資格情報ではない。 + identity が欠落した旧バックアップは新しい保存先と扱い、再検証・移行する。 + +### SQL 差異と対策 + +| 差異 | 対策 | +| --- | --- | +| `?` / `$1` | repository の dialect/query builder で生成。文字列の単純置換はしない | +| PRAGMA / sqlite_master | schema inspector を backend ごとに分離 | +| `rowid` による順序 | 明示 sequence または `(created_at_ms, uuid)` 等の仕様化した順序へ移行 | +| INSERT OR IGNORE/REPLACE / UPSERT | conflict target と更新列を明示。REPLACE の delete+insert を模倣しない | +| 時刻関数 | 保存時刻はアプリ生成 UTC epoch ms / BIGINT、必要な DB 時刻用途は別途定義 | +| LIKE / 照合順序 | Path/UUID/ハッシュの大小文字・正規化規則を共有。DB の既定照合に依存しない | +| TEXT を含む複合 UNIQUE | MySQL の索引長を検討。長い Path は正規化 UTF-8 の digest を索引候補にし、原文比較で衝突検出 | +| FTS / 日本語 / ranking | SearchIndex 契約へ分離。DB ごとの全文検索結果が完全一致すると仮定しない | +| エラーコード | unique / FK / deadlock / serialization / busy を共通分類 | +| job claim / pagination | SQLite single writer、PostgreSQL/MySQL row lock + claim token、安定した keyset cursor | + +SQL を全機能一度に書き換えず、スキーマ → Object/Path/Revision/Asset → 認証/ACL → +タスク/CalDAV → event/outbox/通知 → City/検索/管理の順に契約テストを追加する。 +全 repository の通過前は PostgreSQL/MySQL を本番選択可能と表示しない。 +外部 RDB を導入しただけでは HA ではない。job lease/fencing、可変正本、鍵共有が別の条件。 + +### DB 移行 + +既存データを上書きするのではなく、新規 target DB へ schema 作成 → snapshot import → +UUID/件数/参照整合性/ACL検証 → 停止時間内の差分反映 → 接続先切替を行う。 +最初は書き込み停止で一貫性を確保し、オンライン dual-write は採用しない。 +移行元は保持。新 DB に書き込み開始後の単純な DSN 戻しはデータ喪失となるため禁止する。 + +## 6. Redis / KV + +初期用途は TTL 付き共有キャッシュ、短時間の同一処理集約、通知配信の補助。 +永続ジョブ・通知 Inbox・失効情報・監査 outbox は RDB 側を正本とする。 + +- key は instance / domain / provider / schema / policy epoch を含める。 +- 認可を含む表示 cache は principal と権限 version を分離し、古い ACL で許可しない。 +- 接続 pool、TLS、ACL、タイムアウト、value 上限、TTL jitter、hit/miss/eviction を持つ。 +- cache 障害は安全な DB 参照へ限定的に縮退し、全リクエストの stampede を防止する。 +- Redis Pub/Sub は配送保証に使わない。通知の取りこぼしは DB Inbox から再取得する。 +- 分散ロックだけで正本の排他を保証しない。多ノード更新は DB の fencing token を検証する。 +- 永続化方式 RDB/AOF の運用と、再生成可能 cache のバックアップ不要性を区別する。 + +## 7. ファイルストア別の対応 + +| backend | 導入方法 | 必須条件・制限 | +| --- | --- | --- | +| local | 現在の data/documents と互換 | fsync、原子的な no-replace publish、ルート内アクセス | +| S3 compatible | SDK adapter、bucket/prefix/region/endpoint | 条件付き作成、range、checksum、multipart 中断回収、TLS | +| over SSH | 原則 SFTP adapter。SSHFS mount は別の検証対象 | known_hosts 固定、専用 user/root、host-key 変更を拒否、shell コマンド文字列禁止 | +| NFS | OS/Compose で管理者が mount、アプリはパスを利用 | protocol/mount/lock/durability を環境別認証、切断時に失敗させる | +| SMB | 同上、CIFS mount または後続 adapter | case-sensitive key 契約、rename/link/flush の実環境試験 | + +### S3 + +SDK の標準 credential chain と secret reference を使い、本文や manifest に資格情報を埋め込まない。 +bucket は private を既定とし、公開 URL と物理キーを分離する。 +`If-None-Match: *` による作成、既存時の checksum 検証、`412` と通信障害の区別が必要。 +S3 ETag は Document SHA-256 と見なさない。互換製品ごとに conditional/multipart/checksum を検証する。 +upload は bounded spool または検証付き streaming とし、未検証データを final key へ公開しない。 +LIST をバックアップの整合性境界にせず、DB snapshot の参照 manifest で必要な hash を列挙する。 +障害中は書き込みを失敗させ、プロセスごとのローカル正本へ分岐させない。 + +### ネットワーク filesystem / SSH + +SQLite の DB/WAL/SHM を NFS・SMB・SSHFS に置くことは禁止する。 +共有可能にするのは、契約テストで検証した Document/正本 backend に限る。 +アプリに mount 権限や任意 SSH コマンド実行権限を与えない。 +mount が消えた際に空のローカル mountpoint へ書くことを防ぐ sentinel / device / mount-id 検証が必要。 +context のキャンセルだけで、停止した NFS syscall を必ず中断できるとは扱わない。 +独立 worker、I/O 上限、circuit breaker、管理者向け degraded 状態を設計する。 +hard-link/no-replace や directory fsync のない共有領域を local backend として認定しない。 + +## 8. バックアップと export の区別 + +| 操作 | 目的 | 含めるもの | +| --- | --- | --- | +| backup/restore | 同一インスタンスの災害復旧 | DB snapshot、Document、カレンダー正本、設定、必要な鍵の暗号化保全 | +| export/import | Domain/Path/プロジェクトの移動・交換 | 許可範囲の内容・revision・asset依存・選択したメタ情報。秘密は既定で除外 | +| release/deploy | 実行コードの更新 | SemVer、Git commit、イメージ digest、schema互換条件。ユーザーデータとは別 | + +バックアップ manifest は format_version、instance ID、アプリ SemVer、schema version、 +各 DB snapshot、正本 generation、Document の hash/size、暗号化情報、完了状態を持つ。 +全生成物を検証してから complete marker を発行する。部分生成を復旧可能と表示しない。 + +最初は全書き込み元(Web/API/CalDAV/CLI/同期 worker)を止める maintenance barrier を使い、 +GC を pin して DB と可変正本を snapshot する。不変 blob のコピーはその参照集合で行う。 +「DB はコピーしたが calendar.json は後から別の時点でコピーした」を許さない。 +オンライン化は revision/outbox watermarks と snapshot pin 実装後の段階とする。 + +- SQLite:online backup API / 検証済み snapshot。稼働中の pages.db 単独 rsync はしない。 +- PostgreSQL:論理 dump と必要に応じた physical backup/PITR を別モードにする。 +- MySQL:InnoDB の consistent snapshot と migration/DDL 排他を確認する。 +- Redis:再生成可能 cache は除外。独自永続用途を追加したら manifest へ明示する。 +- 鍵・接続資格情報:別保護領域へ暗号化。復元に必要な鍵を同じ暗号化アーカイブ内だけに入れない。 +- restore:新規領域へ展開・検証・dry-run、互換性確認、切替。通常 Web から稼働 DB を上書きしない。 + +保持期間、容量、転送帯域、暗号化、定期実行、リモート複製、リストア訓練を独立設定にする。 +バックアップ削除は全世代を単純削除せず、増分依存と稼働リリースの復旧条件を確認する。 + +## 9. 手動スクリプト・CLI・Web の共通化 + +将来の `pageswiki-admin` / `pageswiki` 運用サブコマンドに処理を集約し、 +編集可能な shell script は環境変数・設定ファイル・コマンドの薄い組み合わせにする。 +スクリプトから直接各 DB のテーブルへ別々に書く方式は増やさない。 + +候補コマンド(**まだ未実装**): + +```text +storage status / verify / migrate --plan / migrate --apply +backup create / list / verify / restore --plan / restore --apply +export --domain … --path … --format pageswiki-bundle +import --file … --plan / import --file … --apply +release verify / deploy --version vX.Y.Z --plan / deploy --apply +``` + +online CLI は共通 API と権限・監査・event queue を通す。offline は保守モード専用で、 +直接 DB を開く現行 Python CLI は当面の互換/repair 経路として残す。 +import はサイズ・件数・展開率上限、zip-slip、symlink/hardlink を検証し、試行計画を先に表示する。 +UUID保持/新規採番、Path衝突、ACL継承、Cityサジェスト、部分失敗の再試行を明示選択にする。 + +Web バックアップは instance/campus 管理者向け。Domain export 権限と混同しない。 +設定パネルには履歴・状態・検証結果をコンパクトに表示し、作成条件はダイアログに収める。 +作成は既存 scheduler の `backup.create` / `backup.verify` event としてキューへ入れる。 +API は `/_/api/v1/backup/`、ジョブ状態は既存 `/_/api/v1/event/` / `queue` を利用する案とする。 +ダウンロードにも現在の認可、有効期限、監査、no-store を適用する。秘密付き archive を公開しない。 +Web から任意の保存先パス・任意コマンド・任意外部 URL を指定できるようにしない。 + +release script は v0.1.0 で整備した digest 昇格を再利用する。 +deploy は backup 検証 → image pull → maintenance/migration → health check → 切替。 +DB migration 後の旧イメージへの自動 rollback は互換性が明示された場合だけ許可する。 + +## 10. 初回ようこそ画面 + +状態は `uninitialized → claimed → configuring → validating → ready`。 +既存データのあるインスタンスには表示せず、DB 到達不能を未初期化と判定しない。 +初期 DB 未作成でも使える bootstrap 設定は local protected file / secret mount に置く。 + +1. ホスト側で生成した短寿命・一回限りの setup token で管理者が claim(競合時は一人だけ成功)。 +2. 単独 / controller / 外部 Campus 管理モードを選択。外部管理の名前等は従来どおり継承する。 +3. Metadata DB、Document backend、canonical state、任意 Redis を設定。 +4. TLS・資格情報・必要な保存操作を管理者指定 backend 内の専用 probe 領域で検証。 +5. 最初の管理者、言語・タイムゾーン、FQDN、バックアップ先/保持方針を確認。 +6. 設定を原子的に保存し migration/初期化を実行。成功後 setup token を失効し ready にする。 + +公開ネットワークで最初にアクセスした人を無条件に管理者にしない。 +setup 完了後は token 再利用不可。機密欄の再表示・ログ出力をしない。 +外部 Campus 自動参加など既存の非対話起動を維持する。 + +## 11. 段階別 TODO と受入条件 + +### S0:契約とローカル保存の是正 + +- [x] 不変 Document Store interface と既存キー規則を実装。 +- [x] local の streaming 検証、no-replace publish、directory fsync、root 内アクセスを実装。 +- [x] 現行 Store の保存・読み取り・サイズ・部分読み取りを接続。 +- [x] 並行書き込み、既存破損、サイズ/hash不一致、キャンセル、パス逸脱をテスト。 +- [ ] backup inventory / canonical state / secret / scratch の設定と寿命をコード上で分離。 + +### S1:移植準備 + +- [x] Open/Size/Window の context 付き API と主要 HTTP 配信への伝搬。 +- [x] archive/preview/一覧サイズの直ファイルアクセスを置換し Materialize を導入。 +- [x] ZIP 個別配信・画像/ZIP サムネイルを Reader/ReaderAt ベースへ変更。 +- [x] 共有取得、キャンセル、容量/件数制限、破損、最終利用後の削除をテスト。 +- [ ] 従来の Revision hydration/保存、一覧 DB クエリを含む全 payload API へ context を伝搬。 +- [ ] Document の byte size を DB のメタデータとして確定し、外部 backend で一覧ごとの + Stat/HEAD が N+1 呼び出しにならないようにする(現状は既存のサイズ補完動作を維持)。 +- [ ] Materialize の固定上限を設定化し、クラッシュ残骸の安全な回収・メトリクスを接続。 +- [x] Document backfill の保存先 identity、UUID cursor、バッチ checkpoint、完了後の起動時走査省略。 +- [x] SQLite migration runner の version/checksum/排他/rollback と旧 hash 移行のメモリ使用改善。 +- [x] projection/resource/broadcast/browser-navigation/adapter-session/import の schema 定義集約、 + 既存データを保持した採用、条件付き列追加、schema assertion の契約テスト。 +- [x] event queue/log schema の管理、起動時ジョブ復旧と schema 処理の分離。 +- [x] 読み取り専用 `schema status|check` CLI(JSON/終了コード/履歴・契約検査/対象外範囲の明示)。 +- [ ] backfill を保守 CLI / event queue / 進捗 UI へ接続し、非同期実行時の配信契約を整備。 +- [ ] 既存 Ensure*Schema を migration runner へ段階移行し repository 別 SQL 契約テストを整備。 +- [ ] deployment profile で SQL / Document / canonical / cache / secret の設定を分離。 + +### S2:外部 backend + +- [ ] PostgreSQL の全 repository 移植と SQLite との共通機能テスト。 +- [ ] MySQL の DDL/索引/照合順序/同時書き込み試験。 +- [ ] S3 adapter と互換サービスで range/条件付き作成/破損/切断/容量制限を検証。 +- [ ] SFTP adapter と host-key 検証、NFS/SMB/SSHFS の契約検査と mount 喪失検知。 +- [ ] Redis cache adapter と停止時の制限付き縮退、policy epoch による失効。 +- [ ] mutable canonical CAS と metadata 参照の回復プロトコル、HA の fencing。 + +### S3:運用と移行 + +- [ ] 編集可能な設定例と薄い backup / release / deploy script。 +- [ ] DB/Document/calendar/keys を束ねる backup manifest、pin、verify、restore rehearsal。 +- [ ] 共通 online CLI、portable bundle export/import、plan/apply/retry。 +- [ ] Web バックアップ管理、初回ようこそ画面、ジョブ状態とリソース監視の接続。 + +完了条件は接続成功だけではなく、機能契約・故障注入・再起動・データ移行・ +バックアップからの復旧試験まで通ること。未実装 backend を「対応済み」と表示しない。 + +## 12. 根拠資料 + +- [SQLite WAL:network filesystem の制約](https://sqlite.org/wal.html) +- [S3 の conditional writes](https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-writes.html) +- [Redis persistence](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/) +- [PostgreSQL backup / restore](https://www.postgresql.org/docs/current/backup.html) +- [MySQL 8.4 backup / recovery](https://dev.mysql.com/doc/refman/8.4/en/backup-and-recovery.html) diff --git a/docs/design/system-extensions.md b/docs/design/system-extensions.md index e70998d..cdfbb10 100644 --- a/docs/design/system-extensions.md +++ b/docs/design/system-extensions.md @@ -51,6 +51,9 @@ installed → configured → enabled → disabled/failed の状態遷移を監 ## ビルトインのリソース監視 +収集と履歴の期間・保持ルールは [Seheweit](ja/seheweit.md) に初期分離済み。 +serverが認可とキュー状態を扱い、storeが保存する。URI・schema・監視周期は変更しない。 + - 定義ID `builtin.resource.sample`、イベント型 `resource.sample`。 - 起動時に未登録の場合のみ追加。既定60秒、変更可能範囲30〜86400秒。 - 既存キューのpriority=100、通常のclaim/履歴/ログ/キャンセルを利用する。 @@ -59,7 +62,8 @@ installed → configured → enabled → disabled/failed の状態遷移を監 同一jobの再実行は重複登録しない。書き込みと期限切れ削除は同一トランザクション。 - 7日保持。期限切れ削除は収集時に実行する。停止中に過去データを捏造して補間しない。 - `GET /_/api/v1/queue/resources/history?hours=1|24|168` は最大361点。 - timestamp索引で期間を絞り、バケット内の最終サンプルを返す。平均/最大値ではない。 + timestamp索引で期間を絞り、バケット内の最後に挿入されたサンプル(MAX(id))を返す。 + 時刻が逆転した入力では最大timestampとは限らず、平均/最大値ではない。 - 現在値APIと履歴を区別する。グラフの更新停止やブラウザを閉じる操作は収集を止めない。 収集停止は監視イベントのenabledまたはキューのpausedによる。 - ビルトイン定義は削除不可。無効化と間隔変更は再起動後も維持する。 diff --git a/docs/operations/deployments/2026-11-01-city-pages-cli.md b/docs/operations/deployments/2026-11-01-city-pages-cli.md new file mode 100644 index 0000000..88b76a7 --- /dev/null +++ b/docs/operations/deployments/2026-11-01-city-pages-cli.md @@ -0,0 +1,64 @@ +# 2026-11-01:City閲覧導線とpages-cliの反映 + +## 反映対象 + +2026-11-01 12:39 JST頃、Compose `app` / `app2` を切り替えた。連携アダプタは再作成していない。 + +- City検索結果とサジェストから、設定画面ではなく対応Pageまたは配下のブラウザ一覧へ遷移。 +- 生成された未整理Spotの既定名を親Streetの表示名へ射影。内部名・UUID・Path境界と独自タイトルは維持。 +- 親階層を含む閲覧権限の確認、匿名表示名の再計算、検索画面のモバイル幅調整。 +- `pages-cli` ソースをイメージの `/src/pages-cli/` に収録。CLIはホストの `/home/anya/.local/bin/pages` に配置。 + +直前のcommand版をベースに選択した31ファイルのみをCOPYしてビルドした。作業ツリー全体は含めない。電話連絡先、SQLite初期接続の後続変更は引き続き未反映。DBスキーマ・正本形式・認証設定の変更はない。 + +## イメージ・復旧点 + +- 反映版:`pageswiki-app:city-pages-cli-20261101` +- ID:`sha256:bed6b2d3e497e72c26feba4edf225fd9c57d49546fec43db2f88f297fcd989ee` +- 直前版:`pageswiki-app:rollback-before-city-pages-cli-20261101` +- 直前版ID:`sha256:d6c059486c238a1af0c9ebb2b2f3ed51c005a9670a703c55c077ef5a56ca1ec2` +- 限定ビルドレシピと配布ソース:`.deployment-backups/prod-city-pages-cli-20261101/{Dockerfile,overlay/}` +- 同ディレクトリの `reference/` は直前イメージのソース、`verify/` は反映対象を合成したテスト用ソース。どちらも本番データではない。 +- 既存の停止時整合バックアップは保持。今回はデータ契約が不変のため、新たな全データコピーは取得していない。 + +`pageswiki-app:latest` を反映版へ変更。Git commit/push、レジストリpush、バージョンタグ追加は行っていない。 + +## 検証 + +- 限定ソースと候補イメージそれぞれで `go test ./... -count=1` 成功。イメージの試験はネットワーク無効、CPU 2、メモリ2GiB。 +- Python単体テスト12件成功。ホストの限定ソースで `TestRemoteCLIHTTPIntegration` の実行・成功を個別に確認。ログイン/既定設定/一覧/編集/競合拒否/ログアウトを合成データで検証。イメージ内にPythonは追加しておらず、イメージ内の同テストはskipする。 +- 限定ソースのCity系・検索結果・コマンドバー/ハイライトのChromium試験成功。検索結果は1440px/390pxを検査。 +- 既存バックアップのリハーサル専用コピー2系統を、ネットワーク無効・workers無効で起動。healthおよび管理対象8コンポーネントのschema check成功。リハーサルコンテナは切替前に削除。 +- 本番2台のイメージID一致、両方running/再起動回数0、app healthy。 +- localhost3000/3001と公開HTTPSのhealth、公開ログインが200。 +- 公開HTTPSのCity関連JS3ファイルと検索CSSのSHA-256が配布元と一致。 +- 本番2台の管理対象schema check成功。切替直後のログでpanic/fatal/job failed/database locked/scheduler claim failedの該当0件。 + +外部取り込みカレンダーの内容にはアクセスしていない。上記は長期負荷試験や本番の全データ完全性検査を意味しない。 + +## CLIの利用 + +配置前に既存 `pages` コマンドがないことを確認。ユーザーPATHに含まれる `/home/anya/.local/bin/pages` に0755で配置し、helpとソースとのSHA-256一致を確認した。 + +SHA-256:`1a6feb7349a5b1b5b25fbb46cab1c7ac7dd7a46cbaab5aea803151305ac9da77` + +```sh +pages configure --origin https://pages.anya.jp --username anya +pages login +pages +``` + +利用者のプロファイルや資格情報は作成していない。本番資格情報による編集試験も行っていない。クライアントはPython 3.9以降のUnix系環境向け。別端末へのインストールは [pages-cli README](../../../pages-cli/README.md) を参照。OIDC引き継ぎ・Ctrl+P fuzzy UI・新規Page作成は後続。 + +## 今回の切り戻し + +今回の変更にDB/正本の移行はない。データを戻さず、直前のcommand版イメージへ切り替える。 + +```sh +docker image tag pageswiki-app:rollback-before-city-pages-cli-20261101 pageswiki-app:latest +docker compose up -d --no-deps --no-build app app2 +``` + +両イメージID、health、ログイン、再起動回数を確認する。これは今回の変更専用の手順で、保存基盤変更前の古い版には適用しない。 + +CLIはサーバから独立しているため、サーバ切り戻しで削除する必要はない。利用停止時も保存済みセッションや設定を無断で削除しない。 diff --git a/docs/operations/deployments/2026-11-01-command-ux.md b/docs/operations/deployments/2026-11-01-command-ux.md new file mode 100644 index 0000000..63dabf3 --- /dev/null +++ b/docs/operations/deployments/2026-11-01-command-ux.md @@ -0,0 +1,45 @@ +# 2026-11-01:コマンドバーと共通シンタックス色の本番反映 + +## 反映対象 + +2026-11-01 07:14 JST頃、Compose `app` / `app2` に反映。関連アダプタのコンテナは再作成しない。 + +- ブラウザ左下の `cd` / `vi` / `view` バー、文字入力によるフォーカス、TabによるPath補完。 +- 共通ナビがあるホーム以外の画面での `g` コマンドモード。編集中の入力/Vim操作は保護。 +- ブラウザのPathアドレス欄をtopbarへ統合。狭い画面では非表示。 +- エディタ、読み取りコードビューア、言語指定付きコードブロックのテーマ連動シンタックス色。 + +イメージは稼働版fix2をベースに、限定した26ファイルをCOPYして再ビルド。全作業ツリーをCOPYしない。電話連絡先およびSQLite初期接続の後続変更は含まない。今回の変更はスキーマ、正本形式、認証設定を変更しない。 + +## イメージと復旧点 + +- 反映版:`pageswiki-app:command-20261101` +- 反映版ID:`sha256:d6c059486c238a1af0c9ebb2b2f3ed51c005a9670a703c55c077ef5a56ca1ec2` +- 直前版:`pageswiki-app:rollback-before-command-20261101`(元タグ `pageswiki-app:ux-20261101-fix2`) +- 直前版ID:`sha256:aa4336dd677b27966df8732054a27b3544560c6fffb18b232e89d4f764b1551e` +- 限定ビルドレシピ:`.deployment-backups/prod-command-20261101/Dockerfile` +- 直前版ソース参照コピー:同ディレクトリの `reference/`。本番データではない。 +- 前回の停止時整合バックアップ/オンラインDBバックアップは保持。今回はDB変更がないため、新たな全データバックアップは取得していない。 + +現在の `pageswiki-app:latest` は反映版。Git commit/push、レジストリpush、リリースタグ追加は行っていない。 + +## 検証 + +- 候補イメージ内で `go test ./... -count=1` 成功。ネットワーク無効、CPU 2、メモリ2GiBで実行し、テストは合成データを使用。 +- 既存の本番バックアップのリハーサル用コピー2系統を使用。ネットワーク無効、workers無効で起動。両方のhealthと管理対象8コンポーネントのschema check成功。外部取り込みカレンダーの内容は開かない。検証コンテナは切替前に削除。 +- 実装時にChromiumの1440px/375px操作試験、ホーム・エディタ回帰試験済み。実機iPhone Safariやソフトウェアキーボードは未確認。 +- 本番両コンテナのイメージIDが反映版と一致。app healthy、両方running/再起動回数0。 +- localhost3000/3001および公開HTTPSのhealthが200。公開ログイン200。匿名の `/e/` は正規化後にログインページへ遷移。 +- 公開URLのコマンドJS、アドレスJS、ハイライトJS、コマンドCSS、テーマbase CSS、CodeMirror bundleのSHA-256がローカル配布ファイルと一致。 +- 切替後の両インスタンスで管理対象スキーマ検査成功。これは全データ完全性検査や長期負荷試験を意味しない。 + +## この反映だけを切り戻す場合 + +DB/正本の変更がないため、fix2へはデータを巻き戻さずイメージだけで戻す。ユーザの新しい書き込みを削除しない。 + +```sh +docker image tag pageswiki-app:rollback-before-command-20261101 pageswiki-app:latest +docker compose up -d --no-deps --no-build app app2 +``` + +切り戻し後は両イメージID、health、公開ログイン、再起動回数を確認する。この手順は今回のcommand版からfix2への切り戻し専用であり、保存基盤変更前の古い版には適用しない。 diff --git a/docs/operations/deployments/2026-11-01-priority-ux.md b/docs/operations/deployments/2026-11-01-priority-ux.md new file mode 100644 index 0000000..14bb5d1 --- /dev/null +++ b/docs/operations/deployments/2026-11-01-priority-ux.md @@ -0,0 +1,58 @@ +# 2026-11-01:優先UXの本番反映 + +## 対象と復旧点 + +- 対象:Compose `app` / `app2`。最新のUXに加え、作業ツリーにあった保存基盤・スキーマ管理・DR・エディタ復旧の変更を含む。 +- 旧イメージ:`sha256:3ee1b6b5ff24154c05990f02f112db7052b5a9bfb9894e5e9020a55fbb539d8c`。 +- 旧版タグ:`pageswiki-app:rollback-before-ux-20261101`。 +- 初回候補:`pageswiki-app:ux-20261101`、`sha256:1efb6fa94ae447e4769fa1056eb88fd34274cd17264646ad22a13327e05567b7`。隔離検証で旧索引との不一致を検出したため本番へは適用しない。 +- 修正候補:`pageswiki-app:ux-20261101-fix1`、`sha256:1cae0988fdc2a9d008fe971ed7f92bff9e4831b1798d080e4551b9efeca31958`。 +- 停止時整合バックアップ:`.deployment-backups/prod-ux-20261101/before/{data,data2}`。親ディレクトリ0700。アプリ2台を停止して全データをコピーし、DB本体の一致とsyncを確認。 +- ソースの復旧点:`.deployment-backups/dev-priority-20261031/`。Gitの未コミット変更は保持し、commit/push/tagは行わない。 + +## 実行基盤の障害と復旧 + +整合コピー後、旧コンテナの再開が `failed to create TTRPC connection: unsupported protocol` で失敗した。新コード起動・新migration適用の前に発生。 + +起動中のDocker 29.5.0 / containerd 2.2.3と、インストール済みDocker 29.8.2 / containerd-shim 2.3.6が不一致だった。実行中の `/proc/.../exe` は削除済み旧バイナリを参照していた。 + +このrootless Dockerの稼働コンテナがPagesWiki関連のみであることを確認し、`systemctl --user restart docker` で更新済み実行基盤へ切り替え、既存コンテナを再開。rootful Dockerやホスト全体の再起動は行っていない。旧アプリの起動中、一時的にapp2のenroll接続が拒否されたが、app起動後に回復した。 + +旧版復旧後、公開URLとlocalhostの3000/3001双方の `/health` が200、appのDocker healthがhealthyであることを確認した。 + +## 検証・反映の進捗 + +- [x] 全Goテスト成功、候補イメージのビルド成功。 +- [x] 旧版・停止時の正本/DB/鍵を保全。 +- [x] バックアップのコピーを `--network none`、workers無効で起動し、migration/backfillを検証。 +- [x] 管理対象8 componentがcurrent。双方のSQLite quick_check、メインDBのintegrity_check成功。本文backfillはメイン1,928件・外部側0件で完了。 +- [x] オブジェクト1,598/Revision1,928/Path1,093/Principal1件が移行前後で一致(外部側は各0件)。認証画面と新規静的アセットの応答を確認。 +- [x] 05:55 JST頃、停止時の追加バックアップ `cutover/{data,data2}` を取得(beforeから変更のないファイルのみrsync --link-destで共有、ライブデータへのhardlinkではない)。DB一致とsync後、fix1をlatestに付け替え、app/app2のみ再作成。 +- [x] 両インスタンスのヘルス200、app healthy、公開HTTPSのヘルス200・ログイン200・匿名エディタ303。ホーム入力検索/ブラウザアドレスJSの公開URLとローカルのSHA-256一致。両DBのschema check成功。 +- [x] 切替直後の監視で両アプリの再起動回数0。外部カレンダーの内容は開かずに確認。継続監視では05:59:59 JSTにscheduler claimのSQLITE_BUSYを1件検出(下記)。 + +外部取り込み済みカレンダーの内容は検証対象にしない。リハーサル用データから外部同期・通知送信を行わない。 + +この反映に電話連絡先の後続開発は含めない。最終反映済みイメージはfix2に固定し、以後の作業ツリーと区別する。追加機能の本番反映は別途検証して行う。 + +### スケジューラの継続検証 + +05:59:59 JSTに `event scheduler claim failed: database is locked (5)` が1件発生。以後の直近5分では38ジョブが成功しており、停止/ジョブ失敗ではなかった。旧版にもあった、deferred read transactionからwriteへの昇格競合を除去するため、`ClaimDueEvent` を `UPDATE ... WHERE uuid=(SELECT ...) RETURNING ...` の単一statementへ変更。優先度/実行可能時刻/attemptsを保持し、同時取得をSQLiteが直列化する。 + +8つの独立DBハンドル+並行する別書き込み、64ジョブの重複なし全件取得を10回検証。race検査も成功。新規schemaなし。fix1ベースに `internal/store/event_jobs.go` だけを差し替えた `pageswiki-app:ux-20261101-fix2`(`sha256:aa4336dd677b27966df8732054a27b3544560c6fffb18b232e89d4f764b1551e`)を構築。イメージ内の全Goテストと隔離データ2系統の起動・schema checkが成功。 + +fix2には電話連絡先/新規SQLite初期接続の後続変更は含まない。`Dockerfile.scheduler` はバックアップディレクトリに保存。切替前のオンラインSQLiteバックアップは `before-scheduler/` に保管(DB単体であり、全正本の整合復旧点はcutover)。fix2→fix1は同一schema/データ契約のためイメージのみで戻せる。cutoverへの全データ復旧とは区別する。 + +06:06 JST頃、fix2をlatestとしてapp/app2を再作成。両方の実行イメージIDを照合し、公開HTTPS/localhost3001のヘルス200、app healthy、再起動回数0を確認。初回確認期間でpanic/job failed/database locked該当ログ0件。長期負荷検証を完了した意味ではなく、過長な別writerによるSQLITE_BUSYまで根絶したとの主張ではない。 + +後続開発ソースの保全先:`.deployment-backups/dev-followup-20261101/source-after-contacts.tar.gz`。全Goテスト、電話連絡先のChromium操作、初期接続の反復試験はローカルで実施。アプリケーションDBと外部取り込みカレンダーの実データを後続機能テストには使用しない。 + +### リハーサルで修正した問題 + +旧DBの `idx_event_jobs_due(status,run_at_ms,created_at_ms)` が残るため、`CREATE INDEX IF NOT EXISTS` だけではpriority付き索引にならず、`event-jobs/2 assertion 12` に失敗した。未リリースの採用migrationで、この非一意索引をtransaction内で一度再構築する。両本番バックアップに既存の `schema_migration_steps` がないことを確認し、適用履歴を削除する回避は行っていない。 + +`TestEventJobsAdoptLegacyDueIndex` で旧索引からの移行、再実行、job本文・status・attempts・時刻の保持を検証。変更後の全Goテストも成功。 + +## ロールバック + +データ形式の互換性を確認する前に旧イメージだけへ戻さない。書き込みを停止し、新版稼働後のデータも別途保全したうえで、切替直前のDB・正本・鍵と旧イメージの組み合わせへ復元する。ユーザの新規書き込みを無断で破棄しない。詳細は [保存基盤反映手順](../storage-schema-rollout.md)。 diff --git a/docs/operations/storage-schema-rollout.md b/docs/operations/storage-schema-rollout.md new file mode 100644 index 0000000..ce83f1b --- /dev/null +++ b/docs/operations/storage-schema-rollout.md @@ -0,0 +1,124 @@ +# 保存基盤のスキーマ変更・反映手順 + +2026-09-27。保存基盤の段階移行に関する運用メモ。 +本書の作成時点では開発用の一時 DB で検証しており、本番 DB への適用を意味しない。 +詳細は [保存基盤の設計](../design/storage-portability-and-operations.md) を参照。 + +## 今回の変更と別の DB 操作との関係 + +別の DB 修正、復旧、import、DB-only CLI 書き込みと同時に適用しない。 +先に使用するバイナリ・スキーマ・保存先の組み合わせを決め、整合バックアップを取得する。 +ソースの実装完了と、本番への migration 適用完了は別の状態である。 + +アプリケーションを新しいコードで起動すると、以下を実行する。 + +1. `schema_migration_steps` に component/version/name/checksum/適用時刻を記録する。 +2. `document_backfills` と保存先の `.document-store-id` により、本文移行の進捗を管理する。 +3. プレビューのメタデータ、リソース履歴、アナウンス、ブラウザナビゲーション、 + アダプタセッション、インポート履歴、イベントキュー/ログを migration 管理下へ取り込む。 +4. 古い DB で不足していれば `browser_navigation_links.is_directory`、 + `adapter_sessions.root_path` / `principal_uuid`、`import_journal.completed` を追加する。 + `event_jobs` の既知の追加列も存在を検査してから追加し、ALTER の失敗を握りつぶさない。 +5. イベントキューのスキーマ検証後、通常起動時のみ中断ジョブを再キューイングする。 + 読み取り専用の事前チェックではこの復旧処理を呼ばない。 + +現在の既存コードですでに追加済みの列は重複して追加しない。 +対象テーブルの再作成、既存行の削除、本文の一括書き換えは行わない。 +旧式の `schema_migrations` は削除/リネームしない。 +認証・ACL、City、タスク、カレンダーを今回の 7 component に含めない。 +別に `document-store` component が本文移行の進捗テーブルを管理する(合計 8 component)。 +これらの既存起動時処理まで無変更になる、という意味ではない。 + +## 反映前の確認 + +- SQLite の整合バックアップと Document / calendar 正本 / 鍵を同じ復旧点として保全する。 + 稼働中の DB ファイルだけをコピーして完了としない。 +- DB とファイルを隔離したコピーを作り、新しいバイナリで起動試験する。 + 本番の FQDN、外部同期先、通知送信先へ接続しない環境で実施する。 +- 新旧スキーマからの採用テストと全体回帰テストの結果を確認する。 +- migration 定義の checksum 不一致、必要な列/索引/一意性制約の欠落があれば、 + 適用履歴を削除して回避せず、定義の変更か既存 DB の不整合かを調べる。 +- 長い SQL transaction、DB-only writer、旧プロセスによる保守操作を停止してから反映する。 + SQLite の `BEGIN IMMEDIATE` による短時間の排他が必要で、ロック競合はエラーになり得る。 +- 初回 backfill は保存先内の未移行データを検証するため時間がかかることがある。 + 起動前に完了させ、通常の配信を先に始めない。完了後の起動では全 Revision を走査しない。 + +## 読み取り専用で確認できる情報 + +### schema CLI + +新しいバイナリを用意した段階で、Web サーバーを起動せず実行できる。 + +```sh +./pageswiki schema status --data-dir /path/to/data +./pageswiki schema check --db /path/to/data/pages.db --json --timeout 30s +``` + +`--data-dir` を省略した場合は `PAGESWIKI_DATA_DIR`、未設定なら `./data` を使う。 +`--db` と明示的な `--data-dir` は同時指定しない。 + +- `status`:管理対象 component の適用済み/最新 version、未適用 step、契約検査結果を表示する。 + 読み取りに成功すれば終了コード 0。監視などで判定する場合は `check` を使う。 +- `check`:管理対象すべてが current なら 0、pending があれば 2、 + blocked/unknown があれば 1。DB 読み取り失敗や不正な引数も 1。 +- `blocked`:履歴の欠落・checksum 不一致・対応 version より新しい履歴、 + または適用済みスキーマが最新契約を満たしていない。 +- `unknown`:DB に記録された component をこのバイナリが認識しない。 +- `pending`:未適用の定義がある。まだ作られていない列/テーブルによる assertion の失敗も + 併記するが、DDL の実行成功・失敗を予測した結果ではない。 + +`mode=ro` と `query_only` による読み取り専用接続、一つの読み取り transaction を使う。 +DB/データディレクトリの作成、通常起動処理、鍵生成、本文 backfill、ジョブ復旧を実行しない。 +稼働中 DB の committed WAL も参照する。SQLite の WAL/SHM ファイルのアクセス要件は必要で、 +ファイルシステム上の一切のメタデータ変更がないことまで保証するモードではない。 + +**このコマンドは完全な移行リハーサルでも、全 DB の健全性検査でもない。** +出力の `coverage=managed-components-only` と `legacy_areas` を確認する。 +終了コード 0 でも、未移行の従来初期化処理、正本ファイル、データ依存の移行、 +外部接続の正常性は証明しない。隔離コピー上での起動試験を省略しない。 + +### SQL による補助確認 + +DB のパスは実際の起動設定で確認し、SQLite クライアントを read-only モードで使用する。 +次の SQL は状態確認用で、テーブルを作成しない。 + +```sql +SELECT name FROM sqlite_master +WHERE type='table' + AND name IN ('schema_migration_steps', 'document_backfills'); +``` + +テーブルが存在する場合のみ、続けて確認する。 + +```sql +SELECT component, version, name, checksum, applied_at_ms +FROM schema_migration_steps ORDER BY component, version; + +SELECT store_id, last_uuid, processed, completed, updated_at_ms +FROM document_backfills ORDER BY updated_at_ms DESC; +``` + +`completed=1` は「その保存先への初回移行が完了した」ことを示す。 +すべての正本の現在の健全性を証明する値ではない。欠落や破損は verify/repair の対象とする。 +`.document-store-id` は復旧対象に含める。空の保存先へ入れ替えた場合に、このファイルだけを +コピーして完了状態を引き継がせない。 + +## ロールバック・失敗時 + +DDL と migration 履歴の記録は同じ transaction で rollback する。 +Document backfill はバッチ単位で進捗を確定し、失敗したバッチは次回再実行する。 +すでに保存した不変 Document が残っていても、検証して再利用する。 + +起動失敗時には component/version とエラーを保存し、旧バイナリへの切り戻しを +無条件に行わない。旧バイナリが新しい migration 履歴を理解しない可能性があるため、 +DB と正本のバックアップからの復旧か、互換性を検証済みの切り戻しを選ぶ。 +DR standby ではこれらの schema 作成・採用を実行しない。 + +## 残件 + +- 全 Ensure*Schema の集約と、型/CHECK/照合順序/外部キーまで含む schema 契約の拡充。 +- DB ごとの差異を切り分け、PostgreSQL / MySQL の共通機能テストを整備。 +- 本文の健全性/backfill 状態の専用診断、保守 CLI、進捗 UI と event queue の接続。 +- 初回 backfill の非同期化に先立つ、未移行データの配信と readiness の設計。 + +外部 DB への接続や、本番 DB の移行は上記を完了・検証した段階で別途実施する。 diff --git a/internal/compiler/authoring.go b/internal/compiler/authoring.go new file mode 100644 index 0000000..eb3d96e --- /dev/null +++ b/internal/compiler/authoring.go @@ -0,0 +1,15 @@ +package compiler + +import "pageswiki/internal/grammaratur" + +// HTMLToMarkdownAuthoring delegates loss-preserving conversion to Grammaratur. +// Deprecated: use grammaratur.HTMLToMarkdownAuthoring in new application code. +func HTMLToMarkdownAuthoring(source string) (string, error) { + return grammaratur.HTMLToMarkdownAuthoring(source) +} + +// MarkdownAuthoringHTML preserves the existing authoring entry point. +// Deprecated: use grammaratur.MarkdownAuthoringHTML in new application code. +func MarkdownAuthoringHTML(source string) string { + return grammaratur.MarkdownAuthoringHTML(source) +} diff --git a/internal/compiler/diagram.go b/internal/compiler/diagram.go index 9046192..3ddfa7f 100644 --- a/internal/compiler/diagram.go +++ b/internal/compiler/diagram.go @@ -1,24 +1,8 @@ package compiler -import ( - "crypto/sha256" - "encoding/hex" -) +import "pageswiki/internal/kunstzeal" -func DiagramHash( - diagramType string, - source string, - format string, -) string { - - sum := - sha256.Sum256( - []byte( - diagramType + "\x00" + format + "\x00" + source, - ), - ) - - return hex.EncodeToString( - sum[:], - ) +// DiagramHash preserves the legacy key format through the rendering component. +func DiagramHash(diagramType, source, format string) string { + return kunstzeal.DiagramHash(diagramType, source, format) } diff --git a/internal/compiler/markdown.go b/internal/compiler/markdown.go index d74e420..f9eb17e 100644 --- a/internal/compiler/markdown.go +++ b/internal/compiler/markdown.go @@ -1,14 +1,14 @@ package compiler -import ( - htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2" -) +import "pageswiki/internal/grammaratur" +// HTMLToMarkdown preserves the legacy conversion contract (without protection +// fences). Deprecated: use grammaratur.HTMLToMarkdown in new application code. func HTMLToMarkdown( html string, ) (string, error) { - return htmltomarkdown.ConvertString( + return grammaratur.HTMLToMarkdown( html, ) } diff --git a/internal/db/db.go b/internal/db/db.go index 59a3cbe..cc80692 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -1,9 +1,12 @@ package db import ( + "context" "database/sql" + "fmt" "net/url" "path/filepath" + "time" _ "modernc.org/sqlite" ) @@ -24,7 +27,20 @@ func Open(path string) (*sql.DB, error) { }, }.Encode(), }).String() - return sql.Open("sqlite", dsn) + database, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, err + } + // sql.Open is lazy. Establish WAL once before exposing the pool, otherwise + // simultaneous first borrowers race the initial journal_mode transition in + // their connection pragmas (SQLITE_BUSY before BEGIN IMMEDIATE is reached). + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := database.PingContext(ctx); err != nil { + database.Close() + return nil, fmt.Errorf("initialize SQLite connection: %w", err) + } + return database, nil } func Init(db *sql.DB) error { diff --git a/internal/db/migrate.go b/internal/db/migrate.go index a61ee3c..77c6057 100644 --- a/internal/db/migrate.go +++ b/internal/db/migrate.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "database/sql" "encoding/hex" + "fmt" ) func Migrate( @@ -96,54 +97,21 @@ func Migrate( } } - alterStmts := []string{ - - ` - ALTER TABLE paths - ADD COLUMN created_at INTEGER - `, - - ` - ALTER TABLE objects - ADD COLUMN created_at INTEGER - `, - - ` - ALTER TABLE revisions - ADD COLUMN created_at INTEGER - `, - - ` - ALTER TABLE revisions - ADD COLUMN document_hash TEXT - `, - - ` - ALTER TABLE revisions - ADD COLUMN lang TEXT NOT NULL DEFAULT 'und' - `, - - ` - ALTER TABLE revisions - ADD COLUMN created_at_ms INTEGER - `, - ` - ALTER TABLE revisions - ADD COLUMN content_type TEXT NOT NULL DEFAULT 'text/html; charset=utf-8' - `, - ` - ALTER TABLE revisions - ADD COLUMN filename TEXT NOT NULL DEFAULT '' - `, - - ` - ALTER TABLE tags - ADD COLUMN created_at INTEGER - `, + columns := []struct{ table, name, declaration string }{ + {"paths", "created_at", "INTEGER"}, + {"objects", "created_at", "INTEGER"}, + {"revisions", "created_at", "INTEGER"}, + {"revisions", "document_hash", "TEXT"}, + {"revisions", "lang", "TEXT NOT NULL DEFAULT 'und'"}, + {"revisions", "created_at_ms", "INTEGER"}, + {"revisions", "content_type", "TEXT NOT NULL DEFAULT 'text/html; charset=utf-8'"}, + {"revisions", "filename", "TEXT NOT NULL DEFAULT ''"}, + {"tags", "created_at", "INTEGER"}, } - - for _, stmt := range alterStmts { - _, _ = db.Exec(stmt) + for _, column := range columns { + if err := ensureSQLiteColumn(db, column.table, column.name, column.declaration); err != nil { + return err + } } if err := migrateRevisionMetadata(db); err != nil { @@ -185,54 +153,47 @@ func migrateRevisionMetadata(db *sql.DB) error { return nil } - rows, err := db.Query(` - SELECT - uuid, - COALESCE(body, '') - FROM revisions - WHERE document_hash IS NULL - OR document_hash = '' - `) - if err != nil { - return err - } - - type legacyRevision struct { - uuid string - body string - } - - var revisions []legacyRevision - - for rows.Next() { - var revision legacyRevision - - if err := rows.Scan(&revision.uuid, &revision.body); err != nil { + // Fetch only identifiers in batches, then hydrate one legacy body at a + // time. The old implementation accumulated all payloads in memory. + lastUUID := "" + for { + rows, err := db.Query(`SELECT uuid FROM revisions WHERE uuid>? AND (document_hash IS NULL OR document_hash='') ORDER BY uuid LIMIT 128`, lastUUID) + if err != nil { + return err + } + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + rows.Close() + return err + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { rows.Close() return err } - - revisions = append(revisions, revision) - } - - if err := rows.Close(); err != nil { - return err - } - - for _, revision := range revisions { - sum := sha256.Sum256([]byte(revision.body)) - documentHash := hex.EncodeToString(sum[:]) - - if _, err := db.Exec(` - UPDATE revisions - SET document_hash = ? - WHERE uuid = ? - `, documentHash, revision.uuid); err != nil { + if err := rows.Close(); err != nil { return err } + for _, id := range ids { + var body string + if err := db.QueryRow(`SELECT COALESCE(body,'') FROM revisions WHERE uuid=?`, id).Scan(&body); err != nil { + return err + } + sum := sha256.Sum256([]byte(body)) + if _, err := db.Exec(`UPDATE revisions SET document_hash=? WHERE uuid=? AND (document_hash IS NULL OR document_hash='')`, hex.EncodeToString(sum[:]), id); err != nil { + return err + } + lastUUID = id + } + if len(ids) < 128 { + break + } } - if _, err = db.Exec(` + if _, err := db.Exec(` UPDATE revisions SET lang = COALESCE(NULLIF(lang, ''), 'und'), @@ -247,7 +208,7 @@ func migrateRevisionMetadata(db *sql.DB) error { return err } - if _, err = db.Exec(` + if _, err := db.Exec(` CREATE INDEX IF NOT EXISTS idx_revisions_object_created ON revisions ( object_uuid, @@ -257,9 +218,45 @@ func migrateRevisionMetadata(db *sql.DB) error { return err } - _, err = db.Exec(` + _, err := db.Exec(` INSERT OR IGNORE INTO schema_migrations (name, applied_at) VALUES ('revision_metadata_v1', strftime('%s','now')) `) return err } + +func sqliteColumnExists(database *sql.DB, table, column string) (bool, error) { + // All identifiers originate in the fixed legacy bootstrap above. + rows, err := database.Query(fmt.Sprintf("PRAGMA table_info(%q)", table)) + if err != nil { + return false, err + } + defer rows.Close() + found := false + for rows.Next() { + var cid, notNull, primaryKey int + var name, kind string + var defaultValue any + if err := rows.Scan(&cid, &name, &kind, ¬Null, &defaultValue, &primaryKey); err != nil { + return false, err + } + found = found || name == column + } + return found, rows.Err() +} + +func ensureSQLiteColumn(database *sql.DB, table, column, declaration string) error { + exists, err := sqliteColumnExists(database, table, column) + if err != nil || exists { + return err + } + _, err = database.Exec(fmt.Sprintf("ALTER TABLE %q ADD COLUMN %q %s", table, column, declaration)) + if err != nil { + // Only tolerate another bootstrap adding the exact column concurrently. + if exists, checkErr := sqliteColumnExists(database, table, column); checkErr == nil && exists { + return nil + } + return fmt.Errorf("add column %s.%s: %w", table, column, err) + } + return nil +} diff --git a/internal/db/migration_inspect.go b/internal/db/migration_inspect.go new file mode 100644 index 0000000..b5f9af6 --- /dev/null +++ b/internal/db/migration_inspect.go @@ -0,0 +1,139 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + "sort" +) + +type MigrationPlanStep struct { + Version int `json:"version"` + Name string `json:"name"` + Statements int `json:"statements"` + AddColumns []SQLiteColumn `json:"add_columns,omitempty"` +} + +type MigrationInspection struct { + Component string `json:"component"` + State string `json:"state"` // current, pending, blocked, unknown + AppliedVersion int `json:"applied_version"` + LatestVersion int `json:"latest_version"` + Pending []MigrationPlanStep `json:"pending,omitempty"` + ContractChecked bool `json:"contract_checked"` + Issues []string `json:"issues,omitempty"` +} + +// InspectSQLiteMigrations takes a read transaction and never applies DDL/DML. +// Pending definitions are a plan, not a successful rehearsal: data-dependent +// DDL and assertions on not-yet-created objects require a disposable DB copy. +func InspectSQLiteMigrations(ctx context.Context, database *sql.DB, plans map[string][]SQLMigration) ([]MigrationInspection, error) { + tx, err := database.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return nil, err + } + defer tx.Rollback() + var ledger bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_migration_steps')`).Scan(&ledger); err != nil { + return nil, err + } + type record struct { + version int + name, checksum string + } + history := map[string][]record{} + if ledger { + rows, err := tx.QueryContext(ctx, `SELECT component,version,name,checksum FROM schema_migration_steps ORDER BY component,version`) + if err != nil { + return nil, err + } + for rows.Next() { + var component string + var r record + if err := rows.Scan(&component, &r.version, &r.name, &r.checksum); err != nil { + rows.Close() + return nil, err + } + history[component] = append(history[component], r) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + if err := rows.Close(); err != nil { + return nil, err + } + } + names := make([]string, 0, len(plans)+len(history)) + for name := range plans { + names = append(names, name) + } + for name := range history { + if _, ok := plans[name]; !ok { + names = append(names, name) + } + } + sort.Strings(names) + result := make([]MigrationInspection, 0, len(names)) + for _, name := range names { + if err := ctx.Err(); err != nil { + return nil, err + } + steps, known := plans[name] + item := MigrationInspection{Component: name, State: "current", LatestVersion: len(steps)} + for i, step := range steps { + if step.Version != i+1 || step.Name == "" || (len(step.Statements) == 0 && len(step.AddColumns) == 0 && len(step.Assertions) == 0) { + return nil, fmt.Errorf("invalid migration definition %s/%d", name, step.Version) + } + } + if !known { + item.State = "unknown" + item.Issues = []string{"component is not registered in this binary"} + } + for i, record := range history[name] { + item.AppliedVersion = record.version + if known && (record.version != i+1 || record.version > len(steps) || record.name != steps[record.version-1].Name || record.checksum != migrationChecksum(steps[record.version-1])) { + item.State = "blocked" + item.Issues = append(item.Issues, fmt.Sprintf("migration history mismatch at version %d", record.version)) + } + } + if item.State == "current" { + for _, step := range steps[len(history[name]):] { + item.Pending = append(item.Pending, MigrationPlanStep{Version: step.Version, Name: step.Name, Statements: len(step.Statements), AddColumns: step.AddColumns}) + } + if len(item.Pending) > 0 { + item.State = "pending" + } + if len(steps) > 0 && len(steps[len(steps)-1].Assertions) > 0 { + item.ContractChecked = true + for i, assertion := range steps[len(steps)-1].Assertions { + rows, err := tx.QueryContext(ctx, assertion) + if err != nil { + item.Issues = append(item.Issues, fmt.Sprintf("assertion %d: %v", i+1, err)) + continue + } + violated := rows.Next() + err = rows.Err() + closeErr := rows.Close() + if err != nil { + return nil, err + } + if closeErr != nil { + return nil, closeErr + } + if violated { + item.Issues = append(item.Issues, fmt.Sprintf("assertion %d failed", i+1)) + } + } + if len(item.Issues) > 0 && item.State == "current" { + item.State = "blocked" + } + } + } + result = append(result, item) + } + if err := ctx.Err(); err != nil { + return nil, err + } + return result, tx.Commit() +} diff --git a/internal/db/migration_inspect_test.go b/internal/db/migration_inspect_test.go new file mode 100644 index 0000000..f22475e --- /dev/null +++ b/internal/db/migration_inspect_test.go @@ -0,0 +1,121 @@ +package db + +import ( + "context" + "errors" + "path/filepath" + "testing" +) + +func TestInspectMigrationsReadOnlyLifecycle(t *testing.T) { + path := filepath.Join(t.TempDir(), "inspect.db") + writer, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer writer.Close() + if err := writer.Ping(); err != nil { + t.Fatal(err) + } + reader, err := OpenReadOnly(path) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample(id INTEGER PRIMARY KEY, value TEXT)`, `INSERT INTO sample VALUES(1,'keep')`}, Assertions: []string{`SELECT id,value FROM sample LIMIT 0`}}} + plans := map[string][]SQLMigration{"sample": steps} + inspect := func(state string, issues bool) { + t.Helper() + got, err := InspectSQLiteMigrations(context.Background(), reader, plans) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].State != state || (len(got[0].Issues) > 0) != issues || !got[0].ContractChecked { + t.Fatalf("report: %+v", got) + } + } + inspect("pending", true) + var count int + if err := writer.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name IN ('sample','schema_migration_steps')`).Scan(&count); err != nil || count != 0 { + t.Fatalf("inspection created tables: %d %v", count, err) + } + if _, err := reader.Exec(`CREATE TABLE forbidden(id INTEGER)`); err == nil { + t.Fatal("connection is writable") + } + // Keep the writer open: the reader must see committed WAL contents too. + if err := ApplySQLiteMigrations(context.Background(), writer, "sample", steps); err != nil { + t.Fatal(err) + } + inspect("current", false) + if _, err := writer.Exec(`ALTER TABLE sample RENAME COLUMN value TO lost`); err != nil { + t.Fatal(err) + } + inspect("blocked", true) + var value string + if err := writer.QueryRow(`SELECT lost FROM sample WHERE id=1`).Scan(&value); err != nil || value != "keep" { + t.Fatalf("inspection changed data: %q %v", value, err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := InspectSQLiteMigrations(ctx, reader, plans); !errors.Is(err, context.Canceled) { + t.Fatalf("cancel: %v", err) + } +} + +func TestInspectMigrationsRejectHistoryMismatch(t *testing.T) { + for _, tc := range []struct{ name, mutation string }{ + {"checksum", `UPDATE schema_migration_steps SET checksum='changed'`}, + {"name", `UPDATE schema_migration_steps SET name='changed'`}, + {"zero", `UPDATE schema_migration_steps SET version=0`}, + {"negative", `UPDATE schema_migration_steps SET version=-1`}, + {"newer", `UPDATE schema_migration_steps SET version=2`}, + } { + t.Run(tc.name, func(t *testing.T) { + database, err := Open(filepath.Join(t.TempDir(), "inspect.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample(id INTEGER)`}}} + if err := ApplySQLiteMigrations(context.Background(), database, "sample", steps); err != nil { + t.Fatal(err) + } + if _, err := database.Exec(tc.mutation); err != nil { + t.Fatal(err) + } + got, err := InspectSQLiteMigrations(context.Background(), database, map[string][]SQLMigration{"sample": steps}) + if err != nil || len(got) != 1 || got[0].State != "blocked" { + t.Fatalf("report=%+v err=%v", got, err) + } + }) + } +} + +func TestInspectMigrationsUnknownAndPendingPlan(t *testing.T) { + database, err := Open(filepath.Join(t.TempDir(), "inspect.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample(id INTEGER)`}}} + if err := ApplySQLiteMigrations(context.Background(), database, "sample", steps); err != nil { + t.Fatal(err) + } + got, err := InspectSQLiteMigrations(context.Background(), database, nil) + if err != nil || len(got) != 1 || got[0].State != "unknown" { + t.Fatalf("unknown=%+v %v", got, err) + } + steps = append(steps, SQLMigration{Version: 2, Name: "add_value", AddColumns: []SQLiteColumn{{Table: "sample", Name: "value", Declaration: "TEXT NOT NULL DEFAULT ''"}}, Assertions: []string{`SELECT value FROM sample LIMIT 0`}}) + got, err = InspectSQLiteMigrations(context.Background(), database, map[string][]SQLMigration{"sample": steps}) + if err != nil || len(got) != 1 { + t.Fatalf("pending=%+v %v", got, err) + } + item := got[0] + if item.State != "pending" || item.AppliedVersion != 1 || item.LatestVersion != 2 || len(item.Pending) != 1 || len(item.Pending[0].AddColumns) != 1 || len(item.Issues) == 0 { + t.Fatalf("pending=%+v", item) + } + var count int + if err := database.QueryRow(`SELECT count(*) FROM pragma_table_info('sample') WHERE name='value'`).Scan(&count); err != nil || count != 0 { + t.Fatalf("pending DDL applied: %d %v", count, err) + } +} diff --git a/internal/db/migration_runner.go b/internal/db/migration_runner.go new file mode 100644 index 0000000..1586016 --- /dev/null +++ b/internal/db/migration_runner.go @@ -0,0 +1,188 @@ +package db + +import ( + "context" + "crypto/sha256" + "database/sql" + "database/sql/driver" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + "time" +) + +// SQLMigration is append-only once released. Edit a new version, never an +// applied statement. This runner is SQLite-specific; other DBs need their own +// locking/DDL implementations rather than a driver-name substitution. +type SQLMigration struct { + Version int + Name string + Statements []string + // Omit empty new fields to preserve checksums of previously released steps. + AddColumns []SQLiteColumn `json:",omitempty"` + // Each assertion must execute successfully and return no rows. The latest + // version defines the whole current contract; older assertions are superseded + // to permit deliberate column removal in a later, immutable migration. + Assertions []string `json:",omitempty"` +} + +type SQLiteColumn struct { + Table string + Name string + Declaration string +} + +func migrationChecksum(step SQLMigration) string { + encoded, _ := json.Marshal(step) + sum := sha256.Sum256(encoded) + return hex.EncodeToString(sum[:]) +} + +// ApplySQLiteMigrations serializes processes on a dedicated connection with +// BEGIN IMMEDIATE. DDL, data statements and ledger updates commit together. +// The legacy schema_migrations table remains untouched for old installations. +func ApplySQLiteMigrations(ctx context.Context, database *sql.DB, component string, steps []SQLMigration) error { + if component == "" { + return fmt.Errorf("migration component is required") + } + for i, step := range steps { + if step.Version != i+1 || step.Name == "" || (len(step.Statements) == 0 && len(step.AddColumns) == 0 && len(step.Assertions) == 0) { + return fmt.Errorf("invalid migration sequence for %s", component) + } + } + conn, err := database.Conn(ctx) + if err != nil { + return fmt.Errorf("migration %s acquire connection: %w", component, err) + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil { + return fmt.Errorf("migration %s begin immediate: %w", component, err) + } + committed := false + defer func() { + if committed { + return + } + cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := conn.ExecContext(cleanup, "ROLLBACK"); err != nil { + // Never return a connection with uncertain transaction state to + // the pool. A failed/ambiguous COMMIT also takes this path. + _ = conn.Raw(func(any) error { return driver.ErrBadConn }) + } + }() + if _, err := conn.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_migration_steps ( + component TEXT NOT NULL, version INTEGER NOT NULL, name TEXT NOT NULL, + checksum TEXT NOT NULL, applied_at_ms INTEGER NOT NULL, + PRIMARY KEY(component, version))`); err != nil { + return err + } + rows, err := conn.QueryContext(ctx, `SELECT version,name,checksum FROM schema_migration_steps WHERE component=? ORDER BY version`, component) + if err != nil { + return err + } + applied := 0 + for rows.Next() { + var version int + var name, checksum string + if err := rows.Scan(&version, &name, &checksum); err != nil { + rows.Close() + return err + } + if version != applied+1 || version > len(steps) || steps[version-1].Name != name || migrationChecksum(steps[version-1]) != checksum { + rows.Close() + return fmt.Errorf("migration history mismatch: component=%s version=%d (changed, missing or newer migration)", component, version) + } + applied++ + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + if err := rows.Close(); err != nil { + return err + } + for _, step := range steps[applied:] { + for i, statement := range step.Statements { + if _, err := conn.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("migration %s/%d %s statement %d: %w", component, step.Version, step.Name, i+1, err) + } + } + for _, column := range step.AddColumns { + if err := addSQLiteColumn(ctx, conn, column); err != nil { + return fmt.Errorf("migration %s/%d column %s.%s: %w", component, step.Version, column.Table, column.Name, err) + } + } + if _, err := conn.ExecContext(ctx, `INSERT INTO schema_migration_steps(component,version,name,checksum,applied_at_ms) VALUES(?,?,?,?,?)`, component, step.Version, step.Name, migrationChecksum(step), time.Now().UnixMilli()); err != nil { + return err + } + } + if len(steps) > 0 { + step := steps[len(steps)-1] + for i, assertion := range step.Assertions { + rows, err := conn.QueryContext(ctx, assertion) + if err != nil { + return fmt.Errorf("schema contract %s/%d assertion %d: %w", component, step.Version, i+1, err) + } + violated := rows.Next() + err = rows.Err() + closeErr := rows.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + if violated { + return fmt.Errorf("schema contract %s/%d assertion %d failed", component, step.Version, i+1) + } + } + } + _, err = conn.ExecContext(ctx, "COMMIT") + committed = err == nil + return err +} + +func addSQLiteColumn(ctx context.Context, conn *sql.Conn, column SQLiteColumn) error { + // Identifiers are trusted migration definitions, but validate before SQL + // interpolation so accidental punctuation cannot change the statement. + for _, identifier := range []string{column.Table, column.Name} { + if identifier == "" || strings.IndexFunc(identifier, func(r rune) bool { + return !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_') + }) >= 0 { + return fmt.Errorf("invalid schema identifier") + } + } + if strings.TrimSpace(column.Declaration) == "" { + return fmt.Errorf("column declaration is required") + } + rows, err := conn.QueryContext(ctx, `SELECT name FROM pragma_table_info(?)`, column.Table) + if err != nil { + return err + } + found, tableExists := false, false + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + return err + } + found, tableExists = found || name == column.Name, true + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + if err := rows.Close(); err != nil { + return err + } + if !tableExists { + return fmt.Errorf("table %s is missing", column.Table) + } + if found { + return nil + } + _, err = conn.ExecContext(ctx, fmt.Sprintf("ALTER TABLE %q ADD COLUMN %q %s", column.Table, column.Name, column.Declaration)) + return err +} diff --git a/internal/db/migration_runner_test.go b/internal/db/migration_runner_test.go new file mode 100644 index 0000000..26efd73 --- /dev/null +++ b/internal/db/migration_runner_test.go @@ -0,0 +1,150 @@ +package db + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "path/filepath" + "strings" + "sync" + "testing" +) + +func TestSQLMigrationLedgerConcurrencyAndChecksum(t *testing.T) { + database, err := Open(filepath.Join(t.TempDir(), "migration.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + steps := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample (id INTEGER PRIMARY KEY)`, `INSERT INTO sample VALUES (1)`}}} + var wg sync.WaitGroup + errors := make(chan error, 6) + for i := 0; i < 6; i++ { + wg.Add(1) + go func() { + defer wg.Done() + errors <- ApplySQLiteMigrations(context.Background(), database, "test", steps) + }() + } + wg.Wait() + close(errors) + for err := range errors { + if err != nil { + t.Fatal(err) + } + } + var count int + if err := database.QueryRow(`SELECT count(*) FROM sample`).Scan(&count); err != nil || count != 1 { + t.Fatalf("count=%d err=%v", count, err) + } + changed := []SQLMigration{{Version: 1, Name: "initial", Statements: []string{`CREATE TABLE sample (id TEXT)`}}} + if err := ApplySQLiteMigrations(context.Background(), database, "test", changed); err == nil || !strings.Contains(err.Error(), "history mismatch") { + t.Fatalf("changed migration accepted: %v", err) + } + if err := ApplySQLiteMigrations(context.Background(), database, "test", nil); err == nil { + t.Fatal("downgrade accepted") + } + if err := ApplySQLiteMigrations(context.Background(), database, "other", []SQLMigration{{Version: 2, Name: "gap", Statements: []string{`SELECT 1`}}}); err == nil { + t.Fatal("version gap accepted") + } +} + +func TestSQLMigrationFailureRollsBackDDLAndLedger(t *testing.T) { + database, err := Open(filepath.Join(t.TempDir(), "migration.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + steps := []SQLMigration{{Version: 1, Name: "broken", Statements: []string{`CREATE TABLE partial (id INTEGER)`, `NOT VALID SQL`}}} + if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err == nil { + t.Fatal("DDL failure ignored") + } + var count int + if err := database.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name IN ('partial','schema_migration_steps')`).Scan(&count); err != nil || count != 0 { + t.Fatalf("partial migration remains count=%d err=%v", count, err) + } + steps[0].Statements = steps[0].Statements[:1] + if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil { + t.Fatal("retry after rollback", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := ApplySQLiteMigrations(ctx, database, "test", steps); err == nil { + t.Fatal("cancel ignored") + } +} + +func TestLegacyRevisionHashMigrationSpansBoundedBatches(t *testing.T) { + database, err := Open(filepath.Join(t.TempDir(), "legacy.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if _, err := database.Exec(`CREATE TABLE revisions (uuid TEXT PRIMARY KEY,object_uuid TEXT,body TEXT,message TEXT)`); err != nil { + t.Fatal(err) + } + for i := 0; i < 260; i++ { + if _, err := database.Exec(`INSERT INTO revisions(uuid,body) VALUES(?,?)`, fmt.Sprintf("revision-%04d", i), "body"); err != nil { + t.Fatal(err) + } + } + if err := Migrate(database); err != nil { + t.Fatal(err) + } + var count int + if err := database.QueryRow(`SELECT count(*) FROM revisions WHERE length(document_hash)=64 AND lang='und' AND created_at_ms=0`).Scan(&count); err != nil || count != 260 { + t.Fatalf("migrated=%d err=%v", count, err) + } + if err := ensureSQLiteColumn(database, "does_not_exist", "created_at", "INTEGER"); err == nil { + t.Fatal("ALTER failure swallowed") + } +} + +func TestMigrationChecksumPreservesLegacyFormat(t *testing.T) { + step := SQLMigration{Version: 1, Name: "legacy", Statements: []string{"SELECT 1"}} + sum := sha256.Sum256([]byte(`{"Version":1,"Name":"legacy","Statements":["SELECT 1"]}`)) + if got := migrationChecksum(step); got != hex.EncodeToString(sum[:]) { + t.Fatalf("legacy checksum changed: %s", got) + } +} + +func TestMigrationConditionalColumnsAndAssertionsAreAtomic(t *testing.T) { + database, err := Open(filepath.Join(t.TempDir(), "schema.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if _, err := database.Exec(`CREATE TABLE sample(id INTEGER PRIMARY KEY); INSERT INTO sample VALUES(1)`); err != nil { + t.Fatal(err) + } + steps := []SQLMigration{{Version: 1, Name: "adopt", AddColumns: []SQLiteColumn{{Table: "sample", Name: "value", Declaration: "TEXT NOT NULL DEFAULT ''"}}, Assertions: []string{`SELECT missing FROM sample LIMIT 0`}}} + if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err == nil { + t.Fatal("invalid schema accepted") + } + var count int + if err := database.QueryRow(`SELECT count(*) FROM pragma_table_info('sample') WHERE name='value'`).Scan(&count); err != nil || count != 0 { + t.Fatalf("ALTER was not rolled back: %d %v", count, err) + } + steps[0].Assertions = []string{`SELECT id,value FROM sample LIMIT 0`} + if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil { + t.Fatal(err) + } + if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil { + t.Fatal("repeated apply", err) + } + var value string + if err := database.QueryRow(`SELECT value FROM sample WHERE id=1`).Scan(&value); err != nil || value != "" { + t.Fatalf("legacy row lost: %q %v", value, err) + } + steps = append(steps, SQLMigration{Version: 2, Name: "remove_value", Statements: []string{`ALTER TABLE sample DROP COLUMN value`}, Assertions: []string{`SELECT id FROM sample LIMIT 0`}}) + if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err != nil { + t.Fatal("new contract must supersede old contract", err) + } + if _, err := database.Exec(`DROP TABLE sample`); err != nil { + t.Fatal(err) + } + if err := ApplySQLiteMigrations(context.Background(), database, "test", steps); err == nil { + t.Fatal("recorded history masked missing table") + } +} diff --git a/internal/db/readonly.go b/internal/db/readonly.go new file mode 100644 index 0000000..a297dbd --- /dev/null +++ b/internal/db/readonly.go @@ -0,0 +1,25 @@ +package db + +import ( + "database/sql" + "net/url" + "path/filepath" +) + +// OpenReadOnly deliberately does not migrate, set journal_mode, or create a DB. +func OpenReadOnly(path string) (*sql.DB, error) { + path, err := filepath.Abs(path) + if err != nil { + return nil, err + } + dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: url.Values{"mode": {"ro"}, "_pragma": {"query_only(1)", "busy_timeout(5000)"}}.Encode()}).String() + database, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, err + } + if err = database.Ping(); err != nil { + database.Close() + return nil, err + } + return database, nil +} diff --git a/internal/documentstore/compatibility_test.go b/internal/documentstore/compatibility_test.go new file mode 100644 index 0000000..75d0991 --- /dev/null +++ b/internal/documentstore/compatibility_test.go @@ -0,0 +1,176 @@ +package documentstore_test + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "pageswiki/internal/documentstore" + "pageswiki/internal/geduldh" +) + +// These assignments intentionally fail compilation if aliases become wrappers +// with different type identities or constructor signatures. +var ( + _ func(string) (*geduldh.Local, error) = documentstore.NewLocal + _ func(documentstore.Source, documentstore.MaterializeOptions) (*documentstore.Materializer, error) = geduldh.NewMaterializer + _ documentstore.Store = (*geduldh.Local)(nil) + _ geduldh.Store = (*documentstore.Local)(nil) +) + +func TestGeduldhLegacyLayoutAndSharedIdentity(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + body := []byte("existing Document 日本語\x00\xff") + sum := sha256.Sum256(body) + hash := hex.EncodeToString(sum[:]) + // Seed the pre-extraction layout independently of either Key implementation. + filename := filepath.Join(root, "sha256", hash[:2], hash) + if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filename, body, 0644); err != nil { + t.Fatal(err) + } + before, err := os.Stat(filename) + if err != nil { + t.Fatal(err) + } + old, err := documentstore.NewLocal(root) + if err != nil { + t.Fatal(err) + } + current, err := geduldh.NewLocal(root) + if err != nil { + t.Fatal(err) + } + for _, backend := range []geduldh.Store{old, current} { + // The old normalization rules and payload bytes stay unchanged. + inputHash := " " + strings.ToUpper(hash) + " " + info, err := backend.Stat(ctx, inputHash) + if err != nil || info.Hash != hash || info.Size != int64(len(body)) { + t.Fatalf("stat=%+v err=%v", info, err) + } + reader, err := backend.Open(ctx, inputHash) + if err != nil { + t.Fatal(err) + } + func() { + defer reader.Close() + if _, err := reader.Seek(2, io.SeekStart); err != nil { + t.Fatal(err) + } + buf := make([]byte, 4) + if _, err := reader.ReadAt(buf, 5); err != nil || !bytes.Equal(buf, body[5:9]) { + t.Fatalf("ReadAt=%q err=%v", buf, err) + } + // ReaderAt must not move the sequential stream cursor (ZIP and Range). + data, err := io.ReadAll(reader) + if err != nil || !bytes.Equal(data, body[2:]) { + t.Fatalf("stream=%q err=%v", data, err) + } + }() + if err := backend.PutIfAbsent(ctx, hash, int64(len(body)), bytes.NewReader(body)); err != nil { + t.Fatal(err) + } + if err := backend.Verify(ctx, hash); err != nil { + t.Fatal(err) + } + } + after, err := os.Stat(filename) + if err != nil || !os.SameFile(before, after) { + t.Fatalf("canonical inode replaced: %v", err) + } + oldKey, err := documentstore.Key(hash) + if err != nil { + t.Fatal(err) + } + newKey, err := geduldh.Key(hash) + if err != nil || oldKey != newKey || oldKey != "sha256/"+hash[:2]+"/"+hash { + t.Fatalf("key changed: %s %s %v", oldKey, newKey, err) + } + if documentstore.ErrInvalidHash != geduldh.ErrInvalidHash || documentstore.ErrHashMismatch != geduldh.ErrHashMismatch || documentstore.ErrSizeMismatch != geduldh.ErrSizeMismatch || documentstore.ErrMaterializeLimit != geduldh.ErrMaterializeLimit { + t.Fatal("legacy error identity changed") + } + if err := os.WriteFile(filename, []byte("corrupt"), 0644); err != nil { + t.Fatal(err) + } + for _, backend := range []geduldh.Store{old, current} { + if err := backend.PutIfAbsent(ctx, hash, int64(len(body)), bytes.NewReader(body)); !errors.Is(err, documentstore.ErrHashMismatch) { + t.Fatalf("corruption repaired or error changed: %v", err) + } + } + corrupt, err := os.ReadFile(filename) + if err != nil || string(corrupt) != "corrupt" { + t.Fatalf("corrupt canonical overwritten: %v", err) + } +} + +func TestGeduldhCompatibilityMaterializer(t *testing.T) { + root, scratch := t.TempDir(), t.TempDir() + backend, err := geduldh.NewLocal(root) + if err != nil { + t.Fatal(err) + } + body := []byte("shared scratch bytes") + sum := sha256.Sum256(body) + hash := hex.EncodeToString(sum[:]) + ctx := context.Background() + if err := backend.PutIfAbsent(ctx, hash, int64(len(body)), bytes.NewReader(body)); err != nil { + t.Fatal(err) + } + var materializer *geduldh.Materializer + materializer, err = documentstore.NewMaterializer(backend, documentstore.MaterializeOptions{TempDir: scratch, MaxBytes: 1024, Concurrent: 1, Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + var first *documentstore.Lease + first, err = materializer.Acquire(ctx, hash, 1024) + if err != nil { + t.Fatal(err) + } + defer first.Close() + second, err := materializer.Acquire(ctx, hash, 1024) + if err != nil { + t.Fatal(err) + } + defer second.Close() + if first.Path != second.Path { + t.Fatal("legacy/current leases no longer share the same copy") + } + first.Close() + data, err := os.ReadFile(second.Path) + if err != nil || !bytes.Equal(data, body) { + t.Fatalf("active lease lost: %v", err) + } + second.Close() + if _, err := os.Stat(second.Path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("scratch not reclaimed: %v", err) + } + if err := backend.Verify(ctx, hash); err != nil { + t.Fatal("lease cleanup affected canonical content", err) + } +} + +func TestGeduldhConstructorsDoNotProvisionRoots(t *testing.T) { + root := filepath.Join(t.TempDir(), "missing") + for _, create := range []func(string) (*geduldh.Local, error){documentstore.NewLocal, geduldh.NewLocal} { + if _, err := create(root); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(root); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("constructor wrote to storage root: %v", err) + } + if _, err := create(" "); err == nil { + t.Fatal("empty root accepted") + } + } +} diff --git a/internal/documentstore/store.go b/internal/documentstore/store.go new file mode 100644 index 0000000..744c870 --- /dev/null +++ b/internal/documentstore/store.go @@ -0,0 +1,34 @@ +// Package documentstore preserves the original Document Store entry points. +// Payload storage and scratch materialization are implemented by Geduldh. +// New consumers should use pageswiki/internal/geduldh directly. +package documentstore + +import "pageswiki/internal/geduldh" + +// Aliases preserve interface/type identity, method sets and error matching. +// This package must not acquire its own persistence or validation logic. +type ( + Info = geduldh.Info + Reader = geduldh.Reader + Store = geduldh.Store + Local = geduldh.Local + Source = geduldh.Source + MaterializeOptions = geduldh.MaterializeOptions + Materializer = geduldh.Materializer + Lease = geduldh.Lease +) + +var ( + ErrInvalidHash = geduldh.ErrInvalidHash + ErrHashMismatch = geduldh.ErrHashMismatch + ErrSizeMismatch = geduldh.ErrSizeMismatch + ErrMaterializeLimit = geduldh.ErrMaterializeLimit +) + +func Key(hash string) (string, error) { return geduldh.Key(hash) } + +func NewLocal(root string) (*Local, error) { return geduldh.NewLocal(root) } + +func NewMaterializer(source Source, options MaterializeOptions) (*Materializer, error) { + return geduldh.NewMaterializer(source, options) +} diff --git a/internal/dr/dr_test.go b/internal/dr/dr_test.go new file mode 100644 index 0000000..8a94836 --- /dev/null +++ b/internal/dr/dr_test.go @@ -0,0 +1,294 @@ +package dr + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "pageswiki/internal/db" +) + +func fixture(t *testing.T) string { + t.Helper() + root := t.TempDir() + database, err := db.Open(filepath.Join(root, "pages.db")) + if err != nil { + t.Fatal(err) + } + if _, err = database.Exec("CREATE TABLE revisions(document_hash TEXT); INSERT INTO revisions(rowid,document_hash) VALUES(42,'')"); err != nil { + t.Fatal(err) + } + if err = database.Close(); err != nil { + t.Fatal(err) + } + if _, err = Init(root); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(root, "auth.key"), make([]byte, 32), 0600); err != nil { + t.Fatal(err) + } + return root +} + +func TestSnapshotRestorePromote(t *testing.T) { + root := fixture(t) + ctx := context.Background() + dest := filepath.Join(t.TempDir(), "snapshot") + body := []byte("synthetic DR test document") + sum := sha256.Sum256(body) + hash := hex.EncodeToString(sum[:]) + path := filepath.Join(root, "documents", "sha256", hash[:2], hash) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, body, 0600); err != nil { + t.Fatal(err) + } + database, err := db.Open(filepath.Join(root, "pages.db")) + if err != nil { + t.Fatal(err) + } + // Hold a connection open to keep committed WAL pages present. No writer is + // active during snapshot; this covers recovery after an uncheckpointed stop. + if _, err = database.Exec("PRAGMA wal_autocheckpoint=0; UPDATE revisions SET document_hash=? WHERE rowid=42", hash); err != nil { + t.Fatal(err) + } + defer database.Close() + if err = os.MkdirAll(filepath.Join(root, "previews"), 0700); err != nil { + t.Fatal(err) + } + os.WriteFile(filepath.Join(root, "previews", "ignored"), []byte("cache"), 0600) + m, digest, err := Snapshot(ctx, root, dest) + if err != nil { + t.Fatal(err) + } + if _, err = Verify(ctx, dest, digest); err != nil { + t.Fatal(err) + } + if _, err = Verify(ctx, dest, digest); err != nil { + t.Fatalf("verification must not mutate snapshot: %v", err) + } + restored := filepath.Join(t.TempDir(), "standby") + node, err := Restore(ctx, dest, digest, restored) + if err != nil { + t.Fatal(err) + } + if node.Role != "standby" || node.NodeID == node.PrimaryNodeID || node.SnapshotID != m.ID || !node.Readable(time.Now()) { + t.Fatalf("bad standby: %+v", node) + } + if _, err = os.Stat(filepath.Join(restored, "previews")); !os.IsNotExist(err) { + t.Fatal("copied cache") + } + ro, err := db.OpenReadOnly(filepath.Join(restored, "pages.db")) + if err != nil { + t.Fatal(err) + } + defer ro.Close() + var got string + if err = ro.QueryRow("SELECT document_hash FROM revisions WHERE rowid=42").Scan(&got); err != nil || got != hash { + t.Fatalf("WAL or rowid lost: %s %v", got, err) + } + if _, err = ro.Exec("DELETE FROM revisions"); err == nil { + t.Fatal("standby accepted write") + } + shared, err := Lock(restored, false) + if err != nil { + t.Fatal(err) + } + if _, err = Promote(restored, node.PrimaryNodeID, "test fence", true); err == nil { + t.Fatal("promoted running standby") + } + shared.Close() + if _, err = Promote(restored, "wrong", "test fence", true); err == nil { + t.Fatal("accepted wrong primary") + } + if _, err = Promote(restored, node.PrimaryNodeID, "test fence", false); err == nil { + t.Fatal("accepted implicit data loss") + } + promoted, err := Promote(restored, node.PrimaryNodeID, "test fence", true) + if err != nil { + t.Fatal(err) + } + if promoted.Role != "primary" || promoted.Epoch != node.Epoch+1 || len(promoted.History) != 1 { + t.Fatalf("bad promotion: %+v", promoted) + } + if promoted.WorkersEnabled { + t.Fatal("promotion replayed jobs without review") + } + if _, err = ResumeWorkers(restored, false); err == nil { + t.Fatal("resumed jobs without review") + } + resumed, err := ResumeWorkers(restored, true) + if err != nil { + t.Fatal(err) + } + if !resumed.WorkersEnabled || resumed.WorkersResumedAt.IsZero() { + t.Fatal("worker resumption not audited") + } + if _, err = Restore(ctx, dest, digest, restored); err == nil { + t.Fatal("overwrote existing restore") + } +} + +func TestSnapshotCalendarReferencesAndRestoreSafety(t *testing.T) { + root := fixture(t) + dir := filepath.Join(root, "documents", "calendars", "synthetic-domain", "synthetic-calendar") + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + body := []byte("synthetic opaque monthly generation; no external calendar data") + sum := sha256.Sum256(body) + hash := hex.EncodeToString(sum[:]) + metadata, _ := json.Marshal(map[string]any{"months": map[string]string{"2026-09": hash}}) + if err := os.WriteFile(filepath.Join(dir, "calendar.json"), metadata, 0600); err != nil { + t.Fatal(err) + } + dest := filepath.Join(t.TempDir(), "snapshot") + if _, _, err := Snapshot(context.Background(), root, dest); err == nil { + t.Fatal("accepted missing calendar month") + } + if err := os.WriteFile(filepath.Join(dir, "2026-09."+hash+".ics"), body, 0600); err != nil { + t.Fatal(err) + } + _, digest, err := Snapshot(context.Background(), root, dest) + if err != nil { + t.Fatal(err) + } + if _, err = Verify(context.Background(), dest, digest); err != nil { + t.Fatal(err) + } + if _, _, err = Snapshot(context.Background(), root, filepath.Join(root, "inside")); err == nil { + t.Fatal("snapshot inside source accepted") + } + if _, err = Restore(context.Background(), dest, digest, root); err == nil { + t.Fatal("restore overwrote existing data") + } + if _, err = Restore(context.Background(), dest, digest, filepath.Join(dest, "nested")); err == nil { + t.Fatal("restore corrupted source snapshot") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err = Verify(ctx, dest, digest); err == nil { + t.Fatal("verification ignored cancellation") + } +} + +func TestSnapshotRefusesActiveMissingAndSymlink(t *testing.T) { + root := fixture(t) + dest := filepath.Join(t.TempDir(), "snapshot") + ctx := context.Background() + shared, err := Lock(root, false) + if err != nil { + t.Fatal(err) + } + if _, _, err = Snapshot(ctx, root, dest); err == nil { + t.Fatal("snapshotted running instance") + } + shared.Close() + database, err := db.Open(filepath.Join(root, "pages.db")) + if err != nil { + t.Fatal(err) + } + _, err = database.Exec("UPDATE revisions SET document_hash=?", string(make([]byte, 64))) + if err != nil { + t.Fatal(err) + } + database.Close() + if _, _, err = Snapshot(ctx, root, dest); err == nil { + t.Fatal("accepted missing canonical document") + } + if _, err = os.Stat(dest); !os.IsNotExist(err) { + t.Fatal("published failed snapshot") + } + if err = os.Symlink(t.TempDir(), filepath.Join(root, "themes")); err != nil { + t.Fatal(err) + } + if _, _, err = Snapshot(ctx, root, dest); err == nil { + t.Fatal("followed source symlink") + } +} + +func TestSnapshotTamperAndPathValidation(t *testing.T) { + root := fixture(t) + dest := filepath.Join(t.TempDir(), "snapshot") + ctx := context.Background() + _, digest, err := Snapshot(ctx, root, dest) + if err != nil { + t.Fatal(err) + } + if _, err = Verify(ctx, dest, ""); err == nil { + t.Fatal("accepted unauthenticated manifest") + } + manifestPath := filepath.Join(dest, "manifest.json") + encoded, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatal(err) + } + var m Manifest + if err = json.Unmarshal(encoded, &m); err != nil { + t.Fatal(err) + } + m.Files[0].Path = "../outside" + altered, _ := json.Marshal(m) + sum := sha256.Sum256(altered) + os.WriteFile(manifestPath, altered, 0600) + if _, err = Verify(ctx, dest, digest); err == nil { + t.Fatal("accepted changed manifest") + } + if _, err = Verify(ctx, dest, hex.EncodeToString(sum[:])); err == nil { + t.Fatal("accepted traversal even with matching manifest digest") + } + os.WriteFile(manifestPath, encoded, 0600) + os.WriteFile(filepath.Join(dest, "data", "auth.key"), []byte("tampered"), 0600) + if _, err = Verify(ctx, dest, digest); err == nil { + t.Fatal("accepted payload tamper") + } + for _, path := range []string{"/etc/passwd", "documents/../auth.key", "documents\\foo", "dr-node.json", "pages.db-wal", "cache/foo"} { + if canonical(path) { + t.Errorf("accepted %q", path) + } + } +} + +func TestConfigFailsClosedAndExpiry(t *testing.T) { + root := fixture(t) + c, err := Load(root) + if err != nil { + t.Fatal(err) + } + c.Role = "standby" + if err = Save(root, *c); err == nil { + t.Fatal("accepted inconsistent role") + } + os.WriteFile(filepath.Join(root, ConfigFile), []byte(`{"role":"broken"}`), 0600) + if _, err = Load(root); err == nil { + t.Fatal("invalid config accepted") + } + c.Role = "standby" + c.CapturedAt = time.Now().Add(-2 * time.Hour) + c.MaxReadAgeSeconds = 3600 + if c.Readable(time.Now()) { + t.Fatal("stale snapshot is readable") + } + c.CapturedAt = time.Now().Add(time.Hour) + if c.Readable(time.Now()) { + t.Fatal("future snapshot is readable") + } + if err = RequireRole(nil, "standby"); err == nil { + t.Fatal("empty standby volume would start writable") + } + if err = RequireRole(c, "primary"); err == nil { + t.Fatal("mismatched deployment role accepted") + } + if err = RequireRole(c, "standby"); err != nil { + t.Fatal(err) + } + if err = RequireRole(nil, ""); err != nil { + t.Fatal("legacy startup broken") + } +} diff --git a/internal/dr/geduldh_test.go b/internal/dr/geduldh_test.go new file mode 100644 index 0000000..791bde0 --- /dev/null +++ b/internal/dr/geduldh_test.go @@ -0,0 +1,70 @@ +package dr + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "testing" + + "pageswiki/internal/db" + "pageswiki/internal/geduldh" +) + +func TestGeduldhSnapshotRestoreCompatibility(t *testing.T) { + root := fixture(t) + documents := filepath.Join(root, "documents") + if err := os.MkdirAll(documents, 0755); err != nil { + t.Fatal(err) + } + backend, err := geduldh.NewLocal(documents) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + body := []byte("Geduldh DR compatibility 日本語\x00\xff") + sum := sha256.Sum256(body) + hash := hex.EncodeToString(sum[:]) + if err := backend.PutIfAbsent(ctx, hash, int64(len(body)), bytes.NewReader(body)); err != nil { + t.Fatal(err) + } + database, err := db.Open(filepath.Join(root, "pages.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if _, err := database.Exec(`UPDATE revisions SET document_hash=? WHERE rowid=42`, hash); err != nil { + t.Fatal(err) + } + snapshot := filepath.Join(t.TempDir(), "snapshot") + _, digest, err := Snapshot(ctx, root, snapshot) + if err != nil { + t.Fatal(err) + } + if _, err := Verify(ctx, snapshot, digest); err != nil { + t.Fatal(err) + } + restored := filepath.Join(t.TempDir(), "standby") + if _, err := Restore(ctx, snapshot, digest, restored); err != nil { + t.Fatal(err) + } + recovered, err := geduldh.NewLocal(filepath.Join(restored, "documents")) + if err != nil { + t.Fatal(err) + } + if err := recovered.Verify(ctx, hash); err != nil { + t.Fatal(err) + } + reader, err := recovered.Open(ctx, hash) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + data, err := io.ReadAll(reader) + if err != nil || !bytes.Equal(data, body) { + t.Fatalf("restored payload changed: %v", err) + } +} diff --git a/internal/dr/node.go b/internal/dr/node.go new file mode 100644 index 0000000..264df58 --- /dev/null +++ b/internal/dr/node.go @@ -0,0 +1,273 @@ +// Package dr implements offline, operator-fenced recovery, not leader election. +package dr + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/google/uuid" +) + +const ConfigFile = "dr-node.json" +const LockFile = ".pageswiki-instance.lock" + +// RequireRole is an optional deployment guard against mounting an empty/wrong +// data volume on a DR host and accidentally starting a new writable instance. +func RequireRole(c *Config, expected string) error { + if expected == "" { + return nil + } + if expected != "primary" && expected != "standby" { + return errors.New("PAGESWIKI_DR_EXPECT_ROLE must be primary or standby") + } + if c == nil || c.Role != expected { + return errors.New("DR role does not match PAGESWIKI_DR_EXPECT_ROLE; refusing startup") + } + return nil +} + +type Promotion struct { + At time.Time `json:"at"` + PreviousNode string `json:"previous_node"` + FenceReference string `json:"fence_reference"` + SnapshotID string `json:"snapshot_id"` + Epoch uint64 `json:"epoch"` +} + +// Config is node-local and MUST NOT be overwritten by a primary's snapshot. +// Epoch is an audit generation only; it is NOT a distributed fencing token. +type Config struct { + Version int `json:"version"` + ClusterID string `json:"cluster_id"` + NodeID string `json:"node_id"` + PrimaryNodeID string `json:"primary_node_id"` + Role string `json:"role"` + Epoch uint64 `json:"epoch"` + SnapshotID string `json:"snapshot_id,omitempty"` + CapturedAt time.Time `json:"captured_at,omitempty"` + MaxReadAgeSeconds int64 `json:"max_read_age_seconds"` + WorkersEnabled bool `json:"workers_enabled"` + WorkersResumedAt time.Time `json:"workers_resumed_at,omitempty"` + History []Promotion `json:"promotion_history,omitempty"` +} + +func (c Config) Validate() error { + for _, id := range []string{c.ClusterID, c.NodeID, c.PrimaryNodeID} { + u, err := uuid.Parse(id) + if err != nil || u == uuid.Nil || u.String() != id { + return errors.New("DR identity must be a canonical nonzero UUID") + } + } + if c.Version != 1 || c.Epoch == 0 || c.MaxReadAgeSeconds < 1 || c.MaxReadAgeSeconds > 86400 { + return errors.New("invalid DR version, epoch or max_read_age_seconds (1..86400)") + } + if c.Role != "primary" && c.Role != "standby" { + return errors.New("DR role must be primary or standby") + } + if (c.Role == "primary") != (c.PrimaryNodeID == c.NodeID) { + return errors.New("DR role and primary identity disagree") + } + if c.Role == "standby" && (c.SnapshotID == "" || c.CapturedAt.IsZero()) { + return errors.New("standby requires a verified snapshot") + } + if c.Role == "standby" && c.WorkersEnabled { + return errors.New("standby cannot enable background workers") + } + if c.SnapshotID != "" { + u, err := uuid.Parse(c.SnapshotID) + if err != nil || u == uuid.Nil || u.String() != c.SnapshotID { + return errors.New("invalid snapshot identity") + } + } + return nil +} + +func (c Config) Readable(now time.Time) bool { + return c.Role == "standby" && !c.CapturedAt.After(now.Add(time.Minute)) && now.Sub(c.CapturedAt) <= time.Duration(c.MaxReadAgeSeconds)*time.Second +} + +func (c Config) ReadUntil() time.Time { + if c.CapturedAt.After(time.Now().Add(time.Minute)) { + return time.Time{} + } + return c.CapturedAt.Add(time.Duration(c.MaxReadAgeSeconds) * time.Second) +} + +func Load(dataDir string) (*Config, error) { + f, err := os.Open(filepath.Join(dataDir, ConfigFile)) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() || info.Size() > 1<<20 { + return nil, errors.New("invalid or oversized DR configuration") + } + var c Config + d := json.NewDecoder(io.LimitReader(f, 1<<20)) + d.DisallowUnknownFields() + if err = d.Decode(&c); err != nil { + return nil, err + } + if err = d.Decode(new(any)); err != io.EOF { + return nil, errors.New("trailing DR configuration data") + } + return &c, c.Validate() +} + +func Save(dataDir string, c Config) error { + if err := c.Validate(); err != nil { + return err + } + b, err := json.MarshalIndent(c, "", " ") + if err != nil { + return err + } + f, err := os.CreateTemp(dataDir, ".dr-node-") + if err != nil { + return err + } + defer os.Remove(f.Name()) + if _, err = f.Write(append(b, '\n')); err != nil { + f.Close() + return err + } + if err = f.Sync(); err != nil { + f.Close() + return err + } + if err = f.Close(); err != nil { + return err + } + if err = os.Rename(f.Name(), filepath.Join(dataDir, ConfigFile)); err != nil { + return err + } + return syncDir(dataDir) +} + +// Lock is shared for a running server/CLI, exclusive for offline maintenance. +// Advisory locking cannot fence older binaries or an independent SQLite client. +func Lock(dataDir string, exclusive bool) (*os.File, error) { + // Shared locks also work with a read-only standby mount. + path := filepath.Join(dataDir, LockFile) + f, err := os.Open(path) + if exclusive || errors.Is(err, os.ErrNotExist) { + if f != nil { + f.Close() + } + f, err = os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600) + } + if err != nil { + return nil, err + } + mode := syscall.LOCK_SH + if exclusive { + mode = syscall.LOCK_EX + } + if err = syscall.Flock(int(f.Fd()), mode|syscall.LOCK_NB); err != nil { + f.Close() + return nil, fmt.Errorf("DR data directory is in use; stop all writers/readers before maintenance: %w", err) + } + return f, nil +} + +func Init(dataDir string) (*Config, error) { + l, err := Lock(dataDir, true) + if err != nil { + return nil, err + } + defer l.Close() + if c, err := Load(dataDir); err != nil || c != nil { + return nil, errors.New("DR configuration exists or cannot be read; refusing initialization") + } + id := uuid.NewString() + c := Config{Version: 1, ClusterID: uuid.NewString(), NodeID: id, PrimaryNodeID: id, Role: "primary", Epoch: 1, MaxReadAgeSeconds: 3600, WorkersEnabled: true} + return &c, Save(dataDir, c) +} + +// Promote records an operator's fencing attestation. It does not contact, +// power off, or independently verify the former primary. No automatic promotion. +func Promote(dataDir, fencedNode, reference string, acceptLoss bool) (*Config, error) { + l, err := Lock(dataDir, true) + if err != nil { + return nil, err + } + defer l.Close() + c, err := Load(dataDir) + if err != nil { + return nil, err + } + if c == nil || c.Role != "standby" { + return nil, errors.New("only an offline standby can be promoted") + } + if err = checkAuthKey(dataDir); err != nil { + return nil, err + } + if fencedNode != c.PrimaryNodeID || !acceptLoss || len(strings.TrimSpace(reference)) < 4 || len(reference) > 1024 { + return nil, errors.New("promotion requires exact former primary ID, fencing reference and explicit acceptance of snapshot data loss") + } + if c.Epoch == ^uint64(0) { + return nil, errors.New("DR epoch exhausted") + } + c.Epoch++ + c.History = append(c.History, Promotion{At: time.Now().UTC(), PreviousNode: fencedNode, FenceReference: strings.TrimSpace(reference), SnapshotID: c.SnapshotID, Epoch: c.Epoch}) + c.Role, c.PrimaryNodeID = "primary", c.NodeID + // Replaying pre-snapshot jobs can duplicate notifications or external writes. + // Resume them only after an operator reviews the restored queue. + c.WorkersEnabled = false + return c, Save(dataDir, *c) +} + +func checkAuthKey(dataDir string) error { + if strings.TrimSpace(os.Getenv("PAGESWIKI_AUTH_KEY")) != "" { + return nil + } + key, err := os.ReadFile(filepath.Join(dataDir, "auth.key")) + if err != nil || len(key) != 32 { + return errors.New("original auth.key or PAGESWIKI_AUTH_KEY is required; refusing implicit key regeneration") + } + return nil +} + +func ResumeWorkers(dataDir string, reviewed bool) (*Config, error) { + if !reviewed { + return nil, errors.New("review restored jobs and federation state before resuming workers") + } + l, err := Lock(dataDir, true) + if err != nil { + return nil, err + } + defer l.Close() + c, err := Load(dataDir) + if err != nil { + return nil, err + } + if c == nil || c.Role != "primary" { + return nil, errors.New("only an offline primary can resume workers") + } + c.WorkersEnabled = true + c.WorkersResumedAt = time.Now().UTC() + return c, Save(dataDir, *c) +} + +func syncDir(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} diff --git a/internal/dr/snapshot.go b/internal/dr/snapshot.go new file mode 100644 index 0000000..37f42b8 --- /dev/null +++ b/internal/dr/snapshot.go @@ -0,0 +1,579 @@ +package dr + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" + "golang.org/x/sys/unix" + "modernc.org/sqlite" + "pageswiki/internal/db" +) + +type File struct { + Path string `json:"path"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` +} +type Manifest struct { + Version int `json:"version"` + ID string `json:"id"` + ClusterID string `json:"cluster_id"` + PrimaryNodeID string `json:"primary_node_id"` + Epoch uint64 `json:"epoch"` + CapturedAt time.Time `json:"captured_at"` + CompletedAt time.Time `json:"completed_at"` + Scope string `json:"scope"` + Files []File `json:"files"` +} + +// Include canonical stores, auth/signing keys and managed-campus enrollment. +// Caches, temporary uploads, editor workspaces and node-local DR state are NOT +// recovery authorities. Extra mounted storage/environment secrets are external. +func canonical(path string) bool { + if !filepath.IsLocal(path) || filepath.ToSlash(path) != path || strings.Contains(path, "\\") || path != filepath.Clean(path) { + return false + } + root := strings.Split(path, "/")[0] + switch root { + case "documents", "themes", "federation": + return true + case "pages.db", "auth.key", "instance-ed25519.key": + return path == root + } + return false +} + +func fileHash(path string) (File, error) { + f, err := os.Open(path) + if err != nil { + return File{}, err + } + defer f.Close() + i, err := f.Stat() + if err != nil || !i.Mode().IsRegular() { + return File{}, errors.New("snapshot entry is not a regular file") + } + h := sha256.New() + n, err := io.CopyBuffer(h, f, make([]byte, 256<<10)) + return File{Size: n, SHA256: hex.EncodeToString(h.Sum(nil))}, err +} + +func copyFile(source, target string) error { + i, err := os.Lstat(source) + if err != nil { + return err + } + if !i.Mode().IsRegular() { + return fmt.Errorf("refusing nonregular source: %s", source) + } + in, err := os.Open(source) + if err != nil { + return err + } + defer in.Close() + opened, err := in.Stat() + if err != nil || !os.SameFile(i, opened) { + return errors.New("source changed during snapshot") + } + if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil { + return err + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + _, err = io.CopyBuffer(out, in, make([]byte, 256<<10)) + if err == nil { + err = out.Sync() + } + closeErr := out.Close() + if err == nil { + err = closeErr + } + return err +} + +// Backup API preserves rowids (unlike VACUUM INTO) and includes committed WAL. +func backupDB(ctx context.Context, source, target string) error { + database, err := db.OpenReadOnly(source) + if err != nil { + return err + } + defer database.Close() + conn, err := database.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + err = conn.Raw(func(raw any) error { + b, err := raw.(interface { + NewBackup(string) (*sqlite.Backup, error) + }).NewBackup((&url.URL{Scheme: "file", Path: target}).String()) + if err != nil { + return err + } + for { + if err = ctx.Err(); err != nil { + b.Finish() + return err + } + more, e := b.Step(256) + if e != nil { + b.Finish() + return e + } + if !more { + return b.Finish() + } + } + }) + if err != nil { + return err + } + // Do not leave a WAL-mode snapshot that creates sidecars during verification. + dst, err := sql.Open("sqlite", (&url.URL{Scheme: "file", Path: target, RawQuery: "mode=rw"}).String()) + if err != nil { + return err + } + _, err = dst.ExecContext(ctx, "PRAGMA journal_mode=DELETE") + closeErr := dst.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + // Finish leaves a standalone DB; fsync before publishing the generation. + f, err := os.OpenFile(target, os.O_RDWR, 0600) + if err != nil { + return err + } + defer f.Close() + if err = f.Chmod(0600); err != nil { + return err + } + return f.Sync() +} + +// Snapshot requires every writer stopped, including older binaries and CLIs. +// The lock enforces this for upgraded PagesWiki processes, not arbitrary tools. +func Snapshot(ctx context.Context, dataDir, target string) (*Manifest, string, error) { + source, err := filepath.EvalSymlinks(dataDir) + if err != nil { + return nil, "", err + } + source, err = filepath.Abs(source) + if err != nil { + return nil, "", err + } + target, err = filepath.Abs(target) + if err != nil { + return nil, "", err + } + parent, err := filepath.EvalSymlinks(filepath.Dir(target)) + if err != nil { + return nil, "", err + } + target = filepath.Join(parent, filepath.Base(target)) + if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) { + return nil, "", errors.New("snapshot destination must be outside the data directory") + } + l, err := Lock(source, true) + if err != nil { + return nil, "", err + } + defer l.Close() + c, err := Load(source) + if err != nil { + return nil, "", err + } + if c == nil || c.Role != "primary" { + return nil, "", errors.New("initialize primary DR configuration before snapshot") + } + if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) { + return nil, "", errors.New("snapshot destination must not exist") + } + stage, err := os.MkdirTemp(parent, ".dr-snapshot-") + if err != nil { + return nil, "", err + } + defer os.RemoveAll(stage) + m := &Manifest{Version: 1, ID: uuid.NewString(), ClusterID: c.ClusterID, PrimaryNodeID: c.NodeID, Epoch: c.Epoch, CapturedAt: time.Now().UTC(), Scope: "instance-offline-v1"} + root := filepath.Join(stage, "data") + if err = os.Mkdir(root, 0700); err != nil { + return nil, "", err + } + info, err := os.Lstat(filepath.Join(source, "pages.db")) + if err != nil { + return nil, "", err + } + if !info.Mode().IsRegular() { + return nil, "", errors.New("snapshot database must be a regular file, not a symlink") + } + if err = checkAuthKey(source); err != nil { + return nil, "", err + } + if err = backupDB(ctx, filepath.Join(source, "pages.db"), filepath.Join(root, "pages.db")); err != nil { + return nil, "", err + } + for _, name := range []string{"documents", "themes", "federation", "auth.key", "instance-ed25519.key"} { + base := filepath.Join(source, name) + if _, e := os.Lstat(base); errors.Is(e, os.ErrNotExist) { + continue + } else if e != nil { + return nil, "", e + } + err = filepath.WalkDir(base, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := ctx.Err(); err != nil { + return err + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("snapshot refuses symlink: %s", path) + } + rel, _ := filepath.Rel(source, path) + if entry.IsDir() { + return os.MkdirAll(filepath.Join(root, rel), 0700) + } + return copyFile(path, filepath.Join(root, rel)) + }) + if err != nil { + return nil, "", err + } + } + err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, e error) error { + if e != nil { + return e + } + if entry.IsDir() { + return syncDir(path) + } + if len(m.Files) >= 100000 { + return errors.New("snapshot exceeds 100000 files; partitioned snapshots require a later format") + } + f, e := fileHash(path) + if e != nil { + return e + } + f.Path, _ = filepath.Rel(root, path) + m.Files = append(m.Files, f) + return nil + }) + if err != nil { + return nil, "", err + } + m.CompletedAt = time.Now().UTC() + encoded, err := json.MarshalIndent(m, "", " ") + if err != nil { + return nil, "", err + } + if err = os.WriteFile(filepath.Join(stage, "manifest.json"), encoded, 0600); err != nil { + return nil, "", err + } + manifestFile, err := os.OpenFile(filepath.Join(stage, "manifest.json"), os.O_RDWR, 0600) + if err != nil { + return nil, "", err + } + err = manifestFile.Sync() + manifestFile.Close() + if err != nil { + return nil, "", err + } + hash := sha256.Sum256(encoded) + if _, err = Verify(ctx, stage, hex.EncodeToString(hash[:])); err != nil { + return nil, "", err + } + if err = syncDir(stage); err != nil { + return nil, "", err + } + if err = publish(stage, target); err != nil { + return nil, "", err + } + return m, hex.EncodeToString(hash[:]), syncDir(parent) +} + +func Verify(ctx context.Context, root, expectedHash string) (*Manifest, error) { + if len(expectedHash) != 64 { + return nil, errors.New("expected manifest SHA-256 is required from a trusted channel") + } + root, err := filepath.Abs(root) + if err != nil { + return nil, err + } + info, err := os.Lstat(filepath.Join(root, "manifest.json")) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() || info.Size() > 64<<20 { + return nil, errors.New("invalid or oversized snapshot manifest") + } + encoded, err := os.ReadFile(filepath.Join(root, "manifest.json")) + if err != nil { + return nil, err + } + sum := sha256.Sum256(encoded) + if hex.EncodeToString(sum[:]) != expectedHash { + return nil, errors.New("manifest SHA-256 mismatch") + } + var m Manifest + if err = json.Unmarshal(encoded, &m); err != nil { + return nil, err + } + if m.Version != 1 || m.Scope != "instance-offline-v1" || m.Epoch == 0 || len(m.Files) == 0 || len(m.Files) > 100000 || m.CapturedAt.IsZero() || m.CompletedAt.Before(m.CapturedAt) || m.CompletedAt.After(time.Now().Add(time.Minute)) { + return nil, errors.New("invalid snapshot manifest") + } + for _, id := range []string{m.ID, m.ClusterID, m.PrimaryNodeID} { + u, e := uuid.Parse(id) + if e != nil || u == uuid.Nil || u.String() != id { + return nil, errors.New("invalid snapshot identity") + } + } + seen := map[string]bool{} + hashes := map[string]string{} + for _, item := range m.Files { + if err = ctx.Err(); err != nil { + return nil, err + } + if !canonical(item.Path) || seen[item.Path] { + return nil, errors.New("invalid/duplicate snapshot path") + } + seen[item.Path] = true + hashes[item.Path] = item.SHA256 + path := filepath.Join(root, "data", item.Path) + // Refuse symlinks in every component, including parent directories. + for p := path; p != filepath.Dir(root) && p != "."; p = filepath.Dir(p) { + info, e := os.Lstat(p) + if e != nil { + return nil, e + } + if info.Mode()&os.ModeSymlink != 0 { + return nil, errors.New("snapshot contains symlink") + } + if p == root || filepath.Dir(p) == p { + break + } + } + actual, e := fileHash(path) + if e != nil { + return nil, e + } + if actual.Size != item.Size || actual.SHA256 != item.SHA256 { + return nil, fmt.Errorf("snapshot payload mismatch: %s", item.Path) + } + if strings.HasPrefix(item.Path, "documents/sha256/") && filepath.Base(item.Path) != actual.SHA256 { + return nil, errors.New("CAS filename does not match payload hash") + } + } + if !seen["pages.db"] { + return nil, errors.New("snapshot database missing") + } + if err = verifyCalendarFiles(root, hashes); err != nil { + return nil, err + } + err = filepath.WalkDir(filepath.Join(root, "data"), func(path string, entry fs.DirEntry, e error) error { + if e != nil { + return e + } + if entry.IsDir() { + return nil + } + rel, _ := filepath.Rel(filepath.Join(root, "data"), path) + if !seen[rel] { + return errors.New("unlisted snapshot payload") + } + return nil + }) + if err != nil { + return nil, err + } + database, err := db.OpenReadOnly(filepath.Join(root, "data", "pages.db")) + if err != nil { + return nil, err + } + defer database.Close() + var integrity string + if err = database.QueryRowContext(ctx, "PRAGMA quick_check").Scan(&integrity); err != nil { + return nil, err + } + if integrity != "ok" { + return nil, errors.New("snapshot database integrity check failed") + } + if err = verifyReferences(ctx, database, seen); err != nil { + return nil, err + } + return &m, nil +} + +// Restore never merges into or overwrites a running/existing data directory. +// It publishes a new generation with a fresh physical node ID, always standby. +func Restore(ctx context.Context, snapshot, expectedHash, target string) (*Config, error) { + m, err := Verify(ctx, snapshot, expectedHash) + if err != nil { + return nil, err + } + target, err = filepath.Abs(target) + if err != nil { + return nil, err + } + parent, err := filepath.EvalSymlinks(filepath.Dir(target)) + if err != nil { + return nil, err + } + target = filepath.Join(parent, filepath.Base(target)) + source, err := filepath.EvalSymlinks(snapshot) + if err != nil { + return nil, err + } + source, err = filepath.Abs(source) + if err != nil { + return nil, err + } + if target == source || strings.HasPrefix(target, source+string(filepath.Separator)) { + return nil, errors.New("restore destination must be outside the snapshot") + } + if _, err = os.Lstat(target); !errors.Is(err, os.ErrNotExist) { + return nil, errors.New("restore target must not exist; use a new generation") + } + stage, err := os.MkdirTemp(filepath.Dir(target), ".dr-restore-") + if err != nil { + return nil, err + } + defer os.RemoveAll(stage) + for _, item := range m.Files { + if err = ctx.Err(); err != nil { + return nil, err + } + if err = copyFile(filepath.Join(snapshot, "data", item.Path), filepath.Join(stage, item.Path)); err != nil { + return nil, err + } + actual, e := fileHash(filepath.Join(stage, item.Path)) + if e != nil { + return nil, e + } + if actual.Size != item.Size || actual.SHA256 != item.SHA256 { + return nil, errors.New("snapshot changed during restore") + } + } + c := &Config{Version: 1, ClusterID: m.ClusterID, NodeID: uuid.NewString(), PrimaryNodeID: m.PrimaryNodeID, Role: "standby", Epoch: m.Epoch, SnapshotID: m.ID, CapturedAt: m.CapturedAt, MaxReadAgeSeconds: 3600} + if err = Save(stage, *c); err != nil { + return nil, err + } + if err = os.MkdirAll(filepath.Join(stage, "documents", "sha256"), 0700); err != nil { + return nil, err + } + // Create the lock before a read-only data mount is applied. + l, err := Lock(stage, true) + if err != nil { + return nil, err + } + l.Close() + err = filepath.WalkDir(stage, func(path string, entry fs.DirEntry, e error) error { + if e != nil { + return e + } + if entry.IsDir() { + return syncDir(path) + } + return nil + }) + if err != nil { + return nil, err + } + if err = publish(stage, target); err != nil { + return nil, err + } + return c, syncDir(filepath.Dir(target)) +} + +// Linux deployment: atomically publish without replacing even an empty directory. +func publish(stage, target string) error { + return unix.Renameat2(unix.AT_FDCWD, stage, unix.AT_FDCWD, target, unix.RENAME_NOREPLACE) +} + +func verifyReferences(ctx context.Context, database *sql.DB, seen map[string]bool) error { + // Only authoritative references, not derived projection/preview caches. Some + // installations predate optional modules, hence the explicit table check. + for _, ref := range [][2]string{{"revisions", "document_hash"}, {"assets", "head_document_hash"}, {"asset_head_history", "document_hash"}, {"organizer_items", "document_hash"}, {"organizer_revisions", "document_hash"}} { + var exists bool + if err := database.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?)", ref[0]).Scan(&exists); err != nil { + return err + } + if !exists { + continue + } + rows, err := database.QueryContext(ctx, "SELECT DISTINCT "+ref[1]+" FROM "+ref[0]+" WHERE "+ref[1]+" IS NOT NULL AND "+ref[1]+"<>''") + if err != nil { + return err + } + for rows.Next() { + var hash string + if err = rows.Scan(&hash); err != nil { + rows.Close() + return err + } + if len(hash) != 64 || !seen["documents/sha256/"+hash[:2]+"/"+hash] { + rows.Close() + return fmt.Errorf("snapshot lacks canonical payload referenced by %s", ref[0]) + } + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + } + return nil +} + +// Verify file-based calendar authority without parsing event bodies or making +// external calendar requests. Derived calendar indexes may be repaired only +// after promotion, never by a standby read request. +func verifyCalendarFiles(root string, hashes map[string]string) error { + for path := range hashes { + if !strings.HasPrefix(path, "documents/calendars/") || filepath.Base(path) != "calendar.json" { + continue + } + filename := filepath.Join(root, "data", path) + info, err := os.Stat(filename) + if err != nil { + return err + } + if info.Size() > 16<<20 { + return errors.New("calendar metadata exceeds snapshot verification limit") + } + encoded, err := os.ReadFile(filename) + if err != nil { + return err + } + var metadata struct { + Months map[string]string `json:"months"` + } + if err = json.Unmarshal(encoded, &metadata); err != nil { + return errors.New("invalid calendar metadata in snapshot") + } + for month, hash := range metadata.Months { + if _, err = time.Parse("2006-01", month); err != nil { + return errors.New("invalid calendar month in snapshot") + } + if len(hash) != 64 || hashes[filepath.ToSlash(filepath.Join(filepath.Dir(path), month+"."+hash+".ics"))] != hash { + return errors.New("snapshot lacks a valid calendar monthly generation") + } + } + } + return nil +} diff --git a/internal/dregarnuhr/boundary_test.go b/internal/dregarnuhr/boundary_test.go new file mode 100644 index 0000000..c0c4f21 --- /dev/null +++ b/internal/dregarnuhr/boundary_test.go @@ -0,0 +1,40 @@ +package dregarnuhr + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestComponentBoundary(t *testing.T) { + if Component != "dregarnuhr" { + t.Fatal("component identity changed") + } + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, spec := range ast.Imports { + path, err := strconv.Unquote(spec.Path.Value) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"pageswiki/internal", "database", "net", "os"} { + if path == forbidden || strings.HasPrefix(path, forbidden+"/") { + t.Errorf("%s imports %s: scheduler core must not own persistence or business I/O", file, path) + } + } + } + } +} diff --git a/internal/dregarnuhr/clock.go b/internal/dregarnuhr/clock.go new file mode 100644 index 0000000..131b2ae --- /dev/null +++ b/internal/dregarnuhr/clock.go @@ -0,0 +1,20 @@ +package dregarnuhr + +import "time" + +type Timer interface { + C() <-chan time.Time + Stop() bool +} +type Clock interface { + Now() time.Time + NewTimer(time.Duration) Timer +} +type SystemClock struct{} + +func (SystemClock) Now() time.Time { return time.Now() } +func (SystemClock) NewTimer(d time.Duration) Timer { return systemTimer{time.NewTimer(d)} } + +type systemTimer struct{ *time.Timer } + +func (t systemTimer) C() <-chan time.Time { return t.Timer.C } diff --git a/internal/dregarnuhr/clock_test.go b/internal/dregarnuhr/clock_test.go new file mode 100644 index 0000000..a54b3f3 --- /dev/null +++ b/internal/dregarnuhr/clock_test.go @@ -0,0 +1,65 @@ +package dregarnuhr + +import ( + "sync" + "testing" + "time" +) + +type fakeClock struct { + mu sync.Mutex + now time.Time + timers map[*fakeTimer]time.Time + created chan struct{} +} +type fakeTimer struct { + clock *fakeClock + ch chan time.Time +} + +func newFakeClock() *fakeClock { + return &fakeClock{now: time.Unix(100000, 0), timers: map[*fakeTimer]time.Time{}, created: make(chan struct{}, 100)} +} +func (c *fakeClock) Now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.now } +func (c *fakeClock) NewTimer(delay time.Duration) Timer { + c.mu.Lock() + defer c.mu.Unlock() + t := &fakeTimer{clock: c, ch: make(chan time.Time, 1)} + if delay <= 0 { + t.ch <- c.now + } else { + c.timers[t] = c.now.Add(delay) + } + c.created <- struct{}{} + return t +} +func (t *fakeTimer) C() <-chan time.Time { return t.ch } +func (t *fakeTimer) Stop() bool { + t.clock.mu.Lock() + defer t.clock.mu.Unlock() + _, ok := t.clock.timers[t] + delete(t.clock.timers, t) + return ok +} +func (c *fakeClock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(d) + for timer, at := range c.timers { + if !at.After(c.now) { + timer.ch <- c.now + delete(c.timers, timer) + } + } +} +func receive[T any](t *testing.T, ch <-chan T) T { + t.Helper() + select { + case v := <-ch: + return v + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for test worker") + var zero T + return zero + } +} diff --git a/internal/dregarnuhr/loop.go b/internal/dregarnuhr/loop.go new file mode 100644 index 0000000..bcc429f --- /dev/null +++ b/internal/dregarnuhr/loop.go @@ -0,0 +1,104 @@ +package dregarnuhr + +import ( + "context" + "errors" + "fmt" + "time" +) + +type Trigger struct { + Name string + After, Every time.Duration + Run func(context.Context, time.Time) error +} +type Loop struct { + Clock Clock + Every time.Duration + Wake <-chan struct{} + OnWake func(context.Context, time.Time) error + Triggers []Trigger + Step func(context.Context, time.Time) error + Report func(string, error) +} + +// Run coalesces overdue ticks (no catch-up storm), stops its timer on +// cancellation, and invokes caller-supplied hooks. It knows no event types. +func (l Loop) Run(ctx context.Context) error { + if l.Every <= 0 || l.Step == nil { + return errors.New("loop requires positive cadence and step") + } + clock := l.Clock + if clock == nil { + clock = SystemClock{} + } + now := clock.Now() + next := now.Add(l.Every) + due := make([]time.Time, len(l.Triggers)) + seen := map[string]bool{} + for i, t := range l.Triggers { + if t.Name == "" || seen[t.Name] || t.Run == nil || t.After < 0 || t.Every < 0 || (t.After == 0 && t.Every == 0) { + return fmt.Errorf("invalid trigger %q", t.Name) + } + seen[t.Name] = true + delay := t.After + if delay == 0 { + delay = t.Every + } + due[i] = now.Add(delay) + } + wake := l.Wake + call := func(name string, fn func(context.Context, time.Time) error, now time.Time) { + if fn != nil { + if err := fn(ctx, now); err != nil && l.Report != nil { + l.Report(name, err) + } + } + } + for ctx.Err() == nil { + at := next + for _, d := range due { + if !d.IsZero() && d.Before(at) { + at = d + } + } + timer := clock.NewTimer(max(time.Duration(0), at.Sub(clock.Now()))) + woken := false + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case _, ok := <-wake: + if !ok { + wake = nil + timer.Stop() + continue + } + woken = true + case <-timer.C(): + } + timer.Stop() + if ctx.Err() != nil { + return ctx.Err() + } + now = clock.Now() + if woken { + call("wake", l.OnWake, now) + } + for i, t := range l.Triggers { + if !due[i].IsZero() && !due[i].After(now) { + call(t.Name, t.Run, now) + if t.Every > 0 { + due[i] = now.Add(t.Every) + } else { + due[i] = time.Time{} + } + } + } + call("step", l.Step, now) + if !next.After(now) { + next = now.Add(l.Every) + } + } + return ctx.Err() +} diff --git a/internal/dregarnuhr/loop_test.go b/internal/dregarnuhr/loop_test.go new file mode 100644 index 0000000..8763335 --- /dev/null +++ b/internal/dregarnuhr/loop_test.go @@ -0,0 +1,77 @@ +package dregarnuhr + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" +) + +func TestLoopFakeClockWakeAndCoalescing(t *testing.T) { + clock := newFakeClock() + wake := make(chan struct{}, 1) + steps := make(chan time.Time, 20) + reports := make(chan string, 10) + var once, periodic, awakened atomic.Int64 + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + loop := Loop{Clock: clock, Every: time.Second, Wake: wake, + OnWake: func(context.Context, time.Time) error { awakened.Add(1); return nil }, + Triggers: []Trigger{ + {Name: "once", After: 5 * time.Second, Run: func(context.Context, time.Time) error { once.Add(1); return nil }}, + {Name: "periodic", Every: 10 * time.Second, Run: func(context.Context, time.Time) error { periodic.Add(1); return errors.New("synthetic") }}, + }, + Step: func(_ context.Context, at time.Time) error { steps <- at; return nil }, + Report: func(stage string, _ error) { reports <- stage }, + } + done := make(chan error, 1) + go func() { done <- loop.Run(ctx) }() + receive(t, clock.created) + wake <- struct{}{} + receive(t, steps) + receive(t, clock.created) + if awakened.Load() != 1 || once.Load() != 0 { + t.Fatal("wake changed timer deadlines") + } + clock.advance(5 * time.Second) + receive(t, steps) + receive(t, clock.created) + if once.Load() != 1 || periodic.Load() != 0 { + t.Fatal("initial cadence") + } + clock.advance(time.Hour) + receive(t, steps) + receive(t, clock.created) + if once.Load() != 1 || periodic.Load() != 1 { + t.Fatal("catch-up storm", once.Load(), periodic.Load()) + } + if receive(t, reports) != "periodic" { + t.Fatal("hook failure missing") + } + close(wake) + receive(t, clock.created) // closed wake channel must not spin + clock.advance(10 * time.Second) + receive(t, steps) + receive(t, clock.created) + if periodic.Load() != 2 { + t.Fatal("periodic stopped after error") + } + cancel() + if !errors.Is(receive(t, done), context.Canceled) { + t.Fatal("loop did not stop") + } + clock.mu.Lock() + defer clock.mu.Unlock() + if len(clock.timers) != 0 { + t.Fatal("timer leaked") + } +} + +func TestLoopRejectsInvalidHooks(t *testing.T) { + for _, l := range []Loop{{}, {Every: time.Second, Step: func(context.Context, time.Time) error { return nil }, Triggers: []Trigger{{Name: "invalid"}}}} { + if l.Run(context.Background()) == nil { + t.Fatal("invalid loop accepted") + } + } +} diff --git a/internal/dregarnuhr/policy.go b/internal/dregarnuhr/policy.go new file mode 100644 index 0000000..6e335ba --- /dev/null +++ b/internal/dregarnuhr/policy.go @@ -0,0 +1,58 @@ +package dregarnuhr + +import ( + "errors" + "time" +) + +// CanStart retains a single background worker under foreground load, avoiding +// starvation by long-lived requests while protecting foreground capacity. +func CanStart(s Settings, active int64, foreground bool) bool { + return !s.Paused && active < int64(s.Concurrency) && (!foreground || active == 0) +} + +// RetryDelay doubles per completed attempt and saturates without overflowing. +func RetryDelay(baseSeconds, attempt int) time.Duration { + if baseSeconds < 1 { + baseSeconds = 1 + } + if baseSeconds >= 86400 { + return 24 * time.Hour + } + delay := time.Duration(baseSeconds) * time.Second + for i := 1; i < attempt; i++ { + if delay >= 12*time.Hour { + return 24 * time.Hour + } + delay *= 2 + } + return delay +} + +type Completion struct { + Status, Message, Level string + RetryAt time.Time +} + +// Complete is independent of DB access. The repository persists this decision. +func Complete(job Job, runErr error, baseSeconds int, now time.Time) Completion { + out := Completion{Status: "succeeded", Level: "info"} + if runErr == nil { + return out + } + out.Message = runErr.Error() + switch { + case errors.Is(runErr, ErrSkipped): + out.Status = "skipped" + case job.CancelRequested: + out.Status = "cancelled" + case job.Attempts < job.MaxAttempts: + out.Status = "queued" + out.Level = "warning" + out.RetryAt = now.Add(RetryDelay(baseSeconds, job.Attempts)) + default: + out.Status = "failed" + out.Level = "error" + } + return out +} diff --git a/internal/dregarnuhr/policy_test.go b/internal/dregarnuhr/policy_test.go new file mode 100644 index 0000000..9898f58 --- /dev/null +++ b/internal/dregarnuhr/policy_test.go @@ -0,0 +1,63 @@ +package dregarnuhr + +import ( + "errors" + "fmt" + "math" + "testing" + "time" +) + +func TestRetryDelay(t *testing.T) { + for _, tc := range []struct { + base, attempt int + want time.Duration + }{ + {30, 1, 30 * time.Second}, {30, 2, time.Minute}, {30, 3, 2 * time.Minute}, + {30, 1000000, 24 * time.Hour}, {86401, 1, 24 * time.Hour}, {math.MaxInt, math.MaxInt, 24 * time.Hour}, {0, 1, time.Second}, + } { + if got := RetryDelay(tc.base, tc.attempt); got != tc.want { + t.Errorf("%+v got=%s", tc, got) + } + } +} + +func TestCompletionPolicy(t *testing.T) { + now := time.Unix(1000, 0) + for _, tc := range []struct { + name string + job Job + err error + status, level string + }{ + {"success", Job{Attempts: 1, MaxAttempts: 3}, nil, "succeeded", "info"}, + {"retry", Job{Attempts: 2, MaxAttempts: 3}, errors.New("failed"), "queued", "warning"}, + {"exhausted", Job{Attempts: 3, MaxAttempts: 3}, errors.New("failed"), "failed", "error"}, + {"cancel", Job{Attempts: 1, MaxAttempts: 3, CancelRequested: true}, errors.New("cancelled"), "cancelled", "info"}, + {"skip-before-cancel", Job{Attempts: 1, MaxAttempts: 3, CancelRequested: true}, fmt.Errorf("wrapped: %w", ErrSkipped), "skipped", "info"}, + } { + t.Run(tc.name, func(t *testing.T) { + got := Complete(tc.job, tc.err, 30, now) + if got.Status != tc.status || got.Level != tc.level { + t.Fatalf("%+v", got) + } + if tc.status == "queued" && !got.RetryAt.Equal(now.Add(time.Minute)) { + t.Fatal(got) + } + if tc.status != "queued" && !got.RetryAt.IsZero() { + t.Fatal("terminal job retried") + } + }) + } +} + +func TestAdmissionPolicy(t *testing.T) { + s := Settings{Concurrency: 4} + if !CanStart(s, 0, true) || CanStart(s, 1, true) || !CanStart(s, 3, false) || CanStart(s, 4, false) { + t.Fatal("admission policy changed") + } + s.Paused = true + if CanStart(s, 0, false) { + t.Fatal("paused worker admitted") + } +} diff --git a/internal/dregarnuhr/registry.go b/internal/dregarnuhr/registry.go new file mode 100644 index 0000000..aa33133 --- /dev/null +++ b/internal/dregarnuhr/registry.go @@ -0,0 +1,49 @@ +package dregarnuhr + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" +) + +type Handler func(context.Context, *Job) error + +// Registry holds existing event identifiers, not new provider or consumer IDs. +type Registry struct { + mu sync.RWMutex + handlers map[string]Handler +} + +func (r *Registry) Register(kind string, handler Handler) error { + if strings.TrimSpace(kind) == "" || handler == nil { + return errors.New("event type and handler are required") + } + r.mu.Lock() + defer r.mu.Unlock() + if r.handlers == nil { + r.handlers = make(map[string]Handler) + } + if _, ok := r.handlers[kind]; ok { + return fmt.Errorf("duplicate event type %q", kind) + } + r.handlers[kind] = handler + return nil +} + +func (r *Registry) Run(ctx context.Context, job *Job) error { + if job == nil { + return errors.New("job is required") + } + if err := ctx.Err(); err != nil { + return err + } + r.mu.RLock() + handler := r.handlers[job.Type] + r.mu.RUnlock() + if handler == nil { + return fmt.Errorf("unknown event type %q", job.Type) + } + return handler(ctx, job) +} diff --git a/internal/dregarnuhr/runner.go b/internal/dregarnuhr/runner.go new file mode 100644 index 0000000..3825736 --- /dev/null +++ b/internal/dregarnuhr/runner.go @@ -0,0 +1,127 @@ +package dregarnuhr + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "time" +) + +// Queue owns atomic priority-ordered claims and persisted completion. ErrNoJob +// means no eligible job. The initial adapter is single-writer-process, not HA. +type Queue interface { + Settings(context.Context) (Settings, error) + Claim(context.Context, time.Time) (*Job, error) + Cancellation(context.Context, string) (bool, error) + Started(context.Context, *Job) error + Finish(context.Context, *Job, error, time.Time) error +} + +type RunnerOptions struct { + Clock Clock + // Active is an optional compatibility counter for an existing monitor. + Active *atomic.Int64 + Foreground func() bool + Wake func() + Report func(stage string, job *Job, err error) +} + +type Runner struct { + queue Queue + registry *Registry + opts RunnerOptions + mu sync.Mutex +} + +func NewRunner(queue Queue, registry *Registry, opts RunnerOptions) *Runner { + if opts.Clock == nil { + opts.Clock = SystemClock{} + } + if opts.Active == nil { + opts.Active = new(atomic.Int64) + } + return &Runner{queue: queue, registry: registry, opts: opts} +} + +func (r *Runner) report(stage string, job *Job, err error) { + if err != nil && r.opts.Report != nil { + r.opts.Report(stage, job, err) + } +} + +func (r *Runner) Running() int64 { return r.opts.Active.Load() } + +// Dispatch serializes admission plus claim across concurrent callers. Each +// invocation is bounded even if workers finish immediately and refill slots. +func (r *Runner) Dispatch(ctx context.Context) { + r.mu.Lock() + defer r.mu.Unlock() + for n := 0; n < 32 && ctx.Err() == nil; n++ { + settings, err := r.queue.Settings(ctx) + if err != nil { + r.report("settings", nil, err) + return + } + foreground := r.opts.Foreground != nil && r.opts.Foreground() + if !CanStart(settings, r.Running(), foreground) { + return + } + job, err := r.queue.Claim(ctx, r.opts.Clock.Now()) + if errors.Is(err, ErrNoJob) || errors.Is(err, context.Canceled) { + return + } + if err != nil { + r.report("claim", nil, err) + return + } + if job == nil { + r.report("claim", nil, errors.New("queue returned nil job")) + return + } + r.opts.Active.Add(1) + r.report("start-log", job, r.queue.Started(ctx, job)) + go r.execute(ctx, job) + } +} + +func (r *Runner) execute(ctx context.Context, job *Job) { + defer func() { + r.opts.Active.Add(-1) + if r.opts.Wake != nil { + r.opts.Wake() + } + }() + runErr := r.invoke(ctx, job) + // Completion must still be attempted when the worker context is cancelled. + finishCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + r.report("finish", job, r.queue.Finish(finishCtx, job, runErr, r.opts.Clock.Now())) + r.report("execute", job, runErr) +} + +func (r *Runner) invoke(ctx context.Context, job *Job) (err error) { + defer func() { + if recover() != nil { + err = errors.New("event handler panicked") + } + }() + cancelled, err := r.queue.Cancellation(ctx, job.UUID) + if err != nil { + return err + } + if cancelled { + return errors.New("cancelled before execution") + } + if err = r.registry.Run(ctx, job); err != nil { + return err + } + cancelled, err = r.queue.Cancellation(ctx, job.UUID) + if err != nil { + return err + } + if cancelled { + return errors.New("cancelled during execution") + } + return nil +} diff --git a/internal/dregarnuhr/runner_test.go b/internal/dregarnuhr/runner_test.go new file mode 100644 index 0000000..3b0959f --- /dev/null +++ b/internal/dregarnuhr/runner_test.go @@ -0,0 +1,185 @@ +package dregarnuhr + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" +) + +type finishedJob struct { + id string + err error + at time.Time +} +type fakeQueue struct { + mu sync.Mutex + settings Settings + jobs []Job + claimed int + cancelled bool + cancelErr error + finished chan finishedJob + claimAt time.Time +} + +func newFakeQueue(concurrency int) *fakeQueue { + q := &fakeQueue{settings: Settings{Concurrency: concurrency}, finished: make(chan finishedJob, 100)} + for n := 0; n < 64; n++ { + q.jobs = append(q.jobs, Job{UUID: fmt.Sprint(n), Type: "synthetic"}) + } + return q +} +func (q *fakeQueue) Settings(context.Context) (Settings, error) { + q.mu.Lock() + defer q.mu.Unlock() + return q.settings, nil +} +func (q *fakeQueue) Claim(_ context.Context, at time.Time) (*Job, error) { + q.mu.Lock() + defer q.mu.Unlock() + if len(q.jobs) == 0 { + return nil, ErrNoJob + } + j := q.jobs[0] + q.jobs = q.jobs[1:] + q.claimed++ + q.claimAt = at + return &j, nil +} +func (q *fakeQueue) Cancellation(context.Context, string) (bool, error) { + q.mu.Lock() + defer q.mu.Unlock() + return q.cancelled, q.cancelErr +} +func (q *fakeQueue) Started(context.Context, *Job) error { return nil } +func (q *fakeQueue) Finish(ctx context.Context, j *Job, err error, at time.Time) error { + if ctx.Err() != nil { + return ctx.Err() + } + q.finished <- finishedJob{j.UUID, err, at} + return nil +} + +func TestRunnerConcurrentAdmission(t *testing.T) { + for _, foreground := range []bool{false, true} { + t.Run(fmt.Sprint(foreground), func(t *testing.T) { + q := newFakeQueue(4) + reg := &Registry{} + block := make(chan struct{}) + started := make(chan struct{}, 4) + wakes := make(chan struct{}, 4) + clock := newFakeClock() + defer close(block) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := reg.Register("synthetic", func(ctx context.Context, _ *Job) error { + started <- struct{}{} + select { + case <-ctx.Done(): + return ctx.Err() + case <-block: + return nil + } + }); err != nil { + t.Fatal(err) + } + r := NewRunner(q, reg, RunnerOptions{Clock: clock, Foreground: func() bool { return foreground }, Wake: func() { wakes <- struct{}{} }}) + var wg sync.WaitGroup + for n := 0; n < 16; n++ { + wg.Add(1) + go func() { defer wg.Done(); r.Dispatch(ctx) }() + } + wg.Wait() + want := 4 + if foreground { + want = 1 + } + for n := 0; n < want; n++ { + receive(t, started) + } + if r.Running() != int64(want) || q.claimed != want || !q.claimAt.Equal(clock.Now()) { + t.Fatalf("active=%d claimed=%d want=%d", r.Running(), q.claimed, want) + } + cancel() + for n := 0; n < want; n++ { + receive(t, q.finished) + receive(t, wakes) + } + if r.Running() != 0 { + t.Fatal("worker slots leaked") + } + }) + } +} + +func TestRunnerOutcomesAndCancellation(t *testing.T) { + for _, mode := range []string{"success", "cancel-before", "cancel-after", "cancellation-read-error", "panic", "unknown", "failure"} { + t.Run(mode, func(t *testing.T) { + q := newFakeQueue(1) + q.jobs = q.jobs[:1] + q.cancelled = mode == "cancel-before" + if mode == "cancellation-read-error" { + q.cancelErr = errors.New("database unavailable") + } + reg := &Registry{} + called := false + if mode != "unknown" { + if err := reg.Register("synthetic", func(context.Context, *Job) error { + called = true + if mode == "panic" { + panic("sensitive panic payload") + } + if mode == "failure" { + return errors.New("handler failure") + } + if mode == "cancel-after" { + q.mu.Lock() + q.cancelled = true + q.mu.Unlock() + } + return nil + }); err != nil { + t.Fatal(err) + } + } + wakes := make(chan struct{}, 1) + clock := newFakeClock() + r := NewRunner(q, reg, RunnerOptions{Clock: clock, Wake: func() { wakes <- struct{}{} }}) + r.Dispatch(context.Background()) + out := receive(t, q.finished) + receive(t, wakes) + if (out.err == nil) != (mode == "success") { + t.Fatal(mode, out.err) + } + if (mode == "cancel-before" || mode == "cancellation-read-error") && called { + t.Fatal("handler ran without cancellation check") + } + if mode == "panic" && out.err.Error() != "event handler panicked" { + t.Fatal("panic payload leaked") + } + if r.Running() != 0 || !out.at.Equal(clock.Now()) { + t.Fatal("completion/clock failure") + } + }) + } +} + +func TestRegistryAndPausedRunner(t *testing.T) { + r := &Registry{} + h := func(context.Context, *Job) error { return nil } + if r.Register("", h) == nil || r.Register("nil", nil) == nil { + t.Fatal("invalid registration") + } + if r.Register("synthetic", h) != nil || r.Register("synthetic", h) == nil { + t.Fatal("duplicate replaced") + } + q := newFakeQueue(3) + q.settings.Paused = true + NewRunner(q, r, RunnerOptions{}).Dispatch(context.Background()) + if q.claimed != 0 { + t.Fatal("paused claim") + } +} diff --git a/internal/dregarnuhr/types.go b/internal/dregarnuhr/types.go new file mode 100644 index 0000000..1b02f2b --- /dev/null +++ b/internal/dregarnuhr/types.go @@ -0,0 +1,40 @@ +// Package dregarnuhr owns in-process scheduling, worker admission and retry +// policy. Queue provides persistence; Registry provides business operations. +// Atomic claims are not cross-instance expiring leases. +package dregarnuhr + +import "errors" + +const Component = "dregarnuhr" + +var ( + ErrNoJob = errors.New("no due job") + ErrSkipped = errors.New("event is not applicable") +) + +type Job struct { + UUID string `json:"uuid"` + EventUUID string `json:"event_uuid,omitempty"` + Type string `json:"event_type"` + Key string `json:"event_key"` + Payload string `json:"payload,omitempty"` + Status string `json:"status"` + Priority int `json:"priority"` + Attempts int `json:"attempts"` + MaxAttempts int `json:"max_attempts"` + RunAtMS int64 `json:"run_at_ms"` + StartedAtMS int64 `json:"started_at_ms,omitempty"` + FinishedAtMS int64 `json:"finished_at_ms,omitempty"` + CancelRequested bool `json:"cancel_requested"` + Error string `json:"error,omitempty"` + CreatedAt int64 `json:"created_at_ms"` + UpdatedAt int64 `json:"updated_at_ms"` +} + +type Settings struct { + Paused bool `json:"paused"` + Concurrency int `json:"concurrency"` + MaxDepth int `json:"max_depth"` + RetryBaseSeconds int `json:"retry_base_seconds"` + RetentionHours int `json:"retention_hours"` +} diff --git a/internal/geduldh/boundary_test.go b/internal/geduldh/boundary_test.go new file mode 100644 index 0000000..742f5f3 --- /dev/null +++ b/internal/geduldh/boundary_test.go @@ -0,0 +1,50 @@ +package geduldh + +import ( + "go/parser" + "go/token" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" +) + +func TestComponentBoundary(t *testing.T) { + if Component != "geduldh" { + t.Fatal("component identity changed") + } + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, spec := range ast.Imports { + path, err := strconv.Unquote(spec.Path.Value) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"pageswiki/internal", "database", "net/http"} { + if path == forbidden || strings.HasPrefix(path, forbidden+"/") { + t.Errorf("%s imports %s: payload storage must not own metadata, HTTP, authorization or the compatibility facade", file, path) + } + } + } + } + // Ordinary payload access cannot delete objects or make retention decisions. + contract := reflect.TypeFor[Store]() + var methods []string + for i := 0; i < contract.NumMethod(); i++ { + methods = append(methods, contract.Method(i).Name) + } + if want := []string{"Open", "PutIfAbsent", "Stat", "Verify"}; !reflect.DeepEqual(methods, want) { + t.Fatalf("Document Store contract changed: %v", methods) + } +} diff --git a/internal/geduldh/local.go b/internal/geduldh/local.go new file mode 100644 index 0000000..aba9da0 --- /dev/null +++ b/internal/geduldh/local.go @@ -0,0 +1,264 @@ +package geduldh + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "math" + "os" + "path" + "path/filepath" + "strings" +) + +// Local requires hard-link no-replace publication and directory fsync support. +// A mounted NFS/SMB/SSHFS share is not certified just because it has a path. +// Failed persistence primitives return errors; there is no unsafe rename fallback. +type Local struct{ root string } + +var _ Store = (*Local)(nil) + +// NewLocal has no I/O side effects. The root must be provisioned by the caller. +func NewLocal(root string) (*Local, error) { + if strings.TrimSpace(root) == "" { + return nil, errors.New("Document root is required") + } + absolute, err := filepath.Abs(root) + if err != nil { + return nil, err + } + return &Local{root: absolute}, nil +} + +func (s *Local) Open(ctx context.Context, hash string) (Reader, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + key, err := Key(hash) + if err != nil { + return nil, err + } + root, err := os.OpenRoot(s.root) + if err != nil { + return nil, err + } + defer root.Close() + file, err := root.Open(key) + if err != nil { + return nil, err + } + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() { + file.Close() + if err != nil { + return nil, err + } + return nil, errors.New("Document is not a regular file") + } + return &localReader{file: file, ctx: ctx}, nil +} + +func (s *Local) Stat(ctx context.Context, hash string) (Info, error) { + reader, err := s.Open(ctx, hash) + if err != nil { + return Info{}, err + } + defer reader.Close() + info, err := reader.(*localReader).file.Stat() + if err != nil { + return Info{}, err + } + return Info{Hash: strings.ToLower(strings.TrimSpace(hash)), Size: info.Size()}, nil +} + +func verify(ctx context.Context, hash string, reader io.Reader) error { + sum := sha256.New() + if _, err := io.CopyBuffer(sum, contextReader{ctx, reader}, make([]byte, 64<<10)); err != nil { + return err + } + if hex.EncodeToString(sum.Sum(nil)) != strings.ToLower(strings.TrimSpace(hash)) { + return ErrHashMismatch + } + return ctx.Err() +} + +func (s *Local) Verify(ctx context.Context, hash string) error { + reader, err := s.Open(ctx, hash) + if err != nil { + return err + } + defer reader.Close() + return verify(ctx, hash, reader) +} + +func (s *Local) PutIfAbsent(ctx context.Context, hash string, size int64, input io.Reader) error { + if err := ctx.Err(); err != nil { + return err + } + key, err := Key(hash) + if err != nil { + return err + } + if size < 0 || size == math.MaxInt64 { + return ErrSizeMismatch + } + if input == nil { + return errors.New("Document reader is required") + } + root, err := os.OpenRoot(s.root) + if err != nil { + return err + } + defer root.Close() + // The common retry/backfill path must not rewrite or stage existing blobs. + // Still validate the supplied bytes, then verify the stored content. + if existing, err := root.Open(key); err == nil { + defer existing.Close() + info, err := existing.Stat() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return errors.New("Document is not a regular file") + } + if err := copyValidated(ctx, hash, size, input, io.Discard); err != nil { + return err + } + if err := verify(ctx, hash, existing); err != nil { + return err + } + return syncPublishedDirectories(root, path.Dir(key)) + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + dir := path.Dir(key) + if err := root.MkdirAll(dir, 0755); err != nil { + return err + } + var nonce [16]byte + if _, err := rand.Read(nonce[:]); err != nil { + return err + } + tmp := dir + "/.document-" + hex.EncodeToString(nonce[:]) + file, err := root.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + defer root.Remove(tmp) + defer file.Close() + if err := copyValidated(ctx, hash, size, input, file); err != nil { + return err + } + if err := file.Chmod(0644); err != nil { + return err + } + if err := file.Sync(); err != nil { + return err + } + if err := file.Close(); err != nil { + return err + } + // Link is atomic and fails if the destination exists. Rename would silently + // replace an existing payload between an existence check and publication. + if err := root.Link(tmp, key); err != nil { + if !errors.Is(err, os.ErrExist) { + return err + } + existing, err := root.Open(key) + if err != nil { + return err + } + defer existing.Close() + info, err := existing.Stat() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return errors.New("Document is not a regular file") + } + if err := verify(ctx, hash, existing); err != nil { + return fmt.Errorf("existing Document: %w", err) + } + } + // Persist the new entry and newly created shard directories before allowing + // a metadata transaction to reference the payload. Errors are not swallowed. + if err := syncPublishedDirectories(root, dir); err != nil { + return err + } + return ctx.Err() +} + +func syncPublishedDirectories(root *os.Root, shard string) error { + for _, name := range []string{shard, "sha256", "."} { + directory, err := root.Open(name) + if err != nil { + return err + } + err = directory.Sync() + closeErr := directory.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + } + return nil +} + +func copyValidated(ctx context.Context, hash string, size int64, input io.Reader, output io.Writer) error { + sum := sha256.New() + n, err := io.CopyBuffer(io.MultiWriter(output, sum), contextReader{ctx, io.LimitReader(input, size+1)}, make([]byte, 64<<10)) + if err != nil { + return err + } + if n != size { + return ErrSizeMismatch + } + if hex.EncodeToString(sum.Sum(nil)) != strings.ToLower(strings.TrimSpace(hash)) { + return ErrHashMismatch + } + return ctx.Err() +} + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(p) +} + +type localReader struct { + file *os.File + ctx context.Context +} + +func (r *localReader) Close() error { return r.file.Close() } + +func (r *localReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.file.Read(p) +} +func (r *localReader) ReadAt(p []byte, offset int64) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.file.ReadAt(p, offset) +} +func (r *localReader) Seek(offset int64, whence int) (int64, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.file.Seek(offset, whence) +} diff --git a/internal/geduldh/local_test.go b/internal/geduldh/local_test.go new file mode 100644 index 0000000..9e27337 --- /dev/null +++ b/internal/geduldh/local_test.go @@ -0,0 +1,264 @@ +package geduldh + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +func digest(data []byte) string { sum := sha256.Sum256(data); return hex.EncodeToString(sum[:]) } +func local(t *testing.T) (*Local, string) { + t.Helper() + root := t.TempDir() + s, err := NewLocal(root) + if err != nil { + t.Fatal(err) + } + return s, root +} + +func TestLocalRoundTripAndRange(t *testing.T) { + for _, payload := range [][]byte{nil, []byte("text 日本語"), {0, 0xff, 1, 0, 3}} { + s, _ := local(t) + ctx := context.Background() + hash := digest(payload) + if err := s.PutIfAbsent(ctx, hash, int64(len(payload)), bytes.NewReader(payload)); err != nil { + t.Fatal(err) + } + info, err := s.Stat(ctx, hash) + if err != nil || info.Hash != hash || info.Size != int64(len(payload)) { + t.Fatalf("info=%+v err=%v", info, err) + } + reader, err := s.Open(ctx, hash) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(reader) + if err != nil || !bytes.Equal(data, payload) { + t.Fatalf("data=%v err=%v", data, err) + } + if size, err := reader.Seek(0, io.SeekEnd); err != nil || size != int64(len(payload)) { + t.Fatalf("size=%d err=%v", size, err) + } + if len(payload) > 2 { + buf := make([]byte, 2) + if _, err := reader.ReadAt(buf, 1); err != nil || !bytes.Equal(buf, payload[1:3]) { + t.Fatalf("range=%v err=%v", buf, err) + } + } + reader.Close() + if err := s.Verify(ctx, hash); err != nil { + t.Fatal(err) + } + } +} + +func TestLocalRejectsIncompleteOrCorruptInput(t *testing.T) { + payload := []byte("good") + for _, tc := range []struct { + name string + data []byte + size int64 + want error + }{ + {"hash", []byte("evil"), 4, ErrHashMismatch}, + {"short", []byte("goo"), 4, ErrSizeMismatch}, + {"long", []byte("good!"), 4, ErrSizeMismatch}, + {"negative", payload, -1, ErrSizeMismatch}, + } { + t.Run(tc.name, func(t *testing.T) { + s, root := local(t) + err := s.PutIfAbsent(context.Background(), digest(payload), tc.size, bytes.NewReader(tc.data)) + if !errors.Is(err, tc.want) { + t.Fatalf("got %v want %v", err, tc.want) + } + if _, err := s.Stat(context.Background(), digest(payload)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("published bad content: %v", err) + } + assertNoTemps(t, root) + }) + } +} + +func TestLocalExistingAndConcurrentWritesNeverReplace(t *testing.T) { + s, root := local(t) + payload := []byte("immutable payload") + hash := digest(payload) + ctx := context.Background() + var wg sync.WaitGroup + errorsCh := make(chan error, 12) + for range 12 { + wg.Add(1) + go func() { + defer wg.Done() + errorsCh <- s.PutIfAbsent(ctx, hash, int64(len(payload)), bytes.NewReader(payload)) + }() + } + wg.Wait() + close(errorsCh) + for err := range errorsCh { + if err != nil { + t.Fatal(err) + } + } + key, _ := Key(hash) + filename := filepath.Join(root, filepath.FromSlash(key)) + before, _ := os.Stat(filename) + if err := s.PutIfAbsent(ctx, hash, int64(len(payload)), bytes.NewReader(payload)); err != nil { + t.Fatal(err) + } + after, _ := os.Stat(filename) + if !os.SameFile(before, after) { + t.Fatal("existing inode replaced") + } + if err := os.WriteFile(filename, []byte("corrupt"), 0644); err != nil { + t.Fatal(err) + } + if err := s.Verify(ctx, hash); !errors.Is(err, ErrHashMismatch) { + t.Fatalf("verify=%v", err) + } + if err := s.PutIfAbsent(ctx, hash, int64(len(payload)), bytes.NewReader(payload)); !errors.Is(err, ErrHashMismatch) { + t.Fatalf("put=%v", err) + } + content, _ := os.ReadFile(filename) + if string(content) != "corrupt" { + t.Fatal("corruption silently overwritten") + } + assertNoTemps(t, root) +} + +func TestLocalCancellationAndReaderFailure(t *testing.T) { + s, root := local(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + hash := digest([]byte("data")) + if err := s.PutIfAbsent(ctx, hash, 4, strings.NewReader("data")); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if _, err := s.Open(ctx, hash); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + failure := errors.New("source failed") + if err := s.PutIfAbsent(context.Background(), hash, 4, failingReader{failure}); !errors.Is(err, failure) { + t.Fatal(err) + } + if _, err := s.Stat(context.Background(), hash); !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + if err := s.PutIfAbsent(context.Background(), hash, 4, strings.NewReader("data")); err != nil { + t.Fatal(err) + } + ctx, cancel = context.WithCancel(context.Background()) + r, err := s.Open(ctx, hash) + if err != nil { + t.Fatal(err) + } + defer r.Close() + cancel() + if _, err := r.Read(make([]byte, 1)); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if _, err := r.ReadAt(make([]byte, 1), 0); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + assertNoTemps(t, root) +} + +type failingReader struct{ err error } + +func (r failingReader) Read([]byte) (int, error) { return 0, r.err } + +func TestLocalStreamsWithBoundedReadsAndMidStreamCancellation(t *testing.T) { + s, root := local(t) + payload := bytes.Repeat([]byte("large document\x00"), 100000) + hash := digest(payload) + input := &boundedReader{reader: bytes.NewReader(payload)} + if err := s.PutIfAbsent(context.Background(), hash, int64(len(payload)), input); err != nil { + t.Fatal(err) + } + if input.reads < 2 { + t.Fatal("expected multiple bounded reads") + } + // Cancellation while staging must never publish the final content address. + s2, root2 := local(t) + ctx, cancel := context.WithCancel(context.Background()) + input = &boundedReader{reader: bytes.NewReader(payload), cancel: cancel} + if err := s2.PutIfAbsent(ctx, hash, int64(len(payload)), input); !errors.Is(err, context.Canceled) { + t.Fatalf("put=%v", err) + } + if _, err := s2.Stat(context.Background(), hash); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("published canceled upload: %v", err) + } + assertNoTemps(t, root) + assertNoTemps(t, root2) +} + +type boundedReader struct { + reader io.Reader + reads int + cancel context.CancelFunc +} + +func (r *boundedReader) Read(p []byte) (int, error) { + if len(p) > 64<<10 { + return 0, errors.New("unbounded input read") + } + r.reads++ + n, err := r.reader.Read(p) + if r.cancel != nil { + r.cancel() + } + return n, err +} + +func TestLocalKeysAndSymlinkEscape(t *testing.T) { + for _, hash := range []string{"", "../../etc/passwd", strings.Repeat("g", 64), strings.Repeat("a", 63)} { + if _, err := Key(hash); !errors.Is(err, ErrInvalidHash) { + t.Fatal(err) + } + } + hash := digest([]byte("data")) + key, err := Key(" " + strings.ToUpper(hash) + " ") + if err != nil || key != "sha256/"+hash[:2]+"/"+hash { + t.Fatalf("key=%q err=%v", key, err) + } + s, root := local(t) + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "sha256")); err != nil { + t.Skip(err) + } + if err := s.PutIfAbsent(context.Background(), hash, 4, strings.NewReader("data")); err == nil { + t.Fatal("escaped storage root") + } + if _, err := s.Open(context.Background(), hash); err == nil { + t.Fatal("opened escaped path") + } + entries, err := os.ReadDir(outside) + if err != nil || len(entries) != 0 { + t.Fatalf("outside modified: %v %v", entries, err) + } +} + +func assertNoTemps(t *testing.T, root string) { + t.Helper() + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if strings.HasPrefix(entry.Name(), ".document-") { + t.Errorf("temporary file left: %s", path) + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/geduldh/materialize.go b/internal/geduldh/materialize.go new file mode 100644 index 0000000..53052a5 --- /dev/null +++ b/internal/geduldh/materialize.go @@ -0,0 +1,219 @@ +package geduldh + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +var ErrMaterializeLimit = errors.New("Document materialization capacity exceeded") + +// Source deliberately excludes mutation. Authorization precedes Acquire. +type Source interface { + Stat(context.Context, string) (Info, error) + Open(context.Context, string) (Reader, error) +} + +type MaterializeOptions struct { + // TempDir must be node-local scratch, never canonical storage. Empty uses TMPDIR. + TempDir string + MaxBytes int64 + Concurrent int + // MaxEntries bounds queued/leased objects too, including empty payloads. + // Zero selects 64. + MaxEntries int + Timeout time.Duration +} + +// Materializer shares in-flight/leased copies only. The last lease removes the +// copy; it is not a persistent cache and needs no idle cache eviction goroutine. +// Limits include both copies being downloaded and copies in use by tools. +type Materializer struct { + source Source + options MaterializeOptions + slots chan struct{} + mu sync.Mutex + entries map[string]*materializedEntry + reserved int64 + active int +} + +type materializedEntry struct { + hash string + size int64 + refs int + ready chan struct{} + cancel context.CancelFunc + finished bool + dir, path string + err error +} + +// Lease pins an immutable scratch copy until Close. Tools must finish before +// Close; never give its path to a detached process. Close is idempotent. +type Lease struct { + Path string + Size int64 + once sync.Once + release func() +} + +func (l *Lease) Close() { l.once.Do(l.release) } + +func NewMaterializer(source Source, options MaterializeOptions) (*Materializer, error) { + if source == nil || options.MaxBytes <= 0 || options.Concurrent <= 0 || options.Timeout <= 0 { + return nil, errors.New("invalid Document materializer configuration") + } + if options.MaxEntries == 0 { + options.MaxEntries = 64 + } + if options.MaxEntries < 0 { + return nil, errors.New("invalid materializer entry limit") + } + return &Materializer{source: source, options: options, slots: make(chan struct{}, options.Concurrent), entries: make(map[string]*materializedEntry)}, nil +} + +func (m *Materializer) Acquire(ctx context.Context, hash string, maxBytes int64) (*Lease, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if _, err := Key(hash); err != nil { + return nil, err + } + hash = strings.ToLower(strings.TrimSpace(hash)) + info, err := m.source.Stat(ctx, hash) + if err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + if info.Size < 0 || maxBytes < 0 || info.Size > maxBytes || info.Size > m.options.MaxBytes { + return nil, ErrMaterializeLimit + } + m.mu.Lock() + e := m.entries[hash] + if e != nil && e.size != info.Size { + m.mu.Unlock() + return nil, ErrSizeMismatch + } + if e == nil { + if info.Size > m.options.MaxBytes-m.reserved || m.active >= m.options.MaxEntries { + m.mu.Unlock() + return nil, ErrMaterializeLimit + } + // One caller leaving must not cancel another caller's transfer. The + // shared deadline is propagated to backend operations. Backends must + // honor cancellation; it cannot interrupt an uninterruptible OS syscall. + loadCtx, cancel := context.WithTimeout(context.Background(), m.options.Timeout) + e = &materializedEntry{hash: hash, size: info.Size, ready: make(chan struct{}), cancel: cancel} + m.entries[hash] = e + m.reserved += info.Size + m.active++ + go m.load(loadCtx, e) + } + e.refs++ + m.mu.Unlock() + select { + case <-ctx.Done(): + m.release(e) + return nil, ctx.Err() + case <-e.ready: + if err := ctx.Err(); err != nil { + m.release(e) + return nil, err + } + if e.err != nil { + m.release(e) + return nil, e.err + } + return &Lease{Path: e.path, Size: e.size, release: func() { m.release(e) }}, nil + } +} + +func (m *Materializer) load(ctx context.Context, e *materializedEntry) { + defer e.cancel() + var dir, filename string + var err error + select { + case m.slots <- struct{}{}: + dir, filename, err = m.copy(ctx, e.hash, e.size) + <-m.slots + case <-ctx.Done(): + err = ctx.Err() + } + m.mu.Lock() + e.dir, e.path, e.err, e.finished = dir, filename, err, true + close(e.ready) + if e.refs == 0 { + m.remove(e) + } + m.mu.Unlock() +} + +func (m *Materializer) release(e *materializedEntry) { + m.mu.Lock() + defer m.mu.Unlock() + e.refs-- + if e.refs == 0 { + e.cancel() + // A later caller must not join an already-cancelled download. + if m.entries[e.hash] == e { + delete(m.entries, e.hash) + } + if e.finished { + m.remove(e) + } + } +} + +func (m *Materializer) remove(e *materializedEntry) { + if m.entries[e.hash] == e { + delete(m.entries, e.hash) + } + // Failed removal keeps the byte reservation: do not pretend disk space + // was reclaimed. Recovery of crash leftovers is an operator/scratch task. + if e.dir != "" { + if err := os.RemoveAll(e.dir); err != nil { + return + } + } + m.reserved -= e.size + m.active-- +} + +func (m *Materializer) copy(ctx context.Context, hash string, size int64) (dir, filename string, err error) { + reader, err := m.source.Open(ctx, hash) + if err != nil { + return "", "", err + } + defer reader.Close() + dir, err = os.MkdirTemp(m.options.TempDir, "pageswiki-document-*") + if err != nil { + return "", "", err + } + // Failed partial copies are retained in dir until load performs cleanup. + filename = filepath.Join(dir, hash) + file, err := os.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return dir, "", err + } + err = copyValidated(ctx, hash, size, reader, file) + if err == nil { + err = file.Chmod(0400) + } + closeErr := file.Close() + if err == nil { + err = closeErr + } + if err != nil { + return dir, "", fmt.Errorf("materialize Document: %w", err) + } + // Scratch is reproducible, not durable canonical storage: no fsync here. + return dir, filename, ctx.Err() +} diff --git a/internal/geduldh/materialize_test.go b/internal/geduldh/materialize_test.go new file mode 100644 index 0000000..0414208 --- /dev/null +++ b/internal/geduldh/materialize_test.go @@ -0,0 +1,283 @@ +package geduldh + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" +) + +type materializeSource struct { + body []byte + size int64 + gate chan struct{} + started chan struct{} + opens atomic.Int64 +} + +func (s *materializeSource) Stat(ctx context.Context, hash string) (Info, error) { + return Info{Hash: hash, Size: s.size}, ctx.Err() +} +func (s *materializeSource) Open(ctx context.Context, hash string) (Reader, error) { + s.opens.Add(1) + if s.started != nil { + s.started <- struct{}{} + } + if s.gate != nil { + select { + case <-s.gate: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return materializeReader{bytes.NewReader(s.body)}, ctx.Err() +} + +type materializeReader struct{ *bytes.Reader } + +func (materializeReader) Close() error { return nil } + +func newTestMaterializer(t *testing.T, source Source, quota int64) (*Materializer, string) { + t.Helper() + dir := t.TempDir() + m, err := NewMaterializer(source, MaterializeOptions{TempDir: dir, MaxBytes: quota, Concurrent: 1, Timeout: 5 * time.Second}) + if err != nil { + t.Fatal(err) + } + return m, dir +} + +func waitMaterializer(t *testing.T, m *Materializer, predicate func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + m.mu.Lock() + ok := predicate() + m.mu.Unlock() + if ok { + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("materializer condition did not become true") +} + +func TestMaterializerSharesPinnedCopiesAndCleansLastLease(t *testing.T) { + body := []byte("materialized 日本語") + s := &materializeSource{body: body, size: int64(len(body))} + m, root := newTestMaterializer(t, s, 1024) + a, err := m.Acquire(context.Background(), digest(body), 1024) + if err != nil { + t.Fatal(err) + } + b, err := m.Acquire(context.Background(), digest(body), 1024) + if err != nil { + t.Fatal(err) + } + if a.Path != b.Path || s.opens.Load() != 1 { + t.Fatal("copy was not shared") + } + data, err := os.ReadFile(a.Path) + if err != nil || !bytes.Equal(data, body) { + t.Fatalf("body=%q err=%v", data, err) + } + info, _ := os.Stat(a.Path) + if info.Mode().Perm() != 0400 { + t.Fatalf("permissions=%v", info.Mode()) + } + dirInfo, _ := os.Stat(filepath.Dir(a.Path)) + if dirInfo.Mode().Perm() != 0700 { + t.Fatalf("directory permissions=%v", dirInfo.Mode()) + } + a.Close() + a.Close() + if _, err := os.Stat(b.Path); err != nil { + t.Fatal("active lease lost its file", err) + } + b.Close() + items, _ := os.ReadDir(root) + if len(items) != 0 || m.reserved != 0 || m.active != 0 { + t.Fatal("scratch was not reclaimed") + } +} + +func TestMaterializerCoalescesDownloadsAndCancelsOnlyLastWaiter(t *testing.T) { + body := []byte("shared") + s := &materializeSource{body: body, size: int64(len(body)), gate: make(chan struct{}), started: make(chan struct{}, 2)} + m, _ := newTestMaterializer(t, s, 1024) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + errA := make(chan error, 1) + go func() { + lease, err := m.Acquire(ctx, digest(body), 1024) + if lease != nil { + lease.Close() + } + errA <- err + }() + <-s.started + leaseB := make(chan *Lease, 1) + errB := make(chan error, 1) + go func() { + lease, err := m.Acquire(context.Background(), digest(body), 1024) + leaseB <- lease + errB <- err + }() + waitMaterializer(t, m, func() bool { return m.entries[digest(body)].refs == 2 }) + cancel() + if err := <-errA; !errors.Is(err, context.Canceled) { + t.Fatalf("first waiter: %v", err) + } + close(s.gate) + b := <-leaseB + if err := <-errB; err != nil { + t.Fatal(err) + } + if s.opens.Load() != 1 { + t.Fatal("concurrent transfer not coalesced") + } + b.Close() + + s.gate = make(chan struct{}) + ctx, cancel = context.WithCancel(context.Background()) + go func() { + lease, err := m.Acquire(ctx, digest(body), 1024) + if lease != nil { + lease.Close() + } + errA <- err + }() + <-s.started + cancel() + if err := <-errA; !errors.Is(err, context.Canceled) { + t.Fatalf("last waiter: %v", err) + } + waitMaterializer(t, m, func() bool { return m.reserved == 0 && m.active == 0 }) +} + +func TestMaterializerRejectsCorruptionSizeAndCapacity(t *testing.T) { + for _, tc := range []struct { + name string + body []byte + size int64 + hash string + want error + }{ + {"hash", []byte("bad"), 3, digest([]byte("yes")), ErrHashMismatch}, + {"short", []byte("x"), 2, digest([]byte("x")), ErrSizeMismatch}, + {"long", []byte("xyz"), 2, digest([]byte("xyz")), ErrSizeMismatch}, + {"oversize", []byte("xyz"), 3, digest([]byte("xyz")), ErrMaterializeLimit}, + } { + t.Run(tc.name, func(t *testing.T) { + s := &materializeSource{body: tc.body, size: tc.size} + m, root := newTestMaterializer(t, s, 10) + limit := int64(10) + if tc.name == "oversize" { + limit = 2 + } + lease, err := m.Acquire(context.Background(), tc.hash, limit) + if lease != nil || !errors.Is(err, tc.want) { + t.Fatalf("lease=%v err=%v", lease, err) + } + items, _ := os.ReadDir(root) + if len(items) != 0 || m.reserved != 0 { + t.Fatal("failed input leaked scratch") + } + }) + } + s := &materializeSource{body: []byte("x"), size: 1} + m, _ := newTestMaterializer(t, s, 1) + lease, err := m.Acquire(context.Background(), digest(s.body), 1) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + if _, err := m.Acquire(context.Background(), digest([]byte("y")), 1); !errors.Is(err, ErrMaterializeLimit) { + t.Fatalf("quota: %v", err) + } + if s.opens.Load() != 1 { + t.Fatal("over-quota transfer started") + } +} + +func TestMaterializerTimeoutAndEntryBound(t *testing.T) { + s := &materializeSource{body: []byte("x"), size: 1, gate: make(chan struct{}), started: make(chan struct{}, 3)} + m, err := NewMaterializer(s, MaterializeOptions{TempDir: t.TempDir(), MaxBytes: 100, Concurrent: 1, MaxEntries: 1, Timeout: 50 * time.Millisecond}) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { _, err := m.Acquire(context.Background(), digest(s.body), 10); done <- err }() + <-s.started + if _, err := m.Acquire(context.Background(), digest([]byte("y")), 10); !errors.Is(err, ErrMaterializeLimit) { + t.Fatalf("entry bound: %v", err) + } + if err := <-done; !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("timeout: %v", err) + } + waitMaterializer(t, m, func() bool { return m.active == 0 }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := m.Acquire(ctx, digest(s.body), 10); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} + +func TestMaterializerConcurrencyBoundAndCanonicalIsolation(t *testing.T) { + s := &materializeSource{body: []byte("x"), size: 1, gate: make(chan struct{}), started: make(chan struct{}, 3)} + m, _ := newTestMaterializer(t, s, 100) + done := make(chan error, 2) + for _, data := range []string{"x", "y"} { + go func(data string) { + lease, err := m.Acquire(context.Background(), digest([]byte(data)), 100) + if lease != nil { + lease.Close() + } + done <- err + }(data) + } + <-s.started + waitMaterializer(t, m, func() bool { return m.active == 2 }) + if s.opens.Load() != 1 { + t.Fatal("concurrency limit exceeded") + } + close(s.gate) + for i := 0; i < 2; i++ { + if err := <-done; err != nil && !errors.Is(err, ErrHashMismatch) { + t.Fatal(err) + } + } + waitMaterializer(t, m, func() bool { return m.active == 0 }) + + canonical, root := local(t) + body := []byte("canonical") + hash := digest(body) + if err := canonical.PutIfAbsent(context.Background(), hash, int64(len(body)), bytes.NewReader(body)); err != nil { + t.Fatal(err) + } + m, _ = newTestMaterializer(t, canonical, 100) + lease, err := m.Acquire(context.Background(), hash, 100) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + key, _ := Key(hash) + sourceInfo, _ := os.Stat(filepath.Join(root, filepath.FromSlash(key))) + tempInfo, _ := os.Stat(lease.Path) + if os.SameFile(sourceInfo, tempInfo) { + t.Fatal("canonical inode exposed to external tools") + } + if err := os.Chmod(lease.Path, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(lease.Path, []byte("modified!"), 0600); err != nil { + t.Fatal(err) + } + if err := canonical.Verify(context.Background(), hash); err != nil { + t.Fatal("scratch mutation affected canonical data", err) + } +} diff --git a/internal/geduldh/store.go b/internal/geduldh/store.go new file mode 100644 index 0000000..170d5f4 --- /dev/null +++ b/internal/geduldh/store.go @@ -0,0 +1,58 @@ +// Package geduldh stores immutable, content-addressed Document payloads. +// Authorization and reference/retention decisions belong to the application, +// not to a storage backend. A hash or storage key is never an access token. +package geduldh + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "strings" +) + +const Component = "geduldh" + +var ( + ErrInvalidHash = errors.New("invalid Document SHA-256") + ErrHashMismatch = errors.New("Document payload does not match SHA-256") + ErrSizeMismatch = errors.New("Document payload does not match declared size") +) + +type Info struct { + Hash string + Size int64 +} + +// Reader supports bounded and seekable access, including HTTP Range and ZIP. +// A remote backend must not download the entire object merely to open it. +type Reader interface { + io.ReadSeekCloser + io.ReaderAt +} + +// Store is the immutable payload boundary, not a filesystem abstraction. +// PutIfAbsent must verify the size and SHA-256 before publishing a complete +// object, never replace an existing object, and reject corrupt existing bytes. +// A successful write must precede the metadata transaction which references it. +// Delete is deliberately absent: GC requires reference and retention protection. +type Store interface { + Stat(context.Context, string) (Info, error) + Open(context.Context, string) (Reader, error) + PutIfAbsent(context.Context, string, int64, io.Reader) error + Verify(context.Context, string) error +} + +// Key preserves the existing on-disk layout and is also suitable for S3 keys. +// Backend placement never changes Page/Revision identity or public URLs. +func Key(hash string) (string, error) { + hash = strings.ToLower(strings.TrimSpace(hash)) + if len(hash) != sha256.Size*2 { + return "", ErrInvalidHash + } + if _, err := hex.DecodeString(hash); err != nil { + return "", ErrInvalidHash + } + return "sha256/" + hash[:2] + "/" + hash, nil +} diff --git a/internal/grammaratur/authoring.go b/internal/grammaratur/authoring.go new file mode 100644 index 0000000..c88e93f --- /dev/null +++ b/internal/grammaratur/authoring.go @@ -0,0 +1,130 @@ +package grammaratur + +import ( + "bytes" + "encoding/json" + "sort" + "strings" + + "github.com/google/uuid" + "golang.org/x/net/html" +) + +// HTMLToMarkdownAuthoring keeps source HTML when conversion loses structure, +// attributes, comments or text. This is not a sanitization boundary. +func HTMLToMarkdownAuthoring(source string) (string, error) { + converted, err := HTMLToMarkdown(source) + if err != nil { + return "", err + } + if htmlStructure(source) != htmlStructure(MarkdownToHTML(converted)) { + return rawHTMLBlock(source), nil + } + return converted, nil +} + +// {=html} is an explicit authoring fence, not an ordinary code sample. The +// closing delimiter is longer than any run in the source, so arbitrary HTML +// (including Markdown examples) cannot accidentally terminate it. +func rawHTMLBlock(source string) string { + longest, run := 2, 0 + for _, c := range source { + if c == '`' { + run++ + if run > longest { + longest = run + } + } else { + run = 0 + } + } + fence := strings.Repeat("`", longest+1) + return fence + "{=html}\n" + source + "\n" + fence +} + +// MarkdownAuthoringHTML preserves protected HTML byte-for-byte while rendering +// ordinary Markdown. Callers must still apply projection sanitization for UI. +func MarkdownAuthoringHTML(source string) string { + lines := strings.SplitAfter(source, "\n") + var markdown strings.Builder + type rawBlock struct{ token, html string } + var blocks []rawBlock + var ordinary byte + ordinaryLen := 0 + for i := 0; i < len(lines); i++ { + line := strings.TrimSuffix(strings.TrimSuffix(lines[i], "\n"), "\r") + char, n := byte(0), 0 + if len(line) > 0 && (line[0] == '`' || line[0] == '~') { + char = line[0] + for n < len(line) && line[n] == char { + n++ + } + } + if ordinary != 0 { + markdown.WriteString(lines[i]) + if char == ordinary && n >= ordinaryLen && strings.TrimSpace(line[n:]) == "" { + ordinary = 0 + } + continue + } + if char == '`' && n >= 3 && strings.TrimSpace(line[n:]) == "{=html}" { + end := i + 1 + for ; end < len(lines); end++ { + close := strings.TrimSpace(lines[end]) + if len(close) >= n && strings.Trim(close, "`") == "" { + break + } + } + if end < len(lines) { + raw := strings.TrimSuffix(strings.Join(lines[i+1:end], ""), "\n") + token := "PAGESWIKIRAW" + strings.ReplaceAll(uuid.NewString(), "-", "") + blocks = append(blocks, rawBlock{token, raw}) + markdown.WriteString("\n\n" + token + "\n\n") + i = end + continue + } + } + if n >= 3 { + ordinary, ordinaryLen = char, n + } + markdown.WriteString(lines[i]) + } + if len(blocks) == 1 && strings.TrimSpace(markdown.String()) == blocks[0].token { + return blocks[0].html + } + out := MarkdownToHTML(markdown.String()) + for _, block := range blocks { + out = strings.Replace(out, "

"+block.token+"

", block.html, 1) + } + return out +} + +func htmlStructure(source string) string { + n, err := html.Parse(strings.NewReader(source)) + if err != nil { + return source + } + var out bytes.Buffer + var visit func(*html.Node, bool) + visit = func(n *html.Node, literal bool) { + literal = literal || n.Data == "pre" || n.Data == "code" || n.Data == "script" || n.Data == "style" + data := n.Data + if n.Type == html.TextNode && !literal { + if strings.TrimSpace(data) == "" && (n.Parent == nil || n.Parent.Data == "body" || n.Parent.Data == "html") { + return + } + // Inline whitespace can separate words (including around an inline + // element). Never normalize it away when deciding conversion safety. + } + attrs := append([]html.Attribute(nil), n.Attr...) + sort.Slice(attrs, func(i, j int) bool { return attrs[i].Namespace+":"+attrs[i].Key < attrs[j].Namespace+":"+attrs[j].Key }) + b, _ := json.Marshal([]any{n.Type, n.Namespace, data, attrs}) + out.Write(b) + for c := n.FirstChild; c != nil; c = c.NextSibling { + visit(c, literal) + } + out.WriteByte(')') + } + visit(n, false) + return out.String() +} diff --git a/internal/grammaratur/authoring_test.go b/internal/grammaratur/authoring_test.go new file mode 100644 index 0000000..2e0efba --- /dev/null +++ b/internal/grammaratur/authoring_test.go @@ -0,0 +1,67 @@ +package grammaratur + +import ( + "strings" + "testing" +) + +func TestAuthoringRoundTrip(t *testing.T) { + for _, source := range []string{ + "

Heading

Hello world.

", + "

{{children:/guide}}

", + "
\"Photo\"
A caption
", + "

text

", + "
ab
c
", + "

a b c

", + "
  print('x')\n
", + "opaque", + "
\n

one

\n\n

two

\n
", + "", + } { + t.Run(source, func(t *testing.T) { + md, err := HTMLToMarkdownAuthoring(source) + if err != nil { + t.Fatal(err) + } + if htmlStructure(MarkdownAuthoringHTML(md)) != htmlStructure(source) { + t.Fatalf("lossy conversion: %q", md) + } + }) + } + md, err := HTMLToMarkdownAuthoring("

Hello

") + if err != nil || md != "# Hello" { + t.Fatalf("simple Markdown conversion: %q %v", md, err) + } + for _, source := range []string{`

Hello

`, `

Title

`, `

a b c

`} { + md, err := HTMLToMarkdownAuthoring(source) + if err != nil { + t.Fatal(err) + } + if htmlStructure(MarkdownAuthoringHTML(md)) != htmlStructure(source) { + t.Fatal("whitespace/attributes lost") + } + } +} + +func TestAuthoringProtectedHTML(t *testing.T) { + for _, raw := range []string{"
a
", "
```\n{=html}\n````
\n", "", "

line\r\n

"} { + md := rawHTMLBlock(raw) + if got := MarkdownAuthoringHTML(md); got != raw { + t.Fatalf("raw bytes changed: %q want %q", got, raw) + } + mixed := MarkdownAuthoringHTML("# Added\n\n" + md + "\n\nTail") + if !strings.Contains(mixed, raw) || !strings.Contains(mixed, "

Added

") || !strings.Contains(mixed, "

Tail

") { + t.Fatalf("mixed blocks: %q", mixed) + } + } + code := "````text\n```{=html}\n\n```\n````" + if strings.Contains(MarkdownAuthoringHTML(code), ""} { + want, err := htmltomarkdown.ConvertString(source) + if err != nil { + t.Fatal(err) + } + for _, convert := range []func(string) (string, error){grammaratur.HTMLToMarkdown, compiler.HTMLToMarkdown} { + got, err := convert(source) + if err != nil || got != want { + t.Fatalf("ordinary conversion changed: %q %v", got, err) + } + } + protected, err := grammaratur.HTMLToMarkdownAuthoring(source) + if err != nil { + t.Fatal(err) + } + compat, err := compiler.HTMLToMarkdownAuthoring(source) + if err != nil || protected != compat { + t.Fatal("authoring facade changed", err) + } + if compiler.MarkdownAuthoringHTML(compat) != grammaratur.MarkdownAuthoringHTML(protected) { + t.Fatal("protected render facade changed") + } + } +} + +func TestAuthoringIsExplicitAndNotASanitizer(t *testing.T) { + const source = "```{=html}\n\n```" + if strings.Contains(grammaratur.MarkdownToHTML(source), "" { + t.Fatalf("authoring removed source: %q", got) + } + // Authorization, macro expansion and sanitization stay in the projection + // layer. Grammaratur returns source, not HTML trusted for a browser sink. + if got := grammaratur.MarkdownToHTML("{{children}}"); !strings.Contains(got, "{{children}}") { + t.Fatal("macro unexpectedly expanded") + } +} diff --git a/internal/grammaratur/doc.go b/internal/grammaratur/doc.go new file mode 100644 index 0000000..93e68fd --- /dev/null +++ b/internal/grammaratur/doc.go @@ -0,0 +1,16 @@ +// Package grammaratur owns document-format conversion and authoring fidelity. +// +// Ordinary conversion and protected authoring conversion are distinct contracts: +// callers must opt into {=html} fences. Neither path authorizes content, expands +// macros, sanitizes a display projection, persists revisions, or fetches URLs. +// Those responsibilities remain with the caller. In particular, preserving +// source HTML never grants permission to execute its scripts or event handlers. +// +// Grammaratur is an in-process component, not a service or a new public API. +// The name denotes the words/format-conversion responsibility in PagesWiki's +// architecture; see docs/design/ja/component-names.md. +package grammaratur + +// Component is the stable component label for diagnostics and instrumentation. +// It is not a replacement for API routes, provider IDs or persisted settings. +const Component = "grammaratur" diff --git a/internal/grammaratur/markdown.go b/internal/grammaratur/markdown.go new file mode 100644 index 0000000..d434343 --- /dev/null +++ b/internal/grammaratur/markdown.go @@ -0,0 +1,18 @@ +package grammaratur + +import ( + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2" + "github.com/gomarkdown/markdown" +) + +// HTMLToMarkdown is the existing ordinary conversion, used by integrations +// which do not understand protected authoring blocks. It can be lossy. +func HTMLToMarkdown(source string) (string, error) { + return htmltomarkdown.ConvertString(source) +} + +// MarkdownToHTML preserves the existing Markdown dialect and renderer options. +// It neither interprets authoring fences nor sanitizes the returned HTML. +func MarkdownToHTML(source string) string { + return string(markdown.ToHTML([]byte(source), nil, nil)) +} diff --git a/internal/kunstzeal/adapters.go b/internal/kunstzeal/adapters.go new file mode 100644 index 0000000..3d560f4 --- /dev/null +++ b/internal/kunstzeal/adapters.go @@ -0,0 +1,230 @@ +// Package kunstzeal selects projections and executes bounded rendering providers. +// Authentication, source provenance and canonical storage remain caller-owned. +package kunstzeal + +import ( + "path/filepath" + "strings" +) + +type Delivery string + +const ( + DeliveryHTML Delivery = "html" + DeliveryText Delivery = "text" + DeliveryRawMedia Delivery = "raw-media" + DeliveryRawFrame Delivery = "raw-frame" + DeliveryDownload Delivery = "download" +) + +type Descriptor struct { + MediaType string + Filename string + SizeBytes int64 +} + +type Adapter struct { + ID string + Kind string + Delivery Delivery + NeedsText bool + WindowableText bool + Match func(Descriptor) bool +} + +const InlineTextWindowThreshold = int64(2 << 20) + +var projectionAdapters = []Adapter{ + { + ID: "html", + Kind: "html", + Delivery: DeliveryHTML, + Match: mediaTypes("text/html"), + }, + { + ID: "excalidraw", + Kind: "excalidraw", + Delivery: DeliveryText, + NeedsText: true, + WindowableText: true, + Match: extensions(".excalidraw"), + }, + { + ID: "csv", + Kind: "csv", + Delivery: DeliveryText, + NeedsText: true, + WindowableText: true, + Match: matchAny( + mediaTypes("text/csv", "application/csv"), + extensions(".csv"), + ), + }, + { + ID: "archive", + Kind: "archive", + Delivery: DeliveryDownload, + Match: matchAny( + mediaTypes( + "application/zip", "application/x-zip-compressed", + "application/x-rar-compressed", "application/vnd.rar", + "application/x-7z-compressed", "application/x-tar", + "application/gzip", "application/x-xz", + ), + extensions(".zip", ".rar", ".7z", ".tar", ".gz", ".tgz", ".xz", ".txz"), + ), + }, + { + ID: "debian-package", + Kind: "metadata", + Delivery: DeliveryDownload, + Match: matchAny( + mediaTypes( + "application/vnd.debian.binary-package", + "application/x-debian-package", + ), + extensions(".deb"), + ), + }, + { + ID: "eml", + Kind: "email", + Delivery: DeliveryText, + Match: matchAny( + mediaTypes("message/rfc822"), + extensions(".eml"), + ), + }, + { + ID: "pe", + Kind: "metadata", + Delivery: DeliveryDownload, + Match: extensions(".exe", ".dll", ".sys"), + }, + { + ID: "json", + Kind: "json", + Delivery: DeliveryText, + WindowableText: true, + Match: matchAny(mediaTypes("application/json", "application/ld+json"), extensions(".json", ".jsonl")), + }, + { + ID: "text", + Kind: "code", + Delivery: DeliveryText, + NeedsText: true, + WindowableText: true, + Match: textMediaType, + }, + { + ID: "image", + Kind: "image", + Delivery: DeliveryRawMedia, + Match: matchAny( + mediaTypePrefix("image/"), + extensions(".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif", ".svg"), + ), + }, + { + ID: "audio", + Kind: "audio", + Delivery: DeliveryRawMedia, + Match: matchAny( + mediaTypePrefix("audio/"), + extensions(".mp3", ".flac", ".wav", ".ogg", ".m4a", ".aac", ".opus"), + ), + }, + { + ID: "video", + Kind: "video", + Delivery: DeliveryRawMedia, + Match: matchAny( + mediaTypePrefix("video/"), + extensions(".mp4", ".webm", ".mkv", ".mov", ".m4v", ".ogv"), + ), + }, + { + ID: "pdf", + Kind: "pdf", + Delivery: DeliveryRawFrame, + Match: matchAny( + mediaTypes("application/pdf"), + extensions(".pdf"), + ), + }, + { + ID: "download", + Kind: "download", + Delivery: DeliveryDownload, + Match: func(Descriptor) bool { return true }, + }, +} + +func SelectAdapter(descriptor Descriptor) Adapter { + for _, adapter := range projectionAdapters { + if adapter.Match(descriptor) { + return adapter + } + } + return projectionAdapters[len(projectionAdapters)-1] +} + +func mediaTypes(types ...string) func(Descriptor) bool { + return func(descriptor Descriptor) bool { + for _, candidate := range types { + if descriptor.MediaType == candidate { + return true + } + } + return false + } +} + +func mediaTypePrefix(prefix string) func(Descriptor) bool { + return func(descriptor Descriptor) bool { + return strings.HasPrefix(descriptor.MediaType, prefix) + } +} + +func extensions(values ...string) func(Descriptor) bool { + return func(descriptor Descriptor) bool { + extension := strings.ToLower(filepath.Ext(descriptor.Filename)) + for _, candidate := range values { + if extension == candidate { + return true + } + } + return false + } +} + +func matchAny(matchers ...func(Descriptor) bool) func(Descriptor) bool { + return func(descriptor Descriptor) bool { + for _, matcher := range matchers { + if matcher(descriptor) { + return true + } + } + return false + } +} + +func textMediaType(descriptor Descriptor) bool { + switch descriptor.MediaType { + case "application/json", + "application/xml", + "application/javascript", + "application/x-javascript", + "application/x-sh", + "application/x-shellscript", + "application/x-yaml", + "application/yaml", + "application/toml": + return true + default: + return strings.HasPrefix(descriptor.MediaType, "text/") + } +} + +// Catalog returns a copy; callers cannot reorder or disable the shared registry. +func Catalog() []Adapter { return append([]Adapter(nil), projectionAdapters...) } diff --git a/internal/kunstzeal/adapters_test.go b/internal/kunstzeal/adapters_test.go new file mode 100644 index 0000000..d912bbf --- /dev/null +++ b/internal/kunstzeal/adapters_test.go @@ -0,0 +1,105 @@ +package kunstzeal + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" +) + +func TestAdapterSelectionAndOrdering(t *testing.T) { + for _, test := range []struct { + mime, filename, id, kind string + delivery Delivery + needsText, window bool + }{ + {"text/html", "scene.excalidraw", "html", "html", DeliveryHTML, false, false}, + {"application/json", "scene.EXCALIDRAW", "excalidraw", "excalidraw", DeliveryText, true, true}, + {"text/plain", "data.CSV", "csv", "csv", DeliveryText, true, true}, + {"application/zip", "archive", "archive", "archive", DeliveryDownload, false, false}, + {"application/octet-stream", "package.deb", "debian-package", "metadata", DeliveryDownload, false, false}, + {"text/plain", "mail.eml", "eml", "email", DeliveryText, false, false}, + {"application/octet-stream", "program.exe", "pe", "metadata", DeliveryDownload, false, false}, + {"application/json", "data.json", "json", "json", DeliveryText, false, true}, + {"text/x-python", "code.py", "text", "code", DeliveryText, true, true}, + {"image/png", "image", "image", "image", DeliveryRawMedia, false, false}, + {"audio/flac", "music", "audio", "audio", DeliveryRawMedia, false, false}, + {"video/mp4", "movie", "video", "video", DeliveryRawMedia, false, false}, + {"application/pdf", "manual", "pdf", "pdf", DeliveryRawFrame, false, false}, + {"application/octet-stream", "unknown", "download", "download", DeliveryDownload, false, false}, + } { + got := SelectAdapter(Descriptor{MediaType: test.mime, Filename: test.filename, SizeBytes: InlineTextWindowThreshold + 1}) + if got.ID != test.id || got.Kind != test.kind || got.Delivery != test.delivery || got.NeedsText != test.needsText || got.WindowableText != test.window { + t.Fatalf("%s/%s: %+v", test.mime, test.filename, got) + } + } + if got := SelectAdapter(Descriptor{}); got.ID != "download" { + t.Fatal(got) + } +} + +func TestAdapterCatalogIsCopyAndConcurrent(t *testing.T) { + catalog := Catalog() + ids := []string{} + for _, a := range catalog { + ids = append(ids, a.ID) + } + if strings.Join(ids, ",") != "html,excalidraw,csv,archive,debian-package,eml,pe,json,text,image,audio,video,pdf,download" { + t.Fatal("catalog order changed", ids) + } + catalog[0].ID = "corrupt" + catalog[0].Match = func(Descriptor) bool { return false } + if Catalog()[0].ID != "html" || SelectAdapter(Descriptor{MediaType: "text/html"}).ID != "html" { + t.Fatal("catalog mutation changed shared selection") + } + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + Catalog()[0].ID = "copy" + if SelectAdapter(Descriptor{MediaType: "text/html"}).ID != "html" { + t.Error("registry changed") + } + } + }() + } + wg.Wait() +} + +func TestComponentBoundary(t *testing.T) { + if Component != "kunstzeal" { + t.Fatal("component identity changed") + } + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, spec := range ast.Imports { + path, err := strconv.Unquote(spec.Path.Value) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"pageswiki/internal", "database", "os/exec"} { + if path == forbidden || strings.HasPrefix(path, forbidden+"/") { + t.Errorf("%s imports %s: providers must not own application state or arbitrary process execution", file, path) + } + } + if file == "adapters.go" && (strings.HasPrefix(path, "net") || path == "os") { + t.Error("adapter selection acquired I/O") + } + } + } +} diff --git a/internal/kunstzeal/cache.go b/internal/kunstzeal/cache.go new file mode 100644 index 0000000..b71de64 --- /dev/null +++ b/internal/kunstzeal/cache.go @@ -0,0 +1,55 @@ +package kunstzeal + +import ( + "errors" + "io" + "os" + "path/filepath" +) + +var ErrDiagramCacheLimit = errors.New("diagram cache exceeds limit") + +// ReadDiagramCache reads only a bounded derived artifact. filename must be +// constructed by the application after validating kind/hash and authorizing +// source provenance. A cache hit is not an access grant or a sanitized SVG. +func ReadDiagramCache(filename string) ([]byte, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + b, err := io.ReadAll(io.LimitReader(f, DiagramOutputLimit+1)) + if len(b) > DiagramOutputLimit { + return nil, ErrDiagramCacheLimit + } + return b, err +} + +// WriteDiagramCache publishes a complete artifact with same-directory rename. +// Concurrent readers observe the old or new file, never a truncated write. +// This is disposable cache storage, not a durable canonical Document commit. +func WriteDiagramCache(filename string, data []byte) error { + if len(data) > DiagramOutputLimit { + return ErrDiagramCacheLimit + } + dir := filepath.Dir(filename) + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + f, err := os.CreateTemp(dir, ".diagram-*.tmp") + if err != nil { + return err + } + defer os.Remove(f.Name()) + defer f.Close() + if _, err = f.Write(data); err != nil { + return err + } + if err = f.Chmod(0644); err != nil { + return err + } + if err = f.Close(); err != nil { + return err + } + return os.Rename(f.Name(), filename) +} diff --git a/internal/kunstzeal/cache_test.go b/internal/kunstzeal/cache_test.go new file mode 100644 index 0000000..8b2a2f2 --- /dev/null +++ b/internal/kunstzeal/cache_test.go @@ -0,0 +1,84 @@ +package kunstzeal + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "sync" + "testing" +) + +func TestDiagramCacheAtomicReplacement(t *testing.T) { + filename := filepath.Join(t.TempDir(), "mermaid", "fixture.svg") + a, b := bytes.Repeat([]byte("A"), 64<<10), bytes.Repeat([]byte("B"), 64<<10) + if err := WriteDiagramCache(filename, a); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + for n := 0; n < 20; n++ { + if i%2 == 0 { + data := a + if n%2 == 0 { + data = b + } + if err := WriteDiagramCache(filename, data); err != nil { + t.Error(err) + } + } else { + data, err := ReadDiagramCache(filename) + if err != nil || (!bytes.Equal(data, a) && !bytes.Equal(data, b)) { + t.Errorf("partial cache: length=%d err=%v", len(data), err) + return + } + } + } + }(i) + } + wg.Wait() + files, err := filepath.Glob(filepath.Join(filepath.Dir(filename), ".diagram-*.tmp")) + if err != nil || len(files) != 0 { + t.Fatal("temporary artifacts leaked", files, err) + } + info, err := os.Stat(filename) + if err != nil || info.Mode().Perm() != 0644 { + t.Fatal(info, err) + } +} + +func TestDiagramCacheBudgetsAndFailures(t *testing.T) { + root := t.TempDir() + filename := filepath.Join(root, "fixture.svg") + if err := WriteDiagramCache(filename, []byte("complete")); err != nil { + t.Fatal(err) + } + large := bytes.Repeat([]byte("x"), DiagramOutputLimit+1) + if err := WriteDiagramCache(filename, large); !errors.Is(err, ErrDiagramCacheLimit) { + t.Fatal(err) + } + b, err := ReadDiagramCache(filename) + if err != nil || string(b) != "complete" { + t.Fatal("failed write destroyed cache", err) + } + if err = os.WriteFile(filename, large, 0644); err != nil { + t.Fatal(err) + } + if b, err = ReadDiagramCache(filename); !errors.Is(err, ErrDiagramCacheLimit) || b != nil { + t.Fatal("oversized cache returned", err) + } + if _, err = ReadDiagramCache(filepath.Join(root, "missing")); !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + if err = WriteDiagramCache(root, []byte("cannot replace a directory")); err == nil { + t.Fatal("rename failure hidden") + } + // Failed rename removes its temporary artifact too. + files, err := filepath.Glob(filepath.Join(filepath.Dir(root), ".diagram-*.tmp")) + if err != nil || len(files) != 0 { + t.Fatal(files, err) + } +} diff --git a/internal/kunstzeal/diagram.go b/internal/kunstzeal/diagram.go new file mode 100644 index 0000000..7898e84 --- /dev/null +++ b/internal/kunstzeal/diagram.go @@ -0,0 +1,116 @@ +package kunstzeal + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +const Component = "kunstzeal" +const DiagramSourceLimit = 1 << 20 +const DiagramOutputLimit = 4 << 20 +const DiagramInlineLimit = 32 << 10 +const DiagramTimeout = 15 * time.Second + +var ( + ErrDiagramKind = errors.New("unsupported diagram type") + ErrDiagramSource = errors.New("diagram source exceeds limit") + ErrDiagramOutput = errors.New("diagram output unavailable or exceeds limit") + ErrProviderConfig = errors.New("diagram renderer configuration invalid") + ErrProviderUnavailable = errors.New("diagram renderer unavailable") + ErrProviderFailed = errors.New("diagram renderer failed") +) + +// DiagramProvider does not authorize a request or trust its SVG output. +// Callers must retain source access checks and the sandbox response policy. +type DiagramProvider interface { + Render(context.Context, string, string) ([]byte, error) +} + +func DiagramHash(kind, source, format string) string { + sum := sha256.Sum256([]byte(kind + "\x00" + format + "\x00" + source)) + return hex.EncodeToString(sum[:]) +} + +func AllowedDiagramType(kind string) bool { + switch kind { + case "mermaid", "plantuml", "graphviz", "dot", "d2": + return true + } + return false +} + +func NormalizeDiagramType(kind string) string { + if kind == "dot" { + return "graphviz" + } + return kind +} + +type HTTPClient interface { + Do(*http.Request) (*http.Response, error) +} + +// Kroki receives an administrator-configured service address, never a URL from +// document content. Private service-network endpoints are intentional here. +// Client must honor context cancellation and reject redirects. No default +// client is used, and provider failures never expose its URL or response body. +type Kroki struct { + Endpoint string + Client HTTPClient +} + +func (k Kroki) Render(parent context.Context, kind, source string) ([]byte, error) { + if err := parent.Err(); err != nil { + return nil, err + } + kind = NormalizeDiagramType(kind) + if !AllowedDiagramType(kind) { + return nil, ErrDiagramKind + } + if len(source) > DiagramSourceLimit { + return nil, ErrDiagramSource + } + u, err := url.Parse(k.Endpoint) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || k.Client == nil { + return nil, ErrProviderConfig + } + ctx, cancel := context.WithTimeout(parent, DiagramTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(k.Endpoint, "/")+"/"+kind+"/svg", strings.NewReader(source)) + if err != nil { + return nil, ErrProviderConfig + } + req.Header.Set("Content-Type", "text/plain; charset=utf-8") + resp, err := k.Client.Do(req) + if err != nil { + if resp != nil && resp.Body != nil { + resp.Body.Close() + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, ErrProviderUnavailable + } + if resp == nil || resp.Body == nil { + return nil, ErrProviderUnavailable + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, ErrProviderFailed + } + b, err := io.ReadAll(io.LimitReader(resp.Body, DiagramOutputLimit+1)) + if ctx.Err() != nil { + return nil, ctx.Err() + } + if err != nil || len(b) > DiagramOutputLimit { + return nil, ErrDiagramOutput + } + return b, nil +} diff --git a/internal/kunstzeal/diagram_test.go b/internal/kunstzeal/diagram_test.go new file mode 100644 index 0000000..9f18d3e --- /dev/null +++ b/internal/kunstzeal/diagram_test.go @@ -0,0 +1,167 @@ +package kunstzeal + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "strings" + "testing" + "time" +) + +var _ DiagramProvider = Kroki{} + +type clientFixture func(*http.Request) (*http.Response, error) + +func (f clientFixture) Do(r *http.Request) (*http.Response, error) { return f(r) } + +type trackedReader struct { + io.Reader + closed bool +} + +func (r *trackedReader) Close() error { r.closed = true; return nil } + +func TestDiagramIdentityAndKinds(t *testing.T) { + for _, kind := range []string{"mermaid", "plantuml", "graphviz", "dot", "d2"} { + if !AllowedDiagramType(kind) { + t.Fatal(kind) + } + for _, source := range []string{"", "graph LR; A-->B;", "日本語\x00source"} { + sum := sha256.Sum256([]byte(kind + "\x00svg\x00" + source)) + if DiagramHash(kind, source, "svg") != hex.EncodeToString(sum[:]) { + t.Fatal("cache identity changed") + } + } + } + for _, kind := range []string{"", "nwdiag", "../mermaid", "Mermaid", "svg", "graphviz/svg"} { + if AllowedDiagramType(kind) { + t.Fatal("unknown kind", kind) + } + } + if NormalizeDiagramType("dot") != "graphviz" || NormalizeDiagramType("mermaid") != "mermaid" { + t.Fatal("normalization changed") + } +} + +func TestKrokiBoundedRequestAndOutput(t *testing.T) { + for _, kind := range []string{"mermaid", "dot", "d2"} { + body := &trackedReader{Reader: strings.NewReader("fixture")} + k := Kroki{Endpoint: "http://renderer.example/prefix/", Client: clientFixture(func(r *http.Request) (*http.Response, error) { + b, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + if r.Method != "POST" || r.URL.String() != "http://renderer.example/prefix/"+NormalizeDiagramType(kind)+"/svg" || string(b) != "source" || r.Header.Get("Content-Type") != "text/plain; charset=utf-8" { + t.Fatal("request contract changed") + } + if r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" { + t.Fatal("credentials forwarded") + } + deadline, ok := r.Context().Deadline() + if !ok || time.Until(deadline) > DiagramTimeout { + t.Fatal("unbounded provider") + } + return &http.Response{StatusCode: 200, Body: body}, nil + })} + b, err := k.Render(context.Background(), kind, "source") + if err != nil || string(b) != "fixture" || !body.closed { + t.Fatal(string(b), err, body.closed) + } + } + for _, size := range []int{DiagramOutputLimit, DiagramOutputLimit + 1} { + body := &trackedReader{Reader: strings.NewReader(strings.Repeat("x", size))} + k := Kroki{Endpoint: "https://renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) { return &http.Response{StatusCode: 200, Body: body}, nil })} + b, err := k.Render(context.Background(), "mermaid", strings.Repeat("x", DiagramSourceLimit)) + if size == DiagramOutputLimit { + if err != nil || len(b) != size { + t.Fatal(len(b), err) + } + } else if !errors.Is(err, ErrDiagramOutput) || b != nil { + t.Fatal("oversized output escaped", err) + } + if !body.closed { + t.Fatal("response leaked") + } + } +} + +func TestKrokiFailsClosedBeforeIO(t *testing.T) { + client := clientFixture(func(*http.Request) (*http.Response, error) { t.Fatal("invalid request performed I/O"); return nil, nil }) + for _, endpoint := range []string{"", "file:///etc/passwd", "https://user:secret@renderer.example", "https://renderer.example?token=secret", "https://renderer.example#fragment", "https://renderer.example?", "https:///path", "http://%zz"} { + b, err := (Kroki{Endpoint: endpoint, Client: client}).Render(context.Background(), "mermaid", "source") + if !errors.Is(err, ErrProviderConfig) || b != nil { + t.Fatal("bad provider accepted", err) + } + } + k := Kroki{Endpoint: "http://renderer.example", Client: client} + if _, err := k.Render(context.Background(), "../../secret", "source"); !errors.Is(err, ErrDiagramKind) { + t.Fatal(err) + } + if _, err := k.Render(context.Background(), "mermaid", strings.Repeat("x", DiagramSourceLimit+1)); !errors.Is(err, ErrDiagramSource) { + t.Fatal(err) + } + if _, err := (Kroki{Endpoint: k.Endpoint}).Render(context.Background(), "mermaid", "source"); !errors.Is(err, ErrProviderConfig) { + t.Fatal("default HTTP client used", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := k.Render(ctx, "mermaid", "source"); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} + +func TestKrokiFailuresAreOpaqueAndCloseBodies(t *testing.T) { + for _, code := range []int{301, 302, 307, 308, 400, 401, 403, 404, 429, 500} { + body := &trackedReader{Reader: strings.NewReader("PRIVATE SOURCE OR INTERNAL DIAGNOSTICS")} + k := Kroki{Endpoint: "https://private-renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) { return &http.Response{StatusCode: code, Body: body}, nil })} + b, err := k.Render(context.Background(), "mermaid", "PRIVATE SOURCE") + if !errors.Is(err, ErrProviderFailed) || b != nil || !body.closed { + t.Fatal(code, err, body.closed) + } + } + for _, mode := range []string{"network", "nil-response", "nil-body", "read-error"} { + k := Kroki{Endpoint: "https://private-renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) { + switch mode { + case "network": + return nil, errors.New("https://private-renderer.example: secret") + case "nil-response": + return nil, nil + case "nil-body": + return &http.Response{StatusCode: 200}, nil + default: + return &http.Response{StatusCode: 200, Body: io.NopCloser(errorReader{})}, nil + } + })} + b, err := k.Render(context.Background(), "mermaid", "PRIVATE SOURCE") + if b != nil || err == nil || strings.Contains(err.Error(), "secret") || strings.Contains(err.Error(), "private-renderer") { + t.Fatal(mode, err) + } + } +} + +type errorReader struct{} + +func (errorReader) Read([]byte) (int, error) { return 0, errors.New("secret body read failure") } + +func TestKrokiCancellationDuringSendAndRead(t *testing.T) { + for _, duringRead := range []bool{false, true} { + ctx, cancel := context.WithCancel(context.Background()) + body := &trackedReader{Reader: strings.NewReader("")} + k := Kroki{Endpoint: "https://renderer.example", Client: clientFixture(func(*http.Request) (*http.Response, error) { + cancel() + if duringRead { + return &http.Response{StatusCode: 200, Body: body}, nil + } + return &http.Response{StatusCode: 200, Body: body}, errors.New("cancelled network") + })} + b, err := k.Render(ctx, "mermaid", "source") + cancel() + if !errors.Is(err, context.Canceled) || b != nil || !body.closed { + t.Fatal(duringRead, err, body.closed) + } + } +} diff --git a/internal/mestionora/boundary_test.go b/internal/mestionora/boundary_test.go new file mode 100644 index 0000000..3024c05 --- /dev/null +++ b/internal/mestionora/boundary_test.go @@ -0,0 +1,44 @@ +package mestionora + +import ( + "go/parser" + "go/token" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" +) + +func TestComponentBoundary(t *testing.T) { + if Component != "mestionora" { + t.Fatal("component identity changed") + } + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, spec := range ast.Imports { + path, err := strconv.Unquote(spec.Path.Value) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"pageswiki/internal", "net", "os"} { + if path == forbidden || strings.HasPrefix(path, forbidden+"/") { + t.Errorf("%s imports %s: search must not own HTTP, filesystem payloads or application policy", file, path) + } + } + } + } + if _, exists := reflect.TypeFor[*SQLite]().MethodByName("SearchPathsInDomain"); exists { + t.Fatal("implicit allow-all belongs only in the legacy trusted-caller facade") + } +} diff --git a/internal/mestionora/city_display.go b/internal/mestionora/city_display.go new file mode 100644 index 0000000..8ecc8b4 --- /dev/null +++ b/internal/mestionora/city_display.go @@ -0,0 +1,50 @@ +package mestionora + +import ( + "path" + "strings" +) + +// A display projection only: names, boundaries and manual titles remain stored +// unchanged. Legacy generated Spots have no nested Spot semantics. +func IsUnclassifiedCitySpot(name, boundary string) bool { + return strings.HasSuffix(name, "-undefined") || strings.HasSuffix(boundary, "/_undefined") +} + +func CitySpotDisplayTitle(name, title, boundary, parentTitle string) string { + if !IsUnclassifiedCitySpot(name, boundary) { + if title != "" { + return title + } + return name + } + switch strings.ToLower(strings.TrimSpace(title)) { + case "", "undefined", "未定義", "未分類のpage", "unclassified pages", strings.ToLower(name): + if parentTitle != "" { + return parentTitle + } + if parent := path.Base(path.Dir(boundary)); parent != "/" && parent != "." { + return parent + } + return "Spot" + default: + return title // A manual display title takes precedence. + } +} + +func (r CitySearchResult) DisplayTitle() string { + if r.Kind == "spot" { + return CitySpotDisplayTitle(r.Name, r.Title, r.PathPrefix, r.StreetTitle) + } + if r.Title != "" { + return r.Title + } + return r.Name +} + +func (r CitySearchResult) ContentPath() string { + if r.NavigationPath != "" { + return r.NavigationPath + } + return r.PathPrefix +} diff --git a/internal/mestionora/city_query_plan_test.go b/internal/mestionora/city_query_plan_test.go new file mode 100644 index 0000000..942ba5a --- /dev/null +++ b/internal/mestionora/city_query_plan_test.go @@ -0,0 +1,38 @@ +package mestionora + +import ( + "database/sql" + "strings" + "testing" +) + +func TestCityContentExistenceUsesPathIndex(t *testing.T) { + database := newSearchTestDB(t) + for _, kind := range []string{"quartier", "street", "spot"} { + rows, err := database.Query("EXPLAIN QUERY PLAN "+citySearchSQL(kind, ""), sql.Named("domain", "d"), sql.Named("scope", "/"), sql.Named("prefix", "/"), sql.Named("upper", "0"), sql.Named("query", "needle"), sql.Named("limit", 1000)) + if err != nil { + t.Fatal(err) + } + indexed := false + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + if strings.Contains(detail, "SEARCH cp USING") { + indexed = true + } + if strings.Contains(detail, "SCAN cp") { + t.Fatalf("unbounded Path lookup: %s", detail) + } + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + rows.Close() + if !indexed { + t.Fatalf("no indexed portal lookup for %s", kind) + } + } +} diff --git a/internal/mestionora/city_search.go b/internal/mestionora/city_search.go new file mode 100644 index 0000000..f036f6e --- /dev/null +++ b/internal/mestionora/city_search.go @@ -0,0 +1,212 @@ +package mestionora + +import ( + "context" + "database/sql" + "errors" + "sort" + "strings" + "time" +) + +const ( + CitySearchCandidateLimit = 1000 // Per locality kind, before authorization. + CitySearchResultLimit = 30 + citySearchTimeout = 2 * time.Second +) + +// CitySearchResult describes a locality, not its contained Pages. Description +// is a bounded plain-text excerpt around the match and is never trusted HTML. +type CitySearchResult struct { + UUID, Kind, Name, Title, Description, PathPrefix string + QuartierTitle, StreetTitle, QuartierName string + CreatedAtMS, UpdatedAtMS int64 + NameMatch, PathMatch, DescriptionMatch bool + NavigationPath string + HasPage bool +} + +type citySearchCandidate struct { + CitySearchResult + quartierPath, streetPath string +} + +// SearchCityInDomain searches locality metadata only. It does not construct a +// CityMap, expand descendants, read Page bodies, or count hidden localities. +// Scope selects a locality's own boundary, not an ancestor of the scope. Every +// boundary in its parent chain must be readable before the result is exposed. +func (s *SQLite) SearchCityInDomain(parent context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []CitySearchResult, searchErr error) { + if err := options.Validate(); err != nil { + return nil, err + } + query = strings.TrimSpace(query) + if len(query) > 512 || len(strings.Fields(query)) > 16 { + return nil, errors.New("search query is too long") + } + // File filters are intentionally not applied to locality metadata: a Spot + // containing a PDF is not itself a PDF and must not claim to match one. + if allowed == nil || options.FileType != "" || options.Extension != "" || (query == "" && !options.HasFilters()) { + return nil, nil + } + if limit <= 0 || limit > CitySearchResultLimit { + limit = CitySearchResultLimit + } + ctx, cancel := context.WithTimeout(parent, citySearchTimeout) + defer cancel() + defer func() { + if searchErr != nil && ctx.Err() != nil { + searchErr = ctx.Err() + } + }() + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return nil, err + } + defer tx.Rollback() + scope = normalizePath(scope) + prefix, upper := strings.TrimRight(scope, "/")+"/", strings.TrimRight(scope, "/")+"0" + args := []any{ + sql.Named("domain", domainUUID), sql.Named("scope", scope), + sql.Named("prefix", prefix), sql.Named("upper", upper), + sql.Named("query", query), sql.Named("limit", CitySearchCandidateLimit), + } + dateColumn := "n.updated_at_ms" + if options.DateField == "created" { + dateColumn = "n.created_at_ms" + } + dateWhere := "" + if options.FromMS != nil { + dateWhere += " AND " + dateColumn + ">=:from_ms" + args = append(args, sql.Named("from_ms", *options.FromMS)) + } + if options.BeforeMS != nil { + dateWhere += " AND " + dateColumn + "<:before_ms" + args = append(args, sql.Named("before_ms", *options.BeforeMS)) + } + candidates := make([]citySearchCandidate, 0) + for _, kind := range []string{"quartier", "street", "spot"} { + rows, err := tx.QueryContext(ctx, citySearchSQL(kind, dateWhere), args...) + if err != nil { + return nil, err + } + for rows.Next() { + var candidate citySearchCandidate + candidate.Kind = kind + if err := rows.Scan(&candidate.UUID, &candidate.Name, &candidate.Title, &candidate.Description, + &candidate.PathPrefix, &candidate.CreatedAtMS, &candidate.UpdatedAtMS, + &candidate.NameMatch, &candidate.PathMatch, &candidate.DescriptionMatch, + &candidate.QuartierTitle, &candidate.StreetTitle, &candidate.quartierPath, &candidate.streetPath, &candidate.QuartierName, + &candidate.NavigationPath, &candidate.HasPage); err != nil { + rows.Close() + return nil, err + } + candidates = append(candidates, candidate) + } + err = rows.Err() + closeErr := rows.Close() + if err != nil { + return nil, err + } + if closeErr != nil { + return nil, closeErr + } + } + // Release the sole read connection before calling authorization, which may + // itself need a database connection. HTTP normally passes its cached policy. + if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { + return nil, err + } + sort.SliceStable(candidates, func(i, j int) bool { + a, b := candidates[i], candidates[j] + if a.Kind != b.Kind { + return citySearchKindOrder(a.Kind) < citySearchKindOrder(b.Kind) + } + if a.NameMatch != b.NameMatch { + return a.NameMatch + } + if a.PathMatch != b.PathMatch { + return a.PathMatch + } + if a.PathPrefix != b.PathPrefix { + return a.PathPrefix < b.PathPrefix + } + return a.UUID < b.UUID + }) + readable := make(map[string]bool) + canRead := func(path string) bool { + if value, exists := readable[path]; exists { + return value + } + value := allowed(path) + readable[path] = value + return value + } + for _, candidate := range candidates { + if err := ctx.Err(); err != nil { + return nil, err + } + if !canRead(candidate.PathPrefix) || (candidate.quartierPath != "" && !canRead(candidate.quartierPath)) || (candidate.streetPath != "" && !canRead(candidate.streetPath)) { + continue + } + found = append(found, candidate.CitySearchResult) + if len(found) == limit { + break + } + } + if err := ctx.Err(); err != nil { + return nil, err + } + return found, nil +} + +func citySearchKindOrder(kind string) int { + switch kind { + case "quartier": + return 0 + case "street": + return 1 + default: + return 2 + } +} + +// All interpolated SQL fragments are package-owned constants. User input is +// bound, so %, _, quotes and backslashes are literal substring characters. +func citySearchSQL(kind, dateWhere string) string { + from, description := "zones n", "n.description" + navigationPath := "n.path_prefix" + parentColumns := "'','','','',''" + where := "n.kind='quartier'" + switch kind { + case "street": + from += ` JOIN zones q ON q.uuid=n.parent_uuid AND q.domain_uuid=n.domain_uuid AND q.kind='quartier' AND q.state!='archived'` + parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),'',q.path_prefix,'',q.name" + where = "n.kind='street' AND " + citySearchAbsolutePathSQL("q.path_prefix") + case "spot": + navigationPath = `CASE WHEN substr(n.path_prefix,-11)='/_undefined' THEN st.path_prefix ELSE n.path_prefix END` + from = `spots n JOIN street_spots ss ON ss.spot_uuid=n.uuid + JOIN zones st ON st.uuid=ss.street_uuid AND st.domain_uuid=n.domain_uuid AND st.kind='street' AND st.state!='archived' + JOIN zones q ON q.uuid=st.parent_uuid AND q.domain_uuid=st.domain_uuid AND q.kind='quartier' AND q.state!='archived'` + description = "n.summary" + parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),substr(COALESCE(NULLIF(st.title,''),st.name),1,256),q.path_prefix,st.path_prefix,q.name" + where = citySearchAbsolutePathSQL("q.path_prefix") + " AND " + citySearchAbsolutePathSQL("st.path_prefix") + } + nameMatch := `(:query!='' AND (instr(lower(n.name),lower(:query))>0 OR instr(lower(n.title),lower(:query))>0))` + pathMatch := `(:query!='' AND instr(lower(n.path_prefix),lower(:query))>0)` + descriptionMatch := `(:query!='' AND instr(lower(` + description + `),lower(:query))>0)` + return `SELECT n.uuid,substr(n.name,1,256),substr(n.title,1,256), + substr(` + description + `,max(1,instr(lower(` + description + `),lower(:query))-160),512), + n.path_prefix,n.created_at_ms,n.updated_at_ms,` + nameMatch + `,` + pathMatch + `,` + descriptionMatch + `,` + parentColumns + `,` + navigationPath + `, + EXISTS(SELECT 1 FROM paths cp WHERE cp.domain_uuid=n.domain_uuid AND cp.path=(` + navigationPath + `)) + FROM ` + from + ` WHERE n.domain_uuid=:domain AND n.state!='archived' AND ` + where + ` + AND ` + citySearchAbsolutePathSQL("n.path_prefix") + ` + AND (n.path_prefix COLLATE BINARY=:scope OR (n.path_prefix COLLATE BINARY>=:prefix AND n.path_prefix COLLATE BINARY<:upper)) + AND (:query='' OR ` + nameMatch + ` OR ` + pathMatch + ` OR ` + descriptionMatch + `)` + dateWhere + ` + ORDER BY ` + nameMatch + ` DESC,` + pathMatch + ` DESC,n.path_prefix COLLATE BINARY,n.uuid LIMIT :limit` +} + +func citySearchAbsolutePathSQL(column string) string { + // Keep authorization paths intact rather than truncating them. Malformed + // legacy boundaries and impractically large metadata are not navigable hits. + return `(substr(` + column + `,1,1)='/' AND length(` + column + `)<=4096 AND instr(` + column + `,char(0))=0)` +} diff --git a/internal/mestionora/fixture_test.go b/internal/mestionora/fixture_test.go new file mode 100644 index 0000000..777a109 --- /dev/null +++ b/internal/mestionora/fixture_test.go @@ -0,0 +1,40 @@ +package mestionora + +import ( + "database/sql" + "testing" + + "pageswiki/internal/db" +) + +func newSearchTestDB(t *testing.T) *sql.DB { + t.Helper() + database, err := db.Open(t.TempDir() + "/search.db") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { database.Close() }) + initSearchFixture(t, database) + return database +} + +// Only the application columns/indexes consumed by search, not application +// bootstrap, auth keys, job recovery or external calendar initialization. +func initSearchFixture(t *testing.T, database *sql.DB) { + t.Helper() + _, err := database.Exec(` + CREATE TABLE objects(uuid TEXT PRIMARY KEY, created_at INTEGER); + CREATE TABLE revisions(uuid TEXT PRIMARY KEY,object_uuid TEXT,filename TEXT,body TEXT,content_type TEXT,document_hash TEXT,created_at_ms INTEGER,created_at INTEGER); + CREATE TABLE paths(domain_uuid TEXT NOT NULL,path TEXT NOT NULL,object_uuid TEXT,PRIMARY KEY(domain_uuid,path)); + CREATE INDEX idx_paths_object_domain_page ON paths(object_uuid,domain_uuid,path); + CREATE TABLE zones(uuid TEXT PRIMARY KEY,domain_uuid TEXT,kind TEXT,parent_uuid TEXT,name TEXT,title TEXT,description TEXT,path_prefix TEXT,state TEXT,created_at_ms INTEGER,updated_at_ms INTEGER); + CREATE TABLE spots(uuid TEXT PRIMARY KEY,domain_uuid TEXT,name TEXT,title TEXT,summary TEXT,path_prefix TEXT,state TEXT,created_at_ms INTEGER,updated_at_ms INTEGER); + CREATE TABLE street_spots(street_uuid TEXT,spot_uuid TEXT); + `) + if err != nil { + t.Fatal(err) + } + if err := NewSQLite(database).EnsureSearchIndexSchema(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/mestionora/heading.go b/internal/mestionora/heading.go new file mode 100644 index 0000000..b3356a8 --- /dev/null +++ b/internal/mestionora/heading.go @@ -0,0 +1,36 @@ +package mestionora + +import ( + "golang.org/x/net/html" + "io" + "strings" +) + +// FirstHTMLHeading extracts untrusted title text. Callers bound input before +// extraction and escape output at rendering time; this is not a sanitizer. +func FirstHTMLHeading(source string) string { + t := html.NewTokenizer(strings.NewReader(source)) + in := false + var out strings.Builder + for { + kind := t.Next() + if kind == html.ErrorToken { + if t.Err() == io.EOF { + break + } + break + } + tok := t.Token() + if kind == html.StartTagToken && strings.EqualFold(tok.Data, "h1") { + in = true + continue + } + if kind == html.EndTagToken && strings.EqualFold(tok.Data, "h1") { + break + } + if in && kind == html.TextToken { + out.WriteString(tok.Data) + } + } + return strings.TrimSpace(out.String()) +} diff --git a/internal/mestionora/search.go b/internal/mestionora/search.go new file mode 100644 index 0000000..d9f31a7 --- /dev/null +++ b/internal/mestionora/search.go @@ -0,0 +1,377 @@ +package mestionora + +import ( + "context" + "database/sql" + "errors" + "sort" + "strings" + "time" +) + +const SearchCandidateLimit = 1000 +const SearchExcerptDocumentBytes = 64 << 10 +const SearchExcerptTotalBytes = 1 << 20 +const searchExcerptTimeout = 400 * time.Millisecond + +const searchResultColumns = `p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.revision_uuid,''),COALESCE(r.created_at_ms/1000,0)` +const searchPathBoundary = `(p.path=? OR (p.path>=? AND p.path0 OR instr(lower(COALESCE(r.filename,'')),lower(?))>0)` +const searchPathCandidatesSQL = searchPathCandidatesBaseSQL + searchPathQueryPredicate + ` ORDER BY p.path LIMIT ?` + +// CROSS JOIN is intentional: revision_uuid is an UNINDEXED FTS payload column. +// A Path-first plan re-runs the complete MATCH scan for every Path. Start with +// MATCH exactly once, then seek current Revision metadata and Domain bindings +// through their B-tree indexes. Keep scope filtering before the candidate cap. +const searchFTSCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,revision_search.rowid + FROM revision_search + CROSS JOIN latest_revision_index r ON r.revision_uuid=revision_search.revision_uuid + CROSS JOIN paths p ON p.object_uuid=r.object_uuid + WHERE revision_search MATCH ? AND p.domain_uuid=? AND ` + searchPathBoundary +const searchFTSCandidatesSQL = searchFTSCandidatesBaseSQL + ` LIMIT ?` + +func searchBodyExcerptsSQL(count int) string { + // UUID is not indexed by FTS5. Restrict the MATCH/snippet work using the + // integer rowids collected from the same read snapshot, not payload UUIDs. + return `SELECT revision_uuid,substr(snippet(revision_search,3,'','',' … ',36),1,4096) + FROM revision_search WHERE revision_search MATCH ? AND rowid IN (` + strings.TrimRight(strings.Repeat("?,", count), ",") + `)` +} + +// Excerpts are optional enrichment. Even small documents can have expensive +// prefix hit-position scans, so their time budget is separate from finding +// results. A spent excerpt budget omits snippets, not already-authorized hits. +func loadSearchBodyExcerpts(parent context.Context, tx *sql.Tx, expression string, ids []any, budget time.Duration) (excerpts map[string]string, err error) { + ctx, cancel := context.WithTimeout(parent, budget) + defer cancel() + defer func() { + if err != nil && ctx.Err() != nil { + if parent.Err() != nil { + err = parent.Err() + } else { + excerpts, err = nil, nil + } + } + }() + args := append([]any{"body : (" + expression + ")"}, ids...) + rows, err := tx.QueryContext(ctx, searchBodyExcerptsSQL(len(ids)), args...) + if err != nil { + return nil, err + } + defer rows.Close() + excerpts = make(map[string]string) + for rows.Next() { + var id, excerpt string + if err := rows.Scan(&id, &excerpt); err != nil { + return nil, err + } + excerpts[id] = excerpt + } + return excerpts, rows.Err() +} + +type SearchResult struct { + Path string + ContentType string + Filename string + DocumentHash string + RevisionUUID string + Size int64 // Kept for API compatibility; search does not stat Document files. + UpdatedAt int64 + PathMatch bool + FilenameMatch bool + BodyMatch bool + Excerpt string // Untrusted source text, never HTML-safe markup. + ExcerptOmitted bool // A body match remains visible when snippet work exceeds its budget. +} + +type PathSuggestion struct { + SearchResult + Title string +} + +// SearchPathsInDomainFiltered bounds SQL candidates before materialization and +// authorizes each Path before loading an excerpt. It never computes a global hit +// count or stats Document files. A narrow scope improves both recall and cost. +// allowed runs inside a read snapshot. Prefer a preloaded request-local policy; +// a callback doing DB reads requires spare pool capacity (not a one-connection +// pool). Keeping policy I/O outside this snapshot is separate follow-up work. +func (s *SQLite) SearchPathsInDomainFiltered(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]SearchResult, error) { + return s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}) +} + +// SearchPathsInDomainWithOptions also permits filter-only searches. File/date +// predicates apply before SQL candidate caps, while ACL checks stay ahead of all +// snippet enrichment. Zero options preserve the original indexed query plans. +func (s *SQLite) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, searchErr error) { + return s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options, true) +} + +// SearchPathSuggestionsInDomain shares the current-Revision FTS and ACL path +// with full search, but deliberately skips snippet generation. Typeahead calls +// are frequent and only need a bounded display label, never a body excerpt. +func (s *SQLite) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) { + results, err := s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}, false) + if err != nil || len(results) == 0 { + return nil, err + } + ids := make([]any, 0, len(results)) + seen := make(map[string]bool, len(results)) + for _, result := range results { + if result.RevisionUUID != "" && strings.HasPrefix(strings.ToLower(result.ContentType), "text/html") && !seen[result.RevisionUUID] { + ids = append(ids, result.RevisionUUID) + seen[result.RevisionUUID] = true + } + } + headings := make(map[string]string, len(ids)) + if len(ids) > 0 { + rows, queryErr := s.db.QueryContext(ctx, `SELECT uuid,substr(CAST(body AS TEXT),1,65536) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...) + if queryErr != nil { + return nil, queryErr + } + for rows.Next() { + var revision, source string + if queryErr = rows.Scan(&revision, &source); queryErr != nil { + rows.Close() + return nil, queryErr + } + headings[revision] = FirstHTMLHeading(source) + } + queryErr = rows.Err() + rows.Close() + if queryErr != nil { + return nil, queryErr + } + } + out := make([]PathSuggestion, 0, len(results)) + for _, result := range results { + out = append(out, PathSuggestion{SearchResult: result, Title: headings[result.RevisionUUID]}) + } + return out, nil +} + +func (s *SQLite) searchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions, loadExcerpts bool) (found []SearchResult, searchErr error) { + if err := options.Validate(); err != nil { + return nil, err + } + scope, query = normalizePath(scope), strings.TrimSpace(query) + if query == "" && !options.HasFilters() || allowed == nil { + return nil, nil + } + if len(query) > 512 || len(strings.Fields(query)) > 16 { + return nil, errors.New("search query is too long") + } + if limit <= 0 || limit > 200 { + limit = 100 + } + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + defer func() { + // SQLite can report an interrupted statement instead of wrapping the + // context error. Preserve cancellation/deadline semantics for HTTP. + if ctx.Err() != nil { + found = nil + searchErr = ctx.Err() + } + }() + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return nil, err + } + defer tx.Rollback() + prefix := strings.TrimRight(scope, "/") + "/" + upper := strings.TrimRight(scope, "/") + "0" + results := make(map[string]SearchResult) + ftsRows := make(map[string]int64) + collect := func(statement string, body, filename bool, args ...any) error { + rows, err := tx.QueryContext(ctx, statement, args...) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var result SearchResult + var rowID int64 + if err := rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt, &rowID); err != nil { + return err + } + if !allowed(result.Path) { + continue + } + if rowID != 0 { + ftsRows[result.RevisionUUID] = rowID + } + result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query)) + result.FilenameMatch = filename || results[result.Path].FilenameMatch || query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query)) + result.BodyMatch = body || results[result.Path].BodyMatch + results[result.Path] = result + } + return rows.Err() + } + filterSQL, filterArgs := searchOptionsPredicate(options) + pathSQL := searchPathCandidatesBaseSQL + pathArgs := []any{domainUUID, scope, prefix, upper} + if query != "" { + pathSQL += searchPathQueryPredicate + pathArgs = append(pathArgs, query, query) + } + pathSQL += filterSQL + ` ORDER BY p.path LIMIT ?` + pathArgs = append(pathArgs, filterArgs...) + pathArgs = append(pathArgs, SearchCandidateLimit) + if err := collect(pathSQL, false, false, pathArgs...); err != nil { + return nil, err + } + expression := ftsPrefixExpression(query) + if expression != "" { + // No global ORDER BY or unbounded IN materialization: stop the FTS scan + // after at most this many current, in-Domain, in-scope Path candidates. + for _, field := range []string{"body", "filename"} { + args := []any{field + " : (" + expression + ")", domainUUID, scope, prefix, upper} + args = append(args, filterArgs...) + args = append(args, SearchCandidateLimit) + if err := collect(searchFTSCandidatesBaseSQL+filterSQL+` LIMIT ?`, field == "body", field == "filename", args...); err != nil { + return nil, err + } + } + } + out := make([]SearchResult, 0, len(results)) + for _, result := range results { + out = append(out, result) + } + sort.Slice(out, func(i, j int) bool { + // Direct name/Path matches precede body-only hits; all ties are stable. + iName, jName := out[i].PathMatch || out[i].FilenameMatch, out[j].PathMatch || out[j].FilenameMatch + if iName != jName { + return iName + } + if len(out[i].Path) != len(out[j].Path) { + return len(out[i].Path) < len(out[j].Path) + } + return out[i].Path < out[j].Path + }) + if len(out) > limit { + out = out[:limit] + } + if !loadExcerpts { + return out, nil + } + // Only authorized, displayed results receive text excerpts. Batch the FTS + // lookup so a broad query is not re-scanned once for every visible result. + ids := make([]any, 0, len(out)) + seen := make(map[string]bool) + for _, result := range out { + if result.BodyMatch && !seen[result.RevisionUUID] { + ids = append(ids, ftsRows[result.RevisionUUID]) + seen[result.RevisionUUID] = true + } + } + if len(ids) != 0 { + // snippet's output token/character limit does not bound its input work: + // FTS5 may tokenize the entire matched document. Read byte lengths only + // for authorized displayed rows before spending the snippet budget. + rows, err := tx.QueryContext(ctx, `SELECT rowid,octet_length(body) + FROM revision_search WHERE rowid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...) + if err != nil { + return nil, err + } + sizes := make(map[int64]int64) + for rows.Next() { + var rowID, size int64 + if err := rows.Scan(&rowID, &size); err != nil { + rows.Close() + return nil, err + } + sizes[rowID] = size + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + ids = nil + selected := make(map[string]bool) + remaining := int64(SearchExcerptTotalBytes) + for i := range out { + if !out[i].BodyMatch { + continue + } + revision := out[i].RevisionUUID + include, decided := selected[revision] + if !decided { + rowID := ftsRows[revision] + size, exists := sizes[rowID] + include = exists && size >= 0 && size <= SearchExcerptDocumentBytes && size <= remaining + if include { + ids = append(ids, rowID) + remaining -= size + } + selected[revision] = include + } + out[i].ExcerptOmitted = !include + } + } + if len(ids) != 0 { + excerpts, err := loadSearchBodyExcerpts(ctx, tx, expression, ids, searchExcerptTimeout) + if err != nil { + return nil, err + } + for i := range out { + if out[i].BodyMatch { + var present bool + out[i].Excerpt, present = excerpts[out[i].RevisionUUID] + out[i].ExcerptOmitted = !present + } + } + } + // A filename/Path-only hit still benefits from a short Page summary. Read + // only a bounded prefix from displayed textual revisions, in one batch. + ids, seen = nil, make(map[string]bool) + for _, result := range out { + media := strings.ToLower(result.ContentType) + textual := strings.HasPrefix(media, "text/") || strings.HasPrefix(media, "application/json") || strings.HasPrefix(media, "application/xml") || strings.HasPrefix(media, "application/javascript") || strings.HasPrefix(media, "application/yaml") || strings.HasPrefix(media, "application/x-yaml") + if !result.BodyMatch && textual && result.RevisionUUID != "" && !seen[result.RevisionUUID] { + ids = append(ids, result.RevisionUUID) + seen[result.RevisionUUID] = true + } + } + if len(ids) != 0 { + rows, err := tx.QueryContext(ctx, `SELECT uuid,substr(body,1,1024) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...) + if err != nil { + return nil, err + } + excerpts := make(map[string]string) + for rows.Next() { + var id, excerpt string + if err := rows.Scan(&id, &excerpt); err != nil { + rows.Close() + return nil, err + } + excerpts[id] = excerpt + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + for i := range out { + if !out[i].BodyMatch { + out[i].Excerpt = excerpts[out[i].RevisionUUID] + } + } + } + return out, nil +} + +func ftsPrefixExpression(query string) string { + words := strings.Fields(query) + terms := make([]string, 0, len(words)) + for _, word := range words { + word = strings.ReplaceAll(word, `"`, `""`) + if word != "" { + terms = append(terms, `"`+word+`"*`) + } + } + return strings.Join(terms, " AND ") +} diff --git a/internal/mestionora/search_filters.go b/internal/mestionora/search_filters.go new file mode 100644 index 0000000..4bd9379 --- /dev/null +++ b/internal/mestionora/search_filters.go @@ -0,0 +1,135 @@ +package mestionora + +import ( + "errors" + "fmt" + "strings" +) + +var ErrInvalidSearchOptions = errors.New("invalid search options") + +// SearchOptions applies to current Page metadata, not historical Revisions. +// Dates are Unix milliseconds: FromMS is inclusive and BeforeMS is exclusive. +// The created field refers to Object creation, not creation of a Path alias. +// FileType and Extension are normalized lower-case values; Extension has no dot +// prefix and may contain a compound suffix such as tar.gz. +type SearchOptions struct { + FileType string + Extension string + DateField string + FromMS *int64 + BeforeMS *int64 +} + +func (o SearchOptions) HasFilters() bool { + return o.FileType != "" || o.Extension != "" || o.FromMS != nil || o.BeforeMS != nil +} + +func (o SearchOptions) Validate() error { + switch o.FileType { + case "", "html", "text", "image", "audio", "video", "pdf", "archive", "csv", "json", "code", "office", "other": + default: + return fmt.Errorf("%w: unknown file type", ErrInvalidSearchOptions) + } + switch o.DateField { + case "", "updated", "created": + default: + return fmt.Errorf("%w: unknown date field", ErrInvalidSearchOptions) + } + if len(o.Extension) > 32 { + return fmt.Errorf("%w: extension is too long", ErrInvalidSearchOptions) + } + for i, c := range o.Extension { + alnum := c >= 'a' && c <= 'z' || c >= '0' && c <= '9' + if !alnum && (i == 0 || !strings.ContainsRune("._+-", c)) { + return fmt.Errorf("%w: invalid extension", ErrInvalidSearchOptions) + } + } + if o.FromMS != nil && o.BeforeMS != nil && *o.FromMS >= *o.BeforeMS { + return fmt.Errorf("%w: empty or reversed date range", ErrInvalidSearchOptions) + } + return nil +} + +const searchMIMESQL = `lower(trim(substr(COALESCE(r.content_type,''),1,instr(COALESCE(r.content_type,'')||';',';')-1)))` + +// The original filename wins over the Path suffix when present. Names remain +// data: even extension patterns are bound arguments and do not become SQL. +const searchFilenameSQL = `lower(CASE WHEN COALESCE(r.filename,'')<>'' THEN r.filename ELSE p.path END)` + +func searchSuffixesSQL(suffixes ...string) string { + clauses := make([]string, 0, len(suffixes)) + for _, suffix := range suffixes { + // Only package-owned ASCII constants reach this helper. GLOB has no + // escaping ambiguity for underscores, unlike LIKE. + clauses = append(clauses, searchFilenameSQL+` GLOB '*.`+suffix+`'`) + } + return "(" + strings.Join(clauses, " OR ") + ")" +} + +// Specific MIME types are authoritative. Generic text/binary types may use a +// known suffix (e.g. a Python source uploaded as text/plain); unrecognized, +// specific MIME types stay Other instead of pretending to be a different type. +// This uses metadata only and never hydrates a Document to sniff its bytes. +var searchFileTypeSQL = `CASE + WHEN ` + searchMIMESQL + ` IN ('text/html','application/xhtml+xml') THEN 'html' + WHEN ` + searchMIMESQL + `='application/pdf' THEN 'pdf' + WHEN ` + searchMIMESQL + ` LIKE 'image/%' THEN 'image' + WHEN ` + searchMIMESQL + ` LIKE 'audio/%' THEN 'audio' + WHEN ` + searchMIMESQL + ` LIKE 'video/%' THEN 'video' + WHEN ` + searchMIMESQL + ` IN ('text/csv','application/csv','text/tab-separated-values') THEN 'csv' + WHEN ` + searchMIMESQL + ` IN ('application/json','text/json') OR ` + searchMIMESQL + ` GLOB '*+json' THEN 'json' + WHEN ` + searchMIMESQL + ` IN ('application/zip','application/x-zip-compressed','application/x-7z-compressed','application/vnd.rar','application/x-rar','application/x-rar-compressed','application/x-tar','application/gzip','application/x-gzip','application/x-xz','application/x-bzip2','application/zstd','application/x-zstd','application/x-compress','application/x-cpio') THEN 'archive' + WHEN ` + searchMIMESQL + ` IN ('application/msword','application/vnd.ms-excel','application/vnd.ms-powerpoint','application/rtf','text/rtf') OR ` + searchMIMESQL + ` GLOB 'application/vnd.openxmlformats-officedocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.oasis.opendocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.ms-*.macroenabled.*' THEN 'office' + WHEN ` + searchMIMESQL + ` IN ('text/markdown','text/x-markdown') THEN 'text' + WHEN ` + searchMIMESQL + ` IN ('application/javascript','application/x-javascript','text/javascript','application/typescript','text/typescript','application/xml','text/xml','application/yaml','application/x-yaml','text/yaml','application/toml','application/x-sh','application/x-shellscript','application/x-python','application/x-python-code','application/x-httpd-php','application/sql','text/css') OR ` + searchMIMESQL + ` GLOB 'text/x-*' OR ` + searchMIMESQL + ` GLOB '*+xml' THEN 'code' + WHEN ` + searchMIMESQL + ` NOT IN ('','text/plain','application/octet-stream','binary/octet-stream','application/x-empty') THEN CASE WHEN ` + searchMIMESQL + ` LIKE 'text/%' THEN 'text' ELSE 'other' END + WHEN ` + searchSuffixesSQL("html", "htm", "xhtml") + ` THEN 'html' + WHEN ` + searchSuffixesSQL("pdf") + ` THEN 'pdf' + WHEN ` + searchSuffixesSQL("png", "jpg", "jpeg", "gif", "webp", "avif", "bmp", "svg", "tif", "tiff", "ico", "heic", "heif") + ` THEN 'image' + WHEN ` + searchSuffixesSQL("mp3", "flac", "wav", "ogg", "oga", "opus", "m4a", "aac", "aiff", "alac", "mid", "midi") + ` THEN 'audio' + WHEN ` + searchSuffixesSQL("mp4", "webm", "mkv", "mov", "avi", "m4v", "ogv", "mpeg", "mpg") + ` THEN 'video' + WHEN ` + searchSuffixesSQL("zip", "7z", "rar", "tar", "gz", "tgz", "xz", "txz", "bz2", "tbz", "tbz2", "zst", "tzst", "cbz", "cbr", "cpio") + ` THEN 'archive' + WHEN ` + searchSuffixesSQL("csv", "tsv") + ` THEN 'csv' + WHEN ` + searchSuffixesSQL("json", "jsonl", "ndjson", "excalidraw", "ipynb") + ` THEN 'json' + WHEN ` + searchSuffixesSQL("doc", "docx", "docm", "xls", "xlsx", "xlsm", "ppt", "pptx", "pptm", "odt", "ods", "odp", "rtf") + ` THEN 'office' + WHEN ` + searchSuffixesSQL("py", "pyw", "sh", "bash", "zsh", "fish", "ps1", "bat", "cmd", "js", "mjs", "cjs", "jsx", "ts", "tsx", "go", "rs", "c", "h", "cpp", "hpp", "cc", "cs", "java", "kt", "kts", "swift", "rb", "php", "pl", "pm", "lua", "r", "sql", "css", "scss", "sass", "less", "xml", "xsl", "yaml", "yml", "toml", "ini", "conf", "vue", "svelte", "dart", "ex", "exs", "erl", "hs", "scala", "clj", "lisp", "el", "vim", "wasm") + ` THEN 'code' + WHEN ` + searchMIMESQL + `='text/plain' OR ` + searchSuffixesSQL("txt", "text", "log", "md", "markdown", "rst", "adoc", "org") + ` THEN 'text' + ELSE 'other' END` + +// The returned predicate is inserted before each candidate stream's LIMIT, +// including FTS-first streams. Filtering after their cap silently loses hits. +func searchOptionsPredicate(options SearchOptions) (string, []any) { + var predicate strings.Builder + var args []any + if options.FromMS != nil || options.BeforeMS != nil { + dateSQL := "r.created_at_ms" + if options.DateField == "created" { + // Objects.uuid is a primary-key lookup. Avoid joining Objects in the + // normal search plan when Object creation is not being filtered. + predicate.WriteString(" AND EXISTS (SELECT 1 FROM objects o WHERE o.uuid=p.object_uuid") + dateSQL = "o.created_at*1000" + } + if options.FromMS != nil { + predicate.WriteString(" AND " + dateSQL + ">=?") + args = append(args, *options.FromMS) + } + if options.BeforeMS != nil { + predicate.WriteString(" AND " + dateSQL + "= latest_revision_index.created_at_ms; + END + `); err != nil { + return err + } + if _, err := s.db.Exec(` + CREATE TRIGGER IF NOT EXISTS revision_search_delete + AFTER DELETE ON revisions BEGIN + DELETE FROM revision_search WHERE revision_uuid = old.uuid; + DELETE FROM revision_search_state WHERE revision_uuid = old.uuid; + DELETE FROM latest_revision_index + WHERE object_uuid = old.object_uuid + AND revision_uuid = old.uuid; + INSERT OR REPLACE INTO latest_revision_index ( + object_uuid, revision_uuid, content_type, filename, + document_hash, created_at_ms + ) + SELECT + object_uuid, uuid, COALESCE(content_type, ''), + COALESCE(filename, ''), COALESCE(document_hash, ''), + COALESCE(created_at_ms, created_at * 1000, 0) + FROM revisions + WHERE object_uuid = old.object_uuid + ORDER BY COALESCE(created_at_ms, created_at * 1000) DESC, rowid DESC + LIMIT 1; + END + `); err != nil { + return err + } + _, err := s.db.Exec(` + INSERT INTO revision_search ( + revision_uuid, object_uuid, filename, body + ) + SELECT + r.uuid, + r.object_uuid, + COALESCE(r.filename, ''), + CASE + WHEN lower(COALESCE(r.content_type, '')) LIKE 'text/%' + OR lower(COALESCE(r.content_type, '')) LIKE 'application/json%' + OR lower(COALESCE(r.content_type, '')) LIKE 'application/xml%' + OR lower(COALESCE(r.content_type, '')) LIKE 'application/javascript%' + OR lower(COALESCE(r.content_type, '')) LIKE 'application/x-yaml%' + OR lower(COALESCE(r.content_type, '')) LIKE 'application/yaml%' + THEN COALESCE(r.body, '') + ELSE '' + END + FROM revisions r + WHERE NOT EXISTS ( + SELECT 1 FROM revision_search_state state + WHERE state.revision_uuid = r.uuid + ) + `) + if err != nil { + return err + } + _, err = s.db.Exec(` + INSERT OR IGNORE INTO revision_search_state (revision_uuid) + SELECT uuid FROM revisions + `) + return err +} diff --git a/internal/store/search_query_plan_test.go b/internal/mestionora/search_query_plan_test.go similarity index 95% rename from internal/store/search_query_plan_test.go rename to internal/mestionora/search_query_plan_test.go index 1f00a69..bc41838 100644 --- a/internal/store/search_query_plan_test.go +++ b/internal/mestionora/search_query_plan_test.go @@ -1,4 +1,4 @@ -package store +package mestionora import ( "context" @@ -39,10 +39,7 @@ func TestSearchExcerptDeadlineIsOptionalButParentCancellationIsNot(t *testing.T) t.Fatal(err) } defer database.Close() - if err := db.Migrate(database); err != nil { - t.Fatal(err) - } - New(database) + initSearchFixture(t, database) tx, err := database.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true}) if err != nil { t.Fatal(err) @@ -72,10 +69,7 @@ func TestSearchFTSPlanScansMatchesOnce(t *testing.T) { t.Fatal(err) } defer database.Close() - if err := db.Migrate(database); err != nil { - t.Fatal(err) - } - New(database) + initSearchFixture(t, database) for _, field := range []string{"body", "filename"} { plan := searchQueryPlan(t, database, searchFTSCandidatesSQL, field+` : ("needle"*)`, "domain", "/", "/", "0", SearchCandidateLimit) @@ -96,10 +90,7 @@ func TestSearchFilteredFTSPlanKeepsMatchFirstAndIndexedSeeks(t *testing.T) { t.Fatal(err) } defer database.Close() - if err := db.Migrate(database); err != nil { - t.Fatal(err) - } - New(database) + initSearchFixture(t, database) from, before := int64(1000), int64(2000) for _, options := range []SearchOptions{ {FileType: "code"}, {Extension: "py"}, {FromMS: &from, BeforeMS: &before}, diff --git a/internal/mestionora/security_test.go b/internal/mestionora/security_test.go new file mode 100644 index 0000000..fc4eba1 --- /dev/null +++ b/internal/mestionora/security_test.go @@ -0,0 +1,102 @@ +package mestionora + +import ( + "context" + "errors" + "testing" +) + +func TestMissingPolicyDoesNotReadOrInitialize(t *testing.T) { + // A nil pool makes accidental construction, search or maintenance I/O fail. + engine := NewSQLite(nil) + ctx := context.Background() + if hits, err := engine.SearchPathsInDomainFiltered(ctx, "d", "/", "needle", 20, nil); err != nil || len(hits) != 0 { + t.Fatalf("missing page policy: %v %v", hits, err) + } + if hits, err := engine.SearchPathSuggestionsInDomain(ctx, "d", "/", "needle", 8, nil); err != nil || len(hits) != 0 { + t.Fatalf("missing suggestion policy: %v %v", hits, err) + } + if hits, err := engine.SearchCityInDomain(ctx, "d", "/", "needle", 30, nil, SearchOptions{}); err != nil || len(hits) != 0 { + t.Fatalf("missing city policy: %v %v", hits, err) + } +} + +func TestDeniedPagesDoNotLoadSummaryOrHeading(t *testing.T) { + database := newSearchTestDB(t) + _, err := database.Exec(` + INSERT INTO objects(uuid,created_at) VALUES('o',1); + INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('d','/needle','o'); + INSERT INTO revisions(uuid,object_uuid,filename,body,content_type,document_hash,created_at_ms,created_at) + VALUES('r','o','needle.html','

PRIVATE_HEADING

PRIVATE_SUMMARY

','text/html','hash',1000,1); + DROP TABLE revisions; + `) + if err != nil { + t.Fatal(err) + } + engine := NewSQLite(database) + deny := func(string) bool { return false } + ctx := context.Background() + // Candidates still exist in the independent index. Any attempt to hydrate + // denied summaries/headings now fails because their source table is absent. + if hits, err := engine.SearchPathsInDomainFiltered(ctx, "d", "/", "needle", 20, deny); err != nil || len(hits) != 0 { + t.Fatalf("denied summary loaded: %v %v", hits, err) + } + if hits, err := engine.SearchPathSuggestionsInDomain(ctx, "d", "/", "needle", 8, deny); err != nil || len(hits) != 0 { + t.Fatalf("denied heading loaded: %v %v", hits, err) + } + allow := func(string) bool { return true } + if _, err := engine.SearchPathsInDomainFiltered(ctx, "d", "/", "needle", 20, allow); err == nil { + t.Fatal("fixture did not exercise summary loading") + } + if _, err := engine.SearchPathSuggestionsInDomain(ctx, "d", "/", "needle", 8, allow); err == nil { + t.Fatal("fixture did not exercise heading loading") + } +} + +func TestCancellationDuringPolicyNeverReturnsPartialSuccess(t *testing.T) { + database := newSearchTestDB(t) + if _, err := database.Exec(`INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('d','/needle','o')`); err != nil { + t.Fatal(err) + } + for _, excerpts := range []bool{false, true} { + ctx, cancel := context.WithCancel(context.Background()) + allowed := func(string) bool { cancel(); return true } + // Filter-only search has no later FTS statement to notice cancellation. + hits, err := NewSQLite(database).searchPathsInDomainWithOptions(ctx, "d", "/", "", 20, allowed, SearchOptions{FileType: "other"}, excerpts) + cancel() + if !errors.Is(err, context.Canceled) || len(hits) != 0 { + t.Fatalf("partial success after cancellation: %v %v", hits, err) + } + } +} + +func TestFTSExpressionTreatsOperatorsAsData(t *testing.T) { + for input, want := range map[string]string{ + "": "", "alpha beta": `"alpha"* AND "beta"*`, + `a"b OR`: `"a""b"* AND "OR"*`, + `body:NEAR(x) -secret`: `"body:NEAR(x)"* AND "-secret"*`, + "日本語": `"日本語"*`, + } { + if got := ftsPrefixExpression(input); got != want { + t.Fatalf("%q: got %q want %q", input, got, want) + } + } +} + +func TestSearchDoesNotMaintainIndexOnReadOnlyConnection(t *testing.T) { + database := newSearchTestDB(t) + database.SetMaxOpenConns(1) + if _, err := database.Exec(`INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('d','/needle','o'); PRAGMA query_only=ON`); err != nil { + t.Fatal(err) + } + engine := NewSQLite(database) + results, err := engine.SearchPathsInDomainFiltered(context.Background(), "d", "/", "needle", 20, func(string) bool { return true }) + if err != nil || len(results) != 1 { + t.Fatalf("read-only search: %v %v", results, err) + } + // Schema maintenance is deliberately explicit, not performed by NewSQLite + // or a query. An accidental write in those paths would fail the test above. + if err := engine.EnsureSearchIndexSchema(); err == nil { + t.Fatal("read-only fixture unexpectedly accepted maintenance") + } +} diff --git a/internal/mestionora/sqlite.go b/internal/mestionora/sqlite.go new file mode 100644 index 0000000..a2ddc38 --- /dev/null +++ b/internal/mestionora/sqlite.go @@ -0,0 +1,34 @@ +// Package mestionora owns bounded search, search projections and the SQLite +// search index. It consumes read-policy decisions; it never grants permissions. +// Search results and excerpts are untrusted text, not HTML-safe output. +package mestionora + +import ( + "database/sql" + "strings" +) + +const Component = "mestionora" + +// SQLite borrows a metadata connection pool. Construction performs no I/O or +// schema initialization. EnsureSearchIndexSchema is a separate startup step. +// The caller owns the pool lifetime and provisions the application schema. +type SQLite struct{ db *sql.DB } + +func NewSQLite(db *sql.DB) *SQLite { return &SQLite{db: db} } + +// Match the established Path scope rules, without filesystem path cleaning or +// URL decoding. Percent/underscore characters remain literal search data. +func normalizePath(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "/" + } + if !strings.HasPrefix(value, "/") { + value = "/" + value + } + if value != "/" { + value = strings.TrimRight(value, "/") + } + return value +} diff --git a/internal/ordoschnellic/boundary_test.go b/internal/ordoschnellic/boundary_test.go new file mode 100644 index 0000000..5c7f185 --- /dev/null +++ b/internal/ordoschnellic/boundary_test.go @@ -0,0 +1,40 @@ +package ordoschnellic + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestComponentBoundary(t *testing.T) { + if Component != "ordoschnellic" || PushEventType != "notification.push.deliver" { + t.Fatal("component/event identity changed") + } + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, spec := range ast.Imports { + path, err := strconv.Unquote(spec.Path.Value) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"pageswiki/internal", "database", "os"} { + if path == forbidden || strings.HasPrefix(path, forbidden+"/") { + t.Errorf("%s imports %s: delivery must not own application policy, DB transactions or files", file, path) + } + } + } + } +} diff --git a/internal/ordoschnellic/dispatcher.go b/internal/ordoschnellic/dispatcher.go new file mode 100644 index 0000000..2712f25 --- /dev/null +++ b/internal/ordoschnellic/dispatcher.go @@ -0,0 +1,82 @@ +package ordoschnellic + +import ( + "context" + "errors" + "time" +) + +// Queue rechecks session/recipient eligibility in Claim. Empty Subscription +// means the claimed item is no longer deliverable and should be finalized. +// No outbound network operation runs inside a Queue transaction. +type Queue interface { + Cleanup(context.Context, time.Time) error + Claim(context.Context, time.Time) (*PushDelivery, error) + DeferFocused(context.Context, *PushDelivery, time.Time) error + Finish(context.Context, *PushDelivery, Outcome, time.Time) error +} + +type Sender interface { + Send(context.Context, *PushDelivery) Outcome +} + +type Stats struct{ Sent, Failed int } + +type Dispatcher struct { + Queue Queue + Sender Sender + Now func() time.Time +} + +func (d Dispatcher) Run(parent context.Context) (Stats, error) { + var stats Stats + if d.Queue == nil || d.Sender == nil { + return stats, errors.New("notification delivery dependencies are required") + } + if err := parent.Err(); err != nil { + return stats, err + } + now := d.Now + if now == nil { + now = time.Now + } + ctx, cancel := context.WithTimeout(parent, PushBatchTimeout) + defer cancel() + if err := d.Queue.Cleanup(ctx, now()); err != nil { + return stats, err + } + for i := 0; i < PushBatchLimit && ctx.Err() == nil; i++ { + item, err := d.Queue.Claim(ctx, now()) + if err != nil { + return stats, err + } + if item == nil { + break + } + if item.Focused { + if err := d.Queue.DeferFocused(ctx, item, now()); err != nil { + return stats, err + } + continue + } + var outcome Outcome + if item.Subscription != "" { + outcome = d.Sender.Send(ctx, item) + if outcome.Attempted { + if outcome.Success { + stats.Sent++ + } else { + stats.Failed++ + } + } + } + // Preserve completion when the caller or outbound request is cancelled. + finishCtx, stop := context.WithTimeout(context.Background(), PushFinishTimeout) + err = d.Queue.Finish(finishCtx, item, outcome, now()) + stop() + if err != nil { + return stats, err + } + } + return stats, parent.Err() +} diff --git a/internal/ordoschnellic/dispatcher_test.go b/internal/ordoschnellic/dispatcher_test.go new file mode 100644 index 0000000..0aa79b4 --- /dev/null +++ b/internal/ordoschnellic/dispatcher_test.go @@ -0,0 +1,150 @@ +package ordoschnellic + +import ( + "context" + "errors" + "fmt" + "testing" + "time" +) + +type queueFixture struct { + items []*PushDelivery + calls []string + errAt string + err error + finished []Outcome + finishContext func(context.Context) + at time.Time + t *testing.T +} + +func (q *queueFixture) call(op string, at time.Time) error { + q.t.Helper() + if !at.Equal(q.at) { + q.t.Fatalf("clock for %s: %v", op, at) + } + q.calls = append(q.calls, op) + if op == q.errAt { + return q.err + } + return nil +} +func (q *queueFixture) Cleanup(_ context.Context, at time.Time) error { return q.call("cleanup", at) } +func (q *queueFixture) Claim(_ context.Context, at time.Time) (*PushDelivery, error) { + if err := q.call("claim", at); err != nil { + return nil, err + } + if len(q.items) == 0 { + return nil, nil + } + d := q.items[0] + q.items = q.items[1:] + return d, nil +} +func (q *queueFixture) DeferFocused(_ context.Context, _ *PushDelivery, at time.Time) error { + return q.call("defer", at) +} +func (q *queueFixture) Finish(ctx context.Context, _ *PushDelivery, o Outcome, at time.Time) error { + if q.finishContext != nil { + q.finishContext(ctx) + } + q.finished = append(q.finished, o) + return q.call("finish", at) +} + +type senderFixture func(context.Context, *PushDelivery) Outcome + +func (f senderFixture) Send(ctx context.Context, d *PushDelivery) Outcome { return f(ctx, d) } + +func newQueueFixture(t *testing.T) *queueFixture { + return &queueFixture{t: t, at: time.Unix(1800000000, 0), err: errors.New("queue failure")} +} + +func TestDispatcherBoundedBatchAndCounters(t *testing.T) { + q := newQueueFixture(t) + q.items = append(q.items, &PushDelivery{Focused: true}, &PushDelivery{}) + for i := 0; i < 25; i++ { + q.items = append(q.items, &PushDelivery{Subscription: "valid"}) + } + sends := 0 + sender := senderFixture(func(context.Context, *PushDelivery) Outcome { + sends++ + return Outcome{Attempted: true, Success: sends%2 == 0, Retry: sends%2 != 0} + }) + stats, err := (Dispatcher{Queue: q, Sender: sender, Now: func() time.Time { return q.at }}).Run(context.Background()) + if err != nil || stats != (Stats{Sent: 9, Failed: 9}) || sends != 18 || len(q.items) != 7 || len(q.finished) != 19 { + t.Fatalf("stats=%+v sends=%d pending=%d finished=%d err=%v", stats, sends, len(q.items), len(q.finished), err) + } + if q.finished[0] != (Outcome{}) { + t.Fatal("ineligible delivery was sent/retried") + } + if fmt.Sprint(q.calls[:5]) != "[cleanup claim defer claim finish]" { + t.Fatal(q.calls) + } +} + +func TestDispatcherStopsOnQueueErrors(t *testing.T) { + for _, op := range []string{"cleanup", "claim", "defer", "finish"} { + t.Run(op, func(t *testing.T) { + q := newQueueFixture(t) + q.errAt = op + q.items = []*PushDelivery{{Subscription: "valid", Focused: op == "defer"}, {Subscription: "valid"}} + sends := 0 + _, err := (Dispatcher{Queue: q, Sender: senderFixture(func(context.Context, *PushDelivery) Outcome { sends++; return Outcome{Attempted: true, Success: true} }), Now: func() time.Time { return q.at }}).Run(context.Background()) + if !errors.Is(err, q.err) || q.calls[len(q.calls)-1] != op || sends > 1 { + t.Fatalf("calls=%v sends=%d err=%v", q.calls, sends, err) + } + }) + } +} + +func TestDispatcherCancellationFinalizesClaim(t *testing.T) { + q := newQueueFixture(t) + q.items = []*PushDelivery{{Subscription: "valid"}, {Subscription: "valid"}} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + q.finishContext = func(ctx context.Context) { + if ctx.Err() != nil { + t.Fatal("completion inherited cancellation") + } + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) > PushFinishTimeout { + t.Fatal("completion is not bounded") + } + } + stats, err := (Dispatcher{Queue: q, Now: func() time.Time { return q.at }, Sender: senderFixture(func(ctx context.Context, _ *PushDelivery) Outcome { + if _, ok := ctx.Deadline(); !ok { + t.Fatal("send has no deadline") + } + cancel() + return Outcome{Attempted: true, Retry: true} + })}).Run(ctx) + if !errors.Is(err, context.Canceled) || stats.Failed != 1 || len(q.finished) != 1 || len(q.items) != 1 || !q.finished[0].Retry { + t.Fatalf("%+v %v %+v", stats, err, q.finished) + } + q.calls = nil + _, err = (Dispatcher{Queue: q, Sender: senderFixture(func(context.Context, *PushDelivery) Outcome { t.Fatal("cancelled batch sent"); return Outcome{} })}).Run(ctx) + if !errors.Is(err, context.Canceled) || len(q.calls) != 0 { + t.Fatal("cancelled batch accessed queue", err, q.calls) + } +} + +func TestDeliveryPolicy(t *testing.T) { + at := time.Unix(1800000000, 0) + for _, attempt := range []int{-1, 0, 1, 2, 3, 4, 5, 1000000} { + next, ok := RetryAt(attempt, at) + if ok != (attempt >= 1 && attempt <= 4) { + t.Fatalf("attempt %d", attempt) + } + if ok && next.Sub(at) != time.Duration(1<= 200 && code < 300) || o.Retry != (code == 429 || code >= 500) || o.Expired != (code == 404 || code == 410) { + t.Fatal(code, o) + } + } +} diff --git a/internal/ordoschnellic/policy.go b/internal/ordoschnellic/policy.go new file mode 100644 index 0000000..76a7927 --- /dev/null +++ b/internal/ordoschnellic/policy.go @@ -0,0 +1,26 @@ +package ordoschnellic + +import "time" + +const ( + PushBatchLimit = 20 + PushBatchTimeout = 25 * time.Second + PushFinishTimeout = 3 * time.Second + PushLeaseDuration = time.Minute + PushFocusDelay = time.Minute + PushRetention = 24 * time.Hour + PushAttemptLimit = 5 +) + +// RetryAt preserves 2/4/8/16-minute backoff after attempts 1 through 4. +// Exhausted or invalid attempts are terminal and cannot overflow a shift. +func RetryAt(attempt int, now time.Time) (time.Time, bool) { + if attempt < 1 || attempt >= PushAttemptLimit { + return time.Time{}, false + } + return now.Add(time.Duration(1<= 200 && status < 300, Retry: status == 429 || status >= 500, Expired: status == 404 || status == 410, Attempted: true} +} diff --git a/internal/ordoschnellic/types.go b/internal/ordoschnellic/types.go new file mode 100644 index 0000000..a264f5e --- /dev/null +++ b/internal/ordoschnellic/types.go @@ -0,0 +1,38 @@ +// Package ordoschnellic coordinates notification delivery. Business events, +// recipient authorization, Inbox transactions and durable claims belong to its +// callers. A transport acknowledgement is not proof of end-user receipt. +package ordoschnellic + +import "errors" + +const Component = "ordoschnellic" +const PushEventType = "notification.push.deliver" + +type Notification struct { + UUID string `json:"uuid"` + DomainUUID string `json:"domain_uuid"` + RecipientUUID string `json:"recipient_uuid"` + EventUUID string `json:"event_uuid,omitempty"` + Kind string `json:"kind"` + Status string `json:"status"` + RequestUUID string `json:"request_uuid,omitempty"` + Href string `json:"href,omitempty"` + Message string `json:"message,omitempty"` + CreatedAtMS int64 `json:"created_at_ms"` + ReadAtMS int64 `json:"read_at_ms,omitempty"` +} + +var ErrNotificationInput = errors.New("invalid notification") + +type PushDelivery struct { + ID, Notification, Domain, Owner, Subscription string + Focused bool + Attempt int +} + +type Outcome struct { + Success, Retry, Expired bool + // Attempted includes key/encryption setup failures, but excludes invalid + // stored subscriptions. This preserves the existing delivery counters. + Attempted bool +} diff --git a/internal/ordoschnellic/webpush.go b/internal/ordoschnellic/webpush.go new file mode 100644 index 0000000..25c432e --- /dev/null +++ b/internal/ordoschnellic/webpush.go @@ -0,0 +1,86 @@ +package ordoschnellic + +import ( + "context" + "crypto/ecdh" + "encoding/base64" + "encoding/json" + "net/url" + "strings" + + webpush "github.com/SherClockHolmes/webpush-go" +) + +type StoredBrowserPush struct { + webpush.Subscription + Origin string `json:"origin"` +} + +// Only browser push vendors are destinations, never arbitrary user-provided URLs. +// DNS pinning/public-address checks at dial time provide a second SSRF boundary. +func ValidPushSubscription(sub webpush.Subscription) bool { + u, e := url.Parse(sub.Endpoint) + if e != nil || u.Scheme != "https" || u.User != nil || u.Fragment != "" || u.Port() != "" || len(sub.Endpoint) > 4096 { + return false + } + host := u.Hostname() + if host != "fcm.googleapis.com" && host != "updates.push.services.mozilla.com" && host != "web.push.apple.com" && !strings.HasSuffix(host, ".push.services.mozilla.com") { + return false + } + auth, e := base64.RawURLEncoding.DecodeString(sub.Keys.Auth) + if e != nil || len(auth) != 16 { + return false + } + key, e := base64.RawURLEncoding.DecodeString(sub.Keys.P256dh) + if e != nil { + return false + } + _, e = ecdh.P256().NewPublicKey(key) + return e == nil +} + +type KeyProvider func(context.Context) (private, public string, err error) + +// Client must enforce public-address DNS pinning, TLS, a bounded timeout and +// no redirects. There is deliberately no default HTTP client fallback. +type WebPush struct { + Keys KeyProvider + Client webpush.HTTPClient +} + +// wakePayload intentionally has no access to Inbox titles, bodies or tokens. +func wakePayload(item *PushDelivery) []byte { + payload, _ := json.Marshal(map[string]string{"uuid": item.Notification, "owner": item.Owner, "domain": item.Domain}) + return payload +} + +func (w WebPush) Send(ctx context.Context, item *PushDelivery) Outcome { + if item == nil { + return Outcome{} + } + var sub StoredBrowserPush + if json.Unmarshal([]byte(item.Subscription), &sub) != nil || !ValidPushSubscription(sub.Subscription) { + return Outcome{Expired: true} + } + if w.Keys == nil || w.Client == nil || ctx.Err() != nil { + return Outcome{Retry: true, Attempted: true} + } + private, public, err := w.Keys(ctx) + if err != nil { + return Outcome{Retry: true, Attempted: true} + } + resp, err := webpush.SendNotificationWithContext(ctx, wakePayload(item), &sub.Subscription, &webpush.Options{HTTPClient: w.Client, Subscriber: sub.Origin, VAPIDPrivateKey: private, VAPIDPublicKey: public, TTL: 300, Urgency: webpush.UrgencyNormal, Topic: "pageswiki-inbox"}) + if err != nil { + if resp != nil && resp.Body != nil { + resp.Body.Close() + } + return Outcome{Retry: true, Attempted: true} + } + if resp == nil { + return Outcome{Retry: true, Attempted: true} + } + if resp.Body != nil { + defer resp.Body.Close() + } + return ClassifyPushStatus(resp.StatusCode) +} diff --git a/internal/ordoschnellic/webpush_test.go b/internal/ordoschnellic/webpush_test.go new file mode 100644 index 0000000..ffa99ff --- /dev/null +++ b/internal/ordoschnellic/webpush_test.go @@ -0,0 +1,131 @@ +package ordoschnellic + +import ( + "context" + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "testing" + + webpush "github.com/SherClockHolmes/webpush-go" +) + +type httpFixture func(*http.Request) (*http.Response, error) + +func (f httpFixture) Do(r *http.Request) (*http.Response, error) { return f(r) } + +type trackedBody struct { + io.Reader + closed bool +} + +func (b *trackedBody) Close() error { b.closed = true; return nil } + +func subscriptionFixture(t *testing.T) StoredBrowserPush { + t.Helper() + key, err := ecdh.P256().GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + return StoredBrowserPush{Subscription: webpush.Subscription{Endpoint: "https://fcm.googleapis.com/fcm/send/synthetic", Keys: webpush.Keys{Auth: base64.RawURLEncoding.EncodeToString(make([]byte, 16)), P256dh: base64.RawURLEncoding.EncodeToString(key.PublicKey().Bytes())}}, Origin: "https://wiki.example"} +} +func deliveryFixture(t *testing.T, sub StoredBrowserPush) *PushDelivery { + t.Helper() + b, err := json.Marshal(sub) + if err != nil { + t.Fatal(err) + } + return &PushDelivery{Notification: "synthetic-notification", Owner: "synthetic-owner", Domain: "synthetic-domain", Subscription: string(b)} +} + +func TestWebPushPrivacyAndResponseHandling(t *testing.T) { + private, public, err := webpush.GenerateVAPIDKeys() + if err != nil { + t.Fatal(err) + } + d := deliveryFixture(t, subscriptionFixture(t)) + var payload map[string]string + if err := json.Unmarshal(wakePayload(d), &payload); err != nil || len(payload) != 3 || payload["uuid"] != d.Notification || payload["owner"] != d.Owner || payload["domain"] != d.Domain { + t.Fatal("payload contract", payload, err) + } + for _, code := range []int{201, 302, 400, 404, 410, 429, 503} { + body := &trackedBody{Reader: strings.NewReader("")} + w := WebPush{Keys: func(context.Context) (string, string, error) { return private, public, nil }, Client: httpFixture(func(r *http.Request) (*http.Response, error) { + data, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), d.Notification) || strings.Contains(string(data), d.Owner) || r.Header.Get("Content-Encoding") != "aes128gcm" || r.Header.Get("TTL") != "300" || r.Header.Get("Topic") != "pageswiki-inbox" { + t.Fatal("unprotected/changed payload") + } + return &http.Response{StatusCode: code, Body: body}, nil + })} + if o := w.Send(context.Background(), d); o != ClassifyPushStatus(code) || !body.closed { + t.Fatal(code, o, body.closed) + } + } +} + +func TestWebPushFailsClosedWithoutNetwork(t *testing.T) { + sub := subscriptionFixture(t) + d := deliveryFixture(t, sub) + for _, endpoint := range []string{"http://fcm.googleapis.com/x", "https://localhost/x", "https://127.0.0.1/x", "https://fcm.googleapis.com.evil.example/x", "https://fcm.googleapis.com:443/x", "https://user@fcm.googleapis.com/x", "https://web.push.apple.com/x#f"} { + sub.Endpoint = endpoint + w := WebPush{Keys: func(context.Context) (string, string, error) { + t.Fatal("unsafe target requested keys") + return "", "", nil + }, Client: httpFixture(func(*http.Request) (*http.Response, error) { t.Fatal("unsafe target contacted"); return nil, nil })} + if o := w.Send(context.Background(), deliveryFixture(t, sub)); !o.Expired || o.Attempted { + t.Fatal(endpoint, o) + } + } + for _, w := range []WebPush{{}, {Keys: func(context.Context) (string, string, error) { + t.Fatal("no HTTP client must fail closed") + return "", "", nil + }}, {Client: httpFixture(func(*http.Request) (*http.Response, error) { t.Fatal("no keys must fail closed"); return nil, nil })}} { + if o := w.Send(context.Background(), d); !o.Retry || !o.Attempted { + t.Fatal(o) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + w := WebPush{Keys: func(context.Context) (string, string, error) { + t.Fatal("cancelled request loaded keys") + return "", "", nil + }, Client: httpFixture(func(*http.Request) (*http.Response, error) { t.Fatal("cancelled request sent"); return nil, nil })} + if o := w.Send(ctx, d); !o.Retry { + t.Fatal(o) + } +} + +func TestWebPushTransportFailure(t *testing.T) { + private, public, err := webpush.GenerateVAPIDKeys() + if err != nil { + t.Fatal(err) + } + d := deliveryFixture(t, subscriptionFixture(t)) + for _, mode := range []string{"keys", "network", "nil-response"} { + w := WebPush{Keys: func(context.Context) (string, string, error) { + if mode == "keys" { + return "", "", errors.New("secret must not escape") + } + return private, public, nil + }, Client: httpFixture(func(*http.Request) (*http.Response, error) { + if mode == "keys" { + t.Fatal("sent after key failure") + } + if mode == "network" { + return nil, errors.New("endpoint must not escape") + } + return nil, nil + })} + if o := w.Send(context.Background(), d); o != (Outcome{Retry: true, Attempted: true}) { + t.Fatal(mode, o) + } + } +} diff --git a/internal/render/markdown.go b/internal/render/markdown.go index 5307073..7d2ec26 100644 --- a/internal/render/markdown.go +++ b/internal/render/markdown.go @@ -1,15 +1,9 @@ package render -import ( - "github.com/gomarkdown/markdown" -) +import "pageswiki/internal/grammaratur" +// Markdown retains the renderer entry point while Grammaratur owns conversion. +// It deliberately does not enable authoring fences or sanitize display HTML. func Markdown(input string) string { - out := markdown.ToHTML( - []byte(input), - nil, - nil, - ) - - return string(out) + return grammaratur.MarkdownToHTML(input) } diff --git a/internal/schutzaria/access.go b/internal/schutzaria/access.go new file mode 100644 index 0000000..c7f2369 --- /dev/null +++ b/internal/schutzaria/access.go @@ -0,0 +1,114 @@ +// Package schutzaria evaluates trusted, request-scoped authorization facts. +// It does not authenticate callers, load grants, resolve aliases, or commit +// publication. Callers must supply the correct Domain/principal snapshot. +package schutzaria + +import "strings" + +const Component = "schutzaria" + +// AccessLayer is the union of matching subjects at one locality. Layers at +// the same Path and at ancestors intersect; Grant cannot override a denial. +type AccessLayer struct { + Path string + Read, Write, Grant bool +} + +// LocalityAccess is immutable after construction and must not be cached +// across requests. Admin bypasses caps only; it does not create base grants. +type LocalityAccess struct { + admin bool + paths map[string][]AccessLayer +} + +func NewLocalityAccess(admin bool, layers []AccessLayer) *LocalityAccess { + p := &LocalityAccess{admin: admin, paths: make(map[string][]AccessLayer)} + for _, layer := range layers { + p.paths[layer.Path] = append(p.paths[layer.Path], layer) + } + return p +} + +// RequiresPathChecks permits the existing Object-alias fast path only when +// no local cap applies or the authenticated actor has recovery authority. +// Foreign-Domain restrictions must be checked separately before this shortcut. +func (p *LocalityAccess) RequiresPathChecks() bool { + return p != nil && !p.admin && len(p.paths) != 0 +} + +func (p *LocalityAccess) Decide(path, capability string, basis bool) bool { + if !strings.HasPrefix(path, "/") { + return false + } + if p == nil || p.admin { + return basis + } + grant := false + for at := path; ; { + for _, layer := range p.paths[at] { + allowed := layer.Write + if capability == "read" { + allowed = layer.Read + } + if !allowed { + return false + } + if layer.Grant && (capability == "read" || capability == "write" || capability == "create" || capability == "upload") { + grant = true + } + } + if at == "/" { + break + } + i := strings.LastIndex(at, "/") + if i <= 0 { + at = "/" + } else { + at = at[:i] + } + } + return basis || grant +} + +// ReadPathPolicy contains only positive Path grants plus locality caps. +// Object grants must never expose all semantic aliases. A nil/zero policy denies. +type ReadPathPolicy struct { + access *LocalityAccess + all bool + exact, subtrees map[string]bool +} + +// NewReadPathPolicy copies input slices; subsequent mutations cannot broaden +// an existing snapshot. Paths are already normalized by the application. +func NewReadPathPolicy(all bool, exact, subtrees []string, access *LocalityAccess) *ReadPathPolicy { + p := &ReadPathPolicy{all: all, access: access, exact: make(map[string]bool), subtrees: make(map[string]bool)} + for _, path := range exact { + p.exact[path] = true + } + for _, path := range subtrees { + p.subtrees[path] = true + } + return p +} + +func (p *ReadPathPolicy) Allows(path string) bool { + if p == nil { + return false + } + return p.access.Decide(path, "read", p.allowsBase(path)) +} + +func (p *ReadPathPolicy) allowsBase(path string) bool { + if p == nil || !strings.HasPrefix(path, "/") { + return false + } + if p.all || p.exact[path] { + return true + } + for at := strings.LastIndex(path, "/"); at > 0; at = strings.LastIndex(path[:at], "/") { + if p.subtrees[path[:at]] { + return true + } + } + return false +} diff --git a/internal/schutzaria/access_test.go b/internal/schutzaria/access_test.go new file mode 100644 index 0000000..e511e70 --- /dev/null +++ b/internal/schutzaria/access_test.go @@ -0,0 +1,125 @@ +package schutzaria + +import ( + "sync" + "testing" +) + +func TestLocalityAccessIntersectionAndGrantLimits(t *testing.T) { + for _, parentRead := range []bool{false, true} { + for _, parentWrite := range []bool{false, true} { + for _, childRead := range []bool{false, true} { + for _, childWrite := range []bool{false, true} { + for _, grant := range []bool{false, true} { + for _, basis := range []bool{false, true} { + p := NewLocalityAccess(false, []AccessLayer{{Path: "/life", Read: parentRead, Write: parentWrite}, {Path: "/life/home", Read: childRead, Write: childWrite, Grant: grant}}) + for _, cap := range []string{"read", "write", "create", "upload", "rename", "link", "delete", "manage"} { + allowed := parentWrite && childWrite + if cap == "read" { + allowed = parentRead && childRead + } + positive := grant && (cap == "read" || cap == "write" || cap == "create" || cap == "upload") + want := allowed && (basis || positive) + if got := p.Decide("/life/home/page", cap, basis); got != want { + t.Fatalf("parents=%v/%v children=%v/%v grant=%v base=%v cap=%s got=%v", parentRead, parentWrite, childRead, childWrite, grant, basis, cap, got) + } + } + if got := p.Decide("/lifes/other", "read", basis); got != basis { + t.Fatal("prefix boundary leaked") + } + } + } + } + } + } + } + // Multiple locality nodes may share a boundary; one denial still wins. + p := NewLocalityAccess(false, []AccessLayer{{Path: "/same", Read: true, Write: true, Grant: true}, {Path: "/same"}}) + if p.Decide("/same/child", "read", true) { + t.Fatal("same-Path denial lost") + } +} + +func TestLocalityAdminAndAbsentCapsNeverCreateGrants(t *testing.T) { + for _, p := range []*LocalityAccess{nil, NewLocalityAccess(false, nil), NewLocalityAccess(true, []AccessLayer{{Path: "/"}})} { + for _, basis := range []bool{false, true} { + if p.Decide("/a", "read", basis) != basis || p.Decide("relative", "read", basis) || p.Decide("", "read", basis) { + t.Fatal("admin/absent cap created a grant or allowed non-Path") + } + } + if p.RequiresPathChecks() { + t.Fatal("unexpected alias checks") + } + } + if !NewLocalityAccess(false, []AccessLayer{{Path: "/a"}}).RequiresPathChecks() { + t.Fatal("Object alias checks skipped") + } +} + +func TestReadPolicyLiteralScopesAndImmutability(t *testing.T) { + exact := []string{"/exact", "/group_%"} + subtrees := []string{"/group_%"} + layers := []AccessLayer{{Path: "/group_%/private"}} + access := NewLocalityAccess(false, layers) + p := NewReadPathPolicy(false, exact, subtrees, access) + exact[0] = "/leak" + subtrees[0] = "/" + layers[0] = AccessLayer{Path: "/group_%/private", Read: true, Grant: true} + for path, want := range map[string]bool{"/exact": true, "/exact/child": false, "/group_%": true, "/group_%/a/b": true, "/group_%/private": false, "/group_%/private/child": false, "/groupXX/a": false, "/group_%2": false, "/leak": false, "/": false, "": false, "relative": false} { + if got := p.Allows(path); got != want { + t.Fatalf("path=%q got=%v want=%v", path, got, want) + } + } + for _, deny := range []*ReadPathPolicy{nil, {}} { + if deny.Allows("/anything") { + t.Fatal("empty policy failed open") + } + } + all := NewReadPathPolicy(true, nil, nil, access) + if !all.Allows("/other") || all.Allows("/group_%/private") || all.Allows("relative") { + t.Fatal("broad grant bypassed caps") + } + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + if !p.Allows("/exact") || p.Allows("/leak") { + t.Error("snapshot changed") + } + } + }() + } + wg.Wait() +} + +func TestGroupPermissionProfiles(t *testing.T) { + wants := map[string]map[string]bool{ + "read": {"read": true}, + "read_write_once": {"read": true, "create": true, "upload": true}, + "read_write": {"read": true, "create": true, "write": true, "upload": true, "rename": true, "link": true}, + "upload_only": {"upload": true}, + "unknown": {}, + } + for profile, want := range wants { + for _, cap := range []string{"read", "create", "write", "upload", "rename", "link", "delete", "manage", ""} { + for _, path := range []string{"/a_%", "/a_%/child"} { + if GroupPermissionAllows("/a_%", profile, cap, path) != want[cap] { + t.Fatal(profile, cap, path) + } + } + if GroupPermissionAllows("/a_%", profile, cap, "/aXX/child") { + t.Fatal("wildcard interpreted") + } + } + } + if p, err := NormalizeGroupPermission(" READ_WRITE_ONCE "); err != nil || p != "read_write_once" { + t.Fatal(p, err) + } + for _, p := range []string{"", "delete", "group_admin", "read/write"} { + if _, err := NormalizeGroupPermission(p); err == nil { + t.Fatal("unknown profile", p) + } + } +} diff --git a/internal/schutzaria/approval.go b/internal/schutzaria/approval.go new file mode 100644 index 0000000..593261d --- /dev/null +++ b/internal/schutzaria/approval.go @@ -0,0 +1,22 @@ +package schutzaria + +import "errors" + +var ErrApprovalTransition = errors.New("invalid approval request transition") + +func ValidApprovalDecision(decision string) bool { + return decision == "approved" || decision == "rejected" || decision == "changes_requested" +} + +// ApprovalNextState preserves the existing request state machine only. +// It does not verify reviewer authority, expiry, quorum, source revisions or +// audience. The caller owns those checks and the transactional CAS/write. +func ApprovalNextState(state string, currentGeneration, generation int64, decision string) (string, error) { + if generation < 1 || state != "pending" || currentGeneration != generation || !ValidApprovalDecision(decision) { + return "", ErrApprovalTransition + } + if decision == "changes_requested" { + return "rejected", nil + } + return decision, nil +} diff --git a/internal/schutzaria/boundary_test.go b/internal/schutzaria/boundary_test.go new file mode 100644 index 0000000..aebad02 --- /dev/null +++ b/internal/schutzaria/boundary_test.go @@ -0,0 +1,40 @@ +package schutzaria + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestComponentBoundary(t *testing.T) { + if Component != "schutzaria" { + t.Fatal("component identity changed") + } + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + ast, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, spec := range ast.Imports { + path, err := strconv.Unquote(spec.Path.Value) + if err != nil { + t.Fatal(err) + } + for _, prefix := range []string{"pageswiki/internal", "database", "net", "os", "time"} { + if path == prefix || strings.HasPrefix(path, prefix+"/") { + t.Errorf("%s imports %s: evaluation must not acquire state, perform I/O or read time", file, path) + } + } + } + } +} diff --git a/internal/schutzaria/groups.go b/internal/schutzaria/groups.go new file mode 100644 index 0000000..e10d280 --- /dev/null +++ b/internal/schutzaria/groups.go @@ -0,0 +1,29 @@ +package schutzaria + +import ( + "errors" + "strings" +) + +var groupPermissions = map[string]map[string]bool{ + "read": {"read": true}, + "read_write_once": {"read": true, "create": true, "upload": true}, + "read_write": {"read": true, "create": true, "write": true, "upload": true, "rename": true, "link": true}, + "upload_only": {"upload": true}, +} + +func NormalizeGroupPermission(value string) (string, error) { + value = strings.ToLower(strings.TrimSpace(value)) + if _, ok := groupPermissions[value]; !ok { + return "", errors.New("invalid group permission") + } + return value, nil +} + +// GroupPermissionAllows evaluates a previously authenticated membership's +// profile. Creation-only existence checks and principal state belong to callers. +// This literal prefix rule deliberately preserves the existing stored scopes. +func GroupPermissionAllows(scope, permission, capability, targetPath string) bool { + inside := targetPath == scope || scope == "/" || strings.HasPrefix(targetPath, strings.TrimRight(scope, "/")+"/") + return inside && groupPermissions[permission][capability] +} diff --git a/internal/schutzaria/publication.go b/internal/schutzaria/publication.go new file mode 100644 index 0000000..24c4409 --- /dev/null +++ b/internal/schutzaria/publication.go @@ -0,0 +1,52 @@ +package schutzaria + +import "strings" + +type LocalityPolicy struct { + Kind string `json:"kind"` + UUID string `json:"uuid"` + Policy string `json:"policy"` +} + +// EvaluateLocalityPublication combines a Domain-first chain's ceilings and +// intent. Neither result is an anonymous-read grant or a reviewer approval. +// The caller loads/validates topology and binds immutable revisions separately. +func EvaluateLocalityPublication(chain []LocalityPolicy) (allows, published bool) { + allows = true + for i, p := range chain { + if i == 0 && p.Policy == "inherit" { + return false, false + } + switch p.Policy { + case "authenticated_acl": + allows = false + case "public": + if i > 0 { + published = true + } + case "inherit": + default: + return false, false + } + } + return allows, allows && published +} + +// PublicSemanticPath is a namespace eligibility check, never permission to +// publish content. User-space publication has its own authenticated policy path. +func PublicSemanticPath(path string) bool { + if !strings.HasPrefix(path, "/") || strings.ContainsAny(path, "\\\x00\r\n") || strings.Contains(path, "//") { + return false + } + for _, segment := range strings.Split(path, "/") { + if segment == "." || segment == ".." { + return false + } + } + first := strings.Split(strings.TrimPrefix(path, "/"), "/")[0] + switch first { + case "_", "-", ":", "o", "r", "v", "l", "e", "s", "u", "g", "static", "health", "new": + return false + } + return true +} diff --git a/internal/schutzaria/publication_test.go b/internal/schutzaria/publication_test.go new file mode 100644 index 0000000..9285254 --- /dev/null +++ b/internal/schutzaria/publication_test.go @@ -0,0 +1,89 @@ +package schutzaria + +import ( + "errors" + "testing" +) + +func TestPublicationCeilings(t *testing.T) { + values := []string{"public", "authenticated_acl", "inherit", "invalid"} + for _, domain := range values { + for _, quartier := range values { + for _, street := range values { + for _, spot := range values { + chain := []LocalityPolicy{{Kind: "domain", Policy: domain}, {Kind: "quartier", Policy: quartier}, {Kind: "street", Policy: street}, {Kind: "spot", Policy: spot}} + wantAllows := domain == "public" && quartier != "authenticated_acl" && street != "authenticated_acl" && spot != "authenticated_acl" && quartier != "invalid" && street != "invalid" && spot != "invalid" + wantIntent := wantAllows && (quartier == "public" || street == "public" || spot == "public") + a, p := EvaluateLocalityPublication(chain) + if a != wantAllows || p != wantIntent { + t.Fatalf("%+v got=%v/%v", chain, a, p) + } + } + } + } + } + // Preserve the legacy neutral empty-chain result; it is never a grant. + if a, p := EvaluateLocalityPublication(nil); !a || p { + t.Fatal("empty chain semantics changed") + } + if a, p := EvaluateLocalityPublication([]LocalityPolicy{{Kind: "domain", Policy: "public"}}); !a || p { + t.Fatal("Domain ceiling published content") + } +} + +func TestPublicNamespaceEligibility(t *testing.T) { + for _, p := range []string{"/", "/life/home", "/blog/2026/11/01", "/日本語", "/user"} { + if !PublicSemanticPath(p) { + t.Fatal(p) + } + } + for _, p := range []string{"", "relative", "/a/../b", "/a/./b", "/a//b", "/a\\b", "/a\x00b", "/a\nb", "/a\rb"} { + if PublicSemanticPath(p) { + t.Fatal(p) + } + } + for _, root := range []string{"_", "-", ":", "o", "r", "v", "l", "e", "s", "u", "g", "static", "health", "new"} { + if PublicSemanticPath("/"+root) || PublicSemanticPath("/"+root+"/child") { + t.Fatal(root) + } + } +} + +func TestApprovalStateTransitionNotPublicationAuthorization(t *testing.T) { + for _, state := range []string{"pending", "approved", "rejected", "withdrawn", "expired", "superseded", ""} { + for _, decision := range []string{"approved", "rejected", "changes_requested", "unknown", ""} { + for _, generation := range []int64{-1, 0, 1, 2} { + next, err := ApprovalNextState(state, 1, generation, decision) + valid := state == "pending" && generation == 1 && (decision == "approved" || decision == "rejected" || decision == "changes_requested") + if valid { + want := decision + if want == "changes_requested" { + want = "rejected" + } + if err != nil || next != want { + t.Fatal(state, decision, next, err) + } + } else if !errors.Is(err, ErrApprovalTransition) || next != "" { + t.Fatal(state, decision, generation, next, err) + } + } + } + } +} + +func TestUserSpaceNarrowing(t *testing.T) { + p := UserSpacePolicy{Username: "alice", RootVisibility: "domain", DescendantVisibility: "public", Overrides: []UserSpacePathPolicy{{Path: "/u/alice/work", Visibility: "domain"}, {Path: "/u/alice/work/private", Visibility: "private"}, {Path: "/u/alice/work/private/reopen", Visibility: "public"}}} + for path, want := range map[string]string{"/u/alice": "domain", "/u/alice/blog": "public", "/u/alice/work": "domain", "/u/alice/work/page": "domain", "/u/alice/work/private": "private", "/u/alice/work/private/reopen/page": "private", "/u/bob/blog": "private", "/u/alice/_/edit": "private", "/u/alice/a/../blog": "private", "/u/alice/blog/": "private"} { + if got := EffectiveUserSpaceVisibility(p, path); got != want { + t.Fatal(path, got, want) + } + } + // Existing semantics: root and descendants have independent settings. + p.RootVisibility = "private" + if EffectiveUserSpaceVisibility(p, "/u/alice/blog") != "public" { + t.Fatal("root incorrectly became subtree cap") + } + if UserSpaceVisibilityRank("unknown") != -1 || UserSpaceDocumentPath("alice", "/u/alice/_", true) { + t.Fatal("invalid user-space policy/path") + } +} diff --git a/internal/schutzaria/user_space.go b/internal/schutzaria/user_space.go new file mode 100644 index 0000000..b66aff3 --- /dev/null +++ b/internal/schutzaria/user_space.go @@ -0,0 +1,70 @@ +package schutzaria + +import ( + "path" + "strings" +) + +type UserSpacePathPolicy struct { + Path string `json:"path"` + Visibility string `json:"visibility"` +} + +type UserSpacePolicy struct { + PrincipalUUID string `json:"principal_uuid"` + Username string `json:"username"` + ProfileVisibility string `json:"profile_visibility"` + AvatarVisibility string `json:"avatar_visibility"` + RootVisibility string `json:"root_visibility"` + DescendantVisibility string `json:"descendant_visibility"` + Version int64 `json:"version"` + DomainAllowsPublic bool `json:"domain_allows_public"` + Overrides []UserSpacePathPolicy `json:"overrides"` +} + +func UserSpaceVisibilityRank(value string) int { + switch value { + case "private": + return 0 + case "domain": + return 1 + case "public": + return 2 + default: + return -1 + } +} + +func UserSpaceDocumentPath(username, value string, descendant bool) bool { + root := "/u/" + username + if value != path.Clean(value) || strings.ContainsAny(value, "\\\x00\r\n") { + return false + } + if descendant { + if !strings.HasPrefix(value, root+"/") { + return false + } + rest := strings.TrimPrefix(value, root+"/") + return rest != "" && !strings.HasPrefix(rest, "_/") && rest != "_" + } + return value == root +} + +func EffectiveUserSpaceVisibility(p UserSpacePolicy, target string) string { + root := "/u/" + p.Username + if target == root { + return p.RootVisibility + } + if !UserSpaceDocumentPath(p.Username, target, true) { + return "private" + } + visibility := p.DescendantVisibility + for _, item := range p.Overrides { + if target == item.Path || strings.HasPrefix(target, strings.TrimRight(item.Path, "/")+"/") { + if UserSpaceVisibilityRank(item.Visibility) < UserSpaceVisibilityRank(visibility) { + visibility = item.Visibility + } + } + } + return visibility +} diff --git a/internal/seheweit/history.go b/internal/seheweit/history.go new file mode 100644 index 0000000..2909b25 --- /dev/null +++ b/internal/seheweit/history.go @@ -0,0 +1,39 @@ +package seheweit + +import "time" + +const ( + ResourceEventID = "builtin.resource.sample" + ResourceEventType = "resource.sample" + RetentionDays = 7 + MaxHistoryPoints = 361 +) + +// HistoryWindow describes the existing inclusive, epoch-aligned sample query. +// Persistence chooses the most recently inserted sample in each bucket, not an +// average, a peak or necessarily the largest timestamp in out-of-order data. +type HistoryWindow struct { + FromMS int64 + ToMS int64 + BucketMS int64 +} + +func Window(hours int, now time.Time) HistoryWindow { + if hours < 1 { + hours = 1 + } + if hours > RetentionDays*24 { + hours = RetentionDays * 24 + } + span := int64(time.Duration(hours) * time.Hour / time.Millisecond) + bucket := span / (MaxHistoryPoints - 1) + if bucket < int64(time.Minute/time.Millisecond) { + bucket = int64(time.Minute / time.Millisecond) + } + return HistoryWindow{FromMS: now.UnixMilli() - span, ToMS: now.UnixMilli(), BucketMS: bucket} +} + +// RetentionCutoff is inclusive: only samples strictly older are expired. +func RetentionCutoff(atMS int64) int64 { + return atMS - int64(RetentionDays*24*time.Hour/time.Millisecond) +} diff --git a/internal/seheweit/history_test.go b/internal/seheweit/history_test.go new file mode 100644 index 0000000..7e39078 --- /dev/null +++ b/internal/seheweit/history_test.go @@ -0,0 +1,31 @@ +package seheweit + +import ( + "testing" + "time" +) + +func TestHistoryWindow(t *testing.T) { + now := time.Date(2026, 11, 1, 12, 34, 56, 0, time.UTC) + for _, tc := range []struct { + hours, wantHours int + bucket int64 + }{ + {-1, 1, 60000}, {0, 1, 60000}, {1, 1, 60000}, {24, 24, 240000}, {168, 168, 1680000}, {999, 168, 1680000}, + } { + got := Window(tc.hours, now) + if got.ToMS != now.UnixMilli() || got.FromMS != now.Add(-time.Duration(tc.wantHours)*time.Hour).UnixMilli() || got.BucketMS != tc.bucket { + t.Fatalf("hours=%d %+v", tc.hours, got) + } + buckets := got.ToMS/got.BucketMS - got.FromMS/got.BucketMS + 1 + if buckets > MaxHistoryPoints { + t.Fatalf("too many buckets: %d", buckets) + } + } + if RetentionCutoff(now.UnixMilli()) != now.Add(-7*24*time.Hour).UnixMilli() { + t.Fatal("retention changed") + } + if ResourceEventID != "builtin.resource.sample" || ResourceEventType != "resource.sample" { + t.Fatal("persisted identifiers changed") + } +} diff --git a/internal/seheweit/observe.go b/internal/seheweit/observe.go new file mode 100644 index 0000000..d96f862 --- /dev/null +++ b/internal/seheweit/observe.go @@ -0,0 +1,177 @@ +// Package seheweit collects instance observations without controlling the +// observed workload. Authorization, queue execution and persistence belong to +// its callers. It does not inspect document contents or contact remote services. +package seheweit + +import ( + "errors" + "io" + "math" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "syscall" + "time" + + "github.com/google/uuid" +) + +// Identity survives multiple collectors, but not a process restart/PID reuse. +var processID = uuid.NewString() + +type ProcessIO struct { + Scope string `json:"scope"` + Available bool `json:"available"` + ProcessID string `json:"process_id"` + ReadBytes uint64 `json:"read_bytes"` + WriteBytes uint64 `json:"write_bytes"` +} + +type CPU struct { + LogicalCPUs int `json:"logical_cpus"` + LoadAverage string `json:"load_average"` +} + +type Memory struct { + HeapBytes uint64 `json:"heap_bytes"` + GoSystemBytes uint64 `json:"go_system_bytes"` + GCCycles uint32 `json:"gc_cycles"` + Goroutines int `json:"goroutines"` +} + +type Storage struct { + Path string `json:"path"` + TotalBytes uint64 `json:"total_bytes"` + UsedBytes uint64 `json:"used_bytes"` + AvailableBytes uint64 `json:"available_bytes"` +} + +// Queue contains observations supplied by the application, not execution hooks. +type Queue struct { + Stats QueueStats `json:"stats"` + ActiveWorkers int64 `json:"active_workers"` + Paused bool `json:"paused"` +} + +type QueueStats struct { + Queued int `json:"queued"` + Running int `json:"running"` +} + +// Snapshot preserves the resource API and history JSON field names. +// Sources are sampled sequentially, not in an atomic cross-source transaction. +type Snapshot struct { + TimestampMS int64 `json:"timestamp_ms"` + IO ProcessIO `json:"io"` + CPU CPU `json:"cpu"` + Memory Memory `json:"memory"` + Storage Storage `json:"storage"` + Queue Queue `json:"queue"` +} + +type sources struct { + now func() time.Time + readFile func(string) ([]byte, error) + statfs func(string, *syscall.Statfs_t) error + memory func(*runtime.MemStats) + cpus func() int + goroutines func() int +} + +// Collect reads Go runtime counters, two bounded proc files and Statfs on the +// configured document root. No directory walk, shell or background loop. +func Collect(root string, queue Queue) Snapshot { + return collect(root, queue, sources{ + now: time.Now, readFile: readProcFile, statfs: syscall.Statfs, + memory: runtime.ReadMemStats, cpus: runtime.NumCPU, goroutines: runtime.NumGoroutine, + }) +} + +func collect(root string, queue Queue, src sources) Snapshot { + var mem runtime.MemStats + src.memory(&mem) + if root == "" { + root = "." + } + if absolute, err := filepath.Abs(root); err == nil { + root = absolute + } + storage := Storage{Path: root} + var fs syscall.Statfs_t + if err := src.statfs(root, &fs); err == nil { + storage = storageCounters(root, fs.Blocks, fs.Bavail, fs.Bsize) + } + loadAverage := "" + if data, err := src.readFile("/proc/loadavg"); err == nil { + fields := strings.Fields(string(data)) + if len(fields) >= 3 { + loadAverage = strings.Join(fields[:3], " ") + } + } + var ioData []byte + if data, err := src.readFile("/proc/self/io"); err == nil { + ioData = data + } + return Snapshot{ + TimestampMS: src.now().UnixMilli(), IO: ParseProcessIO(string(ioData)), + CPU: CPU{LogicalCPUs: src.cpus(), LoadAverage: loadAverage}, + Memory: Memory{HeapBytes: mem.HeapAlloc, GoSystemBytes: mem.Sys, GCCycles: mem.NumGC, Goroutines: src.goroutines()}, + Storage: storage, Queue: queue, + } +} + +func storageCounters(path string, blocks, available uint64, blockSize int64) Storage { + result := Storage{Path: path} + // Preserve the existing zero fallback rather than wrapping unsigned values. + if blockSize <= 0 || available > blocks || blocks > math.MaxUint64/uint64(blockSize) { + return result + } + result.TotalBytes = blocks * uint64(blockSize) + result.AvailableBytes = available * uint64(blockSize) + result.UsedBytes = result.TotalBytes - result.AvailableBytes + return result +} + +const maxProcBytes = 16 << 10 + +func readProcFile(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + data, err := io.ReadAll(io.LimitReader(f, maxProcBytes+1)) + if err != nil { + return nil, err + } + if len(data) > maxProcBytes { + return nil, errors.New("resource counter file too large") + } + return data, nil +} + +// ParseProcessIO uses actual storage bytes, not rchar/wchar (syscall traffic). +// Partial, missing or invalid counters are unavailable, not a zero I/O rate. +func ParseProcessIO(data string) ProcessIO { + result := ProcessIO{Scope: "process", ProcessID: processID} + read, write := false, false + for _, line := range strings.Split(data, "\n") { + fields := strings.Fields(line) + if len(fields) != 2 || (fields[0] != "read_bytes:" && fields[0] != "write_bytes:") { + continue + } + value, err := strconv.ParseUint(fields[1], 10, 64) + if err != nil { + return result + } + if fields[0] == "read_bytes:" { + result.ReadBytes, read = value, true + } else { + result.WriteBytes, write = value, true + } + } + result.Available = read && write + return result +} diff --git a/internal/seheweit/observe_test.go b/internal/seheweit/observe_test.go new file mode 100644 index 0000000..c971be1 --- /dev/null +++ b/internal/seheweit/observe_test.go @@ -0,0 +1,171 @@ +package seheweit + +import ( + "encoding/json" + "errors" + "math" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "syscall" + "testing" + "time" +) + +func fixtureSources(t *testing.T) sources { + t.Helper() + return sources{ + now: func() time.Time { return time.UnixMilli(123456789) }, + readFile: func(path string) ([]byte, error) { + switch path { + case "/proc/loadavg": + return []byte("0.01 0.20 3.00 1/99 1\n"), nil + case "/proc/self/io": + return []byte("rchar: 999\nread_bytes: 10\nwrite_bytes: 20\n"), nil + default: + t.Fatalf("unexpected file read: %s", path) + return nil, nil + } + }, + statfs: func(path string, fs *syscall.Statfs_t) error { + if !filepath.IsAbs(path) { + t.Fatalf("relative root %q", path) + } + fs.Blocks, fs.Bavail, fs.Bsize = 100, 30, 4096 + return nil + }, + memory: func(m *runtime.MemStats) { m.HeapAlloc, m.Sys, m.NumGC = 1024, 4096, 9 }, + cpus: func() int { return 8 }, goroutines: func() int { return 17 }, + } +} + +func TestSnapshotWireCompatibility(t *testing.T) { + queue := Queue{Stats: QueueStats{Queued: 2, Running: 1}, ActiveWorkers: 1, Paused: true} + snapshot := collect(t.TempDir(), queue, fixtureSources(t)) + data, err := json.Marshal(snapshot) + if err != nil { + t.Fatal(err) + } + // Exact pre-extraction wire shape, including zeros/false (no omitempty). + want := map[string]any{ + "timestamp_ms": int64(123456789), + "io": map[string]any{"scope": "process", "available": true, "process_id": processID, "read_bytes": 10, "write_bytes": 20}, + "cpu": map[string]any{"logical_cpus": 8, "load_average": "0.01 0.20 3.00"}, + "memory": map[string]any{"heap_bytes": 1024, "go_system_bytes": 4096, "gc_cycles": 9, "goroutines": 17}, + "storage": map[string]any{"path": snapshot.Storage.Path, "total_bytes": 409600, "used_bytes": 286720, "available_bytes": 122880}, + "queue": map[string]any{"stats": map[string]int{"queued": 2, "running": 1}, "active_workers": 1, "paused": true}, + } + wantData, _ := json.Marshal(want) + var gotJSON, wantJSON any + if err := json.Unmarshal(data, &gotJSON); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(wantData, &wantJSON); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(gotJSON, wantJSON) { + t.Fatalf("wire changed: %s != %s", data, wantData) + } + zero, _ := json.Marshal(Snapshot{}) + if strings.Contains(string(zero), "null") || !strings.Contains(string(zero), `"paused":false`) { + t.Fatalf("zero shape %s", zero) + } +} + +func TestCollectionFailureAndIsolation(t *testing.T) { + src := fixtureSources(t) + src.readFile = func(string) ([]byte, error) { + return []byte("read_bytes: 5\nwrite_bytes: 6\n"), errors.New("unavailable") + } + src.statfs = func(_ string, fs *syscall.Statfs_t) error { + fs.Blocks, fs.Bavail, fs.Bsize = 100, 20, 4096 + return errors.New("unavailable") + } + got := collect("", Queue{}, src) + if !filepath.IsAbs(got.Storage.Path) || got.Storage.TotalBytes != 0 || got.IO.Available || got.IO.ReadBytes != 0 || got.CPU.LoadAverage != "" { + t.Fatalf("failed source leaked partial values: %+v", got) + } + if got.Memory.HeapBytes != 1024 || got.CPU.LogicalCPUs != 8 || got.IO.ProcessID == "" { + t.Fatalf("independent sources lost: %+v", got) + } + // Each call has independent values; the process identity alone is shared. + for i := 0; i < 16; i++ { + t.Run("parallel", func(t *testing.T) { + t.Parallel() + other := collect("", Queue{}, fixtureSources(t)) + if other.IO.ProcessID != got.IO.ProcessID || other.Storage.TotalBytes != 409600 { + t.Fatalf("identity/isolation %+v", other) + } + }) + } +} + +func TestStorageInvalidCounters(t *testing.T) { + for _, tc := range []struct { + blocks, available uint64 + size int64 + }{ + {10, 11, 4096}, {10, 1, -1}, {10, 1, 0}, {math.MaxUint64, 1, 4096}, + } { + if got := storageCounters("/root", tc.blocks, tc.available, tc.size); got != (Storage{Path: "/root"}) { + t.Fatalf("invalid storage wrapped: %+v", got) + } + } + if got := storageCounters("/root", math.MaxUint64, 0, 1); got.UsedBytes != math.MaxUint64 { + t.Fatalf("valid boundary %+v", got) + } +} + +func TestProcessIOParser(t *testing.T) { + for _, tc := range []struct { + name, text string + available bool + read, write uint64 + }{ + {"valid", "read_bytes: 12\nwrite_bytes: 34\nrchar: 999\ncancelled_write_bytes: 99", true, 12, 34}, + {"zero", "read_bytes: 0\nwrite_bytes: 0", true, 0, 0}, + {"empty", "", false, 0, 0}, + {"partial", "read_bytes: 12", false, 12, 0}, + {"negative", "read_bytes: -1\nwrite_bytes: 34", false, 0, 0}, + {"overflow", "read_bytes: 18446744073709551616\nwrite_bytes: 0", false, 0, 0}, + {"invalid", "read_bytes: 12\nwrite_bytes: bad", false, 12, 0}, + {"max", "read_bytes: 18446744073709551615\nwrite_bytes: 0", true, math.MaxUint64, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + got := ParseProcessIO(tc.text) + if got.Available != tc.available || got.ReadBytes != tc.read || got.WriteBytes != tc.write || got.Scope != "process" || got.ProcessID != processID { + t.Fatalf("%+v", got) + } + }) + } +} + +func TestProcFileBound(t *testing.T) { + path := filepath.Join(t.TempDir(), "counters") + for _, size := range []int{maxProcBytes, maxProcBytes + 1} { + if err := os.WriteFile(path, []byte(strings.Repeat("x", size)), 0600); err != nil { + t.Fatal(err) + } + data, err := readProcFile(path) + if size == maxProcBytes && (err != nil || len(data) != size) { + t.Fatalf("limit rejected: %v", err) + } + if size > maxProcBytes && (err == nil || data != nil) { + t.Fatal("oversize file accepted") + } + } + if _, err := readProcFile(path + "-missing"); err == nil { + t.Fatal("missing file accepted") + } +} + +func TestCollectLocalRuntime(t *testing.T) { + root := t.TempDir() + before := time.Now().UnixMilli() + got := Collect(root, Queue{Paused: true}) + if got.TimestampMS < before || got.TimestampMS > time.Now().UnixMilli() || got.Storage.Path != root || got.CPU.LogicalCPUs < 1 || got.Memory.Goroutines < 1 || !got.Queue.Paused || got.IO.ProcessID != processID { + t.Fatalf("runtime snapshot %+v", got) + } +} diff --git a/internal/server/adapter_plugins.go b/internal/server/adapter_plugins.go index cfac687..bafe658 100644 --- a/internal/server/adapter_plugins.go +++ b/internal/server/adapter_plugins.go @@ -12,6 +12,8 @@ import ( "regexp" "sort" "strings" + + "pageswiki/internal/kunstzeal" ) const adapterPluginsConfigKey = "projection_adapter_plugins" @@ -50,6 +52,7 @@ type builtinAdapterInfo struct { var adapterIDPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{1,63}$`) func builtinAdapterManifest() []builtinAdapterInfo { + projectionAdapters := kunstzeal.Catalog() out := make([]builtinAdapterInfo, 0, len(projectionAdapters)) for _, adapter := range projectionAdapters { out = append(out, builtinAdapterInfo{ diff --git a/internal/server/adapter_session_api.go b/internal/server/adapter_session_api.go index 3907ca9..df0fd98 100644 --- a/internal/server/adapter_session_api.go +++ b/internal/server/adapter_session_api.go @@ -7,7 +7,7 @@ import ( "path/filepath" "strings" - "pageswiki/internal/compiler" + "pageswiki/internal/grammaratur" "pageswiki/internal/render" "pageswiki/internal/store" ) @@ -63,7 +63,7 @@ func (s *Server) workspaceFile(session *store.AdapterSession, semanticPath strin body := revision.Body markdown := plugin.HTMLMarkdown && strings.HasPrefix(strings.ToLower(revision.ContentType), "text/html") if markdown { - body, err = compiler.HTMLToMarkdown(body) + body, err = grammaratur.HTMLToMarkdown(body) if err != nil { return adapterWorkspaceFile{}, err } diff --git a/internal/server/anonymous_view.go b/internal/server/anonymous_view.go index 1beafb3..2681e9b 100644 --- a/internal/server/anonymous_view.go +++ b/internal/server/anonymous_view.go @@ -91,7 +91,7 @@ func (s *Server) tryAnonymousProjection(w http.ResponseWriter, r *http.Request) } func (s *Server) serveAnonymousRevision(w http.ResponseWriter, r *http.Request, semantic string, raw bool, v *store.Revision) bool { - reader, err := s.store.OpenDocument(v.DocumentHash) + reader, err := s.store.OpenDocumentContext(r.Context(), v.DocumentHash) if err != nil { http.Error(w, "published document unavailable", 503) return true @@ -156,6 +156,7 @@ func (s *Server) serveAnonymousRevision(w http.ResponseWriter, r *http.Request, return true } data["Text"] = string(body) + data["CodeLanguage"] = codeLanguage(contentType, filename, string(body)) } data["SocialMetadata"] = s.socialMetadataForRequest(r, "anonymous_view.gohtml", data) w.Header().Set("Content-Type", "text/html; charset=utf-8") diff --git a/internal/server/api_handlers.go b/internal/server/api_handlers.go index e09adc0..ae8faa1 100644 --- a/internal/server/api_handlers.go +++ b/internal/server/api_handlers.go @@ -59,17 +59,26 @@ func (s *Server) handleSaveRevisionAPI( w http.ResponseWriter, r *http.Request, ) { - - var req struct { - ObjectUUID string `json:"object_uuid"` - Body string `json:"body"` - Message string `json:"message"` - Language string `json:"lang"` - ContentType string `json:"content_type"` - Filename string `json:"filename"` + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Cache-Control", "private, no-store") + if r.Method != http.MethodPost { + w.Header().Set("Allow", "POST") + api.WriteError(w, 405, "method_not_allowed", "POST required") + return } - err := json.NewDecoder(r.Body).Decode(&req) + var req struct { + ObjectUUID string `json:"object_uuid"` + Body string `json:"body"` + Message string `json:"message"` + Language string `json:"lang"` + ContentType string `json:"content_type"` + Filename string `json:"filename"` + BaseRevision *string `json:"base_revision_uuid"` + EditorPrincipal *string `json:"editor_principal_uuid"` + } + + err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4<<20)).Decode(&req) if err != nil { api.WriteError( w, @@ -80,6 +89,17 @@ func (s *Server) handleSaveRevisionAPI( return } domainUUID := s.domainUUIDForRequest(r) + if req.EditorPrincipal != nil { + principal, e := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r)) + id := "" + if e == nil && principal != nil { + id = principal.UUID + } + if id != *req.EditorPrincipal { + api.WriteError(w, 401, "editor_identity_changed", "editor identity changed; reopen with the original account") + return + } + } if mode, _ := s.store.GetConfigInDomain(domainUUID, "authorization_mode"); mode == "enforcing" { principal, authErr := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r)) if authErr != nil { @@ -109,15 +129,16 @@ func (s *Server) handleSaveRevisionAPI( } } - revision, err := - s.store.SaveRevisionContent( - req.ObjectUUID, - []byte(req.Body), - req.Message, - req.Language, - req.ContentType, - req.Filename, - ) + var revision *store.Revision + if req.BaseRevision != nil { + revision, err = s.store.SaveRevisionContentIfHead(req.ObjectUUID, []byte(req.Body), req.Message, req.Language, req.ContentType, req.Filename, *req.BaseRevision) + } else { + revision, err = s.store.SaveRevisionContent(req.ObjectUUID, []byte(req.Body), req.Message, req.Language, req.ContentType, req.Filename) + } + if errors.Is(err, store.ErrRevisionConflict) { + api.WriteError(w, http.StatusConflict, "revision_conflict", "document changed; reload the language head before saving") + return + } if err != nil { status := http.StatusInternalServerError @@ -143,5 +164,6 @@ func (s *Server) handleSaveRevisionAPI( "lang": revision.Language, "content_type": revision.ContentType, "filename": revision.Filename, + "body": revision.Body, }) } diff --git a/internal/server/archive_handler.go b/internal/server/archive_handler.go index d38dd74..5ec6114 100644 --- a/internal/server/archive_handler.go +++ b/internal/server/archive_handler.go @@ -2,7 +2,6 @@ package server import ( "archive/zip" - "bytes" "context" "io" "mime" @@ -47,23 +46,20 @@ func (s *Server) handleArchiveEntry(w http.ResponseWriter, r *http.Request) { return } if strings.EqualFold(filepath.Ext(revision.Filename), ".zip") { - hydrated, err := s.store.GetRevision(revision.UUID) - if err != nil { - http.NotFound(w, r) - return - } - s.serveZIPEntry(w, r, hydrated.Body, entry) - return - } - source, err := s.store.DocumentPath(revision.DocumentHash) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + s.serveZIPDocumentEntry(w, r, revision.DocumentHash, entry) return } ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) defer cancel() + lease, err := s.store.MaterializeDocument(ctx, revision.DocumentHash, 1<<30) + if err != nil { + http.Error(w, "archive source unavailable or exceeds materialization limit", http.StatusServiceUnavailable) + return + } + defer lease.Close() command := exec.CommandContext(ctx, "nice", "-n", "10", - "7z", "x", "-so", "-y", "-spd", "-mmt1", "-p-", "--", source, entry) + "7z", "x", "-so", "-y", "-spd", "-mmt1", "-p-", "--", lease.Path, entry) + command.WaitDelay = time.Second stdout, err := command.StdoutPipe() if err != nil || command.Start() != nil { http.Error(w, "cannot extract archive entry", http.StatusUnprocessableEntity) @@ -85,16 +81,24 @@ func (s *Server) handleArchiveEntry(w http.ResponseWriter, r *http.Request) { } } -func (s *Server) serveZIPEntry( +func (s *Server) serveZIPDocumentEntry( w http.ResponseWriter, r *http.Request, - body string, + hash string, entry string, ) { - reader, err := zip.NewReader( - bytes.NewReader([]byte(body)), - int64(len(body)), - ) + input, err := s.store.OpenDocumentContext(r.Context(), hash) + if err != nil { + http.NotFound(w, r) + return + } + defer input.Close() + size, err := input.Seek(0, io.SeekEnd) + if err != nil { + http.Error(w, "cannot determine archive size", http.StatusUnprocessableEntity) + return + } + reader, err := zip.NewReader(input, size) if err != nil { http.Error(w, err.Error(), http.StatusUnprocessableEntity) return @@ -103,6 +107,10 @@ func (s *Server) serveZIPEntry( if file.Name != entry || file.FileInfo().IsDir() { continue } + if file.UncompressedSize64 > 128<<20 { + http.Error(w, "archive entry exceeds 128 MiB limit", http.StatusRequestEntityTooLarge) + return + } source, err := file.Open() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) diff --git a/internal/server/archive_storage_test.go b/internal/server/archive_storage_test.go new file mode 100644 index 0000000..d845284 --- /dev/null +++ b/internal/server/archive_storage_test.go @@ -0,0 +1,228 @@ +package server + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "image" + "image/color" + "image/png" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "pageswiki/internal/db" + "pageswiki/internal/documentstore" + "pageswiki/internal/store" +) + +func TestArchiveEntryReadsCanonicalDocumentWithoutRevisionHydration(t *testing.T) { + database, err := db.Open(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if err := db.Migrate(database); err != nil { + t.Fatal(err) + } + st := store.NewWithDocumentRoot(database, t.TempDir()) + domain, _ := st.GetDefaultDomain() + object, err := st.CreatePageAtPathInDomain(domain.UUID, "/archive.zip") + if err != nil { + t.Fatal(err) + } + var payload bytes.Buffer + w := zip.NewWriter(&payload) + file, err := w.Create("hello.txt") + if err != nil { + t.Fatal(err) + } + file.Write([]byte("hello 日本語")) + if err := w.Close(); err != nil { + t.Fatal(err) + } + revision, err := st.SaveRevisionContent(object, payload.Bytes(), "test", "und", "application/zip", "archive.zip") + if err != nil { + t.Fatal(err) + } + // The canonical Document must work even after compatibility DB bodies + // have been removed. This never touches production records. + if _, err := database.Exec(`UPDATE revisions SET body='' WHERE uuid=?`, revision.UUID); err != nil { + t.Fatal(err) + } + s := &Server{store: st} + r := httptest.NewRequest(http.MethodGet, "/_/archive?path=/archive.zip&entry=hello.txt", nil) + response := httptest.NewRecorder() + s.serveZIPDocumentEntry(response, r, revision.DocumentHash, "hello.txt") + if response.Code != http.StatusOK || response.Body.String() != "hello 日本語" { + t.Fatalf("status=%d body=%q", response.Code, response.Body.String()) + } + if response.Header().Get("Vary") != "Authorization, Cookie" { + t.Fatal("lost authorization cache partition") + } + missing := httptest.NewRecorder() + s.serveZIPDocumentEntry(missing, r, revision.DocumentHash, "missing") + if missing.Code != http.StatusNotFound { + t.Fatalf("missing status=%d", missing.Code) + } + ctx, cancel := context.WithCancel(r.Context()) + cancel() + cancelled := httptest.NewRecorder() + s.serveZIPDocumentEntry(cancelled, r.WithContext(ctx), revision.DocumentHash, "hello.txt") + if cancelled.Code == http.StatusOK { + t.Fatal("cancelled request read Document") + } +} + +// Measures the per-request cost of fetching a tiny entry from a large stored +// ZIP. Setup is outside the timer; no DB body hydration or scratch copy occurs. +func BenchmarkZIPDocumentEntry(b *testing.B) { + var payload bytes.Buffer + w := zip.NewWriter(&payload) + file, err := w.Create("hello.txt") + if err != nil { + b.Fatal(err) + } + file.Write([]byte("hello")) + padding, err := w.CreateHeader(&zip.FileHeader{Name: "padding.bin", Method: zip.Store}) + if err != nil { + b.Fatal(err) + } + padding.Write(bytes.Repeat([]byte{0xaa}, 16<<20)) + if err := w.Close(); err != nil { + b.Fatal(err) + } + root := b.TempDir() + backend, err := documentstore.NewLocal(root) + if err != nil { + b.Fatal(err) + } + sum := sha256.Sum256(payload.Bytes()) + hash := hex.EncodeToString(sum[:]) + if err := backend.PutIfAbsent(context.Background(), hash, int64(payload.Len()), bytes.NewReader(payload.Bytes())); err != nil { + b.Fatal(err) + } + s := &Server{store: &store.Store{DocumentRoot: root}} + r := httptest.NewRequest(http.MethodGet, "/archive", nil) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + response := httptest.NewRecorder() + s.serveZIPDocumentEntry(response, r, hash, "hello.txt") + if response.Code != http.StatusOK || response.Body.String() != "hello" { + b.Fatal("entry read failed") + } + } +} + +func TestPreviewImageAndZIPWorkWithReaderOnlyLegacyStore(t *testing.T) { + database, err := db.Open(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if err := db.Migrate(database); err != nil { + t.Fatal(err) + } + st := store.New(database) + domain, _ := st.GetDefaultDomain() + s := &Server{store: st, previewRoot: t.TempDir()} + var picture bytes.Buffer + im := image.NewRGBA(image.Rect(0, 0, 10, 10)) + im.Set(0, 0, color.White) + if err := png.Encode(&picture, im); err != nil { + t.Fatal(err) + } + var archive bytes.Buffer + w := zip.NewWriter(&archive) + for _, name := range []string{"1.png", "2.png", "3.png"} { + f, err := w.Create(name) + if err != nil { + t.Fatal(err) + } + f.Write(picture.Bytes()) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + kind, filename string + body []byte + }{ + {"image", "picture.png", picture.Bytes()}, {"archive", "comic.zip", archive.Bytes()}, + } { + object, err := st.CreatePageAtPathInDomain(domain.UUID, "/"+tc.filename) + if err != nil { + t.Fatal(err) + } + revision, err := st.SaveRevisionContent(object, tc.body, "test", "und", "application/octet-stream", tc.filename) + if err != nil { + t.Fatal(err) + } + if err := s.generatePreview(previewEventPayload{DocumentHash: revision.DocumentHash, Kind: tc.kind, Filename: tc.filename}); err != nil { + t.Fatal(err) + } + if info, err := os.Stat(s.previewCachePath(revision.DocumentHash, tc.kind)); err != nil || info.Size() == 0 { + t.Fatalf("preview: %v", err) + } + } +} + +func TestProjectionMetadataMaterializesAndCleansSource(t *testing.T) { + database, err := db.Open(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if err := db.Migrate(database); err != nil { + t.Fatal(err) + } + st := store.NewWithDocumentRoot(database, t.TempDir()) + domain, _ := st.GetDefaultDomain() + object, err := st.CreatePageAtPathInDomain(domain.UUID, "/test.pdf") + if err != nil { + t.Fatal(err) + } + revision, err := st.SaveRevisionContent(object, []byte("synthetic PDF fixture"), "test", "und", "application/pdf", "test.pdf") + if err != nil { + t.Fatal(err) + } + // Synthetic tools validate only the storage handoff, not PDF rendering. + bin := t.TempDir() + for name, script := range map[string]string{ + "nice": "#!/bin/sh\nshift\nshift\nexec \"$@\"\n", + "pdfinfo": "#!/bin/sh\ntest -r \"$3\" || exit 1\nprintf '%s' \"$3\" > \"$PAGESWIKI_TEST_SOURCE\"\nprintf 'Pages: 3\\n'\n", + } { + if err := os.WriteFile(filepath.Join(bin, name), []byte(script), 0700); err != nil { + t.Fatal(err) + } + } + logFile := filepath.Join(t.TempDir(), "source") + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("PAGESWIKI_TEST_SOURCE", logFile) + s := &Server{store: st} + result, err := s.inspectProjectionMetadata(metadataProjectionPayload{DocumentHash: revision.DocumentHash, Kind: "pdf", Version: metadataProjectionVersion}) + if err != nil || result.PDFPages != 3 { + t.Fatalf("result=%+v err=%v", result, err) + } + source, err := os.ReadFile(logFile) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(source), "pageswiki-document-") || strings.HasPrefix(string(source), st.DocumentRoot+string(os.PathSeparator)) { + t.Fatalf("tool received canonical path: %s", source) + } + if _, err := os.Stat(string(source)); !os.IsNotExist(err) { + t.Fatalf("scratch leaked: %v", err) + } + reader, err := st.OpenDocument(revision.DocumentHash) + if err != nil { + t.Fatal("canonical document removed", err) + } + reader.Close() +} diff --git a/internal/server/auth_admin.go b/internal/server/auth_admin.go index 913cb55..161d7bd 100644 --- a/internal/server/auth_admin.go +++ b/internal/server/auth_admin.go @@ -5,6 +5,7 @@ import ( "log" "net/http" "pageswiki/internal/store" + "strconv" "time" ) @@ -34,6 +35,17 @@ func (s *Server) handleAuthAdmin(w http.ResponseWriter, r *http.Request) { } panel := "auth" switch r.FormValue("action") { + case "save_session_policy": + panel = "auth-sessions" + minutes, err := strconv.Atoi(r.FormValue("session_minutes")) + if err != nil || minutes < 5 || minutes > store.MaxAuthSessionMinutes { + http.Error(w, "session lifetime must be between 5 and 43200 minutes", http.StatusBadRequest) + return + } + if err := s.store.SetAuthSessionLifetimeMinutes(domainUUID, minutes); err != nil { + http.Error(w, "could not save session policy", http.StatusInternalServerError) + return + } case "revoke_session": panel = "auth-sessions" if err := s.store.RevokeAuthSessionByID(domainUUID, r.FormValue("session_uuid")); err != nil { diff --git a/internal/server/auth_handler.go b/internal/server/auth_handler.go index cdef042..0275fb6 100644 --- a/internal/server/auth_handler.go +++ b/internal/server/auth_handler.go @@ -62,7 +62,7 @@ func requestIsHTTPS(r *http.Request) bool { } func (s *Server) setAuthCookie(w http.ResponseWriter, r *http.Request, token string) { - http.SetCookie(w, &http.Cookie{Name: authCookieName, Value: token, Path: "/", HttpOnly: true, Secure: requestIsHTTPS(r), SameSite: http.SameSiteLaxMode, MaxAge: 24 * 60 * 60}) + http.SetCookie(w, &http.Cookie{Name: authCookieName, Value: token, Path: "/", HttpOnly: true, Secure: requestIsHTTPS(r), SameSite: http.SameSiteLaxMode, MaxAge: s.store.AuthSessionLifetimeMinutes(s.domainUUIDForRequest(r)) * 60}) } func (s *Server) authTokenFromRequest(r *http.Request) string { @@ -205,7 +205,7 @@ func (s *Server) handleAuth(w http.ResponseWriter, r *http.Request) { } if err == nil { if strings.Contains(r.Header.Get("Accept"), "application/json") { - writeEventJSON(w, http.StatusOK, map[string]any{"access_token": token, "token_type": "Bearer", "expires_in": 86400}) + writeEventJSON(w, http.StatusOK, map[string]any{"access_token": token, "token_type": "Bearer", "expires_in": s.store.AuthSessionLifetimeMinutes(domainUUID) * 60}) return } s.setAuthCookie(w, r, token) diff --git a/internal/server/auth_ldap.go b/internal/server/auth_ldap.go index 5806ff1..f61c93a 100644 --- a/internal/server/auth_ldap.go +++ b/internal/server/auth_ldap.go @@ -245,7 +245,7 @@ func (s *Server) handleLDAPLogin(w http.ResponseWriter, r *http.Request) { return } if strings.Contains(r.Header.Get("Accept"), "application/json") { - writeEventJSON(w, http.StatusOK, map[string]any{"access_token": token, "token_type": "Bearer", "expires_in": 86400}) + writeEventJSON(w, http.StatusOK, map[string]any{"access_token": token, "token_type": "Bearer", "expires_in": s.store.AuthSessionLifetimeMinutes(domainUUID) * 60}) return } s.setAuthCookie(w, r, token) diff --git a/internal/server/auth_session_policy_test.go b/internal/server/auth_session_policy_test.go new file mode 100644 index 0000000..bf053d9 --- /dev/null +++ b/internal/server/auth_session_policy_test.go @@ -0,0 +1,50 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +func TestSessionPolicyAdminUIAndCookie(t *testing.T) { + s, token, domain := configFixture(t) + form := url.Values{"action": {"save_session_policy"}, "session_minutes": {"60"}} + w := configRequest(s, token, "POST", "/-/sys/config/auth", form) + if w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/-/sys/config?panel=auth-sessions&saved=1" { + t.Fatalf("policy save %d %s", w.Code, w.Body.String()) + } + if s.store.AuthSessionLifetimeMinutes(domain) != 60 { + t.Fatal("policy not saved") + } + w = configRequest(s, token, "GET", "/-/sys/config?panel=auth-sessions", nil) + if w.Code != 200 || !strings.Contains(w.Body.String(), `name="session_minutes" value="60"`) { + t.Fatal("policy control missing") + } + for _, bad := range []string{"", "abc", "0", "99999999999999999999999"} { + form.Set("session_minutes", bad) + if w = configRequest(s, token, "POST", "/-/sys/config/auth", form); w.Code != 400 { + t.Fatalf("bad lifetime %q accepted", bad) + } + } + user, err := s.store.CreatePasswordAccount(domain, "reader", "a long test password", "Reader", "") + if err != nil { + t.Fatal(err) + } + reader, err := s.store.CreateAuthSession(domain, user.UUID) + if err != nil { + t.Fatal(err) + } + form.Set("session_minutes", "120") + if w = configRequest(s, reader, "POST", "/-/sys/config/auth", form); w.Code != 403 { + t.Fatalf("non-admin policy change: %d", w.Code) + } + w = httptest.NewRecorder() + r := httptest.NewRequest("GET", "https://localhost/", nil) + s.setAuthCookie(w, r, "fixture-token") + cookies := w.Result().Cookies() + if len(cookies) != 1 || cookies[0].MaxAge != 3600 || !cookies[0].HttpOnly || !cookies[0].Secure { + t.Fatalf("cookie policy %#v", cookies) + } +} diff --git a/internal/server/browser_handler.go b/internal/server/browser_handler.go index ab38716..7690fec 100644 --- a/internal/server/browser_handler.go +++ b/internal/server/browser_handler.go @@ -58,6 +58,14 @@ func (s *Server) handleBrowser( if !strings.HasPrefix(currentPath, "/") { currentPath = "/" + currentPath } + // Browser navigation is scoped to this semantic Path, not a Domain-wide + // read grant. Filter each child separately below as well. + if !s.requestCanReadPath(r, currentPath) { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + w.Header().Set("Cache-Control", "private, no-store") + w.Header().Add("Vary", "Cookie, Authorization") entries, err := s.store.ListBrowserEntriesInDomain( diff --git a/internal/server/browser_i18n.go b/internal/server/browser_i18n.go new file mode 100644 index 0000000..9e6fcae --- /dev/null +++ b/internal/server/browser_i18n.go @@ -0,0 +1,36 @@ +package server + +func init() { + for lang, entries := range map[string]map[string]string{ + "en": { + "command.label": "Command · Tab: complete Path · Esc: close", "command.close": "Close command mode", "command.unavailable": "Path completion unavailable. Sign in or retry.", "command.empty": "No matching readable Paths.", "command.more": "More matches exist. Type a longer Path prefix.", + "auth.session_minutes": "Session lifetime (minutes)", "auth.session_policy_help": "5–43200 minutes (30 days); default 1440 (24 hours). Applies to new browser/API logins, including OIDC/LDAP. Existing sessions and CalDAV app passwords are unchanged. This is an absolute expiry, not an inactivity timeout.", + "browser.format": "Document format", "browser.wiki": "Wiki (default editor format)", "browser.plain": "PlainText (text/plain)", + "theme.quick": "Theme", "theme.default": "Domain default", "theme.local": "This browser only", "theme.failed": "Could not load the theme.", + "browser.address": "Path", "browser.go": "Go", "browser.up": "Parent folder", + "browser.command_help": "Enter /path or a relative path · cd .. · vi filename · view filename", + "browser.new": "New document", "browser.folder": "Open virtual folder", "browser.name": "Name or Path", + "browser.create_help": "Open the editor to create a document. If the Path already exists, edit that document.", + "browser.folder_help": "You can enter an empty Path without creating a folder, then create or upload documents there.", + "browser.invalid": "Enter a Wiki Path, not an external URL or a path containing control characters or backslashes.", + "browser.empty": "No visible items. Create a document, upload a file, or enter a different Path.", + }, + "ja": { + "command.label": "コマンド · Tab: Path補完 · Esc: 閉じる", "command.close": "コマンドモードを閉じる", "command.unavailable": "Path補完を取得できません。ログイン状態を確認して再試行してください。", "command.empty": "閲覧可能な一致するPathがありません。", "command.more": "候補がほかにもあります。Pathをさらに入力してください。", + "auth.session_minutes": "セッションの有効期間(分)", "auth.session_policy_help": "5〜43200分(30日)。既定は1440分(24時間)。OIDC・LDAPを含む新規ブラウザ/APIログインに適用します。既存セッションとCalDAVのアプリパスワードは変更しません。無操作時間ではなくログインからの絶対期限です。", + "browser.format": "文書形式", "browser.wiki": "Wiki(既定の編集形式)", "browser.plain": "PlainText(text/plain)", + "theme.quick": "テーマ", "theme.default": "ドメインの既定", "theme.local": "このブラウザだけに適用", "theme.failed": "テーマを読み込めませんでした。", + "browser.address": "Path", "browser.go": "移動", "browser.up": "親フォルダ", + "browser.command_help": "/path または相対パス · cd .. · vi ファイル名 · view ファイル名", + "browser.new": "新規文書", "browser.folder": "仮想フォルダを開く", "browser.name": "名前またはPath", + "browser.create_help": "エディタを開いて文書を作成します。既存のPathでは、その文書を編集します。", + "browser.folder_help": "空のPathにもフォルダを作らず移動できます。移動先で文書の作成やアップロードができます。", + "browser.invalid": "WikiのPathを入力してください。外部URL・制御文字・バックスラッシュは使用できません。", + "browser.empty": "表示できる項目はありません。文書の作成、ファイルのアップロード、別のPathへの移動ができます。", + }, + } { + for key, value := range entries { + translations[lang][key] = value + } + } +} diff --git a/internal/server/city_content_navigation_test.go b/internal/server/city_content_navigation_test.go new file mode 100644 index 0000000..eb2ac9b --- /dev/null +++ b/internal/server/city_content_navigation_test.go @@ -0,0 +1,82 @@ +package server + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "pageswiki/internal/store" +) + +func TestCitySearchContentDestinations(t *testing.T) { + s, token, domain := searchServerFixture(t) + street := seedSearchCity(t, s, token, domain) + createUserSpacePage(t, s, domain, "/city?#%", "

Portal

") + if _, err := s.store.CreateSpotAtPath(domain, "test", street, "nebula-way-undefined", "Undefined", "nebula", "/city?#%/street/_undefined"); err != nil { + t.Fatal(err) + } + principal, err := s.store.PrincipalForSession(domain, token) + if err != nil { + t.Fatal(err) + } + if _, err := s.store.DB.Exec(`INSERT INTO auth_capability_grants(uuid,role_uuid,capability,target_type,target_id,created_at_ms) + SELECT 'entrance-grant',b.role_uuid,'read','path','/city?#%/street/_undefined',1 FROM auth_role_bindings b JOIN domain_memberships m ON m.uuid=b.membership_uuid WHERE m.domain_uuid=? AND m.principal_uuid=? LIMIT 1`, domain, principal.UUID); err != nil { + t.Fatal(err) + } + s.mux.HandleFunc("/_/api/v1/search/suggest", s.handleSearchSuggestAPI) + s.mux.HandleFunc("/v/browser/", s.handleBrowser) + s.mux.HandleFunc("/", s.handleSemanticRoute) + w := configRequest(s, token, "GET", "/_/api/v1/search/suggest?q=nebula", nil) + var payload struct { + Items []searchSuggestionItem `json:"items"` + } + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &payload) != nil { + t.Fatalf("suggest: %d %s", w.Code, w.Body.String()) + } + foundPortal, foundEntrance := false, false + for _, item := range payload.Items { + if strings.HasPrefix(item.Href, "/v/city/") || strings.Contains(item.Href, "_undefined") { + t.Fatalf("management/virtual link: %+v", item) + } + if item.Kind == "quartier" { + foundPortal = item.Href == "/city%3F%23%25" + } + if item.Kind == "spot" && item.Label == "Nebula Street" { + foundEntrance = item.Href == "/v/browser/city%3F%23%25/street" + } + } + if !foundPortal || !foundEntrance { + t.Fatalf("destinations missing: %+v", payload.Items) + } + for _, target := range []struct{ url, marker string }{ + {"/city%3F%23%25", "

Portal

"}, + {"/v/browser/city%3F%23%25/street", "browser-layout"}, + } { + opened := configRequest(s, token, "GET", target.url, nil) + if opened.Code != 200 || !strings.Contains(opened.Body.String(), target.marker) || strings.Contains(opened.Body.String(), `class="city-node-detail"`) { + t.Fatalf("destination %s: %d", target.url, opened.Code) + } + } + if denied := configRequest(s, token, "GET", "/v/browser/private-city", nil); denied.Code != 403 { + t.Fatalf("unreadable folder: %d", denied.Code) + } + w = configRequest(s, token, "GET", "/v/search?q=nebula", nil) + if w.Code != 200 || !strings.Contains(w.Body.String(), `href="/city%3F%23%25"`) || strings.Contains(w.Body.String(), `href="/v/city/`) { + t.Fatalf("search: %d", w.Code) + } +} + +func TestCitySearchViewAndSuggestionsAgree(t *testing.T) { + for _, hasPage := range []bool{false, true} { + result := store.CitySearchResult{Kind: "spot", Name: "notes-undefined", Title: "Undefined", PathPrefix: "/日本語 #?%/_undefined", NavigationPath: "/日本語 #?%", StreetTitle: "Notes", HasPage: hasPage} + view := citySearchResultView(result, "notes", time.UTC) + suggestion := citySuggestion(result, "notes") + if view.Href != suggestion.Href || view.Href != view.PathHref || suggestion.Label != "Notes" { + t.Fatalf("inconsistent: %+v %+v", view, suggestion) + } + if strings.Contains(view.Href, "#") || strings.Contains(view.Href, "?") || strings.Contains(view.Href, "_undefined") { + t.Fatal(view.Href) + } + } +} diff --git a/internal/server/city_handler.go b/internal/server/city_handler.go index 4e0bbab..8f3a6c7 100644 --- a/internal/server/city_handler.go +++ b/internal/server/city_handler.go @@ -80,7 +80,7 @@ func (s *Server) handleCityMap(w http.ResponseWriter, r *http.Request) { if detail.Street != nil { streetHref = cityDetailHref("street", detail.Street.Name, detail.Quartier.Name, detail.Street.UUID) } - s.renderWithContext(w, r, "city_node.gohtml", map[string]any{"Title": detail.Title, "Node": detail, "Pages": pageViews, "QuartierHref": quartierHref, "StreetHref": streetHref, "MapHref": "/v/city?" + kind + "=" + detail.UUID, "CanPlan": canPlan, "Locale": s.getLocale(domainUUID), "GovernanceEndpoint": s.governanceForUI(r, kind, detail.UUID)}) + s.renderWithContext(w, r, "city_node.gohtml", map[string]any{"Title": detail.DisplayTitle(), "Node": detail, "Pages": pageViews, "QuartierHref": quartierHref, "StreetHref": streetHref, "MapHref": "/v/city?" + kind + "=" + detail.UUID, "CanPlan": canPlan, "Locale": s.getLocale(domainUUID), "GovernanceEndpoint": s.governanceForUI(r, kind, detail.UUID)}) return } s.renderWithContext(w, r, "city_map.gohtml", map[string]any{"Title": translate(s.getLocale(domainUUID), "city.map"), "CanPlan": canPlan, "Locale": s.getLocale(domainUUID), "GovernanceEndpoint": s.governanceForUI(r, "domain", domainUUID)}) diff --git a/internal/server/city_links.go b/internal/server/city_links.go index 03a1a47..68fe851 100644 --- a/internal/server/city_links.go +++ b/internal/server/city_links.go @@ -34,6 +34,19 @@ func cityPageHref(path string) string { return (&url.URL{Path: path}).EscapedPath() } +// Search is a reading entry point, not a locality management shortcut. Both +// title and Path links use this projection; no additional per-hit DB reads. +func cityContentHref(result store.CitySearchResult) string { + path := result.ContentPath() + if !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") { + return "/v/browser" + } + if result.HasPage { + return cityPageHref(path) + } + return cityPageHref("/v/browser" + path) +} + type cityPageView struct { store.CityPagePlacement Href string diff --git a/internal/server/city_projection.go b/internal/server/city_projection.go index f6e2740..8cba791 100644 --- a/internal/server/city_projection.go +++ b/internal/server/city_projection.go @@ -29,6 +29,11 @@ func filterCityProjection(city store.CityMap, allowed func(string) bool) store.C continue } spot.DetailHref = cityDetailHref("spot", spot.Name, quartier.Name, spot.UUID) + parentTitle := street.Title + if parentTitle == "" { + parentTitle = street.Name + } + spot.DisplayTitle = store.CitySpotDisplayTitle(spot.Name, spot.Title, spot.PathPrefix, parentTitle) pages := []store.CityPagePlacement{} unique := map[string]bool{} spot.PortalPath, spot.PortalPageUUID = "", "" @@ -113,7 +118,7 @@ func (s *Server) pageSpotAppearancesForRequest(r *http.Request, objectUUID strin out := []store.CitySpotAppearance{} allowed := s.pathReadFilter(r) for _, spot := range spots { - if allowed(spot.PathPrefix) { + if allowed(spot.PathPrefix) && allowed(spot.StreetPath) && allowed(spot.QuartierPath) { out = append(out, spot) } } diff --git a/internal/server/city_projection_test.go b/internal/server/city_projection_test.go index 74718af..abaeabb 100644 --- a/internal/server/city_projection_test.go +++ b/internal/server/city_projection_test.go @@ -49,6 +49,19 @@ func TestCityReadPortalPriorityAfterFiltering(t *testing.T) { } } +func TestCityProjectionDisplayTitleDoesNotMutateCachedIdentity(t *testing.T) { + spot := store.CitySpot{UUID: "spot", Name: "home-undefined", Title: "Undefined", PathPrefix: "/life/home/_undefined"} + city := store.CityMap{Quartiers: []store.CityQuartier{{PathPrefix: "/life", Streets: []store.CityStreet{{Title: "Home", PathPrefix: "/life/home", Spots: []store.CitySpot{spot}}}}}} + projected := filterCityProjection(city, func(string) bool { return true }) + got := projected.Quartiers[0].Streets[0].Spots[0] + if got.DisplayTitle != "Home" || got.Title != "Undefined" || got.Name != spot.Name || got.PathPrefix != spot.PathPrefix { + t.Fatalf("projection: %+v", got) + } + if city.Quartiers[0].Streets[0].Spots[0].DisplayTitle != "" { + t.Fatal("shared navigation cache mutated") + } +} + func TestCityReadProjectionHidesPagesLocalitiesPortalsAndRelations(t *testing.T) { s, _, domain := configFixture(t) s.mux.HandleFunc("/_/api/v1/city/", s.handleCityAPI) diff --git a/internal/server/command_bar_test.go b/internal/server/command_bar_test.go new file mode 100644 index 0000000..90f5030 --- /dev/null +++ b/internal/server/command_bar_test.go @@ -0,0 +1,91 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +func TestCommandCompletionReadACL(t *testing.T) { + s, token, _ := searchServerFixture(t) + s.mux.HandleFunc("/_/api/v1/path-picker", s.handlePathPickerAPI) + w := configRequest(s, token, "GET", "/_/api/v1/path-picker?path=/docs&mode=children", nil) + if w.Code != 200 || strings.Contains(w.Body.String(), "hidden") || !strings.Contains(w.Body.String(), "nebula guide?#%") { + t.Fatalf("ACL completion: %d %s", w.Code, w.Body.String()) + } +} + +func TestCommandPathCompletion(t *testing.T) { + s, token, domain := configFixture(t) + s.mux.HandleFunc("/_/api/v1/path-picker", s.handlePathPickerAPI) + createUserSpacePage(t, s, domain, "/manual", "

Parent document and folder

") + for i := 0; i < 260; i++ { + if _, err := s.store.CreatePageAtPathInDomain(domain, fmt.Sprintf("/manual/a%03d", i)); err != nil { + t.Fatal(err) + } + } + createUserSpacePage(t, s, domain, "/manual/日本語 #?", "plain") + w := configRequest(s, token, "GET", "/_/api/v1/path-picker?path=/manual&mode=children&prefix=日本語", nil) + var result struct { + Path string `json:"path"` + Entries []pathPickerEntry `json:"entries"` + } + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &result) != nil || result.Path != "/manual" || len(result.Entries) != 1 || result.Entries[0].Name != "日本語 #?" { + t.Fatalf("completion: %d %s", w.Code, w.Body.String()) + } + w = configRequest(s, "", "GET", "/_/api/v1/path-picker?path=/manual&mode=children", nil) + if w.Code == 200 { + t.Fatal("anonymous API listed private paths") + } +} + +func TestCommandAndHighlightChromium(t *testing.T) { + if os.Getenv("PAGESWIKI_BROWSER_TEST") != "1" { + t.Skip("opt-in Chromium") + } + s, token, domain, _ := editorFixture(t) + s.mux.HandleFunc("/v/browser/", s.handleBrowser) + s.mux.HandleFunc("/_/api/v1/path-picker", s.handlePathPickerAPI) + s.mux.HandleFunc("/notes", s.handleSemanticRoute) + s.mux.HandleFunc("/", s.handleHome) + createUserSpacePage(t, s, domain, "/notes", `

Code sample

# comment
+def hello():
+    return "<script>literal</script>"
`) + createUserSpacePage(t, s, domain, "/notes/alpha", "

Alpha

") + createUserSpacePage(t, s, domain, "/notes/alpine", "

Alpine

") + createUserSpacePage(t, s, domain, "/日本語 #?/child", "

Unicode

") + script, err := os.ReadFile("internal/server/testdata/command-highlight.js") + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Host = "localhost" + r.Header.Set("Authorization", "Bearer "+token) + if r.URL.Path == "/fixture" { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte("")) + return + } + s.withAuthorization(s.mux).ServeHTTP(w, r) + })) + defer server.Close() + for _, width := range []string{"1440", "375"} { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + shot := "" + if base := os.Getenv("PAGESWIKI_COMMAND_SCREENSHOT"); base != "" { + shot = base + "-" + width + ".png" + } + out, err := exec.CommandContext(ctx, "node", "internal/server/testdata/vim-browser-smoke.mjs", server.URL+"/fixture", t.TempDir(), shot, width, "950").CombinedOutput() + cancel() + if err != nil { + t.Fatalf("%s: %v\n%s", width, err, out) + } + } +} diff --git a/internal/server/compile_handler.go b/internal/server/compile_handler.go index 1512f9a..bc2b634 100644 --- a/internal/server/compile_handler.go +++ b/internal/server/compile_handler.go @@ -4,7 +4,7 @@ import ( "net/http" "strings" - "pageswiki/internal/compiler" + "pageswiki/internal/grammaratur" ) func (s *Server) compileMarkdownFromPath( @@ -46,9 +46,11 @@ func (s *Server) transpileMarkdownFromPath(r *http.Request, path string, mode ma html := latest.Body if mode == markdownProjection { html = s.expandMacrosForRequest(r, html, path, s.domainUUIDForRequest(r)) + } else { + return grammaratur.HTMLToMarkdownAuthoring(html) } - return compiler.HTMLToMarkdown( + return grammaratur.HTMLToMarkdown( html, ) } diff --git a/internal/server/config_handler.go b/internal/server/config_handler.go index 6288153..ac794d2 100644 --- a/internal/server/config_handler.go +++ b/internal/server/config_handler.go @@ -1088,6 +1088,7 @@ func (s *Server) handleConfig( "CampusTokens": campusTokens, "AuthPrincipals": authPrincipals, "AuthSessions": authSessions, + "AuthSessionMinutes": s.store.AuthSessionLifetimeMinutes(s.domainUUIDForRequest(r)), "OIDCConfig": oidcConfig, "SMTPConfig": smtpConfig, "LDAPConfig": ldapConfig, diff --git a/internal/server/contact_i18n.go b/internal/server/contact_i18n.go new file mode 100644 index 0000000..5cb76cf --- /dev/null +++ b/internal/server/contact_i18n.go @@ -0,0 +1,24 @@ +package server + +func init() { + for lang, entries := range map[string]map[string]string{ + "en": { + "contact.title": "Contact details", "contact.phone": "Phone number", + "contact.help": "Private by default, independently of your profile. A tap opens the phone app. Leave blank to remove. Use 3–15 digits with an optional leading +, spaces, hyphens or parentheses; extensions and service codes are not supported.", + "contact.visibility_help": "The profile audience remains the upper limit. Anonymous access also requires the domain to permit public profiles. Administrators do not automatically see private numbers.", + "contact.confirm": "I explicitly agree to make this phone number public.", + "contact.invalid_phone": "Enter a phone number with 3–15 digits and optional leading +; only spaces, hyphens and parentheses may separate digits.", + }, + "ja": { + "contact.title": "連絡先", "contact.phone": "電話番号", + "contact.help": "プロフィールとは別に、既定で非公開です。タップすると電話アプリを開きます。空欄で削除。3〜15桁の数字と先頭の+、空白・ハイフン・括弧を使用できます。内線やサービスコードは未対応です。", + "contact.visibility_help": "プロフィールの公開範囲を上限とします。匿名公開にはドメイン側の許可も必要です。管理者でも非公開の番号は表示されません。", + "contact.confirm": "この電話番号をインターネットに公開することに同意します。", + "contact.invalid_phone": "電話番号は3〜15桁の数字と先頭の+で指定してください。区切りには空白・ハイフン・括弧のみ使用できます。", + }, + } { + for key, value := range entries { + translations[lang][key] = value + } + } +} diff --git a/internal/server/content_adapter.go b/internal/server/content_adapter.go index 4914a0f..5e3d6ab 100644 --- a/internal/server/content_adapter.go +++ b/internal/server/content_adapter.go @@ -1,225 +1,20 @@ package server -import ( - "path/filepath" - "strings" -) +import "pageswiki/internal/kunstzeal" -type projectionDelivery string +type projectionDelivery = kunstzeal.Delivery +type contentDescriptor = kunstzeal.Descriptor +type projectionAdapter = kunstzeal.Adapter const ( - deliveryHTML projectionDelivery = "html" - deliveryText projectionDelivery = "text" - deliveryRawMedia projectionDelivery = "raw-media" - deliveryRawFrame projectionDelivery = "raw-frame" - deliveryDownload projectionDelivery = "download" + deliveryHTML = kunstzeal.DeliveryHTML + deliveryText = kunstzeal.DeliveryText + deliveryRawMedia = kunstzeal.DeliveryRawMedia + deliveryRawFrame = kunstzeal.DeliveryRawFrame + deliveryDownload = kunstzeal.DeliveryDownload + inlineTextWindowThreshold = kunstzeal.InlineTextWindowThreshold ) -type contentDescriptor struct { - MediaType string - Filename string - SizeBytes int64 -} - -type projectionAdapter struct { - ID string - Kind string - Delivery projectionDelivery - NeedsText bool - WindowableText bool - Match func(contentDescriptor) bool -} - -const inlineTextWindowThreshold = int64(2 << 20) - -var projectionAdapters = []projectionAdapter{ - { - ID: "html", - Kind: "html", - Delivery: deliveryHTML, - Match: mediaTypes("text/html"), - }, - { - ID: "excalidraw", - Kind: "excalidraw", - Delivery: deliveryText, - NeedsText: true, - WindowableText: true, - Match: extensions(".excalidraw"), - }, - { - ID: "csv", - Kind: "csv", - Delivery: deliveryText, - NeedsText: true, - WindowableText: true, - Match: matchAny( - mediaTypes("text/csv", "application/csv"), - extensions(".csv"), - ), - }, - { - ID: "archive", - Kind: "archive", - Delivery: deliveryDownload, - Match: matchAny( - mediaTypes( - "application/zip", "application/x-zip-compressed", - "application/x-rar-compressed", "application/vnd.rar", - "application/x-7z-compressed", "application/x-tar", - "application/gzip", "application/x-xz", - ), - extensions(".zip", ".rar", ".7z", ".tar", ".gz", ".tgz", ".xz", ".txz"), - ), - }, - { - ID: "debian-package", - Kind: "metadata", - Delivery: deliveryDownload, - Match: matchAny( - mediaTypes( - "application/vnd.debian.binary-package", - "application/x-debian-package", - ), - extensions(".deb"), - ), - }, - { - ID: "eml", - Kind: "email", - Delivery: deliveryText, - Match: matchAny( - mediaTypes("message/rfc822"), - extensions(".eml"), - ), - }, - { - ID: "pe", - Kind: "metadata", - Delivery: deliveryDownload, - Match: extensions(".exe", ".dll", ".sys"), - }, - { - ID: "json", - Kind: "json", - Delivery: deliveryText, - WindowableText: true, - Match: matchAny(mediaTypes("application/json", "application/ld+json"), extensions(".json", ".jsonl")), - }, - { - ID: "text", - Kind: "code", - Delivery: deliveryText, - NeedsText: true, - WindowableText: true, - Match: textMediaType, - }, - { - ID: "image", - Kind: "image", - Delivery: deliveryRawMedia, - Match: matchAny( - mediaTypePrefix("image/"), - extensions(".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif", ".svg"), - ), - }, - { - ID: "audio", - Kind: "audio", - Delivery: deliveryRawMedia, - Match: matchAny( - mediaTypePrefix("audio/"), - extensions(".mp3", ".flac", ".wav", ".ogg", ".m4a", ".aac", ".opus"), - ), - }, - { - ID: "video", - Kind: "video", - Delivery: deliveryRawMedia, - Match: matchAny( - mediaTypePrefix("video/"), - extensions(".mp4", ".webm", ".mkv", ".mov", ".m4v", ".ogv"), - ), - }, - { - ID: "pdf", - Kind: "pdf", - Delivery: deliveryRawFrame, - Match: matchAny( - mediaTypes("application/pdf"), - extensions(".pdf"), - ), - }, - { - ID: "download", - Kind: "download", - Delivery: deliveryDownload, - Match: func(contentDescriptor) bool { return true }, - }, -} - func selectProjectionAdapter(descriptor contentDescriptor) projectionAdapter { - for _, adapter := range projectionAdapters { - if adapter.Match(descriptor) { - return adapter - } - } - return projectionAdapters[len(projectionAdapters)-1] -} - -func mediaTypes(types ...string) func(contentDescriptor) bool { - return func(descriptor contentDescriptor) bool { - for _, candidate := range types { - if descriptor.MediaType == candidate { - return true - } - } - return false - } -} - -func mediaTypePrefix(prefix string) func(contentDescriptor) bool { - return func(descriptor contentDescriptor) bool { - return strings.HasPrefix(descriptor.MediaType, prefix) - } -} - -func extensions(values ...string) func(contentDescriptor) bool { - return func(descriptor contentDescriptor) bool { - extension := strings.ToLower(filepath.Ext(descriptor.Filename)) - for _, candidate := range values { - if extension == candidate { - return true - } - } - return false - } -} - -func matchAny(matchers ...func(contentDescriptor) bool) func(contentDescriptor) bool { - return func(descriptor contentDescriptor) bool { - for _, matcher := range matchers { - if matcher(descriptor) { - return true - } - } - return false - } -} - -func textMediaType(descriptor contentDescriptor) bool { - switch descriptor.MediaType { - case "application/json", - "application/xml", - "application/javascript", - "application/x-javascript", - "application/x-sh", - "application/x-shellscript", - "application/x-yaml", - "application/yaml", - "application/toml": - return true - default: - return strings.HasPrefix(descriptor.MediaType, "text/") - } + return kunstzeal.SelectAdapter(descriptor) } diff --git a/internal/server/content_view.go b/internal/server/content_view.go index 8dee72c..e3d8733 100644 --- a/internal/server/content_view.go +++ b/internal/server/content_view.go @@ -14,7 +14,7 @@ func (s *Server) windowedContentViewForRequest(r *http.Request, objectUUID, sema if err != nil || metadata.UUID == "" { return contentView{}, false, err } - size, err := s.store.DocumentSize(metadata.DocumentHash) + size, err := s.store.DocumentSizeContext(r.Context(), metadata.DocumentHash) if err != nil { return contentView{}, false, nil } @@ -30,7 +30,7 @@ func (s *Server) windowedContentViewForRequest(r *http.Request, objectUUID, sema if size <= inlineTextWindowThreshold || !adapter.WindowableText { return contentView{}, false, nil } - window, err := s.store.ReadDocumentWindow(metadata.DocumentHash, 0, 128<<10) + window, err := s.store.ReadDocumentWindowContext(r.Context(), metadata.DocumentHash, 0, 128<<10) if err != nil { return contentView{}, false, err } diff --git a/internal/server/diagram_handler.go b/internal/server/diagram_handler.go index 267e8e4..4c5ec0b 100644 --- a/internal/server/diagram_handler.go +++ b/internal/server/diagram_handler.go @@ -1,9 +1,6 @@ package server import ( - "bytes" - "crypto/sha256" - "encoding/hex" "errors" "io" "net/http" @@ -13,6 +10,8 @@ import ( "strings" "sync" "time" + + "pageswiki/internal/kunstzeal" ) var diagramSourceCache = map[string]diagramCacheEntry{} @@ -34,60 +33,15 @@ func (s *Server) diagramCachePath(kind, hash string) string { return filepath.Join(root, kind, hash+".svg") } -func readDiagramCache(path string) ([]byte, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - b, err := io.ReadAll(io.LimitReader(f, (4<<20)+1)) - if len(b) > 4<<20 { - return nil, errors.New("diagram cache exceeds limit") - } - return b, err +func readDiagramCache(path string) ([]byte, error) { return kunstzeal.ReadDiagramCache(path) } + +func diagramHash(kind, source, format string) string { + return kunstzeal.DiagramHash(kind, source, format) } -func diagramHash( - diagramType string, - source string, - format string, -) string { +func allowedDiagramType(kind string) bool { return kunstzeal.AllowedDiagramType(kind) } - sum := - sha256.Sum256( - []byte( - diagramType + "\x00" + format + "\x00" + source, - ), - ) - - return hex.EncodeToString( - sum[:], - ) -} - -func allowedDiagramType( - t string, -) bool { - - switch t { - case "mermaid", "plantuml", "graphviz", "dot", "d2": - return true - default: - return false - } -} - -func normalizeDiagramType( - t string, -) string { - - switch t { - case "dot": - return "graphviz" - default: - return t - } -} +func normalizeDiagramType(kind string) string { return kunstzeal.NormalizeDiagramType(kind) } func krokiURL() string { @@ -137,7 +91,7 @@ func (s *Server) renderDiagramsInHTML( htmlUnescapeString( m[2], ) - if len(source) > 32<<10 || path == "" { + if len(source) > kunstzeal.DiagramInlineLimit || path == "" { return block } @@ -311,11 +265,11 @@ func (s *Server) handleCompileDiagramPost( io.ReadAll( io.LimitReader( r.Body, - (1<<20)+1, + kunstzeal.DiagramSourceLimit+1, ), ) - if err != nil || len(sourceBytes) > 1<<20 { + if err != nil || len(sourceBytes) > kunstzeal.DiagramSourceLimit { http.Error(w, "diagram source exceeds limit", http.StatusRequestEntityTooLarge) return } @@ -352,91 +306,26 @@ func (s *Server) handleCompileDiagramPost( ) } -func (s *Server) renderDiagramSVG( - w http.ResponseWriter, - r *http.Request, - diagramType string, - source string, - cachePath string, -) { - - req, err := - http.NewRequestWithContext( - r.Context(), - http.MethodPost, - krokiURL()+"/"+diagramType+"/svg", - bytes.NewBufferString(source), - ) - +// The configured service may be private (Compose), but it may not redirect +// source content to another destination. Authorization is checked before this. +func diagramHTTPClient() *http.Client { + return &http.Client{Timeout: kunstzeal.DiagramTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} +} + +func (s *Server) renderDiagramSVG(w http.ResponseWriter, r *http.Request, diagramType, source, cachePath string) { + provider := kunstzeal.Kroki{Endpoint: krokiURL(), Client: diagramHTTPClient()} + b, err := provider.Render(r.Context(), diagramType, source) if err != nil { - http.Error(w, err.Error(), 500) - return - } - - req.Header.Set( - "Content-Type", - "text/plain; charset=utf-8", - ) - - client := - http.Client{ - Timeout: 15 * time.Second, + status := http.StatusBadGateway + if errors.Is(err, kunstzeal.ErrProviderConfig) { + status = http.StatusInternalServerError } - - resp, err := - client.Do( - req, - ) - - if err != nil { - http.Error(w, err.Error(), 502) + http.Error(w, err.Error(), status) return } - - defer resp.Body.Close() - - if resp.StatusCode < 200 || - resp.StatusCode >= 300 { - - http.Error(w, "diagram renderer failed", 502) - return - } - - b, err := - io.ReadAll( - io.LimitReader(resp.Body, (4<<20)+1), - ) - - if err != nil || len(b) > 4<<20 { - http.Error(w, "diagram output unavailable or exceeds limit", http.StatusBadGateway) - return - } - - _ = - os.MkdirAll( - filepath.Dir(cachePath), - 0755, - ) - - _ = - os.WriteFile( - cachePath, - b, - 0644, - ) - - w.Header().Set( - "Content-Type", - "image/svg+xml; charset=utf-8", - ) - - w.Header().Set( - "Content-Security-Policy", - "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:", - ) - - _, _ = - w.Write( - b, - ) + // Cache failure must not discard a successfully generated projection. + _ = kunstzeal.WriteDiagramCache(cachePath, b) + w.Header().Set("Content-Type", "image/svg+xml; charset=utf-8") + w.Header().Set("Content-Security-Policy", "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:") + _, _ = w.Write(b) } diff --git a/internal/server/dr_readonly.go b/internal/server/dr_readonly.go new file mode 100644 index 0000000..863ee8f --- /dev/null +++ b/internal/server/dr_readonly.go @@ -0,0 +1,98 @@ +package server + +import ( + "net/http" + "strings" + "time" +) + +// This gate is outside authorization; it never grants access. DB mode=ro and +// query_only enforce a second boundary even if a GET handler attempts a write. +// The data directory should additionally be mounted read-only on standby. +func (s *Server) withStandbyReadOnly(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/health/read" || r.URL.Path == "/health/write" { + w.Header().Set("Cache-Control", "no-store") + if r.Method != "GET" && r.Method != "HEAD" { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + ready := !s.store.ReadOnly || (r.URL.Path == "/health/read" && time.Now().Before(s.store.StandbyReadUntil)) + if ready && s.store.DB != nil { + ready = s.store.DB.PingContext(r.Context()) == nil + } + if !ready { + http.Error(w, "not ready", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + return + } + if !s.store.ReadOnly { + next.ServeHTTP(w, r) + return + } + w = &standbyResponseWriter{ResponseWriter: w} + w.Header().Set("X-PagesWiki-DR-Role", "standby") + w.Header().Set("Cache-Control", "private, no-store") + fail := func(message string) { + w.Header().Set("Retry-After", "60") + http.Error(w, message, http.StatusServiceUnavailable) + } + // Health remains a liveness endpoint, NOT proof that this node can write. + if r.URL.Path == "/health" && (r.Method == "GET" || r.Method == "HEAD") { + next.ServeHTTP(w, r) + return + } + if s.store.StandbyReadUntil.IsZero() || !time.Now().Before(s.store.StandbyReadUntil) { + fail("DR snapshot expired; routing or offline refresh is required") + return + } + if r.Method != "GET" && r.Method != "HEAD" && r.Method != "OPTIONS" && r.Method != "PROPFIND" && r.Method != "REPORT" { + fail("DR standby is read-only; write request was not executed") + return + } + // Authentication issuance, edit launch, federation and external renderers + // can have side effects even on GET. Do not invoke them on a standby. + for _, prefix := range []string{"/_/auth/", "/-/sys/", "/e/", "/o/e/", "/n/", "/:/", "/new", "/v/upload", "/_/adapter-gateway", "/_/adaptor/", "/_/api/v1/federation", "/_/api/v1/adapter", "/v/open", "/v/import", "/v/compile", "/_/api/v1/import"} { + if strings.HasPrefix(r.URL.Path, prefix) { + fail("operation unavailable on DR standby") + return + } + } + // V1 standby does not serve organizer or DAV endpoints: their current + // read paths repair file-backed indexes and update credential metadata. + // They become available after promotion; no false successful sync responses. + for _, prefix := range []string{"/v/plan", "/v/calendar", "/v/event", "/v/tasks", "/_/api/v1/calendar", "/_/api/v1/caldav", "/_/api/v1/task", "/_/api/v1/extension/", "/.well-known/caldav"} { + if strings.HasPrefix(r.URL.Path, prefix) { + fail("organizer/DAV read service requires DR promotion in this version") + return + } + } + next.ServeHTTP(w, r) + }) +} + +// Immutable/raw handlers normally set long cache lifetimes. A standby must not +// let those headers outlive the snapshot's authorization freshness deadline. +type standbyResponseWriter struct { + http.ResponseWriter + wrote bool +} + +func (w *standbyResponseWriter) WriteHeader(status int) { + if w.wrote { + return + } + w.wrote = true + w.Header().Set("Cache-Control", "private, no-store") + w.Header().Set("X-PagesWiki-DR-Role", "standby") + w.ResponseWriter.WriteHeader(status) +} +func (w *standbyResponseWriter) Write(b []byte) (int, error) { + if !w.wrote { + w.WriteHeader(http.StatusOK) + } + return w.ResponseWriter.Write(b) +} +func (w *standbyResponseWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } diff --git a/internal/server/dr_readonly_test.go b/internal/server/dr_readonly_test.go new file mode 100644 index 0000000..f376da7 --- /dev/null +++ b/internal/server/dr_readonly_test.go @@ -0,0 +1,52 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "pageswiki/internal/store" +) + +func TestDRStandbyGate(t *testing.T) { + s := &Server{store: &store.Store{ReadOnly: true, StandbyReadUntil: time.Now().Add(time.Hour)}} + h := s.withStandbyReadOnly(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })) + for _, tc := range []struct { + method, path string + status int + }{ + {"GET", "/readable-page", 200}, {"HEAD", "/readable-page", 200}, {"GET", "/health", 200}, {"GET", "/health/read", 200}, {"GET", "/health/write", 503}, + {"POST", "/_/api/v1/objects", 503}, {"PUT", "/anything", 503}, {"DELETE", "/anything", 503}, {"MKCALENDAR", "/_/api/v1/caldav/new", 503}, {"PATCH", "/anything", 503}, + {"GET", "/_/auth/login", 503}, {"GET", "/e/test", 503}, {"GET", "/:/compile/diagram/mermaid/x", 503}, {"GET", "/v/open-with", 503}, {"GET", "/_/api/v1/federation/manifest", 503}, {"REPORT", "/_/api/v1/caldav/", 503}, + } { + t.Run(tc.method+tc.path, func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(tc.method, tc.path, nil)) + if w.Code != tc.status { + t.Fatalf("%d: %s", w.Code, w.Body.String()) + } + }) + } + s.store.StandbyReadUntil = time.Now().Add(-time.Second) + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest("GET", "/readable-page", nil)) + if w.Code != 503 { + t.Fatal("expired snapshot served") + } + w = httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest("GET", "/health/read", nil)) + if w.Code != 503 { + t.Fatal("expired snapshot ready") + } +} + +func TestDRPrimaryGatePassThrough(t *testing.T) { + s := &Server{store: &store.Store{}} + h := s.withStandbyReadOnly(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(202) })) + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest("POST", "/anything", nil)) + if w.Code != 202 { + t.Fatal("primary write blocked") + } +} diff --git a/internal/server/dr_recovery_test.go b/internal/server/dr_recovery_test.go new file mode 100644 index 0000000..a082cb7 --- /dev/null +++ b/internal/server/dr_recovery_test.go @@ -0,0 +1,138 @@ +package server + +import ( + "context" + "crypto/sha256" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "pageswiki/internal/db" + "pageswiki/internal/dr" + "pageswiki/internal/store" +) + +// Full startup against a real schema and synthetic data, not a mock connection. +// No production data, imported calendars or outbound requests are involved. +func TestDRRecoveryReadOnlyServer(t *testing.T) { + t.Chdir("../..") + root := t.TempDir() + database, err := db.Open(filepath.Join(root, "pages.db")) + if err != nil { + t.Fatal(err) + } + if err = db.Migrate(database); err != nil { + t.Fatal(err) + } + st := store.NewWithDocumentRoot(database, filepath.Join(root, "documents")) + if err = st.EnsureConfigTable(); err != nil { + t.Fatal(err) + } + domain, err := st.GetDefaultDomain() + if err != nil { + t.Fatal(err) + } + owner, err := st.BootstrapOwner(domain.UUID, "dr-fixture", "synthetic long test password", "DR Fixture", "") + if err != nil { + t.Fatal(err) + } + token, err := st.CreateAuthSession(domain.UUID, owner.UUID) + if err != nil { + t.Fatal(err) + } + if err = st.SetConfigInDomain(domain.UUID, "authorization_mode", "enforcing"); err != nil { + t.Fatal(err) + } + object, err := st.CreatePageAtPath("/dr-fixture") + if err != nil { + t.Fatal(err) + } + rev, err := st.SaveRevisionForLanguage(object, "

DR fixture

Canonical body.

", "test", "und") + if err != nil { + t.Fatal(err) + } + if err = database.Close(); err != nil { + t.Fatal(err) + } + if _, err = dr.Init(root); err != nil { + t.Fatal(err) + } + snap := filepath.Join(t.TempDir(), "snapshot") + _, digest, err := dr.Snapshot(context.Background(), root, snap) + if err != nil { + t.Fatal(err) + } + standby := filepath.Join(t.TempDir(), "standby") + node, err := dr.Restore(context.Background(), snap, digest, standby) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(standby, "pages.db") + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + ro, err := db.OpenReadOnly(path) + if err != nil { + t.Fatal(err) + } + defer ro.Close() + readonly, err := store.NewReadOnlyWithDocumentRoot(ro, filepath.Join(standby, "documents")) + if err != nil { + t.Fatal(err) + } + readonly.StandbyReadUntil = node.ReadUntil() + s := New(readonly) + if s.previewRoot != "" || s.importRoot != "" || s.schedulerRunning.Load() != 0 { + t.Fatal("started standby worker/cache writers") + } + request := func(method, path, auth string) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, "http://localhost"+path, nil) + if auth != "" { + r.Header.Set("Authorization", "Bearer "+auth) + } + w := httptest.NewRecorder() + s.handler().ServeHTTP(w, r) + return w + } + for _, path := range []string{"/v/raw/rev:" + rev.UUID, "/dr-fixture"} { + w := request("GET", path, token) + if w.Code != 200 || !strings.Contains(w.Body.String(), "Canonical body.") { + t.Fatalf("read %s: %d %s", path, w.Code, w.Body.String()) + } + if w.Header().Get("Cache-Control") != "private, no-store" { + t.Fatalf("stale authorization cache permitted: %s", w.Header().Get("Cache-Control")) + } + } + if w := request("GET", "/v/raw/rev:"+rev.UUID, ""); w.Code == 200 { + t.Fatal("read-only bypassed authorization") + } + if w := request("PUT", "/_/api/v1/objects/"+object, token); w.Code != 503 { + t.Fatal("standby write not rejected") + } + if _, err = readonly.SaveRevisionForLanguage(object, "must never persist", "test", "und"); err == nil { + t.Fatal("store allowed write") + } + if _, err = store.NewReadOnlyWithDocumentRoot(ro, filepath.Join(t.TempDir(), "documents")); err == nil { + t.Fatal("regenerated missing auth key") + } + readonly.StandbyReadUntil = time.Now().Add(-time.Second) + if w := request("GET", "/dr-fixture", token); w.Code != 503 { + t.Fatal("stale snapshot served") + } + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if sha256.Sum256(before) != sha256.Sum256(after) { + t.Fatal("standby changed database bytes") + } + for _, name := range []string{"previews", "imports", "pages.db-wal", "pages.db-shm"} { + if _, err = os.Stat(filepath.Join(standby, name)); !os.IsNotExist(err) { + t.Fatalf("created standby artifact %s", name) + } + } +} diff --git a/internal/server/dregarnuhr_queue.go b/internal/server/dregarnuhr_queue.go new file mode 100644 index 0000000..0faf3b3 --- /dev/null +++ b/internal/server/dregarnuhr_queue.go @@ -0,0 +1,99 @@ +package server + +import ( + "context" + "database/sql" + "errors" + "log" + "time" + + "pageswiki/internal/dregarnuhr" + "pageswiki/internal/store" +) + +// This adapter preserves SQL/storage ownership and all existing queue IDs. +type schedulerQueue struct{ store *store.Store } + +func (q schedulerQueue) Settings(ctx context.Context) (dregarnuhr.Settings, error) { + if err := ctx.Err(); err != nil { + return dregarnuhr.Settings{}, err + } + if q.store.ReadOnly || q.store.BackgroundJobsDisabled { + return dregarnuhr.Settings{Paused: true}, nil + } + return q.store.GetQueueSettings() +} +func (q schedulerQueue) Claim(ctx context.Context, now time.Time) (*dregarnuhr.Job, error) { + job, err := q.store.ClaimDueEventAt(ctx, now) + if errors.Is(err, sql.ErrNoRows) { + return nil, dregarnuhr.ErrNoJob + } + return job, err +} +func (q schedulerQueue) Cancellation(ctx context.Context, id string) (bool, error) { + return q.store.EventCancellationState(ctx, id) +} +func (q schedulerQueue) Started(ctx context.Context, job *dregarnuhr.Job) error { + return q.store.AddEventLog(job.UUID, job.EventUUID, "info", "execution started") +} +func (q schedulerQueue) Finish(ctx context.Context, job *dregarnuhr.Job, err error, now time.Time) error { + return q.store.FinishEventAt(ctx, job.UUID, err, now) +} + +func (s *Server) eventRunner() *dregarnuhr.Runner { + s.schedulerOnce.Do(func() { + s.schedulerRunner = dregarnuhr.NewRunner(schedulerQueue{s.store}, s.registeredEventHandlers(), dregarnuhr.RunnerOptions{ + Active: &s.schedulerRunning, + Foreground: func() bool { return s.foregroundRequests.Load() > 0 }, + Wake: s.wakeEventScheduler, + Report: func(stage string, job *dregarnuhr.Job, err error) { + if job != nil { + log.Printf("component=%s event scheduler %s failed job=%s type=%s: %v", dregarnuhr.Component, stage, job.UUID, job.Type, err) + } else { + log.Printf("component=%s event scheduler %s failed: %v", dregarnuhr.Component, stage, err) + } + }, + }) + }) + return s.schedulerRunner +} + +func (s *Server) runDueEvents() { + if s.store == nil { + return + } + s.eventRunner().Dispatch(context.Background()) +} + +func (s *Server) schedulerActivity(ctx context.Context, now time.Time) error { + _, err := s.store.ExpireApprovalRequests(ctx, now.UnixMilli(), 100) + // Preserve independent City processing even if approval expiry fails. + s.enqueueCityActivityEvents() + return err +} + +func (s *Server) schedulerLoop() dregarnuhr.Loop { + return dregarnuhr.Loop{ + Every: time.Second, Wake: s.schedulerWake, OnWake: s.schedulerActivity, + Triggers: []dregarnuhr.Trigger{ + {Name: "initial-preview-sweep", After: 5 * time.Second, Run: func(context.Context, time.Time) error { s.enqueueAllPreviewEvents(); return nil }}, + {Name: "preview-sweep", Every: 15 * time.Minute, Run: func(context.Context, time.Time) error { s.enqueueAllPreviewEvents(); return nil }}, + {Name: "history-cleanup", Every: time.Hour, Run: func(context.Context, time.Time) error { + settings, err := s.store.GetQueueSettings() + if err != nil { + return err + } + return s.store.PurgeEventHistory(settings.RetentionHours) + }}, + {Name: "activity-sweep", Every: 10 * time.Second, Run: s.schedulerActivity}, + }, + Step: func(ctx context.Context, now time.Time) error { + _, err := s.store.EnqueueDueDefinitions(now) + s.eventRunner().Dispatch(ctx) + return err + }, + Report: func(stage string, err error) { + log.Printf("component=%s event scheduler %s failed: %v", dregarnuhr.Component, stage, err) + }, + } +} diff --git a/internal/server/dregarnuhr_queue_test.go b/internal/server/dregarnuhr_queue_test.go new file mode 100644 index 0000000..9826417 --- /dev/null +++ b/internal/server/dregarnuhr_queue_test.go @@ -0,0 +1,135 @@ +package server + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "pageswiki/internal/db" + "pageswiki/internal/dregarnuhr" + "pageswiki/internal/store" +) + +func TestDregarnuhrBuiltinsRemainRegistered(t *testing.T) { + s := &Server{} + registry := s.registeredEventHandlers() + // Duplicate registration proves presence without executing a business handler + // or reading any external calendar/source content. + for _, kind := range []string{ + store.NotificationPushEventType, store.TaskFocusEventType, store.TaskReminderEventType, + store.TaskActivityBridgeEventType, calendarSweepType, calendarSyncType, + metadataProjectionEventType, store.ResourceEventType, previewSweepEventType, + previewEventType, cityReconcileEventType, "pdf.page.generate", importAnalyzeEventType, importCommitEventType, + } { + if err := registry.Register(kind, func(context.Context, *dregarnuhr.Job) error { return nil }); err == nil { + t.Errorf("missing builtin: %s", kind) + } + } + if s.registeredEventHandlers() != registry { + t.Fatal("registry recreated") + } + if err := s.runEvent(&store.EventJob{Type: "test.unknown"}); err == nil { + t.Fatal("unknown event accepted") + } + loop := s.schedulerLoop() + if loop.Every != time.Second || loop.OnWake == nil { + t.Fatal("dispatch cadence or activity hook changed") + } + want := map[string][2]time.Duration{ + "initial-preview-sweep": {5 * time.Second, 0}, "preview-sweep": {0, 15 * time.Minute}, + "history-cleanup": {0, time.Hour}, "activity-sweep": {0, 10 * time.Second}, + } + if len(loop.Triggers) != len(want) { + t.Fatal("maintenance hooks changed") + } + for _, trigger := range loop.Triggers { + v, ok := want[trigger.Name] + if !ok || v != [2]time.Duration{trigger.After, trigger.Every} || trigger.Run == nil { + t.Fatalf("unexpected trigger: %+v", trigger) + } + } +} + +func TestDregarnuhrServerQueueIntegration(t *testing.T) { + database, err := db.Open(t.TempDir() + "/queue.db") + if err != nil { + t.Fatal(err) + } + defer database.Close() + st := &store.Store{DB: database} + if err := st.EnsureEventJobsSchema(); err != nil { + t.Fatal(err) + } + settings, err := st.GetQueueSettings() + if err != nil { + t.Fatal(err) + } + settings.Concurrency = 2 + if err := st.SetQueueSettings(settings); err != nil { + t.Fatal(err) + } + s := &Server{store: st, schedulerWake: make(chan struct{}, 8)} + started := make(chan struct{}, 8) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := s.registeredEventHandlers().Register("test.synthetic", func(ctx context.Context, job *dregarnuhr.Job) error { + started <- struct{}{} + <-ctx.Done() + return nil + }); err != nil { + t.Fatal(err) + } + for n := 0; n < 4; n++ { + if _, err := st.EnqueueEvent("test.synthetic", fmt.Sprint(n), "{}", time.Now().Add(-time.Minute)); err != nil { + t.Fatal(err) + } + } + var wg sync.WaitGroup + for range 16 { + wg.Add(1) + go func() { defer wg.Done(); s.eventRunner().Dispatch(ctx) }() + } + wg.Wait() + // Ensure workers have reached the synthetic handler before cancellation. + for range 2 { + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("handler not started") + } + } + if got := s.schedulerRunning.Load(); got != 2 { + t.Errorf("monitor running=%d, want=2", got) + } + cancel() + for range 2 { + select { + case <-s.schedulerWake: + case <-time.After(5 * time.Second): + t.Fatal("completion not persisted") + } + } + if got := s.schedulerRunning.Load(); got != 0 { + t.Fatalf("monitor leaked running=%d", got) + } + var running, attempted int + if err := database.QueryRow(`SELECT count(*) FROM event_jobs WHERE status='running'`).Scan(&running); err != nil { + t.Fatal(err) + } + if err := database.QueryRow(`SELECT count(*) FROM event_jobs WHERE attempts=1`).Scan(&attempted); err != nil { + t.Fatal(err) + } + if running != 0 || attempted != 2 { + t.Fatalf("persisted running=%d attempts=%d", running, attempted) + } + // Both deployment guards must stop claim before any registered handler runs. + for _, mode := range []string{"readonly", "disabled"} { + st.ReadOnly, st.BackgroundJobsDisabled = mode == "readonly", mode == "disabled" + s.runDueEvents() + if got := s.schedulerRunning.Load(); got != 0 { + t.Fatalf("%s started worker", mode) + } + } +} diff --git a/internal/server/editor_i18n.go b/internal/server/editor_i18n.go new file mode 100644 index 0000000..76d8602 --- /dev/null +++ b/internal/server/editor_i18n.go @@ -0,0 +1,12 @@ +package server + +func init() { + for lang, entries := range map[string]map[string]string{ + "en": {"editor.tools": "Insert component / drafts", "editor.language": "Edit / fork language", "editor.restore": "Restore a revision into the editor"}, + "ja": {"editor.tools": "コンポーネント挿入・下書き", "editor.language": "言語の編集・分岐", "editor.restore": "過去のリビジョンをエディタに復元"}, + } { + for key, value := range entries { + translations[lang][key] = value + } + } +} diff --git a/internal/server/editor_plain_test.go b/internal/server/editor_plain_test.go new file mode 100644 index 0000000..7786f3b --- /dev/null +++ b/internal/server/editor_plain_test.go @@ -0,0 +1,86 @@ +package server + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestPlainTextEditorCanonicalAndExistingHTML(t *testing.T) { + s, token, domain, htmlObject := editorFixture(t) + page := configRequest(s, token, "GET", "/e/-/notes.txt?edittype=plain", nil) + if page.Code != 200 || !strings.Contains(page.Body.String(), `value="plain"`) || !strings.Contains(page.Body.String(), `value="text/plain; charset=utf-8"`) { + t.Fatalf("plain editor: %d %s", page.Code, page.Body.String()) + } + object, err := s.store.ResolvePathInDomain(domain, "/notes.txt") + if err != nil { + t.Fatal(err) + } + body := "\n# Not Markdown\n{{children}}\n日本語 & <>\n" + payload, _ := json.Marshal(map[string]any{"object_uuid": object, "body": body, "lang": "und", "content_type": "text/plain; charset=utf-8", "filename": "notes.txt", "base_revision_uuid": ""}) + r := httptest.NewRequest("POST", "http://localhost/_/api/v1/revisions/save", bytes.NewReader(payload)) + r.Header.Set("Authorization", "Bearer "+token) + r.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + s.withAuthorization(s.mux).ServeHTTP(w, r) + if w.Code != 200 { + t.Fatalf("plain save: %d %s", w.Code, w.Body.String()) + } + rev, err := s.store.GetRevisionForExactLanguage(object, "und") + if err != nil { + t.Fatal(err) + } + if rev.Body != body || rev.ContentType != "text/plain; charset=utf-8" { + t.Fatalf("plain content mutated: %#v", rev) + } + if view := revisionContentView(rev, "/notes.txt"); view.Kind != "code" { + t.Fatalf("plain rendered as %s", view.Kind) + } + page = configRequest(s, token, "GET", "/e/-/notes.txt?edittype=markdown", nil) + if page.Code != 200 || !strings.Contains(page.Body.String(), `value="plain"`) || strings.Contains(page.Body.String(), `data-editor-action="markdown"`) { + t.Fatal("reopen switched plain file to HTML") + } + before, _ := s.store.GetRevisionForExactLanguage(htmlObject, "und") + page = configRequest(s, token, "GET", "/e/-/editor-fixture?edittype=plain", nil) + if page.Code != http.StatusConflict { + t.Fatalf("existing HTML converted: %d", page.Code) + } + after, _ := s.store.GetRevisionForExactLanguage(htmlObject, "und") + if before.UUID != after.UUID { + t.Fatal("existing HTML changed") + } + page = configRequest(s, token, "GET", "/e/-/invalid-format?edittype=not-a-format", nil) + if page.Code != http.StatusBadRequest { + t.Fatal("invalid editor format accepted") + } + if _, err = s.store.ResolvePathInDomain(domain, "/invalid-format"); err == nil { + t.Fatal("invalid format created a Page") + } +} + +func TestBrowserEditorCreationRequiresPermission(t *testing.T) { + s, _, domain, _ := editorFixture(t) + user, err := s.store.CreatePasswordAccount(domain, "reader", "a long test password", "Reader", "") + if err != nil { + t.Fatal(err) + } + if err := s.store.SetPrincipalState(domain, user.UUID, "readonly"); err != nil { + t.Fatal(err) + } + token, err := s.store.CreateAuthSession(domain, user.UUID) + if err != nil { + t.Fatal(err) + } + for _, format := range []string{"html", "plain"} { + response := configRequest(s, token, "GET", "/e/-/denied?edittype="+format, nil) + if response.Code != http.StatusForbidden { + t.Fatalf("read-only create: %d", response.Code) + } + if _, err := s.store.ResolvePathInDomain(domain, "/denied"); err == nil { + t.Fatal("denied editor created a Page") + } + } +} diff --git a/internal/server/editor_resilience_test.go b/internal/server/editor_resilience_test.go new file mode 100644 index 0000000..860453a --- /dev/null +++ b/internal/server/editor_resilience_test.go @@ -0,0 +1,147 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +func editorFixture(t *testing.T) (*Server, string, string, string) { + s, token, domain := configFixture(t) + s.mux.HandleFunc("/e/", s.handleEditorRoute) + s.mux.HandleFunc("/_/api/v1/editor/state", s.handleEditorState) + s.mux.HandleFunc("/_/api/v1/editor/convert", s.handleEditorConvert) + s.mux.HandleFunc("/_/api/v1/revisions/save", s.handleSaveRevisionAPI) + s.mux.HandleFunc("/_/api/v1/revisions/list", s.handleRevisionListAPI) + s.mux.HandleFunc("/_/api/v1/render/markdown", s.handleMarkdownRenderAPI) + object, err := s.store.CreatePageAtPathInDomain(domain, "/editor-fixture") + if err != nil { + t.Fatal(err) + } + if _, err = s.store.SaveRevisionContent(object, []byte("

Base

one two three

"), "base", "und", "text/html", ""); err != nil { + t.Fatal(err) + } + return s, token, domain, object +} + +func TestEditorResilienceAPI(t *testing.T) { + s, token, domain, object := editorFixture(t) + principal, err := s.store.PrincipalForSession(domain, token) + if err != nil { + t.Fatal(err) + } + base, err := s.store.GetRevisionForExactLanguage(object, "und") + if err != nil { + t.Fatal(err) + } + payload := map[string]any{"object_uuid": object, "body": "

Saved

", "lang": "und", "content_type": "text/html", "base_revision_uuid": base.UUID, "editor_principal_uuid": principal.UUID} + save := func(token string) *httptest.ResponseRecorder { + b, _ := json.Marshal(payload) + r := httptest.NewRequest("POST", "http://localhost/_/api/v1/revisions/save", bytes.NewReader(b)) + r.Header.Set("Authorization", "Bearer "+token) + r.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + s.withAuthorization(s.mux).ServeHTTP(w, r) + return w + } + if w := save(token); w.Code != 200 { + t.Fatalf("save %d %s", w.Code, w.Body.String()) + } + if w := save(token); w.Code != 409 { + t.Fatalf("stale %d %s", w.Code, w.Body.String()) + } + payload["editor_principal_uuid"] = "different-user" + if w := save(token); w.Code != 401 { + t.Fatalf("identity %d", w.Code) + } + if w := save("expired"); w.Code != 401 { + t.Fatalf("expired %d", w.Code) + } + for _, query := range []string{"&metadata=1", "&revision=" + base.UUID, "&lang=en"} { + w := configRequest(s, token, "GET", "/_/api/v1/editor/state?uuid="+object+query, nil) + if w.Code != 200 { + t.Fatalf("state %s %d %s", query, w.Code, w.Body.String()) + } + var out map[string]any + if err = json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + if out["principal_uuid"] != principal.UUID { + t.Fatal("identity omitted") + } + if query == "&metadata=1" { + if _, ok := out["body"]; ok { + t.Fatal("poll hydrated body") + } + } + if query == "&revision="+base.UUID && out["body"] != base.Body { + t.Fatal("old revision missing") + } + if query == "&lang=en" && out["revision_uuid"] != "" { + t.Fatal("language fallback") + } + } + other, err := s.store.CreatePageAtPathInDomain(domain, "/other") + if err != nil { + t.Fatal(err) + } + old, err := s.store.SaveRevisionForLanguage(other, "private unrelated body", "", "und") + if err != nil { + t.Fatal(err) + } + if w := configRequest(s, token, "GET", "/_/api/v1/editor/state?uuid="+object+"&revision="+old.UUID, nil); w.Code != 404 { + t.Fatalf("other object revision %d", w.Code) + } + w := configRequest(s, token, "GET", "/e/-/editor-fixture?lang=en&edittype=markdown", nil) + if w.Code != 200 || !strings.Contains(w.Body.String(), `data-base-revision=""`) || !strings.Contains(w.Body.String(), `data-owner="`+principal.UUID+`"`) { + t.Fatalf("editor language/owner %d", w.Code) + } + // Legacy API callers can still save without CAS during the migration. + delete(payload, "base_revision_uuid") + payload["editor_principal_uuid"] = principal.UUID + if w := save(token); w.Code != 200 { + t.Fatalf("legacy API %d", w.Code) + } +} + +func TestEditorResilienceChromium(t *testing.T) { + if os.Getenv("PAGESWIKI_BROWSER_TEST") != "1" { + t.Skip("opt-in Chromium") + } + s, token, domain, _ := editorFixture(t) + if err := s.store.SetConfigInDomain(domain, "editor_keymap", "vim"); err != nil { + t.Fatal(err) + } + script, err := os.ReadFile("internal/server/testdata/editor-resilience.js") + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Host = "localhost" + r.Header.Set("Authorization", "Bearer "+token) + if r.URL.Path == "/fixture" { + w.Header().Set("Content-Type", "text/html") + _, _ = io.WriteString(w, ``) + return + } + s.withAuthorization(s.mux).ServeHTTP(w, r) + })) + defer server.Close() + ctx, cancel := context.WithTimeout(context.Background(), 55*time.Second) + defer cancel() + args := []string{"internal/server/testdata/vim-browser-smoke.mjs", server.URL + "/fixture", t.TempDir()} + if shot := os.Getenv("EDITOR_TEST_SCREENSHOT"); shot != "" { + args = append(args, shot) + } + if out, err := exec.CommandContext(ctx, "node", args...).CombinedOutput(); err != nil { + t.Fatalf("browser %v %s", err, out) + } +} diff --git a/internal/server/editor_state.go b/internal/server/editor_state.go new file mode 100644 index 0000000..3563644 --- /dev/null +++ b/internal/server/editor_state.go @@ -0,0 +1,96 @@ +package server + +import ( + "encoding/json" + "net/http" + "strings" + + "pageswiki/internal/grammaratur" +) + +// Converts authoring input only. It never expands macros or writes a Revision. +func (s *Server) handleEditorConvert(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Cache-Control", "private, no-store") + if r.Method != http.MethodPost { + w.Header().Set("Allow", "POST") + http.Error(w, "method not allowed", 405) + return + } + var input struct { + HTML string `json:"html"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 2<<20)).Decode(&input); err != nil { + http.Error(w, "invalid input", 400) + return + } + markdown, err := grammaratur.HTMLToMarkdownAuthoring(input.HTML) + if err != nil { + http.Error(w, "conversion failed", 422) + return + } + _ = json.NewEncoder(w).Encode(map[string]string{"markdown": markdown}) +} + +// Restore loads a working buffer only; it never mutates an old Revision. +func (s *Server) handleEditorState(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Cache-Control", "private, no-store") + w.Header().Set("Vary", "Authorization, Cookie") + if r.Method != http.MethodGet { + w.Header().Set("Allow", "GET") + http.Error(w, "method not allowed", 405) + return + } + domain := s.domainUUIDForRequest(r) + principal, authErr := s.store.PrincipalForSession(domain, s.authTokenFromRequest(r)) + if mode, _ := s.store.GetConfigInDomain(domain, "authorization_mode"); mode == "enforcing" && authErr != nil { + http.Error(w, "authentication required", 401) + return + } + object := r.URL.Query().Get("uuid") + if object == "" || !s.store.ObjectBoundInDomain(domain, object) || s.authorizeObjectForRequest(r, "read", object) != nil { + http.Error(w, "forbidden", 403) + return + } + rev, err := s.store.GetRevisionMetadataForExactLanguage(object, r.URL.Query().Get("lang")) + if err != nil { + http.Error(w, "invalid language or unavailable revision", 400) + return + } + head := rev.UUID + if id := r.URL.Query().Get("revision"); id != "" { + metadata, e := s.store.GetRevisionMetadata(id) + if e != nil || metadata.ObjectUUID != object { + http.Error(w, "revision not found", 404) + return + } + rev = metadata + } + owner := "" + if principal != nil { + owner = principal.UUID + } + out := map[string]any{"revision_uuid": rev.UUID, "head_revision_uuid": head, "lang": rev.Language, "content_type": rev.ContentType, "filename": rev.Filename, "principal_uuid": owner} + if r.URL.Query().Get("metadata") != "1" { + if rev.UUID != "" { + rev, err = s.store.GetRevision(rev.UUID) + } + if err != nil { + http.Error(w, "revision unavailable", 503) + return + } + markdown := rev.Body + if strings.HasPrefix(rev.ContentType, "text/html") { + markdown, err = grammaratur.HTMLToMarkdownAuthoring(rev.Body) + } + if err != nil { + http.Error(w, "conversion unavailable", 503) + return + } + out["body"], out["markdown"] = rev.Body, markdown + kind := revisionContentView(rev, "").Kind + out["editable"] = rev.UUID == "" || kind == "html" || kind == "code" + } + _ = json.NewEncoder(w).Encode(out) +} diff --git a/internal/server/endpoint_policy.go b/internal/server/endpoint_policy.go index 4dc379d..730f119 100644 --- a/internal/server/endpoint_policy.go +++ b/internal/server/endpoint_policy.go @@ -37,6 +37,8 @@ const ( ) var endpointPolicies = map[string]endpointPolicy{ + "GET /_/api/v1/editor/state": {Name: "editor.state", Capability: "read", Target: targetDomain, Resolver: resolveHandler}, + "POST /_/api/v1/editor/convert": {Name: "editor.convert", Capability: "read", Target: targetDomain, Resolver: resolveHandler}, "GET /_/api/v1/extension": {Name: "extension.catalog", Capability: "manage", Target: targetDomain, Resolver: resolveHandler}, "POST /-/sys/config/vim": {Name: "config.vim", Capability: "manage", Target: targetDomain, Resolver: resolveHandler}, "GET /:/new": {Name: "page.create_form", Capability: "create", Target: targetDomain}, @@ -160,6 +162,8 @@ func policyForRequest(r *http.Request) endpointPolicy { func readEndpointResolver(path string) (endpointTargetResolver, string) { switch { + case path == "/v/browser", strings.HasPrefix(path, "/v/browser/"): + return resolveHandler, "read" case path == "/v/children", path == "/v/tree", strings.HasPrefix(path, "/v/tree/"): return resolveHandler, "read" case path == "/v/city", strings.HasPrefix(path, "/v/city/"), strings.HasPrefix(path, "/_/api/v1/city/"): diff --git a/internal/server/event_api.go b/internal/server/event_api.go index 16340d4..365c64a 100644 --- a/internal/server/event_api.go +++ b/internal/server/event_api.go @@ -4,14 +4,11 @@ import ( "encoding/json" "errors" "net/http" - "os" - "path/filepath" - "runtime" "strconv" "strings" - "syscall" "time" + "pageswiki/internal/seheweit" "pageswiki/internal/store" ) @@ -250,7 +247,7 @@ func (s *Server) handleQueueAPI(w http.ResponseWriter, r *http.Request) { writeEventJSON(w, 500, map[string]string{"error": "could not load resource history"}) return } - writeEventJSON(w, 200, map[string]any{"samples": samples, "hours": hours, "retention_days": 7}) + writeEventJSON(w, 200, map[string]any{"samples": samples, "hours": hours, "retention_days": seheweit.RetentionDays}) return } } @@ -258,39 +255,11 @@ func (s *Server) handleQueueAPI(w http.ResponseWriter, r *http.Request) { writeEventJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "unsupported queue operation"}) } -func (s *Server) resourceStatus() map[string]any { - var memory runtime.MemStats - runtime.ReadMemStats(&memory) - root := s.store.DocumentRoot - if root == "" { - root = "." - } - root, _ = filepath.Abs(root) - var fs syscall.Statfs_t - _ = syscall.Statfs(root, &fs) - total := fs.Blocks * uint64(fs.Bsize) - available := fs.Bavail * uint64(fs.Bsize) - loadAverage := "" - if data, err := os.ReadFile("/proc/loadavg"); err == nil { - fields := strings.Fields(string(data)) - if len(fields) >= 3 { - loadAverage = strings.Join(fields[:3], " ") - } - } +func (s *Server) resourceStatus() seheweit.Snapshot { stats, _ := s.store.ActiveEventStats() settings, _ := s.store.GetQueueSettings() - return map[string]any{ - "timestamp_ms": time.Now().UnixMilli(), - "io": readProcessIO(), - "cpu": map[string]any{"logical_cpus": runtime.NumCPU(), "load_average": loadAverage}, - "memory": map[string]any{ - "heap_bytes": memory.HeapAlloc, "go_system_bytes": memory.Sys, - "gc_cycles": memory.NumGC, "goroutines": runtime.NumGoroutine(), - }, - "storage": map[string]any{ - "path": root, "total_bytes": total, "used_bytes": total - available, - "available_bytes": available, - }, - "queue": map[string]any{"stats": map[string]int{"queued": stats.Queued, "running": stats.Running}, "active_workers": s.schedulerRunning.Load(), "paused": settings.Paused}, - } + return seheweit.Collect(s.store.DocumentRoot, seheweit.Queue{ + Stats: seheweit.QueueStats{Queued: stats.Queued, Running: stats.Running}, + ActiveWorkers: s.schedulerRunning.Load(), Paused: settings.Paused, + }) } diff --git a/internal/server/event_handlers.go b/internal/server/event_handlers.go new file mode 100644 index 0000000..f972c0b --- /dev/null +++ b/internal/server/event_handlers.go @@ -0,0 +1,120 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "pageswiki/internal/dregarnuhr" + "pageswiki/internal/store" +) + +// Business handlers stay in the application. Dregarnuhr knows no event types, +// payload schemas, notification recipients or calendar/source configuration. +func (s *Server) registeredEventHandlers() *dregarnuhr.Registry { + s.eventHandlersOnce.Do(func() { + registry := &dregarnuhr.Registry{} + handlers := map[string]dregarnuhr.Handler{ + store.NotificationPushEventType: func(ctx context.Context, job *store.EventJob) error { + return s.deliverNotificationPushContext(ctx, job) + }, + store.TaskFocusEventType: func(ctx context.Context, job *store.EventJob) error { + processed, err := s.store.CompleteTaskFocus(ctx, time.Now(), 100) + if err == nil && processed > 0 { + _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("focus timers completed=%d", processed)) + } + return err + }, + store.TaskReminderEventType: func(ctx context.Context, job *store.EventJob) error { + checked, sent, err := s.store.DeliverTaskReminders(ctx, time.Now(), 100) + if err == nil && checked > 0 { + _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("task reminders checked=%d delivered=%d", checked, sent)) + } + return err + }, + store.TaskActivityBridgeEventType: func(ctx context.Context, job *store.EventJob) error { + processed, published, err := s.store.PublishTaskActivity(ctx, 100) + if err == nil && processed > 0 { + _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("journal entries checked=%d published=%d", processed, published)) + } + return err + }, + calendarSweepType: func(ctx context.Context, job *store.EventJob) error { + return s.sweepCalendarSources() + }, + calendarSyncType: func(ctx context.Context, job *store.EventJob) error { + return s.runCalendarSource(job) + }, + metadataProjectionEventType: func(ctx context.Context, job *store.EventJob) error { + var payload metadataProjectionPayload + if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { + return err + } + return s.generateMetadataProjection(payload) + }, + store.ResourceEventType: func(ctx context.Context, job *store.EventJob) error { + if err := ctx.Err(); err != nil { + return err + } + snapshot := s.resourceStatus() + payload, err := json.Marshal(snapshot) + if err != nil { + return err + } + return s.store.SaveResourceSampleContext(ctx, job.UUID, snapshot.TimestampMS, payload) + }, + previewSweepEventType: func(ctx context.Context, job *store.EventJob) error { + s.enqueueAllPreviewEvents() + return nil + }, + previewEventType: func(ctx context.Context, job *store.EventJob) error { + var payload previewEventPayload + if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { + return err + } + return s.generatePreview(payload) + }, + cityReconcileEventType: func(ctx context.Context, job *store.EventJob) error { + var payload cityReconcilePayload + if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { + return err + } + return s.reconcileCityActivity(payload.DomainUUID) + }, + "pdf.page.generate": func(ctx context.Context, job *store.EventJob) error { + var payload previewEventPayload + if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { + return err + } + return s.generatePDFPage(payload) + }, + importAnalyzeEventType: func(ctx context.Context, job *store.EventJob) error { + var payload importAnalyzePayload + if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { + return err + } + return s.analyzeImport(payload) + }, + importCommitEventType: func(ctx context.Context, job *store.EventJob) error { + var payload importCommitPayload + if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { + return err + } + return s.commitImport(payload) + }, + } + for kind, handler := range handlers { + // A duplicate/invalid built-in is a programmer error, never silently replace. + if err := registry.Register(kind, handler); err != nil { + panic(err) + } + } + s.eventHandlers = registry + }) + return s.eventHandlers +} + +func (s *Server) runEvent(job *store.EventJob) error { + return s.registeredEventHandlers().Run(context.Background(), job) +} diff --git a/internal/server/event_scheduler.go b/internal/server/event_scheduler.go index 63fd8e0..8743c1a 100644 --- a/internal/server/event_scheduler.go +++ b/internal/server/event_scheduler.go @@ -3,7 +3,6 @@ package server import ( "archive/zip" "context" - "database/sql" "encoding/json" "errors" "fmt" @@ -78,44 +77,8 @@ func (s *Server) startEventScheduler() { log.Printf("notification push initialization failed: %v", err) } go func() { - initialPreviewSweep := time.NewTimer(5 * time.Second) - defer initialPreviewSweep.Stop() - previewSweep := time.NewTicker(15 * time.Minute) - defer previewSweep.Stop() - dispatch := time.NewTicker(time.Second) - defer dispatch.Stop() - cleanup := time.NewTicker(time.Hour) - defer cleanup.Stop() - activitySweep := time.NewTicker(10 * time.Second) - defer activitySweep.Stop() - for { - select { - case <-initialPreviewSweep.C: - s.enqueueAllPreviewEvents() - case <-previewSweep.C: - s.enqueueAllPreviewEvents() - case <-dispatch.C: - case <-cleanup.C: - if settings, err := s.store.GetQueueSettings(); err == nil { - if err := s.store.PurgeEventHistory(settings.RetentionHours); err != nil { - log.Printf("event history cleanup failed: %v", err) - } - } - case <-activitySweep.C: - if _, err := s.store.ExpireApprovalRequests(context.Background(), time.Now().UnixMilli(), 100); err != nil { - log.Printf("approval request expiry failed: %v", err) - } - s.enqueueCityActivityEvents() - case <-s.schedulerWake: - if _, err := s.store.ExpireApprovalRequests(context.Background(), time.Now().UnixMilli(), 100); err != nil { - log.Printf("approval request expiry failed: %v", err) - } - s.enqueueCityActivityEvents() - } - if _, err := s.store.EnqueueDueDefinitions(time.Now()); err != nil { - log.Printf("event schedule evaluation failed: %v", err) - } - s.runDueEvents() + if err := s.schedulerLoop().Run(context.Background()); err != nil { + log.Printf("event scheduler loop stopped: %v", err) } }() } @@ -130,132 +93,6 @@ func (s *Server) wakeEventScheduler() { } } -func (s *Server) runDueEvents() { - for { - settings, err := s.store.GetQueueSettings() - if err != nil { - log.Printf("event scheduler settings failed: %v", err) - return - } - if settings.Paused || s.schedulerRunning.Load() >= int64(settings.Concurrency) { - return - } - // Keep one reserved background slot while foreground work is active. - // A global stop lets long-lived editor/WebSocket requests starve jobs. - if s.foregroundRequests.Load() > 0 && s.schedulerRunning.Load() > 0 { - return - } - job, err := s.store.ClaimDueEvent() - if errors.Is(err, sql.ErrNoRows) || errors.Is(err, context.Canceled) { - return - } - if err != nil { - log.Printf("event scheduler claim failed: %v", err) - return - } - s.schedulerRunning.Add(1) - _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", "execution started") - go func(job *store.EventJob) { - defer func() { - s.schedulerRunning.Add(-1) - s.wakeEventScheduler() - }() - var runErr error - if s.store.EventCancellationRequested(job.UUID) { - runErr = errors.New("cancelled before execution") - } else { - runErr = s.runEvent(job) - if runErr == nil && s.store.EventCancellationRequested(job.UUID) { - runErr = errors.New("cancelled during execution") - } - } - if finishErr := s.store.FinishEvent(job.UUID, runErr); finishErr != nil { - log.Printf("event scheduler finish failed job=%s: %v", job.UUID, finishErr) - } - if runErr != nil { - log.Printf("event job failed type=%s key=%s: %v", job.Type, job.Key, runErr) - } - }(job) - } -} - -func (s *Server) runEvent(job *store.EventJob) error { - switch job.Type { - case store.NotificationPushEventType: - return s.deliverNotificationPush(job) - case store.TaskFocusEventType: - processed, err := s.store.CompleteTaskFocus(context.Background(), time.Now(), 100) - if err == nil && processed > 0 { - _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("focus timers completed=%d", processed)) - } - return err - case store.TaskReminderEventType: - checked, sent, err := s.store.DeliverTaskReminders(context.Background(), time.Now(), 100) - if err == nil && checked > 0 { - _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("task reminders checked=%d delivered=%d", checked, sent)) - } - return err - case store.TaskActivityBridgeEventType: - processed, published, err := s.store.PublishTaskActivity(context.Background(), 100) - if err == nil && processed > 0 { - _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("journal entries checked=%d published=%d", processed, published)) - } - return err - case calendarSweepType: - return s.sweepCalendarSources() - case calendarSyncType: - return s.runCalendarSource(job) - case metadataProjectionEventType: - var payload metadataProjectionPayload - if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { - return err - } - return s.generateMetadataProjection(payload) - case store.ResourceEventType: - snapshot := s.resourceStatus() - payload, err := json.Marshal(snapshot) - if err != nil { - return err - } - return s.store.SaveResourceSample(job.UUID, snapshot["timestamp_ms"].(int64), payload) - case previewSweepEventType: - s.enqueueAllPreviewEvents() - return nil - case previewEventType: - var payload previewEventPayload - if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { - return err - } - return s.generatePreview(payload) - case cityReconcileEventType: - var payload cityReconcilePayload - if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { - return err - } - return s.reconcileCityActivity(payload.DomainUUID) - case "pdf.page.generate": - var payload previewEventPayload - if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { - return err - } - return s.generatePDFPage(payload) - case importAnalyzeEventType: - var payload importAnalyzePayload - if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { - return err - } - return s.analyzeImport(payload) - case importCommitEventType: - var payload importCommitPayload - if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { - return err - } - return s.commitImport(payload) - default: - return fmt.Errorf("unknown event type %q", job.Type) - } -} - func (s *Server) enqueueCityActivityEvents() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -300,17 +137,20 @@ func (s *Server) generatePDFPage(payload previewEventPayload) error { if info, err := os.Stat(target); err == nil && info.Size() > 0 { return nil } - source, err := s.store.DocumentPath(payload.DocumentHash) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + lease, err := s.store.MaterializeDocument(ctx, payload.DocumentHash, maxMetadataInputBytes) if err != nil { return err } + defer lease.Close() workDir, err := os.MkdirTemp(s.previewRoot, ".pdf-page-*") if err != nil { return err } defer os.RemoveAll(workDir) base := filepath.Join(workDir, "page") - if err := runPreviewCommand("pdftoppm", "-f", strconv.Itoa(payload.Page), "-l", strconv.Itoa(payload.Page), "-singlefile", "-jpeg", "-scale-to", "1600", source, base); err != nil { + if err := runPreviewCommandContext(ctx, "pdftoppm", "-f", strconv.Itoa(payload.Page), "-l", strconv.Itoa(payload.Page), "-singlefile", "-jpeg", "-scale-to", "1600", lease.Path, base); err != nil { return err } return os.Rename(base+".jpg", target) @@ -428,9 +268,25 @@ func (s *Server) generatePreview(payload previewEventPayload) error { if info, err := os.Stat(target); err == nil && info.Size() > 0 { return nil } - source, err := s.store.DocumentPath(payload.DocumentHash) - if err != nil { - return err + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + // Image and ZIP decoders consume readers directly. Only subprocesses and + // the legacy Excalidraw renderer need a verified local scratch pathname. + var source string + if payload.Kind == "pdf" || payload.Kind == "video" || payload.Kind == "excalidraw" { + limit := int64(maxMetadataInputBytes) + if payload.Kind == "video" { + limit = 1 << 30 + } + if payload.Kind == "excalidraw" { + limit = 16 << 20 + } + lease, err := s.store.MaterializeDocument(ctx, payload.DocumentHash, limit) + if err != nil { + return err + } + defer lease.Close() + source = lease.Path } workDir, err := os.MkdirTemp(s.previewRoot, ".job-*") if err != nil { @@ -442,10 +298,15 @@ func (s *Server) generatePreview(payload previewEventPayload) error { switch payload.Kind { case "image": generated = filepath.Join(workDir, "preview.jpg") - err = generateImageThumbnail(source, generated) + input, openErr := s.store.OpenDocumentContext(ctx, payload.DocumentHash) + if openErr != nil { + return openErr + } + defer input.Close() + err = generateImageThumbnailFromReader(input, generated) case "pdf": generated = filepath.Join(workDir, "preview.jpg") - err = runPreviewCommand( + err = runPreviewCommandContext(ctx, "pdftoppm", "-f", "1", "-singlefile", "-jpeg", "-scale-to", "480", @@ -454,7 +315,7 @@ func (s *Server) generatePreview(payload previewEventPayload) error { ) case "video": generated = filepath.Join(workDir, "preview.jpg") - err = runPreviewCommand( + err = runPreviewCommandContext(ctx, "ffmpeg", "-hide_banner", "-loglevel", "error", "-ss", "1", "-i", source, "-frames:v", "1", "-vf", "scale=480:-2", @@ -465,7 +326,23 @@ func (s *Server) generatePreview(payload previewEventPayload) error { err = generateExcalidrawSVG(source, generated) case "archive": generated = filepath.Join(workDir, "preview.jpg") - err = generateArchiveThumbnail(source, generated, payload.Filename) + if !strings.EqualFold(filepath.Ext(payload.Filename), ".zip") { + return store.ErrEventSkipped + } + input, openErr := s.store.OpenDocumentContext(ctx, payload.DocumentHash) + if openErr != nil { + return openErr + } + defer input.Close() + size, seekErr := input.Seek(0, io.SeekEnd) + if seekErr != nil { + return seekErr + } + reader, zipErr := zip.NewReader(input, size) + if zipErr != nil { + return zipErr + } + err = generateZIPThumbnailFromReader(reader, generated) default: return fmt.Errorf("unsupported preview kind %q", payload.Kind) } @@ -481,13 +358,22 @@ func (s *Server) generatePreview(payload previewEventPayload) error { func runPreviewCommand(name string, args ...string) error { ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() + return runPreviewCommandContext(ctx, name, args...) +} + +func runPreviewCommandContext(ctx context.Context, name string, args ...string) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() command := exec.CommandContext(ctx, "nice", append([]string{"-n", "15", name}, args...)...) - output, err := command.CombinedOutput() + output := &boundedMetadataOutput{limit: 2 << 20, cancel: cancel} + command.Stdout, command.Stderr = output, output + command.WaitDelay = time.Second + err := command.Run() if ctx.Err() != nil { return ctx.Err() } if err != nil { - return fmt.Errorf("%s failed: %w: %s", name, err, strings.TrimSpace(string(output))) + return fmt.Errorf("%s failed: %w: %s", name, err, strings.TrimSpace(output.buffer.String())) } return nil } @@ -514,6 +400,10 @@ func generateZIPThumbnail(source, target string) error { return err } defer reader.Close() + return generateZIPThumbnailFromReader(&reader.Reader, target) +} + +func generateZIPThumbnailFromReader(reader *zip.Reader, target string) error { imageCount, fileCount := 0, 0 for _, file := range reader.File { if file.FileInfo().IsDir() { diff --git a/internal/server/immutable_object.go b/internal/server/immutable_object.go index 041bf0b..ee90c76 100644 --- a/internal/server/immutable_object.go +++ b/internal/server/immutable_object.go @@ -52,7 +52,7 @@ func (s *Server) handleImmutableObject(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) return } - reader, err := s.store.OpenDocument(hash) + reader, err := s.store.OpenDocumentContext(r.Context(), hash) if err != nil { http.NotFound(w, r) return diff --git a/internal/server/json_structure.go b/internal/server/json_structure.go index 1a6b077..89546d8 100644 --- a/internal/server/json_structure.go +++ b/internal/server/json_structure.go @@ -56,7 +56,7 @@ func (s *Server) handleJSONStructure(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) return } - reader, err := s.store.OpenDocument(revision.DocumentHash) + reader, err := s.store.OpenDocumentContext(r.Context(), revision.DocumentHash) if err != nil { http.NotFound(w, r) return diff --git a/internal/server/kunstzeal_test.go b/internal/server/kunstzeal_test.go new file mode 100644 index 0000000..bd0ee1d --- /dev/null +++ b/internal/server/kunstzeal_test.go @@ -0,0 +1,144 @@ +package server + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "pageswiki/internal/compiler" + "pageswiki/internal/kunstzeal" +) + +func TestKunstzealCompatibility(t *testing.T) { + var d contentDescriptor = kunstzeal.Descriptor{MediaType: "application/json", Filename: "scene.excalidraw"} + var adapter projectionAdapter = kunstzeal.SelectAdapter(d) + if adapter.ID != selectProjectionAdapter(d).ID || adapter.Delivery != deliveryText || inlineTextWindowThreshold != kunstzeal.InlineTextWindowThreshold { + t.Fatal("adapter compatibility changed") + } + if len(builtinAdapterManifest()) != len(kunstzeal.Catalog()) { + t.Fatal("discovery catalog diverged") + } + for _, kind := range []string{"mermaid", "dot", "graphviz", "d2", "unknown"} { + if diagramHash(kind, "source", "svg") != compiler.DiagramHash(kind, "source", "svg") || diagramHash(kind, "source", "svg") != kunstzeal.DiagramHash(kind, "source", "svg") { + t.Fatal("hash diverged") + } + if allowedDiagramType(kind) != kunstzeal.AllowedDiagramType(kind) || normalizeDiagramType(kind) != kunstzeal.NormalizeDiagramType(kind) { + t.Fatal("kind facade diverged") + } + } + client := diagramHTTPClient() + if client.Timeout != kunstzeal.DiagramTimeout || client.CheckRedirect(nil, nil) != http.ErrUseLastResponse { + t.Fatal("unsafe provider defaults") + } +} + +func TestKunstzealRedirectAndErrorIsolation(t *testing.T) { + s, token, _ := configFixture(t) + s.diagramRoot = t.TempDir() + s.mux.HandleFunc("/:/compile/diagram/", s.handleCompileDiagram) + var leaked atomic.Int32 + other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { leaked.Add(1); io.WriteString(w, "") })) + defer other.Close() + var status atomic.Int32 + status.Store(http.StatusTemporaryRedirect) + renderer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", other.URL+"/collect-source") + w.WriteHeader(int(status.Load())) + io.WriteString(w, "PRIVATE RENDERER DIAGNOSTICS") + })) + defer renderer.Close() + t.Setenv("KROKI_URL", renderer.URL) + for _, code := range []int{301, 302, 307, 308, 400, 500} { + status.Store(int32(code)) + r := httptest.NewRequest("POST", "http://localhost/:/compile/diagram/mermaid", strings.NewReader("private diagram source")) + r.Header.Set("Authorization", "Bearer "+token) + w := httptest.NewRecorder() + s.withAuthorization(s.mux).ServeHTTP(w, r) + if w.Code != 502 || leaked.Load() != 0 || strings.Contains(w.Body.String(), "PRIVATE") || strings.Contains(w.Body.String(), renderer.URL) || w.Header().Get("Cache-Control") != "private, no-store" || !strings.Contains(w.Header().Get("Content-Security-Policy"), "sandbox") { + t.Fatalf("status=%d response=%d leaked=%d body=%s", code, w.Code, leaked.Load(), w.Body.String()) + } + } + files, err := filepath.Glob(filepath.Join(s.diagramRoot, "mermaid", "*.svg")) + if err != nil || len(files) != 0 { + t.Fatal("failure cached", files, err) + } +} + +func TestKunstzealPostCacheBoundsAndFailureFallback(t *testing.T) { + s, token, _ := configFixture(t) + s.diagramRoot = t.TempDir() + s.mux.HandleFunc("/:/compile/diagram/", s.handleCompileDiagram) + var calls atomic.Int32 + renderer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.URL.Path != "/graphviz/svg" { + t.Error("dot not normalized", r.URL.Path) + } + io.WriteString(w, `fixture`) + })) + defer renderer.Close() + t.Setenv("KROKI_URL", renderer.URL) + post := func(source string) *httptest.ResponseRecorder { + r := httptest.NewRequest("POST", "http://localhost/:/compile/diagram/dot", strings.NewReader(source)) + r.Header.Set("Authorization", "Bearer "+token) + w := httptest.NewRecorder() + s.withAuthorization(s.mux).ServeHTTP(w, r) + return w + } + if w := post(strings.Repeat("x", kunstzeal.DiagramSourceLimit+1)); w.Code != 413 || calls.Load() != 0 { + t.Fatal("oversized input sent", w.Code, calls.Load()) + } + for i := 0; i < 2; i++ { + w := post("digraph { a -> b }") + if w.Code != 200 || !strings.HasPrefix(w.Body.String(), " d }"); w.Code != 200 || calls.Load() != 2 { + t.Fatal("cache failure broke rendering", w.Code, calls.Load()) + } +} + +func TestKunstzealDiskCacheNeedsLiveProvenance(t *testing.T) { + s, token, domain := configFixture(t) + s.diagramRoot = t.TempDir() + s.mux.HandleFunc("/:/compile/diagram/", s.handleCompileDiagram) + hash := diagramHash("mermaid", t.Name(), "svg") + if err := kunstzeal.WriteDiagramCache(s.diagramCachePath("mermaid", hash), []byte("")); err != nil { + t.Fatal(err) + } + url := "/:/compile/diagram/mermaid/" + hash + ".svg" + if w := configRequest(s, token, "GET", url, nil); w.Code != 404 { + t.Fatal("orphan cache was served", w.Code) + } + diagramSourceMu.Lock() + diagramSourceCache[hash] = diagramCacheEntry{Type: "mermaid", Domain: domain, Path: "/guide", Source: "source", At: time.Now().Add(-31 * time.Minute)} + diagramSourceMu.Unlock() + t.Cleanup(func() { diagramSourceMu.Lock(); delete(diagramSourceCache, hash); diagramSourceMu.Unlock() }) + if w := configRequest(s, token, "GET", url, nil); w.Code != 404 { + t.Fatal("expired provenance was served", w.Code) + } + diagramSourceMu.Lock() + entry := diagramSourceCache[hash] + entry.At = time.Now() + diagramSourceCache[hash] = entry + diagramSourceMu.Unlock() + if w := configRequest(s, token, "GET", url, nil); w.Code != 200 || w.Body.String() != "" { + t.Fatal("authorized cache not served", w.Code, w.Body.String()) + } +} diff --git a/internal/server/macro_expand.go b/internal/server/macro_expand.go index a34c652..4fa2f4d 100644 --- a/internal/server/macro_expand.go +++ b/internal/server/macro_expand.go @@ -162,7 +162,7 @@ func (s *Server) expandCityMacrosContext(ctx context.Context, body, currentPath, if context.Quartier != nil { quartierName = context.Quartier.Name } - return `` + html.EscapeString(node.Title) + `` + return `` + html.EscapeString(node.DisplayTitle()) + `` } return cityMacroRe.ReplaceAllStringFunc(body, func(macro string) string { if ctx.Err() != nil { diff --git a/internal/server/mestionora.go b/internal/server/mestionora.go new file mode 100644 index 0000000..cdb2f5b --- /dev/null +++ b/internal/server/mestionora.go @@ -0,0 +1,9 @@ +package server + +import "pageswiki/internal/mestionora" + +// Construction is read-only: schema/index maintenance remains an explicit +// application startup operation, never a search request side effect. +func (s *Server) searchEngine() *mestionora.SQLite { + return mestionora.NewSQLite(s.store.DB) +} diff --git a/internal/server/notification_push.go b/internal/server/notification_push.go index 740e85a..f37b90a 100644 --- a/internal/server/notification_push.go +++ b/internal/server/notification_push.go @@ -2,18 +2,14 @@ package server import ( "context" - "crypto/ecdh" "database/sql" - "encoding/base64" "encoding/json" "errors" - "fmt" "net/http" - "net/url" - "strings" "time" webpush "github.com/SherClockHolmes/webpush-go" + "pageswiki/internal/ordoschnellic" "pageswiki/internal/store" ) @@ -22,32 +18,10 @@ type browserPushInput struct { Focused bool `json:"focused"` Seen []string `json:"seen"` } -type storedBrowserPush struct { - webpush.Subscription - Origin string `json:"origin"` -} +type storedBrowserPush = ordoschnellic.StoredBrowserPush -// Only browser push vendors are destinations, never arbitrary user-provided URLs. -// DNS pinning/public-address checks at dial time provide a second SSRF boundary. func validPushSubscription(sub webpush.Subscription) bool { - u, e := url.Parse(sub.Endpoint) - if e != nil || u.Scheme != "https" || u.User != nil || u.Fragment != "" || u.Port() != "" || len(sub.Endpoint) > 4096 { - return false - } - host := u.Hostname() - if host != "fcm.googleapis.com" && host != "updates.push.services.mozilla.com" && host != "web.push.apple.com" && !strings.HasSuffix(host, ".push.services.mozilla.com") { - return false - } - auth, e := base64.RawURLEncoding.DecodeString(sub.Keys.Auth) - if e != nil || len(auth) != 16 { - return false - } - key, e := base64.RawURLEncoding.DecodeString(sub.Keys.P256dh) - if e != nil { - return false - } - _, e = ecdh.P256().NewPublicKey(key) - return e == nil + return ordoschnellic.ValidPushSubscription(sub) } func (s *Server) notificationPushKeys(ctx context.Context) (private, public string, err error) { @@ -93,7 +67,7 @@ func (s *Server) handleNotificationPush(w http.ResponseWriter, r *http.Request, if r.Method == "DELETE" { err = s.store.DeleteNotificationPush(ctx, domain, owner, input.Subscription.Endpoint) } else { - raw, _ := json.Marshal(storedBrowserPush{input.Subscription, s.socialOrigin(r)}) + raw, _ := json.Marshal(storedBrowserPush{Subscription: input.Subscription, Origin: s.socialOrigin(r)}) err = s.store.SaveNotificationPush(ctx, domain, owner, s.authTokenFromRequest(r), input.Subscription.Endpoint, string(raw), input.Focused) if err == nil && input.Focused { err = s.store.AcknowledgeNotificationPush(ctx, domain, owner, input.Subscription.Endpoint, input.Seen) @@ -110,79 +84,16 @@ func (s *Server) handleNotificationPush(w http.ResponseWriter, r *http.Request, http.Error(w, "method not allowed", 405) } +// Compatibility entry points for existing callers; event registration passes +// its worker context through deliverNotificationPushContext. func (s *Server) deliverNotificationPush(job *store.EventJob) error { - ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) - defer cancel() - // Bounded retention even for abandoned devices. Inbox is not affected. - _, err := s.store.DB.ExecContext(ctx, `DELETE FROM notification_push_subscriptions WHERE id IN (SELECT id FROM notification_push_subscriptions WHERE expires_at_ms 0 { - _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("push sent=%d failed=%d (no payload or endpoint logged)", sent, failed)) - } - return nil + return s.deliverNotificationPushContext(context.Background(), job) } func (s *Server) sendNotificationPush(ctx context.Context, d *store.NotificationPushDelivery, sub storedBrowserPush, client webpush.HTTPClient) (success, retry, expired bool) { - if !validPushSubscription(sub.Subscription) { - return false, false, true - } - private, public, e := s.notificationPushKeys(ctx) - if e != nil { - return false, true, false - } - // No task titles, body text or credentials leave the server in a push payload. - payload, _ := json.Marshal(map[string]string{"uuid": d.Notification, "owner": d.Owner, "domain": d.Domain}) - resp, e := webpush.SendNotificationWithContext(ctx, payload, &sub.Subscription, &webpush.Options{HTTPClient: client, Subscriber: sub.Origin, VAPIDPrivateKey: private, VAPIDPublicKey: public, TTL: 300, Urgency: webpush.UrgencyNormal, Topic: "pageswiki-inbox"}) - if e != nil { - return false, true, false - } - defer resp.Body.Close() - return resp.StatusCode >= 200 && resp.StatusCode < 300, resp.StatusCode == 429 || resp.StatusCode >= 500, resp.StatusCode == 404 || resp.StatusCode == 410 + copy := *d + raw, _ := json.Marshal(sub) + copy.Subscription = string(raw) + outcome := (ordoschnellic.WebPush{Keys: s.notificationPushKeys, Client: client}).Send(ctx, ©) + return outcome.Success, outcome.Retry, outcome.Expired } diff --git a/internal/server/notification_push_test.go b/internal/server/notification_push_test.go index c9d57be..5372041 100644 --- a/internal/server/notification_push_test.go +++ b/internal/server/notification_push_test.go @@ -22,7 +22,7 @@ func (f pushHTTPFixture) Do(r *http.Request) (*http.Response, error) { return f( func TestNotificationPushEncryptedDeliveryResponses(t *testing.T) { s, _, domain, owner := taskFixture(t) - sub := storedBrowserPush{testPushSubscription(t), "https://wiki.example"} + sub := storedBrowserPush{Subscription: testPushSubscription(t), Origin: "https://wiki.example"} d := &store.NotificationPushDelivery{Notification: uuid.NewString(), Domain: domain, Owner: owner} for _, code := range []int{201, 400, 404, 410, 429, 500} { client := pushHTTPFixture(func(r *http.Request) (*http.Response, error) { @@ -99,7 +99,7 @@ func TestNotificationPushQueueAtomicityRevocationAndFocus(t *testing.T) { s, token, domain, owner := taskFixture(t) ctx := context.Background() sub := testPushSubscription(t) - raw, _ := json.Marshal(storedBrowserPush{sub, "https://wiki.example"}) + raw, _ := json.Marshal(storedBrowserPush{Subscription: sub, Origin: "https://wiki.example"}) if err := s.store.SaveNotificationPush(ctx, domain, owner, token, sub.Endpoint, string(raw), true); err != nil { t.Fatal(err) } diff --git a/internal/server/ordoschnellic.go b/internal/server/ordoschnellic.go new file mode 100644 index 0000000..d17b6c8 --- /dev/null +++ b/internal/server/ordoschnellic.go @@ -0,0 +1,49 @@ +package server + +import ( + "context" + "fmt" + "net/http" + "time" + + "pageswiki/internal/ordoschnellic" + "pageswiki/internal/store" +) + +type notificationDeliveryQueue struct{ store *store.Store } + +func (q notificationDeliveryQueue) Cleanup(ctx context.Context, at time.Time) error { + return q.store.CleanupNotificationPush(ctx, at) +} +func (q notificationDeliveryQueue) Claim(ctx context.Context, at time.Time) (*ordoschnellic.PushDelivery, error) { + return q.store.ClaimNotificationPushAt(ctx, at) +} +func (q notificationDeliveryQueue) DeferFocused(ctx context.Context, d *ordoschnellic.PushDelivery, at time.Time) error { + return q.store.DeferFocusedNotificationPushAt(ctx, d, at) +} +func (q notificationDeliveryQueue) Finish(ctx context.Context, d *ordoschnellic.PushDelivery, outcome ordoschnellic.Outcome, at time.Time) error { + return q.store.FinishNotificationPushAt(ctx, d, outcome.Retry, outcome.Expired, at) +} + +func notificationDeliveryClient() *http.Client { + // Preserve pinned public-address dialing and never follow push redirects. + client := calendarSourceClient() + client.Timeout = 5 * time.Second + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + return client +} + +func (s *Server) deliverNotificationPushContext(ctx context.Context, job *store.EventJob) error { + dispatcher := ordoschnellic.Dispatcher{ + Queue: notificationDeliveryQueue{s.store}, + Sender: ordoschnellic.WebPush{Keys: s.notificationPushKeys, Client: notificationDeliveryClient()}, + } + stats, err := dispatcher.Run(ctx) + if err != nil { + return err + } + if stats.Sent+stats.Failed > 0 { + _ = s.store.AddEventLog(job.UUID, job.EventUUID, "info", fmt.Sprintf("component=%s push sent=%d failed=%d (no payload or endpoint logged)", ordoschnellic.Component, stats.Sent, stats.Failed)) + } + return nil +} diff --git a/internal/server/ordoschnellic_test.go b/internal/server/ordoschnellic_test.go new file mode 100644 index 0000000..2dd7a03 --- /dev/null +++ b/internal/server/ordoschnellic_test.go @@ -0,0 +1,235 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "pageswiki/internal/ordoschnellic" + "pageswiki/internal/store" +) + +func queuedPushFixture(t *testing.T, focused bool) (*Server, store.Notification, time.Time) { + t.Helper() + s, token, domain, owner := taskFixture(t) + sub := testPushSubscription(t) + raw, err := json.Marshal(storedBrowserPush{Subscription: sub, Origin: "https://wiki.example"}) + if err != nil { + t.Fatal(err) + } + if err = s.store.SaveNotificationPush(context.Background(), domain, owner, token, sub.Endpoint, string(raw), focused); err != nil { + t.Fatal(err) + } + tx, err := s.store.DB.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + n, _, err := s.store.CreateNotificationTx(tx, store.Notification{DomainUUID: domain, RecipientUUID: owner, EventUUID: uuid.NewString(), Kind: "task.due", Message: "synthetic private fixture"}) + if err != nil { + t.Fatal(err) + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + return s, n, time.Now() +} + +func TestOrdoschnellicQueueLeaseAndStaleCompletion(t *testing.T) { + s, _, at := queuedPushFixture(t, false) + ctx := context.Background() + var wg sync.WaitGroup + claims := make(chan *store.NotificationPushDelivery, 8) + errs := make(chan error, 8) + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { defer wg.Done(); d, err := s.store.ClaimNotificationPushAt(ctx, at); claims <- d; errs <- err }() + } + wg.Wait() + close(claims) + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } + var first *store.NotificationPushDelivery + count := 0 + for d := range claims { + if d != nil { + first = d + count++ + } + } + if count != 1 || first.Attempt != 1 { + t.Fatalf("claims=%d first=%+v", count, first) + } + if d, err := s.store.ClaimNotificationPushAt(ctx, at.Add(ordoschnellic.PushLeaseDuration-time.Millisecond)); err != nil || d != nil { + t.Fatal("lease not held", d, err) + } + second, err := s.store.ClaimNotificationPushAt(ctx, at.Add(ordoschnellic.PushLeaseDuration)) + if err != nil || second == nil || second.Attempt != 2 { + t.Fatal("lease recovery", second, err) + } + if err = s.store.FinishNotificationPushAt(ctx, first, false, false, at); err != nil { + t.Fatal(err) + } + var attempt int + if err = s.store.DB.QueryRow(`SELECT attempts FROM notification_push_queue`).Scan(&attempt); err != nil || attempt != 2 { + t.Fatal("stale completion removed current claim", attempt, err) + } + if err = s.store.FinishNotificationPushAt(ctx, second, true, false, at); err != nil { + t.Fatal(err) + } + var next, lease int64 + if err = s.store.DB.QueryRow(`SELECT next_at_ms,lease_until_ms FROM notification_push_queue`).Scan(&next, &lease); err != nil || next != at.Add(4*time.Minute).UnixMilli() || lease != 0 { + t.Fatal("retry state", next, lease, err) + } +} + +func TestOrdoschnellicDispatcherDatabaseOutcomes(t *testing.T) { + for _, test := range []struct { + name string + code int + focused bool + queue, subscription, attempts int + }{{"sent", 201, false, 0, 1, 0}, {"retry", 429, false, 1, 1, 1}, {"expired", 410, false, 0, 0, 0}, {"focused", 201, true, 1, 1, 0}} { + t.Run(test.name, func(t *testing.T) { + s, n, at := queuedPushFixture(t, test.focused) + sends := 0 + client := pushHTTPFixture(func(*http.Request) (*http.Response, error) { + sends++ + return &http.Response{StatusCode: test.code, Body: io.NopCloser(strings.NewReader(""))}, nil + }) + stats, err := (ordoschnellic.Dispatcher{Queue: notificationDeliveryQueue{s.store}, Sender: ordoschnellic.WebPush{Keys: s.notificationPushKeys, Client: client}, Now: func() time.Time { return at }}).Run(context.Background()) + if err != nil { + t.Fatal(err) + } + var queue, subscription, inbox int + for _, query := range []struct { + sql string + out *int + }{{`SELECT COUNT(*) FROM notification_push_queue`, &queue}, {`SELECT COUNT(*) FROM notification_push_subscriptions`, &subscription}, {`SELECT COUNT(*) FROM notification_inbox WHERE uuid=?`, &inbox}} { + args := []any{} + if query.out == &inbox { + args = append(args, n.UUID) + } + if err = s.store.DB.QueryRow(query.sql, args...).Scan(query.out); err != nil { + t.Fatal(err) + } + } + if queue != test.queue || subscription != test.subscription || inbox != 1 { + t.Fatalf("queue=%d subscription=%d inbox=%d", queue, subscription, inbox) + } + if test.focused { + if sends != 0 || stats != (ordoschnellic.Stats{}) { + t.Fatal("foreground sent", sends, stats) + } + } else if sends != 1 || stats.Sent+stats.Failed != 1 { + t.Fatal(sends, stats) + } + if queue > 0 { + var attempts int + var next int64 + if err = s.store.DB.QueryRow(`SELECT attempts,next_at_ms FROM notification_push_queue`).Scan(&attempts, &next); err != nil { + t.Fatal(err) + } + delay := 2 * time.Minute + if test.focused { + delay = time.Minute + } + if attempts != test.attempts || next != at.Add(delay).UnixMilli() { + t.Fatal("reschedule", attempts, next) + } + } + }) + } +} + +func TestOrdoschnellicHostCancellationAndTransport(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + // No store/job is required: cancellation must prevent all persistence and I/O. + if err := (&Server{}).deliverNotificationPushContext(ctx, nil); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + client := notificationDeliveryClient() + if client.Timeout != 5*time.Second || client.Transport == nil || client.CheckRedirect(nil, nil) != http.ErrUseLastResponse { + t.Fatal("unsafe push HTTP defaults") + } +} + +func TestOrdoschnellicEligibility(t *testing.T) { + for _, test := range []struct{ name, sql string }{ + {"session", `UPDATE auth_sessions SET revoked_at_ms=1`}, + {"principal", `UPDATE principals SET state='suspended'`}, + {"membership", `UPDATE domain_memberships SET state='suspended'`}, + {"domain", `UPDATE domains SET enabled=0`}, + {"read", `UPDATE notification_inbox SET status='read'`}, + {"recipient", `UPDATE notification_inbox SET recipient_uuid='other-owner'`}, + {"scope", `UPDATE notification_inbox SET domain_uuid='other-domain'`}, + } { + t.Run(test.name, func(t *testing.T) { + s, _, at := queuedPushFixture(t, false) + if _, err := s.store.DB.Exec(test.sql); err != nil { + t.Fatal(err) + } + d, err := s.store.ClaimNotificationPushAt(context.Background(), at) + if err != nil || d == nil || d.Subscription != "" { + t.Fatalf("ineligible claim exposed delivery %+v %v", d, err) + } + }) + } +} + +func TestOrdoschnellicCleanupBoundedAndInboxPreserved(t *testing.T) { + s, n, at := queuedPushFixture(t, false) + tx, err := s.store.DB.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + for i := 0; i < 105; i++ { + _, err = tx.Exec(`INSERT INTO notification_push_subscriptions(id,domain_uuid,owner_uuid,session_hash,subscription,expires_at_ms) VALUES(?,?,?,?,?,?)`, fmt.Sprintf("expired-%d", i), n.DomainUUID, n.RecipientUUID, "synthetic-session", "{}", at.Add(-time.Hour).UnixMilli()) + if err != nil { + t.Fatal(err) + } + } + for i := 0; i < 505; i++ { + _, err = tx.Exec(`INSERT INTO notification_push_queue(subscription_id,notification_uuid,created_at_ms,next_at_ms) VALUES(?,?,?,0)`, "expired-0", fmt.Sprintf("old-%d", i), at.Add(-25*time.Hour).UnixMilli()) + if err != nil { + t.Fatal(err) + } + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + if err = s.store.CleanupNotificationPush(context.Background(), at); err != nil { + t.Fatal(err) + } + for _, query := range []struct { + sql string + want int + }{ + {`SELECT COUNT(*) FROM notification_push_subscriptions`, 6}, + {`SELECT COUNT(*) FROM notification_push_queue`, 6}, + {`SELECT COUNT(*) FROM notification_inbox`, 1}, + } { + var count int + if err = s.store.DB.QueryRow(query.sql).Scan(&count); err != nil || count != query.want { + t.Fatalf("%s: count=%d want=%d err=%v", query.sql, count, query.want, err) + } + } + s.store.ReadOnly = true + if err = s.store.CleanupNotificationPush(context.Background(), at); !errors.Is(err, store.ErrReadOnly) { + t.Fatal("readonly cleanup", err) + } +} diff --git a/internal/server/path_picker.go b/internal/server/path_picker.go index 5865c62..2119a81 100644 --- a/internal/server/path_picker.go +++ b/internal/server/path_picker.go @@ -1,9 +1,13 @@ package server import ( + "context" "encoding/json" "net/http" "strings" + "time" + + "pageswiki/internal/store" ) type pathPickerEntry struct { @@ -31,13 +35,13 @@ func (s *Server) handlePathPickerAPI(w http.ResponseWriter, r *http.Request) { } mode := strings.TrimSpace(r.URL.Query().Get("mode")) switch mode { - case "", "any", "file", "folder", "image", "image-or-folder": + case "", "any", "file", "folder", "image", "image-or-folder", "children": default: http.Error(w, "invalid picker mode", http.StatusBadRequest) return } selected := "" - if mode != "folder" && current != "/" { + if mode != "folder" && mode != "children" && current != "/" { if objectUUID, resolveErr := s.store.ResolvePathInDomain(s.domainUUIDForRequest(r), current); resolveErr == nil { if revision, revisionErr := s.latestRevisionMetadataForRequest(r, objectUUID); revisionErr == nil && (mode == "" || mode == "any" || mode == "file" || imageContentType(revision.ContentType)) { selected = current @@ -45,14 +49,30 @@ func (s *Server) handlePathPickerAPI(w http.ResponseWriter, r *http.Request) { } } } - entries, err := s.store.ListBrowserEntriesInDomain(s.domainUUIDForRequest(r), current) + prefix := r.URL.Query().Get("prefix") + if len(prefix) > 1024 { + http.Error(w, "prefix too long", http.StatusBadRequest) + return + } + var entries []store.BrowserEntry + var err error + truncated := false + if mode == "children" { + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + entries, truncated, err = s.store.ListPathCompletionNames(ctx, s.domainUUIDForRequest(r), current, prefix) + } else { + entries, err = s.store.ListBrowserEntriesInDomain(s.domainUUIDForRequest(r), current) + } if err != nil { http.Error(w, "path listing unavailable", http.StatusInternalServerError) return } result := make([]pathPickerEntry, 0, min(len(entries), 250)) - truncated := false for _, entry := range entries { + if prefix != "" && !strings.HasPrefix(strings.ToLower(entry.Name), strings.ToLower(prefix)) { + continue + } if !s.requestCanReadPath(r, entry.Path) { continue } diff --git a/internal/server/print_handler.go b/internal/server/print_handler.go index 113b278..c291e17 100644 --- a/internal/server/print_handler.go +++ b/internal/server/print_handler.go @@ -1,6 +1,7 @@ package server import ( + "context" "database/sql" "errors" "html/template" @@ -28,7 +29,7 @@ func (s *Server) handlePrint(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - view, err := s.printContentView(revision, semanticPath) + view, err := s.printContentViewContext(r.Context(), revision, semanticPath) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -84,7 +85,11 @@ func (s *Server) resolvePrintRevision(r *http.Request, address string) (*store.R } func (s *Server) printContentView(metadata *store.Revision, semanticPath string) (contentView, error) { - size, err := s.store.DocumentSize(metadata.DocumentHash) + return s.printContentViewContext(context.Background(), metadata, semanticPath) +} + +func (s *Server) printContentViewContext(ctx context.Context, metadata *store.Revision, semanticPath string) (contentView, error) { + size, err := s.store.DocumentSizeContext(ctx, metadata.DocumentHash) if err != nil { return contentView{}, err } @@ -108,7 +113,7 @@ func (s *Server) printContentView(metadata *store.Revision, semanticPath string) return view, nil } if adapter.WindowableText && size > maxPrintTextBytes { - window, err := s.store.ReadDocumentWindow(metadata.DocumentHash, 0, maxPrintTextBytes) + window, err := s.store.ReadDocumentWindowContext(ctx, metadata.DocumentHash, 0, maxPrintTextBytes) if err != nil { return contentView{}, err } diff --git a/internal/server/priority_navigation_test.go b/internal/server/priority_navigation_test.go new file mode 100644 index 0000000..b93f436 --- /dev/null +++ b/internal/server/priority_navigation_test.go @@ -0,0 +1,63 @@ +package server + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +func TestBrowserAddressCommandsJS(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node required") + } + t.Chdir("../..") + if out, err := exec.Command(node, "--test", "internal/server/testdata/browser-address.test.cjs").CombinedOutput(); err != nil { + t.Fatalf("%v\n%s", err, out) + } +} + +func TestPriorityNavigationChromium(t *testing.T) { + if os.Getenv("PAGESWIKI_BROWSER_TEST") != "1" { + t.Skip("opt-in Chromium") + } + for _, layout := range []string{"classic", "search", "directory"} { + t.Run(layout, func(t *testing.T) { priorityNavigationChromium(t, layout) }) + } +} + +func priorityNavigationChromium(t *testing.T, layout string) { + s, token, domain := configFixture(t) + s.mux.HandleFunc("/v/browser/", s.handleBrowser) + s.mux.HandleFunc("/e/", s.handleEditorRoute) + s.mux.HandleFunc("/", s.handleHome) + if err := s.store.SetConfigInDomain(domain, "home_layout", layout); err != nil { + t.Fatal(err) + } + script, err := os.ReadFile("internal/server/testdata/priority-navigation.js") + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Host = "localhost" + r.Header.Set("Authorization", "Bearer "+token) + if r.URL.Path == "/fixture" { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte("")) + return + } + s.withAuthorization(s.mux).ServeHTTP(w, r) + })) + defer server.Close() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + target := strings.Replace(server.URL, "127.0.0.1", "localhost", 1) + "/fixture" + if out, err := exec.CommandContext(ctx, "node", "internal/server/testdata/vim-browser-smoke.mjs", target, t.TempDir()).CombinedOutput(); err != nil { + t.Fatalf("navigation: %v\n%s", err, out) + } +} diff --git a/internal/server/profile_contact_test.go b/internal/server/profile_contact_test.go new file mode 100644 index 0000000..ceb3163 --- /dev/null +++ b/internal/server/profile_contact_test.go @@ -0,0 +1,230 @@ +package server + +import ( + "context" + "errors" + "html" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "strings" + "testing" + "time" + + "pageswiki/internal/store" +) + +func TestProfileContactPrivacyAndRevocation(t *testing.T) { + s, token, domain := configFixture(t) + ctx := context.Background() + admin, err := s.store.PrincipalForSession(domain, token) + if err != nil { + t.Fatal(err) + } + owner, err := s.store.CreatePasswordAccount(domain, "phone-owner", "strong-password-for-test", "Phone Owner", "") + if err != nil { + t.Fatal(err) + } + contact := store.ProfileContact{PhoneDisplay: "+81 (90) 1234-5678", Visibility: "private"} + if err := s.store.SaveProfileContact(ctx, domain, owner.UUID, contact, false); err != nil { + t.Fatal(err) + } + get := func(viewer string, want bool) store.ProfileContact { + t.Helper() + got, err := s.store.ProfileContactForViewer(ctx, domain, owner.UUID, viewer) + if err != nil || (got.PhoneNumber != "") != want { + t.Fatalf("viewer=%q visible=%v want=%v err=%v", viewer, got.PhoneNumber != "", want, err) + } + if !want && got != (store.ProfileContact{}) { + t.Fatal("hidden contact metadata leaked") + } + return got + } + contact = get(owner.UUID, true) + get(admin.UUID, false) // domain administrator is not consent + get("", false) + get("foreign-user", false) + stale := contact + stale.Version = 0 + if err := s.store.SaveProfileContact(ctx, domain, owner.UUID, stale, false); !errors.Is(err, store.ErrGovernanceConflict) { + t.Fatalf("stale write: %v", err) + } + contact.Visibility = "domain" + if err := s.store.SaveProfileContact(ctx, domain, owner.UUID, contact, false); err != nil { + t.Fatal(err) + } + get(admin.UUID, true) + get("", false) + contact = get(owner.UUID, true) + contact.Visibility = "public" + if err := s.store.SaveProfileContact(ctx, domain, owner.UUID, contact, true); !errors.Is(err, store.ErrGovernanceCeiling) { + t.Fatalf("ceiling bypass: %v", err) + } + enableDomainPublicCeiling(t, s, domain, admin.UUID) + policy, err := s.store.UserSpacePolicy(ctx, domain, owner.UUID) + if err != nil { + t.Fatal(err) + } + policy.ProfileVisibility = "public" + policy, err = s.store.SaveUserSpacePolicy(ctx, domain, owner.UUID, policy) + if err != nil { + t.Fatal(err) + } + if err := s.store.SaveProfileContact(ctx, domain, owner.UUID, contact, false); err == nil { + t.Fatal("public without confirmation") + } + if err := s.store.SaveProfileContact(ctx, domain, owner.UUID, contact, true); err != nil { + t.Fatal(err) + } + get("", true) + changed := get(owner.UUID, true) + changed.PhoneDisplay = "03-1234-5678" + if err := s.store.SaveProfileContact(ctx, domain, owner.UUID, changed, false); err == nil { + t.Fatal("new public phone did not require renewed confirmation") + } + governance, err := s.store.CityGovernance(ctx, domain, admin.UUID, "domain", domain) + if err != nil { + t.Fatal(err) + } + closed := "authenticated_acl" + if _, err = s.store.ChangeCityGovernance(ctx, domain, admin.UUID, "domain", domain, store.CityGovernanceChange{Epoch: governance.Epoch, TopologyVersion: governance.TopologyVersion, Policy: &closed}, false); err != nil { + t.Fatal(err) + } + get("", false) + enableDomainPublicCeiling(t, s, domain, admin.UUID) + get("", true) + policy.ProfileVisibility = "private" + if _, err = s.store.SaveUserSpacePolicy(ctx, domain, owner.UUID, policy); err != nil { + t.Fatal(err) + } + get("", false) + get(admin.UUID, false) + contact = get(owner.UUID, true) + if err = s.store.SetPrincipalState(domain, owner.UUID, "readonly"); err != nil { + t.Fatal(err) + } + if err = s.store.SaveProfileContact(ctx, domain, owner.UUID, contact, false); err == nil { + t.Fatal("readonly writer accepted") + } + if err = s.store.SetPrincipalState(domain, owner.UUID, "active"); err != nil { + t.Fatal(err) + } + contact.PhoneDisplay = "" + if err = s.store.SaveProfileContact(ctx, domain, owner.UUID, contact, false); err != nil { + t.Fatal(err) + } + cleared, err := s.store.ProfileContactForViewer(ctx, domain, owner.UUID, owner.UUID) + if err != nil || cleared.PhoneNumber != "" || cleared.PhoneDisplay != "" || cleared.Visibility != "private" { + t.Fatalf("not cleared: %+v %v", cleared, err) + } + if err = s.store.SaveProfileContact(ctx, "unrelated-domain", owner.UUID, store.ProfileContact{Visibility: "private", PhoneDisplay: "12345"}, false); err == nil { + t.Fatal("cross-domain write accepted") + } +} + +func TestProfileContactChromium(t *testing.T) { + if os.Getenv("PAGESWIKI_BROWSER_TEST") != "1" { + t.Skip("opt-in Chromium") + } + s, token, _ := configFixture(t) + s.mux.HandleFunc("/u/", s.handleUserProfile) + script, err := os.ReadFile("internal/server/testdata/profile-contact.js") + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Host = "localhost" + r.Header.Set("Authorization", "Bearer "+token) + if r.URL.Path == "/fixture" { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte("")) + return + } + s.withAuthorization(s.mux).ServeHTTP(w, r) + })) + defer server.Close() + for _, width := range []string{"1440", "375"} { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + screenshot := "" + if base := os.Getenv("PAGESWIKI_CONTACT_SCREENSHOT"); base != "" { + screenshot = base + "-" + width + ".png" + } + out, err := exec.CommandContext(ctx, "node", "internal/server/testdata/vim-browser-smoke.mjs", server.URL+"/fixture", t.TempDir(), screenshot, width, "900").CombinedOutput() + cancel() + if err != nil { + t.Fatalf("contact width=%s: %v\n%s", width, err, out) + } + } +} + +func TestProfileContactHTTP(t *testing.T) { + s, token, domain := configFixture(t) + owner, err := s.store.PrincipalForSession(domain, token) + if err != nil { + t.Fatal(err) + } + s.mux.HandleFunc("/u/", s.handleUserProfile) + call := func(method, target string, form url.Values, authenticated bool) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, "http://localhost"+target, strings.NewReader(form.Encode())) + if method == "POST" { + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + if authenticated { + r.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + s.withAuthorization(s.mux).ServeHTTP(w, r) + return w + } + form := url.Values{"action": {"contact"}, "contact_version": {"0"}, "phone": {"+81 (90) 1234-5678"}, "contact_visibility": {"private"}} + w := call("POST", "/u/admin/_/edit", form, true) + if w.Code != 303 || w.Header().Get("Location") != "/u/admin/_/edit?contact=saved#contact" { + t.Fatalf("save: %d %s", w.Code, w.Body.String()) + } + if _, err := s.store.CreatePasswordAccount(domain, "other-phone", "strong-password-test", "Other", ""); err != nil { + t.Fatal(err) + } + if other := call("POST", "/u/other-phone/_/edit", form, true); other.Code != 403 { + t.Fatalf("editing another account: %d", other.Code) + } + w = call("GET", "/u/admin", nil, true) + if w.Code != 200 || !strings.Contains(html.UnescapeString(w.Body.String()), `href="tel:+819012345678"`) || w.Header().Get("Cache-Control") != "private, no-store" { + t.Fatalf("owner view: %d %s", w.Code, w.Body.String()) + } + if w = call("POST", "/u/admin/_/edit", form, true); w.Code != 409 { + t.Fatalf("conflict: %d", w.Code) + } + form.Set("contact_version", "1") + form.Set("phone", "123;ext=456") + if w = call("POST", "/u/admin/_/edit", form, true); w.Code != 400 { + t.Fatalf("invalid: %d", w.Code) + } + enableDomainPublicCeiling(t, s, domain, owner.UUID) + policy, err := s.store.UserSpacePolicy(context.Background(), domain, owner.UUID) + if err != nil { + t.Fatal(err) + } + policy.ProfileVisibility = "public" + if _, err = s.store.SaveUserSpacePolicy(context.Background(), domain, owner.UUID, policy); err != nil { + t.Fatal(err) + } + w = call("GET", "/u/admin", nil, false) + if w.Code != 200 || strings.Contains(w.Body.String(), "1234-5678") || strings.Contains(w.Body.String(), "tel:") { + t.Fatal("private phone leaked in public profile", w.Code) + } + form.Set("phone", "+81 (90) 1234-5678") + form.Set("contact_visibility", "public") + form.Set("contact_confirm_public", "on") + if w = call("POST", "/u/admin/_/edit", form, true); w.Code != 303 { + t.Fatalf("public save: %d %s", w.Code, w.Body.String()) + } + w = call("GET", "/u/admin", nil, false) + if w.Code != 200 || !strings.Contains(html.UnescapeString(w.Body.String()), `href="tel:+819012345678"`) { + t.Fatal("public phone missing", w.Code) + } + if got := profileContactHref(store.ProfileContact{PhoneNumber: "javascript:alert(1)"}); got != "" { + t.Fatal("unsafe phone href") + } +} diff --git a/internal/server/profile_handler.go b/internal/server/profile_handler.go index abff6ef..b261eea 100644 --- a/internal/server/profile_handler.go +++ b/internal/server/profile_handler.go @@ -108,6 +108,24 @@ func (s *Server) handleUserProfile(w http.ResponseWriter, r *http.Request) { return } action := r.FormValue("action") + if action == "contact" { + version, err := strconv.ParseInt(r.FormValue("contact_version"), 10, 64) + if err != nil { + http.Error(w, "invalid contact version", http.StatusBadRequest) + return + } + err = s.store.SaveProfileContact(r.Context(), domainUUID, current.UUID, store.ProfileContact{PhoneDisplay: r.FormValue("phone"), Visibility: r.FormValue("contact_visibility"), Version: version}, r.FormValue("contact_confirm_public") == "on") + if err != nil { + status := http.StatusBadRequest + if errors.Is(err, store.ErrGovernanceConflict) { + status = http.StatusConflict + } + http.Error(w, translate(s.getLocale(domainUUID), err.Error()), status) + return + } + http.Redirect(w, r, "/u/"+url.PathEscape(profile.Username)+"/_/edit?contact=saved#contact", http.StatusSeeOther) + return + } if action == "visibility" { version, parseErr := strconv.ParseInt(r.FormValue("visibility_version"), 10, 64) if parseErr != nil { @@ -235,7 +253,22 @@ func (s *Server) handleUserProfile(w http.ResponseWriter, r *http.Request) { if editing { userPages, _ = s.store.ListUserSpacePages(r.Context(), domainUUID, profile.UUID, 512) } - s.renderWithContext(w, r, "profile.gohtml", map[string]any{"Principal": profile, "Editing": editing, "Saved": r.URL.Query().Get("saved") == "1", "VisibilitySaved": r.URL.Query().Get("visibility") == "saved", "Handle": profile.Username + "@" + profileHost(r), "RenderedHTML": template.HTML(rendered), "TOTPEnabled": totpEnabled, "TOTPSecret": pendingSecret, "TOTPUri": store.TOTPURI(pendingSecret, profile.Username, s.getWikiName(domainUUID)), "TOTPStatus": r.URL.Query().Get("totp"), "UserSpacePolicy": visibility, "UserSpacePages": userPages, "Locale": s.getLocale(domainUUID)}) + contact, contactErr := s.store.ProfileContactForViewer(r.Context(), domainUUID, profile.UUID, viewer.UUID) + if contactErr != nil { + http.Error(w, "contact unavailable", http.StatusServiceUnavailable) + return + } + w.Header().Set("Cache-Control", "private, no-store") + s.renderWithContext(w, r, "profile.gohtml", map[string]any{"Principal": profile, "Editing": editing, "Saved": r.URL.Query().Get("saved") == "1", "VisibilitySaved": r.URL.Query().Get("visibility") == "saved", "Handle": profile.Username + "@" + profileHost(r), "RenderedHTML": template.HTML(rendered), "TOTPEnabled": totpEnabled, "TOTPSecret": pendingSecret, "TOTPUri": store.TOTPURI(pendingSecret, profile.Username, s.getWikiName(domainUUID)), "TOTPStatus": r.URL.Query().Get("totp"), "UserSpacePolicy": visibility, "UserSpacePages": userPages, "Contact": contact, "ContactHref": profileContactHref(contact), "ContactSaved": r.URL.Query().Get("contact") == "saved", "Locale": s.getLocale(domainUUID)}) +} + +func profileContactHref(contact store.ProfileContact) template.URL { + _, number, err := store.NormalizeContactPhone(contact.PhoneNumber) + if err != nil || number == "" { + return "" + } + // Only digits and an optional leading '+' survive validation above. + return template.URL("tel:" + number) } func (s *Server) serveProfileAvatar(w http.ResponseWriter, r *http.Request, principalUUID string) { diff --git a/internal/server/projection_metadata.go b/internal/server/projection_metadata.go index 140a659..8a42397 100644 --- a/internal/server/projection_metadata.go +++ b/internal/server/projection_metadata.go @@ -10,7 +10,6 @@ import ( "io" "net/http" "net/url" - "os" "os/exec" "path/filepath" "strconv" @@ -157,19 +156,14 @@ func (s *Server) generateMetadataProjection(payload metadataProjectionPayload) e func (s *Server) inspectProjectionMetadata(payload metadataProjectionPayload) (metadataProjectionResult, error) { result := metadataProjectionResult{Status: "ready"} - source, err := s.store.DocumentPath(payload.DocumentHash) - if err != nil || source == "" { - return result, errors.New("metadata projection requires immutable Document storage") - } - info, err := os.Stat(source) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + lease, err := s.store.MaterializeDocument(ctx, payload.DocumentHash, maxMetadataInputBytes) if err != nil { return result, err } - if !info.Mode().IsRegular() || info.Size() > maxMetadataInputBytes { - return result, errors.New("metadata source exceeds the 512 MiB limit") - } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() + defer lease.Close() + source := lease.Path if payload.Kind == "pdf" { output, truncated, err := runMetadataCommand(ctx, 64<<10, "pdfinfo", "-enc", "UTF-8", source) if err != nil || truncated { diff --git a/internal/server/public_navigation.go b/internal/server/public_navigation.go index b2888e5..17d5cea 100644 --- a/internal/server/public_navigation.go +++ b/internal/server/public_navigation.go @@ -395,7 +395,7 @@ func (s *Server) expandPublicMacro(r *http.Request, macro, currentPath string) s continue } if match[1] == "spot" { - return link(sp.DetailHref, sp.Title) + return link(sp.DetailHref, sp.DisplayTitle) } var out strings.Builder out.WriteString(`
    `) @@ -458,6 +458,9 @@ func (s *Server) publicCityProjection(r *http.Request) (store.CityMap, error) { sp := &st.Spots[pi] sp.Name = path.Base(sp.PathPrefix) sp.Title = sp.Name + // Recompute from public-safe fields; the authenticated display + // projection can contain a private custom title. + sp.DisplayTitle = store.CitySpotDisplayTitle(sp.Name, sp.Name, sp.PathPrefix, st.Name) sp.Summary = "" sp.State = "active" sp.SortKey = 0 diff --git a/internal/server/public_periodical.go b/internal/server/public_periodical.go index c9ada8b..233687d 100644 --- a/internal/server/public_periodical.go +++ b/internal/server/public_periodical.go @@ -135,7 +135,7 @@ func (s *Server) servePublicPeriodical(w http.ResponseWriter, r *http.Request) b if i >= 20 || titles[e.Path] != "" || !strings.HasPrefix(v.ContentType, "text/html") { continue } - reader, err := s.store.OpenDocument(v.DocumentHash) + reader, err := s.store.OpenDocumentContext(r.Context(), v.DocumentHash) if err != nil { http.Error(w, "published document unavailable", 503) return true diff --git a/internal/server/public_periodical_view.go b/internal/server/public_periodical_view.go index 5a0de16..975f875 100644 --- a/internal/server/public_periodical_view.go +++ b/internal/server/public_periodical_view.go @@ -71,7 +71,7 @@ func (s *Server) renderPublicPeriodical(w http.ResponseWriter, r *http.Request, http.NotFound(w, r) return } - reader, err := s.store.OpenDocument(v.DocumentHash) + reader, err := s.store.OpenDocumentContext(r.Context(), v.DocumentHash) if err != nil { http.Error(w, "published document unavailable", 503) return @@ -291,7 +291,7 @@ func (s *Server) renderPeriodicalRSS(w http.ResponseWriter, r *http.Request, sit http.Error(w, "publication changed; retry", 503) return } - reader, err := s.store.OpenDocument(v.DocumentHash) + reader, err := s.store.OpenDocumentContext(r.Context(), v.DocumentHash) if err != nil { http.Error(w, "published document unavailable", 503) return diff --git a/internal/server/raw_view.go b/internal/server/raw_view.go index 7133552..22341f0 100644 --- a/internal/server/raw_view.go +++ b/internal/server/raw_view.go @@ -144,7 +144,7 @@ func (s *Server) serveAssetHead(w http.ResponseWriter, r *http.Request, asset *s http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } - reader, err := s.store.OpenDocument(asset.HeadDocumentHash) + reader, err := s.store.OpenDocumentContext(r.Context(), asset.HeadDocumentHash) if err != nil { http.NotFound(w, r) return diff --git a/internal/server/remote_cli_test.go b/internal/server/remote_cli_test.go new file mode 100644 index 0000000..ec47bb1 --- /dev/null +++ b/internal/server/remote_cli_test.go @@ -0,0 +1,35 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os/exec" + "testing" + "time" +) + +func TestRemoteCLIHTTPIntegration(t *testing.T) { + if _, err := exec.LookPath("python3"); err != nil { + t.Skip("python3 not installed") + } + s, _, _, _ := editorFixture(t) + s.mux.HandleFunc("/_/auth/", s.handleAuth) + s.mux.HandleFunc("/v/raw/", s.handleRawView) + s.mux.HandleFunc("/_/api/v1/path-picker", s.handlePathPickerAPI) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Host = "localhost" + s.withAuthorization(s.mux).ServeHTTP(w, r) + })) + defer server.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "python3", "pages-cli/test_pages.py", "--integration") + input, _ := json.Marshal(map[string]string{"origin": server.URL}) + cmd.Stdin = bytes.NewReader(input) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("remote CLI: %v\n%s", err, out) + } +} diff --git a/internal/server/render_api.go b/internal/server/render_api.go index fc927be..c73456c 100644 --- a/internal/server/render_api.go +++ b/internal/server/render_api.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" + "pageswiki/internal/grammaratur" "pageswiki/internal/render" ) @@ -17,6 +18,7 @@ func (s *Server) handleMarkdownRenderAPI( Path string `json:"path"` Projection bool `json:"projection"` HTML *string `json:"html"` + Authoring bool `json:"authoring"` } err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 2<<20)).Decode(&req) @@ -33,7 +35,11 @@ func (s *Server) handleMarkdownRenderAPI( } html = *req.HTML } else { - html = render.Markdown(req.Markdown) + if req.Authoring { + html = grammaratur.MarkdownAuthoringHTML(req.Markdown) + } else { + html = render.Markdown(req.Markdown) + } } if req.Projection { path := req.Path diff --git a/internal/server/request_context.go b/internal/server/request_context.go index 912f904..3d06c55 100644 --- a/internal/server/request_context.go +++ b/internal/server/request_context.go @@ -2,6 +2,7 @@ package server import ( "net/http" + "net/url" "pageswiki/internal/store" ) @@ -46,6 +47,24 @@ func (s *Server) renderWithContext( data["AuthUser"] = nil data["AuthIsAdmin"] = false domainUUID := s.domainUUIDForRequest(r) + data["CommandEnabled"] = name != "home.gohtml" && name != "search_home.gohtml" && r.URL.Path != "/" + data["CommandBrowser"] = name == "browser.gohtml" + data["CommandPath"] = "/" + if semantic, ok := data["Path"].(string); ok && len(semantic) > 0 && semantic[0] == '/' { + data["CommandPath"] = semantic + } + data["QuickThemes"] = themeOptions() + data["DefaultThemeCSS"] = s.getThemeCSS(domainUUID) + data["QuickThemeID"] = "" + if cookie, err := r.Cookie("pageswiki_theme"); err == nil { + id, decodeErr := url.PathUnescape(cookie.Value) + if theme, ok := findTheme(id); ok && decodeErr == nil { + // Presentation only: never change the Domain's configured theme. + data["ThemeCSS"] = theme.CSS + data["QuickThemeID"] = theme.ID + } + } + w.Header().Add("Vary", "Cookie") data["CalendarEnabled"] = s.systemExtensionEnabled(domainUUID, "organizer.calendar") data["TasksEnabled"] = s.systemExtensionEnabled(domainUUID, "organizer.tasks") if r.URL.Path == "/" { diff --git a/internal/server/resource_io.go b/internal/server/resource_io.go index d4f52d2..413b86e 100644 --- a/internal/server/resource_io.go +++ b/internal/server/resource_io.go @@ -1,52 +1,10 @@ package server -import ( - "os" - "strconv" - "strings" +import "pageswiki/internal/seheweit" - "github.com/google/uuid" -) - -// A new process gets a new identity, even when a container reuses PID 1. -var resourceProcessID = uuid.NewString() - -type processIOCounters struct { - Scope string `json:"scope"` - Available bool `json:"available"` - ProcessID string `json:"process_id"` - ReadBytes uint64 `json:"read_bytes"` - WriteBytes uint64 `json:"write_bytes"` -} - -func readProcessIO() processIOCounters { - data, _ := os.ReadFile("/proc/self/io") - return parseProcessIO(string(data)) -} +// Compatibility facade; the process identity and parser belong to Seheweit. +type processIOCounters = seheweit.ProcessIO func parseProcessIO(data string) processIOCounters { - result := processIOCounters{Scope: "process", ProcessID: resourceProcessID} - read, write := false, false - for _, line := range strings.Split(data, "\n") { - fields := strings.Fields(line) - if len(fields) != 2 { - continue - } - if fields[0] != "read_bytes:" && fields[0] != "write_bytes:" { - continue - } - value, err := strconv.ParseUint(fields[1], 10, 64) - if err != nil { - return result - } - if fields[0] == "read_bytes:" { - result.ReadBytes = value - read = true - } else { - result.WriteBytes = value - write = true - } - } - result.Available = read && write - return result + return seheweit.ParseProcessIO(data) } diff --git a/internal/server/search_handler.go b/internal/server/search_handler.go index 87e0a30..30864fa 100644 --- a/internal/server/search_handler.go +++ b/internal/server/search_handler.go @@ -169,7 +169,7 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { // not build the full City graph or traverse its Page placements here. ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) defer cancel() - city, err := s.store.SearchCityInDomain(ctx, domain, scope, query, 30, allowed, options) + city, err := s.searchEngine().SearchCityInDomain(ctx, domain, scope, query, 30, allowed, options) if err != nil { s.renderSearchFailure(w, r, query, scope, localityKind, localityID, err) return @@ -177,7 +177,7 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { for _, result := range city { cityResults = append(cityResults, citySearchResultView(result, query, location)) } - found, err := s.store.SearchPathsInDomainWithOptions( + found, err := s.searchEngine().SearchPathsInDomainWithOptions( ctx, domain, scope, query, 100, allowed, options, ) if err != nil { diff --git a/internal/server/search_options.go b/internal/server/search_options.go index 174d1b5..3b09b2a 100644 --- a/internal/server/search_options.go +++ b/internal/server/search_options.go @@ -98,10 +98,7 @@ type searchCityViewResult struct { } func citySearchResultView(result store.CitySearchResult, query string, location *time.Location) searchCityViewResult { - title := result.Title - if title == "" { - title = result.Name - } + title := result.DisplayTitle() context := []string{} if result.QuartierTitle != "" { context = append(context, result.QuartierTitle) @@ -112,12 +109,12 @@ func citySearchResultView(result store.CitySearchResult, query string, location return searchCityViewResult{ CitySearchResult: result, KindKey: "city." + result.Kind, - Href: cityDetailHref(result.Kind, result.Name, result.QuartierName, result.UUID), - PathHref: (&url.URL{Path: result.PathPrefix}).EscapedPath(), + Href: cityContentHref(result), + PathHref: cityContentHref(result), ContextLabel: strings.Join(context, " / "), Updated: time.UnixMilli(result.UpdatedAtMS).In(location).Format("2006-01-02 15:04"), NameParts: highlightSearchText(title, query), - PathParts: highlightSearchText(result.PathPrefix, query), + PathParts: highlightSearchText(result.ContentPath(), query), ExcerptParts: highlightSearchText(searchExcerpt(result.Description, "text/plain", query), query), } } diff --git a/internal/server/search_options_test.go b/internal/server/search_options_test.go index da7d929..ec060ff 100644 --- a/internal/server/search_options_test.go +++ b/internal/server/search_options_test.go @@ -102,13 +102,13 @@ func TestSearchCityResultsPrecedePagesAndHaveSafeLinks(t *testing.T) { } city := body[cityStart:pagesStart] for _, value := range []string{`data-city-search-kind="quartier"`, `data-city-search-kind="street"`, `data-city-search-kind="spot"`, - `href="/v/city/q/nebula-campus"`, `href="/v/city/s/nebula-campus/nebula-way"`, `href="/v/city/p/nebula-spot"`, - `href="/city%3F%23%25"`, `Nebula`, `<img src=x onerror=alert(1)>`} { + `href="/v/browser/city%3F%23%25"`, `href="/v/browser/city%3F%23%25/street"`, `href="/v/browser/city%3F%23%25/street/spot"`, + `Nebula`, `<img src=x onerror=alert(1)>`} { if !strings.Contains(city, value) { t.Errorf("missing City result element %q", value) } } - for _, value := range []string{`data-child-path`, `children-entry`, `CITY_HIDDEN_MARKER`, ` { + assert.equal(destination('/docs/new').href, '/v/browser/docs/new'); + assert.equal(destination('child', '/docs').href, '/v/browser/docs/child'); + assert.equal(destination('cd ../new', '/docs/old').path, '/docs/new'); + assert.equal(destination('cd', '/docs').path, '/'); + assert.equal(destination('cd ../../../../', '/docs').path, '/'); + assert.equal(destination('./a//b/./c/..', '/docs').path, '/docs/a/b'); +}); +test('editor and viewer commands preserve names and avoid adapter-route ambiguity', () => { + assert.equal(destination('vi plain/example', '/').href, '/e/-/plain/example'); + assert.equal(destination('view', '/docs/page').href, '/docs/page'); + assert.equal(destination('vi file name', '/docs').href, '/e/-/docs/file%20name'); + assert.equal(destination('vi 日記 #1?%25', '/docs').href, '/e/-/docs/%E6%97%A5%E8%A8%98%20%231%3F%2525'); + assert.equal(destination('/cd', '/docs').path, '/cd'); + assert.equal(destination('view a;rm -rf', '/').href, '/a%3Brm%20-rf'); +}); +test('schemes, network paths and controls are not accepted', () => { + for (const value of ['https://outside.test', '//outside.test', 'javascript:alert(1)', 'file:///etc/passwd', 'a\\b', '/a\n/b', '/a\u0000b']) { + assert.throws(() => resolvePath(value), /invalid_path/); + } + assert.equal(destination('view /%2f%2fevil.test').href, '/%252f%252fevil.test'); +}); diff --git a/internal/server/testdata/city-labels.js b/internal/server/testdata/city-labels.js index 9906796..88954cd 100644 --- a/internal/server/testdata/city-labels.js +++ b/internal/server/testdata/city-labels.js @@ -44,6 +44,13 @@ window.addEventListener('load', async () => { checkLabels('.city-generated-street-label'); activate(document.querySelector('.city-street-tile')); await wait(() => document.querySelector('.city-undefined-page-label')); + const entrance = document.querySelector('.city-spot-point-label'); + if (!entrance || /undefined|未定義|未分類/i.test(entrance.textContent)) throw new Error('Internal placeholder label is visible'); + activate(document.querySelector('.city-spot-entry')); + await wait(() => document.querySelector('.city-spot-monument')); + if (document.querySelector('[data-city-trail] [aria-current="location"]')?.textContent !== entrance.querySelector('title')?.textContent) throw new Error('Spot breadcrumb differs from display title'); + activate(document.querySelector('[data-city-level="street"]')); + await wait(() => document.querySelector('.city-undefined-page-label')); checkLabels('.city-generated-street-label,.city-undefined-page-label'); toggleLabels(false); if (getComputedStyle(document.querySelector('.city-undefined-page-label')).display !== 'none') diff --git a/internal/server/testdata/command-highlight.js b/internal/server/testdata/command-highlight.js new file mode 100644 index 0000000..38f1674 --- /dev/null +++ b/internal/server/testdata/command-highlight.js @@ -0,0 +1,63 @@ +(async () => { + const assert=(ok,message)=>{if(!ok)throw Error(message)}; + const wait=async test=>{for(let n=0;n<160;n++){if(test())return;await new Promise(r=>setTimeout(r,25))}throw Error('Timed out: '+test)}; + const frame=document.createElement('iframe');frame.style='width:100%;height:930px;border:0';document.body.append(frame); + const load=path=>new Promise(resolve=>{frame.onload=resolve;frame.src=path}); + const key=(d,target,key,extras={})=>target.dispatchEvent(new frame.contentWindow.KeyboardEvent('keydown',{key,bubbles:true,cancelable:true,...extras})); + try { + await load('/');let d=frame.contentDocument; + assert(!d.querySelector('[data-command-bar]'),'Home has command interception'); + await load('/v/browser/notes');d=frame.contentDocument; + let bar=d.querySelector('[data-command-bar]'),input=bar.querySelector('input'); + assert(!bar.hidden && bar.parentElement===d.body,'Command bar not at body level'); + assert(d.querySelector('.topbar [data-browser-address]'),'Address not integrated with topbar'); + assert(!!d.querySelector('[data-browser-address]').getClientRects().length === (innerWidth>1100),'Responsive address visibility'); + key(d,d.body,'c');assert(input===d.activeElement&&input.value==='c','Type-to-command lost first character'); + input.value='cd al';input.dispatchEvent(new Event('input',{bubbles:true}));key(d,input,'Tab'); + await wait(()=>input.value==='cd alpha');key(d,input,'Tab');assert(input.value==='cd alpine','Tab did not cycle'); + key(d,input,'Tab',{shiftKey:true});assert(input.value==='cd alpha','Shift Tab did not cycle back'); + key(d,input,'Escape');assert(!bar.hidden&&input.value==='','Browser Escape must clear, not remove bar'); + input.focus();input.value='cd /not-committed';input.dispatchEvent(new frame.contentWindow.CompositionEvent('compositionstart',{bubbles:true}));bar.requestSubmit();assert(frame.contentWindow.location.pathname==='/v/browser/notes','IME confirmation navigated');input.dispatchEvent(new frame.contentWindow.CompositionEvent('compositionend',{bubbles:true}));input.value=''; + const other=d.createElement('input');d.body.append(other);other.focus();key(d,other,'x');assert(input.value===''&&d.activeElement===other,'Other input intercepted');other.remove(); + const modal=d.createElement('dialog');d.body.append(modal);modal.showModal();key(d,d.body,'x');assert(input.value==='','Modal intercepted');modal.close();modal.remove(); + const fetchBefore=frame.contentWindow.fetch; + let release; + frame.contentWindow.fetch=()=>new Promise(resolve=>{release=resolve}); + input.focus();input.value='vi stale';input.dispatchEvent(new Event('input',{bubbles:true}));key(d,input,'Tab'); + key(d,input,'Escape');release(new Response(JSON.stringify({entries:[{name:'stale',kind:'file'}]}),{headers:{'Content-Type':'application/json'}})); + await new Promise(r=>setTimeout(r,30));assert(input.value===''&&bar.querySelector('[role=listbox]').hidden,'Stale completion reopened list');frame.contentWindow.fetch=fetchBefore; + input.focus();input.value='cd /日本';input.dispatchEvent(new Event('input',{bubbles:true}));key(d,input,'Tab');await wait(()=>input.value==='cd /日本語 #?/'); + key(d,input,'Tab');await wait(()=>input.value==='cd /日本語 #?/child'); + input.value='cd /notes';input.dispatchEvent(new Event('input',{bubbles:true})); + await new Promise(resolve=>{frame.onload=resolve;bar.requestSubmit()});assert(frame.contentWindow.location.pathname==='/v/browser/notes','Command navigation failed'); + await load('/notes');d=frame.contentDocument;bar=d.querySelector('[data-command-bar]');input=bar.querySelector('input'); + assert(bar.hidden,'Viewer command bar initially open');key(d,d.body,'g');assert(!bar.hidden&&d.activeElement===input&&input.value==='','g activation failed'); + key(d,input,'Escape');assert(bar.hidden,'Viewer Escape failed'); + const code=d.querySelector('.page-html pre code'); + await wait(()=>code?.querySelector('.pw-syntax-keyword')); + assert(!code.querySelector('script')&&code.textContent.includes('")) + return + } + s.withAuthorization(s.mux).ServeHTTP(w, r) + })) + defer server.Close() + ctx, cancel := context.WithTimeout(context.Background(), 55*time.Second) + defer cancel() + args := []string{"internal/server/testdata/vim-browser-smoke.mjs", server.URL + "/fixture", t.TempDir()} + if shot := os.Getenv("EDITOR_THEME_SCREENSHOT"); shot != "" { + args = append(args, shot) + } + if out, err := exec.CommandContext(ctx, "node", args...).CombinedOutput(); err != nil { + t.Fatalf("theme: %v\n%s", err, out) + } +} diff --git a/internal/server/user_space_public.go b/internal/server/user_space_public.go index 7678bec..c3e0226 100644 --- a/internal/server/user_space_public.go +++ b/internal/server/user_space_public.go @@ -166,7 +166,7 @@ func (s *Server) servePublicUserProfile(w http.ResponseWriter, r *http.Request, if policy.RootVisibility == "public" { if revision, revisionErr := s.store.PublicUserSpaceRevision(publicRequest.Context(), s.domainUUIDForRequest(publicRequest), owner.Username, root, s.revisionLanguageCandidates(publicRequest)); revisionErr == nil { if view := revisionContentView(revision, root); view.Kind == "html" { - reader, openErr := s.store.OpenDocument(revision.DocumentHash) + reader, openErr := s.store.OpenDocumentContext(publicRequest.Context(), revision.DocumentHash) if openErr == nil { body, readErr := io.ReadAll(io.LimitReader(reader, maxProjectionInput+1)) reader.Close() @@ -178,9 +178,16 @@ func (s *Server) servePublicUserProfile(w http.ResponseWriter, r *http.Request, } } snapshot := publicNavigationFor(publicRequest) + contact, contactErr := s.store.ProfileContactForViewer(publicRequest.Context(), s.domainUUIDForRequest(publicRequest), owner.UUID, "") + if contactErr != nil { + http.Error(w, "contact unavailable", http.StatusServiceUnavailable) + return true + } + w.Header().Set("Cache-Control", "private, no-store") w.Header().Set("Content-Type", "text/html; charset=utf-8") s.renderWithContext(w, publicRequest, "profile.gohtml", map[string]any{ "Principal": display, "Editing": false, "PublicProfile": true, + "Contact": contact, "ContactHref": profileContactHref(contact), "Handle": owner.Username + "@" + profileHost(publicRequest), "RenderedHTML": template.HTML(rendered), "PublicHasChildren": snapshot != nil && snapshot.hasChildren(root), "PublicChildrenHref": cityPageHref("/v/browser" + root), "Locale": s.getLocale(s.domainUUIDForRequest(publicRequest)), diff --git a/internal/store/adapter_sessions.go b/internal/store/adapter_sessions.go index 21dbcb4..13f7e4d 100644 --- a/internal/store/adapter_sessions.go +++ b/internal/store/adapter_sessions.go @@ -38,22 +38,7 @@ func adapterSecret(prefix string) (string, string, error) { } func (s *Store) EnsureAdapterSessionsSchema() error { - _, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS adapter_sessions ( - ticket_hash TEXT PRIMARY KEY, session_hash TEXT NOT NULL DEFAULT '', - domain_uuid TEXT NOT NULL, plugin_id TEXT NOT NULL, object_uuid TEXT NOT NULL, - principal_uuid TEXT NOT NULL DEFAULT '', - base_revision_uuid TEXT NOT NULL, ticket_expires_at_ms INTEGER NOT NULL, - session_expires_at_ms INTEGER NOT NULL DEFAULT 0, exchanged_at_ms INTEGER NOT NULL DEFAULT 0, - created_at_ms INTEGER NOT NULL - )`) - if err != nil { - return err - } - _, _ = s.DB.Exec(`ALTER TABLE adapter_sessions ADD COLUMN root_path TEXT NOT NULL DEFAULT ''`) - _, _ = s.DB.Exec(`ALTER TABLE adapter_sessions ADD COLUMN principal_uuid TEXT NOT NULL DEFAULT ''`) - _, err = s.DB.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_adapter_session_hash - ON adapter_sessions(session_hash) WHERE session_hash != ''`) - return err + return s.ensureSchemaComponent("adapter-sessions") } func (s *Store) IssueAdapterTicket(domainUUID, pluginID, objectUUID, revisionUUID, rootPath string) (string, error) { diff --git a/internal/store/auth.go b/internal/store/auth.go index 01157c2..b3d0f31 100644 --- a/internal/store/auth.go +++ b/internal/store/auth.go @@ -177,7 +177,7 @@ func (s *Store) createTypedAuthSession(domainUUID, principalUUID string, mfaLeve result, err := s.DB.Exec(`INSERT INTO auth_sessions(uuid,token_hash,principal_uuid,domain_uuid,auth_time_ms,expires_at_ms,last_seen_at_ms,mfa_level) SELECT ?,?,p.uuid,m.domain_uuid,?,?,?,? FROM principals p JOIN domain_memberships m ON m.principal_uuid=p.uuid WHERE p.uuid=? AND p.state IN ('active','readonly') AND m.domain_uuid=? AND m.state='active'`, - uuid.NewString(), hash, now, now+int64((24*time.Hour)/time.Millisecond), now, mfaLevel, principalUUID, domainUUID) + uuid.NewString(), hash, now, now+s.AuthSessionLifetime(domainUUID).Milliseconds(), now, mfaLevel, principalUUID, domainUUID) if err != nil { return "", err } @@ -472,7 +472,7 @@ func (s *Store) Authorize(request AuthorizationRequest) error { if err != nil { return err } - if p.decide(request.TargetID, request.Capability, base == nil) { + if p.Decide(request.TargetID, request.Capability, base == nil) { return nil } return ErrAccessDenied diff --git a/internal/store/auth_session_policy.go b/internal/store/auth_session_policy.go new file mode 100644 index 0000000..0974029 --- /dev/null +++ b/internal/store/auth_session_policy.go @@ -0,0 +1,32 @@ +package store + +import ( + "errors" + "strconv" + "time" +) + +const DefaultAuthSessionMinutes = 24 * 60 +const MaxAuthSessionMinutes = 30 * 24 * 60 + +// Absolute lifetime, fixed at issuance. Changing the policy cannot silently +// revive, extend or invalidate existing sessions. App passwords are separate. +func (s *Store) AuthSessionLifetimeMinutes(domain string) int { + raw, err := s.GetConfigInDomain(domain, "auth_session_minutes") + value, parseErr := strconv.Atoi(raw) + if err != nil || parseErr != nil || value < 5 || value > MaxAuthSessionMinutes { + return DefaultAuthSessionMinutes + } + return value +} + +func (s *Store) SetAuthSessionLifetimeMinutes(domain string, minutes int) error { + if minutes < 5 || minutes > MaxAuthSessionMinutes { + return errors.New("session lifetime must be between 5 and 43200 minutes") + } + return s.SetConfigInDomain(domain, "auth_session_minutes", strconv.Itoa(minutes)) +} + +func (s *Store) AuthSessionLifetime(domain string) time.Duration { + return time.Duration(s.AuthSessionLifetimeMinutes(domain)) * time.Minute +} diff --git a/internal/store/auth_session_policy_test.go b/internal/store/auth_session_policy_test.go new file mode 100644 index 0000000..e682f2c --- /dev/null +++ b/internal/store/auth_session_policy_test.go @@ -0,0 +1,84 @@ +package store + +import ( + "database/sql" + "testing" + "time" + + "pageswiki/internal/db" +) + +func TestAuthSessionLifetimeIsDomainScopedAndFixedAtIssuance(t *testing.T) { + database, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + database.SetMaxOpenConns(1) + t.Cleanup(func() { database.Close() }) + if err = db.Migrate(database); err != nil { + t.Fatal(err) + } + s := NewWithDocumentRoot(database, t.TempDir()+"/documents") + if err = s.EnsureAuthSchema(); err != nil { + t.Fatal(err) + } + if err = s.EnsureConfigTable(); err != nil { + t.Fatal(err) + } + p, err := s.BootstrapOwner("domain", "admin", "a long test password", "Admin", "") + if err != nil { + t.Fatal(err) + } + if s.AuthSessionLifetimeMinutes("domain") != 1440 { + t.Fatal("default changed") + } + for _, minutes := range []int{-1, 0, 4, 43201} { + if s.SetAuthSessionLifetimeMinutes("domain", minutes) == nil { + t.Fatalf("accepted %d", minutes) + } + } + if err = s.SetAuthSessionLifetimeMinutes("domain", 60); err != nil { + t.Fatal(err) + } + if s.AuthSessionLifetimeMinutes("other") != 1440 { + t.Fatal("policy leaked across Domains") + } + for _, browser := range []bool{false, true} { + var token string + if browser { + token, err = s.CreateBrowserAuthSession("domain", p.UUID, 0) + } else { + token, err = s.CreateAuthSession("domain", p.UUID) + } + if err != nil { + t.Fatal(err) + } + var duration int64 + if err = database.QueryRow(`SELECT expires_at_ms-auth_time_ms FROM auth_sessions ORDER BY rowid DESC LIMIT 1`).Scan(&duration); err != nil { + t.Fatal(err) + } + if duration != time.Hour.Milliseconds() { + t.Fatalf("lifetime %d", duration) + } + if _, err = s.PrincipalForSession("domain", token); err != nil { + t.Fatal(err) + } + } + if err = s.SetAuthSessionLifetimeMinutes("domain", 5); err != nil { + t.Fatal(err) + } + var changed int + if err = database.QueryRow(`SELECT count(*) FROM auth_sessions WHERE expires_at_ms-auth_time_ms<>?`, time.Hour.Milliseconds()).Scan(&changed); err != nil || changed != 0 { + t.Fatalf("existing sessions changed: %d %v", changed, err) + } + token, err := s.CreateAuthSession("domain", p.UUID) + if err != nil { + t.Fatal(err) + } + if _, err = database.Exec(`UPDATE auth_sessions SET expires_at_ms=?`, time.Now().Add(-time.Second).UnixMilli()); err != nil { + t.Fatal(err) + } + if _, err = s.PrincipalForSession("domain", token); err == nil { + t.Fatal("expired token authenticated") + } +} diff --git a/internal/store/broadcasts.go b/internal/store/broadcasts.go index 995eeda..f8ce65e 100644 --- a/internal/store/broadcasts.go +++ b/internal/store/broadcasts.go @@ -21,12 +21,7 @@ type Broadcast struct { } func (s *Store) EnsureBroadcastSchema() error { - _, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS broadcasts (uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,title TEXT NOT NULL,body TEXT NOT NULL,severity TEXT NOT NULL DEFAULT 'info' CHECK(severity IN ('info','success','warning','critical')),starts_at_ms INTEGER NOT NULL DEFAULT 0,ends_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`) - if err != nil { - return err - } - _, err = s.DB.Exec(`CREATE INDEX IF NOT EXISTS idx_broadcast_active ON broadcasts(domain_uuid,starts_at_ms,ends_at_ms,updated_at_ms DESC)`) - return err + return s.ensureSchemaComponent("broadcasts") } const broadcastColumns = `uuid,domain_uuid,title,body,severity,starts_at_ms,ends_at_ms,created_at_ms,updated_at_ms` diff --git a/internal/store/browser.go b/internal/store/browser.go index 61d833e..63d09c5 100644 --- a/internal/store/browser.go +++ b/internal/store/browser.go @@ -1,11 +1,9 @@ package store import ( - "io" - "os" "strings" - "golang.org/x/net/html" + "pageswiki/internal/mestionora" ) type BrowserEntry struct { @@ -23,32 +21,7 @@ type BrowserEntry struct { Title string } -func firstHTMLHeading(source string) string { - t := html.NewTokenizer(strings.NewReader(source)) - in := false - var out strings.Builder - for { - kind := t.Next() - if kind == html.ErrorToken { - if t.Err() == io.EOF { - break - } - break - } - tok := t.Token() - if kind == html.StartTagToken && strings.EqualFold(tok.Data, "h1") { - in = true - continue - } - if kind == html.EndTagToken && strings.EqualFold(tok.Data, "h1") { - break - } - if in && kind == html.TextToken { - out.WriteString(tok.Data) - } - } - return strings.TrimSpace(out.String()) -} +func firstHTMLHeading(source string) string { return mestionora.FirstHTMLHeading(source) } func normalizeBrowserPath(p string) string { p = strings.TrimSpace(p) @@ -117,10 +90,8 @@ func (s *Store) ListBrowserEntriesInDomain(domainUUID, parent string) ([]Browser } entry.Path = prefix + entry.Name if !entry.IsDir && s.DocumentRoot != "" && entry.DocumentHash != "" { - if filename, err := s.DocumentPath(entry.DocumentHash); err == nil { - if info, err := os.Stat(filename); err == nil { - entry.Size = int(info.Size()) - } + if size, err := s.DocumentSize(entry.DocumentHash); err == nil { + entry.Size = int(size) } } out = append(out, entry) diff --git a/internal/store/browser_navigation.go b/internal/store/browser_navigation.go index c04afb0..cc29fce 100644 --- a/internal/store/browser_navigation.go +++ b/internal/store/browser_navigation.go @@ -23,25 +23,7 @@ type BrowserNavigationLink struct { } func (s *Store) EnsureBrowserNavigationSchema() error { - _, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS browser_navigation_links ( - uuid TEXT PRIMARY KEY, - domain_uuid TEXT NOT NULL, - principal_uuid TEXT NOT NULL, - label TEXT NOT NULL, - path TEXT NOT NULL, - is_directory INTEGER NOT NULL DEFAULT 0, - position INTEGER NOT NULL DEFAULT 0, - created_at_ms INTEGER NOT NULL, - updated_at_ms INTEGER NOT NULL, - UNIQUE(domain_uuid, principal_uuid, path) - )`) - if err != nil { - return err - } - _, _ = s.DB.Exec(`ALTER TABLE browser_navigation_links ADD COLUMN is_directory INTEGER NOT NULL DEFAULT 0`) - _, err = s.DB.Exec(`CREATE INDEX IF NOT EXISTS idx_browser_navigation_owner - ON browser_navigation_links(domain_uuid, principal_uuid, position, created_at_ms)`) - return err + return s.ensureSchemaComponent("browser-navigation") } func (s *Store) ListBrowserNavigationLinks(domainUUID, principalUUID string) ([]BrowserNavigationLink, error) { diff --git a/internal/store/city.go b/internal/store/city.go index 54fe187..015deb6 100644 --- a/internal/store/city.go +++ b/internal/store/city.go @@ -13,6 +13,7 @@ import ( var cityNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{1,63}$`) type CitySpot struct { + DisplayTitle string `json:"display_title,omitempty"` DetailHref string `json:"detail_href,omitempty"` UUID string `json:"uuid"` Name string `json:"name"` @@ -74,6 +75,7 @@ type CityPagePlacement struct { type CitySpotAppearance struct { SpotUUID, SpotName, SpotTitle, EdgeType, Label, Source, PathPrefix string + DisplayTitle, StreetPath, QuartierPath string } type CityNodeDetail struct { @@ -538,13 +540,16 @@ func listSpotPages(db cityQuerier, domainUUID, spotUUID string, limit int) ([]Ci } func (s *Store) ListPageSpotAppearances(domainUUID, pageUUID string) ([]CitySpotAppearance, error) { - rows, err := s.DB.Query(`SELECT uuid,name,title,edge_type,label,source,path_prefix FROM ( + rows, err := s.DB.Query(`SELECT a.uuid,a.name,a.title,a.edge_type,a.label,a.source,a.path_prefix,COALESCE(NULLIF(st.title,''),st.name),st.path_prefix,q.path_prefix FROM ( SELECT s.uuid,s.name,s.title,sp.edge_type,sp.label,sp.source,s.path_prefix FROM spot_pages sp JOIN spots s ON s.uuid=sp.spot_uuid AND s.domain_uuid=? AND s.state!='archived' WHERE sp.page_uuid=? UNION ALL SELECT s.uuid,s.name,s.title,'path','','path',s.path_prefix FROM spots s JOIN paths p ON p.domain_uuid=s.domain_uuid AND p.object_uuid=? AND `+cityPathScopeSQL("s.path_prefix", "p.path")+` WHERE s.domain_uuid=? AND s.state!='archived' AND NOT EXISTS(SELECT 1 FROM spot_pages explicit WHERE explicit.spot_uuid=s.uuid AND explicit.page_uuid=?) AND NOT EXISTS(SELECT 1 FROM city_page_overrides ov JOIN spots dest ON dest.uuid=ov.spot_uuid AND dest.domain_uuid=ov.domain_uuid AND dest.state!='archived' WHERE ov.domain_uuid=p.domain_uuid AND ov.page_uuid=p.object_uuid AND ov.spot_uuid!=s.uuid) - ) ORDER BY lower(title),edge_type`, domainUUID, pageUUID, pageUUID, domainUUID, pageUUID) + ) a JOIN street_spots ss ON ss.spot_uuid=a.uuid + JOIN zones st ON st.uuid=ss.street_uuid AND st.domain_uuid=? AND st.kind='street' AND st.state!='archived' + JOIN zones q ON q.uuid=st.parent_uuid AND q.domain_uuid=st.domain_uuid AND q.kind='quartier' AND q.state!='archived' + ORDER BY lower(a.title),a.edge_type`, domainUUID, pageUUID, pageUUID, domainUUID, pageUUID, domainUUID) if err != nil { return nil, err } @@ -552,9 +557,11 @@ func (s *Store) ListPageSpotAppearances(domainUUID, pageUUID string) ([]CitySpot var out []CitySpotAppearance for rows.Next() { var item CitySpotAppearance - if err := rows.Scan(&item.SpotUUID, &item.SpotName, &item.SpotTitle, &item.EdgeType, &item.Label, &item.Source, &item.PathPrefix); err != nil { + var parentTitle string + if err := rows.Scan(&item.SpotUUID, &item.SpotName, &item.SpotTitle, &item.EdgeType, &item.Label, &item.Source, &item.PathPrefix, &parentTitle, &item.StreetPath, &item.QuartierPath); err != nil { return nil, err } + item.DisplayTitle = CitySpotDisplayTitle(item.SpotName, item.SpotTitle, item.PathPrefix, parentTitle) out = append(out, item) } return out, rows.Err() diff --git a/internal/store/city_display.go b/internal/store/city_display.go new file mode 100644 index 0000000..63b3daa --- /dev/null +++ b/internal/store/city_display.go @@ -0,0 +1,28 @@ +package store + +import "pageswiki/internal/mestionora" + +func IsUnclassifiedCitySpot(name, boundary string) bool { + return mestionora.IsUnclassifiedCitySpot(name, boundary) +} + +func CitySpotDisplayTitle(name, title, boundary, parentTitle string) string { + return mestionora.CitySpotDisplayTitle(name, title, boundary, parentTitle) +} + +func (n CityNodeDetail) DisplayTitle() string { + if n.Kind == "spot" { + parent := "" + if n.Street != nil { + parent = n.Street.Title + if parent == "" { + parent = n.Street.Name + } + } + return CitySpotDisplayTitle(n.Name, n.Title, n.PathPrefix, parent) + } + if n.Title != "" { + return n.Title + } + return n.Name +} diff --git a/internal/store/city_display_test.go b/internal/store/city_display_test.go new file mode 100644 index 0000000..8c55f13 --- /dev/null +++ b/internal/store/city_display_test.go @@ -0,0 +1,50 @@ +package store + +import ( + "context" + "testing" +) + +func TestCitySpotDisplayTitlePreservesIdentity(t *testing.T) { + for _, test := range []struct{ name, title, boundary, parent, want string }{ + {"home-undefined", "Undefined", "/life/home/_undefined", "Home & Garden", "Home & Garden"}, + {"home-undefined", "home-undefined", "/life/home/_undefined", "Home", "Home"}, + {"home-undefined", "My notes", "/life/home/_undefined", "Home", "My notes"}, + {"manual", "Undefined", "/life/home/manual", "Home", "Undefined"}, + {"home-undefined", "", "/life/home/_undefined", "", "home"}, + } { + if got := CitySpotDisplayTitle(test.name, test.title, test.boundary, test.parent); got != test.want { + t.Fatalf("%+v: %q", test, got) + } + } +} + +func TestCitySearchContentPathMetadata(t *testing.T) { + st := cityReadFixture(t) + citySearchInsertZone(t, st, "q", "d", "quartier", "", "campus", "Campus", "needle", "/life") + citySearchInsertZone(t, st, "s", "d", "street", "q", "home", "Home", "needle", "/life/home") + citySearchInsertSpot(t, st, "p", "d", "s", "home-undefined", "Undefined", "needle", "/life/home/_undefined") + // Same Path in another Domain must not be mistaken for a local portal. + cityTestExec(t, st, `INSERT INTO paths(domain_uuid,path,object_uuid) VALUES('other','/life','other-page'),('d','/life/home','home-page')`) + results, err := st.SearchCityInDomain(context.Background(), "d", "/", "needle", 30, citySearchAllowAll, SearchOptions{}) + if err != nil || len(results) != 3 { + t.Fatalf("results: %+v %v", results, err) + } + if results[0].HasPage || !results[1].HasPage || !results[2].HasPage || results[2].ContentPath() != "/life/home" || results[2].DisplayTitle() != "Home" { + t.Fatalf("projection: %+v", results) + } + node, err := st.CityNode("d", "spot", "home-undefined") + if err != nil || node.DisplayTitle() != "Home" || node.Title != "Undefined" || node.Name != "home-undefined" || node.PathPrefix != "/life/home/_undefined" { + t.Fatalf("raw identity mutated: %+v %v", node, err) + } + cityTestExec(t, st, `INSERT INTO spot_pages(spot_uuid,page_uuid,edge_type,label,source,created_at_ms) VALUES('p','home-page','primary','','local',1)`) + appearances, err := st.ListPageSpotAppearances("d", "home-page") + if err != nil || len(appearances) != 1 || appearances[0].DisplayTitle != "Home" || appearances[0].SpotTitle != "Undefined" || appearances[0].StreetPath != "/life/home" || appearances[0].QuartierPath != "/life" { + t.Fatalf("appearance projection: %+v %v", appearances, err) + } + // Both parent and boundary ACLs still gate the result. + results, err = st.SearchCityInDomain(context.Background(), "d", "/", "needle", 30, func(p string) bool { return p != "/life/home" }, SearchOptions{}) + if err != nil || len(results) != 1 { + t.Fatalf("parent ACL: %+v %v", results, err) + } +} diff --git a/internal/store/city_governance.go b/internal/store/city_governance.go index 513cde6..77cb67d 100644 --- a/internal/store/city_governance.go +++ b/internal/store/city_governance.go @@ -7,6 +7,8 @@ import ( "errors" "strings" "time" + + "pageswiki/internal/schutzaria" ) var ( @@ -16,11 +18,7 @@ var ( ErrGovernanceConfirm = errors.New("city.governance.confirm_error") ) -type LocalityPolicy struct { - Kind string `json:"kind"` - UUID string `json:"uuid"` - Policy string `json:"policy"` -} +type LocalityPolicy = schutzaria.LocalityPolicy type LocalityOwner struct { MembershipUUID string `json:"membership_uuid"` @@ -72,24 +70,7 @@ func (s *Store) ensureCityGovernanceSchema() error { // EvaluateLocalityPublication combines ceilings with intent. A public Domain // merely allows publication: it never publishes its root or arbitrary Pages. func EvaluateLocalityPublication(chain []LocalityPolicy) (allows, published bool) { - allows = true - for i, p := range chain { - if i == 0 && p.Policy == "inherit" { - return false, false - } - switch p.Policy { - case "authenticated_acl": - allows = false - case "public": - if i > 0 { - published = true - } - case "inherit": - default: - return false, false - } - } - return allows, allows && published + return schutzaria.EvaluateLocalityPublication(chain) } // All authority, parentage and policy reads use this one snapshot. Mutation diff --git a/internal/store/city_search.go b/internal/store/city_search.go index 0f26530..78eb571 100644 --- a/internal/store/city_search.go +++ b/internal/store/city_search.go @@ -2,205 +2,17 @@ package store import ( "context" - "database/sql" - "errors" - "sort" - "strings" - "time" + + "pageswiki/internal/mestionora" ) +type CitySearchResult = mestionora.CitySearchResult + const ( - CitySearchCandidateLimit = 1000 // Per locality kind, before authorization. - CitySearchResultLimit = 30 - citySearchTimeout = 2 * time.Second + CitySearchCandidateLimit = mestionora.CitySearchCandidateLimit + CitySearchResultLimit = mestionora.CitySearchResultLimit ) -// CitySearchResult describes a locality, not its contained Pages. Description -// is a bounded plain-text excerpt around the match and is never trusted HTML. -type CitySearchResult struct { - UUID, Kind, Name, Title, Description, PathPrefix string - QuartierTitle, StreetTitle, QuartierName string - CreatedAtMS, UpdatedAtMS int64 - NameMatch, PathMatch, DescriptionMatch bool -} - -type citySearchCandidate struct { - CitySearchResult - quartierPath, streetPath string -} - -// SearchCityInDomain searches locality metadata only. It does not construct a -// CityMap, expand descendants, read Page bodies, or count hidden localities. -// Scope selects a locality's own boundary, not an ancestor of the scope. Every -// boundary in its parent chain must be readable before the result is exposed. -func (s *Store) SearchCityInDomain(parent context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []CitySearchResult, searchErr error) { - if err := options.Validate(); err != nil { - return nil, err - } - query = strings.TrimSpace(query) - if len(query) > 512 || len(strings.Fields(query)) > 16 { - return nil, errors.New("search query is too long") - } - // File filters are intentionally not applied to locality metadata: a Spot - // containing a PDF is not itself a PDF and must not claim to match one. - if allowed == nil || options.FileType != "" || options.Extension != "" || (query == "" && !options.HasFilters()) { - return nil, nil - } - if limit <= 0 || limit > CitySearchResultLimit { - limit = CitySearchResultLimit - } - ctx, cancel := context.WithTimeout(parent, citySearchTimeout) - defer cancel() - defer func() { - if searchErr != nil && ctx.Err() != nil { - searchErr = ctx.Err() - } - }() - tx, err := s.DB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) - if err != nil { - return nil, err - } - defer tx.Rollback() - scope = normalizePath(scope) - prefix, upper := strings.TrimRight(scope, "/")+"/", strings.TrimRight(scope, "/")+"0" - args := []any{ - sql.Named("domain", domainUUID), sql.Named("scope", scope), - sql.Named("prefix", prefix), sql.Named("upper", upper), - sql.Named("query", query), sql.Named("limit", CitySearchCandidateLimit), - } - dateColumn := "n.updated_at_ms" - if options.DateField == "created" { - dateColumn = "n.created_at_ms" - } - dateWhere := "" - if options.FromMS != nil { - dateWhere += " AND " + dateColumn + ">=:from_ms" - args = append(args, sql.Named("from_ms", *options.FromMS)) - } - if options.BeforeMS != nil { - dateWhere += " AND " + dateColumn + "<:before_ms" - args = append(args, sql.Named("before_ms", *options.BeforeMS)) - } - candidates := make([]citySearchCandidate, 0) - for _, kind := range []string{"quartier", "street", "spot"} { - rows, err := tx.QueryContext(ctx, citySearchSQL(kind, dateWhere), args...) - if err != nil { - return nil, err - } - for rows.Next() { - var candidate citySearchCandidate - candidate.Kind = kind - if err := rows.Scan(&candidate.UUID, &candidate.Name, &candidate.Title, &candidate.Description, - &candidate.PathPrefix, &candidate.CreatedAtMS, &candidate.UpdatedAtMS, - &candidate.NameMatch, &candidate.PathMatch, &candidate.DescriptionMatch, - &candidate.QuartierTitle, &candidate.StreetTitle, &candidate.quartierPath, &candidate.streetPath, &candidate.QuartierName); err != nil { - rows.Close() - return nil, err - } - candidates = append(candidates, candidate) - } - err = rows.Err() - closeErr := rows.Close() - if err != nil { - return nil, err - } - if closeErr != nil { - return nil, closeErr - } - } - // Release the sole read connection before calling authorization, which may - // itself need a database connection. HTTP normally passes its cached policy. - if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { - return nil, err - } - sort.SliceStable(candidates, func(i, j int) bool { - a, b := candidates[i], candidates[j] - if a.Kind != b.Kind { - return citySearchKindOrder(a.Kind) < citySearchKindOrder(b.Kind) - } - if a.NameMatch != b.NameMatch { - return a.NameMatch - } - if a.PathMatch != b.PathMatch { - return a.PathMatch - } - if a.PathPrefix != b.PathPrefix { - return a.PathPrefix < b.PathPrefix - } - return a.UUID < b.UUID - }) - readable := make(map[string]bool) - canRead := func(path string) bool { - if value, exists := readable[path]; exists { - return value - } - value := allowed(path) - readable[path] = value - return value - } - for _, candidate := range candidates { - if err := ctx.Err(); err != nil { - return nil, err - } - if !canRead(candidate.PathPrefix) || (candidate.quartierPath != "" && !canRead(candidate.quartierPath)) || (candidate.streetPath != "" && !canRead(candidate.streetPath)) { - continue - } - found = append(found, candidate.CitySearchResult) - if len(found) == limit { - break - } - } - if err := ctx.Err(); err != nil { - return nil, err - } - return found, nil -} - -func citySearchKindOrder(kind string) int { - switch kind { - case "quartier": - return 0 - case "street": - return 1 - default: - return 2 - } -} - -// All interpolated SQL fragments are package-owned constants. User input is -// bound, so %, _, quotes and backslashes are literal substring characters. -func citySearchSQL(kind, dateWhere string) string { - from, description := "zones n", "n.description" - parentColumns := "'','','','',''" - where := "n.kind='quartier'" - switch kind { - case "street": - from += ` JOIN zones q ON q.uuid=n.parent_uuid AND q.domain_uuid=n.domain_uuid AND q.kind='quartier' AND q.state!='archived'` - parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),'',q.path_prefix,'',q.name" - where = "n.kind='street' AND " + citySearchAbsolutePathSQL("q.path_prefix") - case "spot": - from = `spots n JOIN street_spots ss ON ss.spot_uuid=n.uuid - JOIN zones st ON st.uuid=ss.street_uuid AND st.domain_uuid=n.domain_uuid AND st.kind='street' AND st.state!='archived' - JOIN zones q ON q.uuid=st.parent_uuid AND q.domain_uuid=st.domain_uuid AND q.kind='quartier' AND q.state!='archived'` - description = "n.summary" - parentColumns = "substr(COALESCE(NULLIF(q.title,''),q.name),1,256),substr(COALESCE(NULLIF(st.title,''),st.name),1,256),q.path_prefix,st.path_prefix,q.name" - where = citySearchAbsolutePathSQL("q.path_prefix") + " AND " + citySearchAbsolutePathSQL("st.path_prefix") - } - nameMatch := `(:query!='' AND (instr(lower(n.name),lower(:query))>0 OR instr(lower(n.title),lower(:query))>0))` - pathMatch := `(:query!='' AND instr(lower(n.path_prefix),lower(:query))>0)` - descriptionMatch := `(:query!='' AND instr(lower(` + description + `),lower(:query))>0)` - return `SELECT n.uuid,substr(n.name,1,256),substr(n.title,1,256), - substr(` + description + `,max(1,instr(lower(` + description + `),lower(:query))-160),512), - n.path_prefix,n.created_at_ms,n.updated_at_ms,` + nameMatch + `,` + pathMatch + `,` + descriptionMatch + `,` + parentColumns + ` - FROM ` + from + ` WHERE n.domain_uuid=:domain AND n.state!='archived' AND ` + where + ` - AND ` + citySearchAbsolutePathSQL("n.path_prefix") + ` - AND (n.path_prefix COLLATE BINARY=:scope OR (n.path_prefix COLLATE BINARY>=:prefix AND n.path_prefix COLLATE BINARY<:upper)) - AND (:query='' OR ` + nameMatch + ` OR ` + pathMatch + ` OR ` + descriptionMatch + `)` + dateWhere + ` - ORDER BY ` + nameMatch + ` DESC,` + pathMatch + ` DESC,n.path_prefix COLLATE BINARY,n.uuid LIMIT :limit` -} - -func citySearchAbsolutePathSQL(column string) string { - // Keep authorization paths intact rather than truncating them. Malformed - // legacy boundaries and impractically large metadata are not navigable hits. - return `(substr(` + column + `,1,1)='/' AND length(` + column + `)<=4096 AND instr(` + column + `,char(0))=0)` +func (s *Store) SearchCityInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) ([]CitySearchResult, error) { + return mestionora.NewSQLite(s.DB).SearchCityInDomain(ctx, domainUUID, scope, query, limit, allowed, options) } diff --git a/internal/store/city_search_test.go b/internal/store/city_search_test.go index 11139fa..c12ec7f 100644 --- a/internal/store/city_search_test.go +++ b/internal/store/city_search_test.go @@ -33,8 +33,8 @@ func citySearchIDs(results []CitySearchResult) []string { func TestCitySearchMetadataRankingAndMatchSources(t *testing.T) { st := cityReadFixture(t) - // No Page table remains: this search must not expand a CityMap or bodies. - cityTestExec(t, st, `DROP TABLE paths`) + // The fixture has only Path metadata, no document/revision bodies. Search + // may test boundary existence but must not expand a CityMap or read bodies. citySearchInsertZone(t, st, "q-description", "d", "quartier", "", "description", "Description", "needle in charter", "/a-description") citySearchInsertZone(t, st, "q-path", "d", "quartier", "", "path", "Path", "", "/b-needle") citySearchInsertZone(t, st, "q-title", "d", "quartier", "", "title", "NeEdLe quartier", "", "/c-title") diff --git a/internal/store/document_backfill.go b/internal/store/document_backfill.go new file mode 100644 index 0000000..8c33510 --- /dev/null +++ b/internal/store/document_backfill.go @@ -0,0 +1,228 @@ +package store + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "pageswiki/internal/db" +) + +var documentBackfillMigrations = []db.SQLMigration{{Version: 1, Name: "document_backfill_progress", Statements: []string{ + `CREATE TABLE document_backfills ( + store_id TEXT PRIMARY KEY, last_uuid TEXT NOT NULL DEFAULT '', + processed INTEGER NOT NULL DEFAULT 0, completed INTEGER NOT NULL DEFAULT 0, + updated_at_ms INTEGER NOT NULL)`, +}}, {Version: 2, Name: "verify_backfill_contract", Assertions: []string{ + schemaColumns("document_backfills", "store_id,last_uuid,processed,completed,updated_at_ms"), + schemaUnique("document_backfills", "store_id"), +}}} + +type DocumentBackfillProgress struct { + StoreID string + LastUUID string + Processed int64 + Completed bool +} + +// BackfillDocuments performs one restartable, bounded batch. It is a maintenance +// operation: legacy DB-only writers must be stopped. Normal application saves +// persist canonical bytes before inserting revisions and need no backfill. +// The UUID cursor is stable across SQLite VACUUM (rowid is not). +func (s *Store) BackfillDocuments(ctx context.Context, batchSize int) (DocumentBackfillProgress, error) { + var progress DocumentBackfillProgress + if s.ReadOnly { + return progress, ErrReadOnly + } + if err := ctx.Err(); err != nil { + return progress, err + } + if s.DocumentRoot == "" { + progress.Completed = true + return progress, nil + } + if batchSize <= 0 || batchSize > 128 { + batchSize = 128 + } + if err := os.MkdirAll(filepath.Join(s.DocumentRoot, "sha256"), 0755); err != nil { + return progress, err + } + root, err := os.OpenRoot(s.DocumentRoot) + if err != nil { + return progress, err + } + defer root.Close() + unlock, err := lockDocumentBackfill(ctx, root) + if err != nil { + return progress, err + } + defer unlock() + progress.StoreID, err = documentStoreIdentity(root) + if err != nil { + return progress, err + } + if err := db.ApplySQLiteMigrations(ctx, s.DB, "document-store", documentBackfillMigrations); err != nil { + return progress, err + } + if _, err := s.DB.ExecContext(ctx, `INSERT INTO document_backfills(store_id,updated_at_ms) VALUES(?,?) ON CONFLICT(store_id) DO NOTHING`, progress.StoreID, time.Now().UnixMilli()); err != nil { + return progress, err + } + if err := s.DB.QueryRowContext(ctx, `SELECT last_uuid,processed,completed FROM document_backfills WHERE store_id=?`, progress.StoreID).Scan(&progress.LastUUID, &progress.Processed, &progress.Completed); err != nil { + return progress, err + } + if progress.Completed { + return progress, nil + } + rows, err := s.DB.QueryContext(ctx, `SELECT uuid,document_hash FROM revisions WHERE uuid>? AND document_hash IS NOT NULL AND document_hash!='' ORDER BY uuid LIMIT ?`, progress.LastUUID, batchSize) + if err != nil { + return progress, err + } + type ref struct{ uuid, hash string } + var refs []ref + for rows.Next() { + var r ref + if err := rows.Scan(&r.uuid, &r.hash); err != nil { + rows.Close() + return progress, err + } + refs = append(refs, r) + } + if err := rows.Err(); err != nil { + rows.Close() + return progress, err + } + if err := rows.Close(); err != nil { + return progress, err + } + backend, err := s.documentStorage() + if err != nil { + return progress, err + } + for _, ref := range refs { + // Existing canonical bytes outrank a stale/missing compatibility body. + // Never overwrite a corrupt canonical file with a cache implicitly. + err := backend.Verify(ctx, ref.hash) + if errors.Is(err, os.ErrNotExist) { + var body string + if err = s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE uuid=? AND document_hash=?`, ref.uuid, ref.hash).Scan(&body); err == nil { + err = backend.PutIfAbsent(ctx, ref.hash, int64(len(body)), strings.NewReader(body)) + } + } + if err != nil { + return progress, fmt.Errorf("Document backfill revision=%s: %w", ref.uuid, err) + } + } + next := progress + if len(refs) != 0 { + next.LastUUID = refs[len(refs)-1].uuid + } + next.Processed += int64(len(refs)) + next.Completed = len(refs) < batchSize + if _, err := s.DB.ExecContext(ctx, `UPDATE document_backfills SET last_uuid=?,processed=?,completed=?,updated_at_ms=? WHERE store_id=?`, next.LastUUID, next.Processed, next.Completed, time.Now().UnixMilli(), next.StoreID); err != nil { + return progress, err + } + return next, nil +} + +func lockDocumentBackfill(ctx context.Context, root *os.Root) (func(), error) { + file, err := root.OpenFile(".document-backfill.lock", os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, err + } + for { + if err := ctx.Err(); err != nil { + file.Close() + return nil, err + } + err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if err == nil { + return func() { _ = syscall.Flock(int(file.Fd()), syscall.LOCK_UN); file.Close() }, nil + } + if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) { + file.Close() + return nil, err + } + timer := time.NewTimer(20 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + file.Close() + return nil, ctx.Err() + case <-timer.C: + } + } +} + +// Identity follows the physical store, not its pathname. A replaced/empty root +// gets a new identity and cannot inherit the DB's previous completion marker. +func documentStoreIdentity(root *os.Root) (string, error) { + file, err := root.Open(".document-store-id") + if err == nil { + defer file.Close() + info, err := file.Stat() + if err != nil { + return "", err + } + if !info.Mode().IsRegular() || info.Size() > 64 { + return "", errors.New("invalid Document store identity file") + } + body, err := io.ReadAll(io.LimitReader(file, 128)) + if err != nil { + return "", err + } + id := strings.TrimSpace(string(body)) + if decoded, err := hex.DecodeString(id); err != nil || len(decoded) != 16 { + return "", errors.New("invalid Document store identity") + } + // Also sync on resume: the previous process may have published the + // identity then crashed before syncing its directory. + return id, syncDocumentRoot(root) + } + if !errors.Is(err, os.ErrNotExist) { + return "", err + } + var nonce [16]byte + if _, err := rand.Read(nonce[:]); err != nil { + return "", err + } + id := hex.EncodeToString(nonce[:]) + tmp := ".document-store-id-" + id + file, err = root.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return "", err + } + defer root.Remove(tmp) + _, err = file.WriteString(id + "\n") + if err == nil { + err = file.Sync() + } + closeErr := file.Close() + if err == nil { + err = closeErr + } + if err != nil { + return "", err + } + if err := root.Link(tmp, ".document-store-id"); err != nil { + return "", err + } + return id, syncDocumentRoot(root) +} + +func syncDocumentRoot(root *os.Root) error { + dir, err := root.Open(".") + if err != nil { + return err + } + err = dir.Sync() + dir.Close() + return err +} diff --git a/internal/store/document_backfill_test.go b/internal/store/document_backfill_test.go new file mode 100644 index 0000000..849e97b --- /dev/null +++ b/internal/store/document_backfill_test.go @@ -0,0 +1,200 @@ +package store_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "path/filepath" + "sync" + "syscall" + "testing" + "time" + + "pageswiki/internal/store" +) + +func backfillFixture(t *testing.T, st *store.Store, id, body string) string { + t.Helper() + sum := sha256.Sum256([]byte(body)) + hash := hex.EncodeToString(sum[:]) + _, err := st.DB.Exec(`INSERT INTO revisions(uuid,object_uuid,body,document_hash,lang,content_type,filename,message,created_at_ms) VALUES(?,'backfill-object',?,?,'und','text/plain','','',0)`, id, body, hash) + if err != nil { + t.Fatal(err) + } + return hash +} + +func TestDocumentBackfillResumesAndDoesNotRescanCompletedStore(t *testing.T) { + st := newTopologyStore(t) + hashA := backfillFixture(t, st, "a", "first") + hashB := backfillFixture(t, st, "b", "second") + st.DocumentRoot = t.TempDir() + first, err := st.BackfillDocuments(context.Background(), 1) + if err != nil || first.LastUUID != "a" || first.Processed != 1 || first.Completed { + t.Fatalf("first=%+v err=%v", first, err) + } + if _, err := st.DB.Exec(`UPDATE revisions SET body='broken cache' WHERE uuid='a'`); err != nil { + t.Fatal(err) + } + if _, err := st.DB.Exec(`VACUUM`); err != nil { + t.Fatal(err) + } + // A new Store instance resumes the durable UUID cursor, not in-memory state. + restarted := &store.Store{DB: st.DB, DocumentRoot: st.DocumentRoot} + second, err := restarted.BackfillDocuments(context.Background(), 1) + if err != nil || second.LastUUID != "b" || second.Processed != 2 { + t.Fatalf("second=%+v err=%v", second, err) + } + final, err := restarted.BackfillDocuments(context.Background(), 1) + if err != nil || !final.Completed { + t.Fatalf("final=%+v err=%v", final, err) + } + for _, hash := range []string{hashA, hashB} { + if _, err := os.Stat(filepath.Join(st.DocumentRoot, "sha256", hash[:2], hash)); err != nil { + t.Fatal(err) + } + } + // Renaming the revisions table proves the completed path does not even + // query it. This is synthetic data in a temporary DB, never production. + if _, err := st.DB.Exec(`ALTER TABLE revisions RENAME TO hidden_revisions`); err != nil { + t.Fatal(err) + } + again, err := restarted.BackfillDocuments(context.Background(), 128) + if err != nil || again != final { + t.Fatalf("completed progress=%+v err=%v", again, err) + } +} + +func TestDocumentBackfillRootReplacementAndFailedBatch(t *testing.T) { + st := newTopologyStore(t) + hash := backfillFixture(t, st, "a", "original") + st.DocumentRoot = filepath.Join(t.TempDir(), "documents") + first, err := st.BackfillDocuments(context.Background(), 128) + if err != nil || !first.Completed { + t.Fatal(err) + } + // Keep exactly the same pathname, but replace its storage. Path-based + // completion markers would incorrectly skip this empty replacement. + if err := os.Rename(st.DocumentRoot, st.DocumentRoot+"-old"); err != nil { + t.Fatal(err) + } + if _, err := st.DB.Exec(`UPDATE revisions SET body='wrong' WHERE uuid='a'`); err != nil { + t.Fatal(err) + } + failed, err := st.BackfillDocuments(context.Background(), 128) + if !errors.Is(err, store.ErrDocumentHashMismatch) || failed.Processed != 0 || failed.Completed || failed.StoreID == first.StoreID { + t.Fatalf("failed=%+v err=%v", failed, err) + } + if _, err := st.DB.Exec(`UPDATE revisions SET body='original' WHERE uuid='a'`); err != nil { + t.Fatal(err) + } + retried, err := st.BackfillDocuments(context.Background(), 128) + if err != nil || !retried.Completed || retried.Processed != 1 { + t.Fatalf("retry=%+v err=%v", retried, err) + } + if _, err := os.Stat(filepath.Join(st.DocumentRoot, "sha256", hash[:2], hash)); err != nil { + t.Fatal(err) + } +} + +func TestDocumentBackfillPrefersCanonicalAndNeverRepairsCorruptionImplicitly(t *testing.T) { + st := newTopologyStore(t) + hash := backfillFixture(t, st, "a", "canonical") + st.DocumentRoot = t.TempDir() + dir := filepath.Join(st.DocumentRoot, "sha256", hash[:2]) + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + filename := filepath.Join(dir, hash) + if err := os.WriteFile(filename, []byte("canonical"), 0644); err != nil { + t.Fatal(err) + } + if _, err := st.DB.Exec(`UPDATE revisions SET body='' WHERE uuid='a'`); err != nil { + t.Fatal(err) + } + progress, err := st.BackfillDocuments(context.Background(), 128) + if err != nil || !progress.Completed { + t.Fatalf("canonical rejected: %v", err) + } + if _, err := st.DB.Exec(`DELETE FROM document_backfills`); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filename, []byte("broken"), 0644); err != nil { + t.Fatal(err) + } + if _, err := st.BackfillDocuments(context.Background(), 128); !errors.Is(err, store.ErrDocumentHashMismatch) { + t.Fatalf("corrupt canonical: %v", err) + } + data, _ := os.ReadFile(filename) + if string(data) != "broken" { + t.Fatal("corruption overwritten") + } +} + +func TestDocumentBackfillCancellationAndReadOnlyDoNotCreateFiles(t *testing.T) { + st := newTopologyStore(t) + st.DocumentRoot = filepath.Join(t.TempDir(), "new-root") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := st.EnsureDocumentStoreContext(ctx); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + st.ReadOnly = true + if err := st.EnsureDocumentStore(); !errors.Is(err, store.ErrReadOnly) { + t.Fatal(err) + } + if _, err := os.Stat(st.DocumentRoot); !os.IsNotExist(err) { + t.Fatal("cancelled/standby operation created root") + } +} + +func TestDocumentBackfillSerializesAndLockWaitHonorsCancellation(t *testing.T) { + st := newTopologyStore(t) + backfillFixture(t, st, "a", "first") + backfillFixture(t, st, "b", "second") + st.DocumentRoot = t.TempDir() + lock, err := os.OpenFile(filepath.Join(st.DocumentRoot, ".document-backfill.lock"), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + defer lock.Close() + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if _, err := st.BackfillDocuments(ctx, 1); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("lock cancellation: %v", err) + } + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_UN); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + errs := make(chan error, 2) + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + progress, err := st.BackfillDocuments(context.Background(), 1) + if err != nil || progress.Completed { + errs <- err + return + } + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } + progress, err := st.BackfillDocuments(context.Background(), 1) + if err != nil || !progress.Completed || progress.Processed != 2 { + t.Fatalf("double-counted migration: %+v %v", progress, err) + } +} diff --git a/internal/store/document_materialize.go b/internal/store/document_materialize.go new file mode 100644 index 0000000..12933e6 --- /dev/null +++ b/internal/store/document_materialize.go @@ -0,0 +1,33 @@ +package store + +import ( + "context" + "time" + + "pageswiki/internal/geduldh" +) + +// MaterializeDocument supplies a verified node-local, read-only scratch copy +// for tools needing a pathname. Callers must authorize the reference first and +// close the lease after all subprocesses finish. DB-only legacy stores work too. +// TMPDIR is the operator-controlled local scratch root, not DocumentRoot. +func (s *Store) MaterializeDocument(ctx context.Context, hash string, maxBytes int64) (*geduldh.Lease, error) { + s.documentMaterializerOnce.Do(func() { + // These fixed safe defaults precede configurable deployment profiles. + s.documentMaterializer, _ = geduldh.NewMaterializer(documentSource{s}, geduldh.MaterializeOptions{ + MaxBytes: 2 << 30, Concurrent: 2, Timeout: 2 * time.Minute, + }) + }) + return s.documentMaterializer.Acquire(ctx, hash, maxBytes) +} + +type documentSource struct{ store *Store } + +func (source documentSource) Stat(ctx context.Context, hash string) (geduldh.Info, error) { + size, err := source.store.DocumentSizeContext(ctx, hash) + return geduldh.Info{Hash: hash, Size: size}, err +} + +func (source documentSource) Open(ctx context.Context, hash string) (geduldh.Reader, error) { + return source.store.OpenDocumentContext(ctx, hash) +} diff --git a/internal/store/document_materialize_test.go b/internal/store/document_materialize_test.go new file mode 100644 index 0000000..0809e43 --- /dev/null +++ b/internal/store/document_materialize_test.go @@ -0,0 +1,74 @@ +package store_test + +import ( + "context" + "errors" + "io" + "os" + "testing" +) + +func TestDocumentMaterializeContextAndDBByteSize(t *testing.T) { + for _, disk := range []bool{false, true} { + name := "database" + if disk { + name = "local" + } + t.Run(name, func(t *testing.T) { + st := newTopologyStore(t) + if disk { + st.DocumentRoot = t.TempDir() + if err := st.EnsureDocumentStore(); err != nil { + t.Fatal(err) + } + } + object, err := st.CreatePageAtPathInDomain(mustDefaultDomainUUID(t, st), "/materialize.txt") + if err != nil { + t.Fatal(err) + } + body := []byte("日本語\x00\xff\n") + revision, err := st.SaveRevisionContent(object, body, "test", "und", "application/octet-stream", "materialize.txt") + if err != nil { + t.Fatal(err) + } + size, err := st.DocumentSizeContext(context.Background(), revision.DocumentHash) + if err != nil || size != int64(len(body)) { + t.Fatalf("size=%d err=%v", size, err) + } + lease, err := st.MaterializeDocument(context.Background(), revision.DocumentHash, 1024) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(lease.Path) + if err != nil || string(data) != string(body) { + t.Fatalf("data=%q err=%v", data, err) + } + lease.Close() + if _, err := os.Stat(lease.Path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("cleanup: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + reader, err := st.OpenDocumentContext(ctx, revision.DocumentHash) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + cancel() + if _, err := reader.Read(make([]byte, 1)); !errors.Is(err, context.Canceled) { + t.Fatalf("read: %v", err) + } + if _, err := reader.ReadAt(make([]byte, 1), 0); !errors.Is(err, context.Canceled) { + t.Fatalf("readat: %v", err) + } + if _, err := reader.Seek(0, io.SeekStart); !errors.Is(err, context.Canceled) { + t.Fatalf("seek: %v", err) + } + if _, err := st.ReadDocumentWindowContext(ctx, revision.DocumentHash, 0, 10); !errors.Is(err, context.Canceled) { + t.Fatalf("window: %v", err) + } + if _, err := st.DocumentSizeContext(ctx, revision.DocumentHash); !errors.Is(err, context.Canceled) { + t.Fatalf("stat: %v", err) + } + }) + } +} diff --git a/internal/store/document_store.go b/internal/store/document_store.go index 26451c2..3dc96a7 100644 --- a/internal/store/document_store.go +++ b/internal/store/document_store.go @@ -2,28 +2,50 @@ package store import ( "bytes" + "context" "crypto/sha256" "database/sql" "encoding/hex" "errors" - "fmt" "io" "os" "path/filepath" "strings" + + "pageswiki/internal/geduldh" ) type memoryDocumentReader struct { - *bytes.Reader + reader *bytes.Reader + ctx context.Context } func (memoryDocumentReader) Close() error { return nil } +func (r memoryDocumentReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(p) +} + +func (r memoryDocumentReader) ReadAt(p []byte, offset int64) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.ReadAt(p, offset) +} + +func (r memoryDocumentReader) Seek(offset int64, whence int) (int64, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Seek(offset, whence) +} + var ( - ErrInvalidDocumentHash = errors.New("invalid Document SHA-256") - ErrDocumentHashMismatch = errors.New( - "Document payload does not match SHA-256", - ) + ErrInvalidDocumentHash = geduldh.ErrInvalidHash + ErrDocumentHashMismatch = geduldh.ErrHashMismatch ) func MigrateLegacyDocumentRoot( @@ -131,18 +153,38 @@ func (s *Store) DocumentPath(hash string) (string, error) { // OpenDocument opens an immutable content-addressed Document. Callers must // perform authorization through a referencing object before invoking it. func (s *Store) OpenDocument(hash string) (io.ReadSeekCloser, error) { - path, err := s.documentPath(hash) - if err != nil { + return s.OpenDocumentContext(context.Background(), hash) +} + +// OpenDocumentContext supports bounded ReaderAt access without hydrating a +// revision body. A request/job context must be used by new storage consumers. +func (s *Store) OpenDocumentContext(ctx context.Context, hash string) (geduldh.Reader, error) { + if err := ctx.Err(); err != nil { return nil, err } - if path != "" { - return os.Open(path) + if _, err := geduldh.Key(hash); err != nil { + return nil, err + } + if s.DocumentRoot != "" { + backend, err := s.documentStorage() + if err != nil { + return nil, err + } + return backend.Open(ctx, hash) } var body string - if err := s.DB.QueryRow(`SELECT COALESCE(body,'') FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, strings.ToLower(strings.TrimSpace(hash))).Scan(&body); err != nil { + if err := s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, strings.ToLower(strings.TrimSpace(hash))).Scan(&body); err != nil { return nil, err } - return memoryDocumentReader{bytes.NewReader([]byte(body))}, nil + return memoryDocumentReader{reader: bytes.NewReader([]byte(body)), ctx: ctx}, nil +} + +// documentStorage is the payload-only backend boundary. DocumentRoot still +// anchors legacy calendars and instance paths; it must not become an S3 URL. +// Remote selection is intentionally not exposed until canonical state, CLI, +// deployment profiles and backend contract tests have been migrated. +func (s *Store) documentStorage() (geduldh.Store, error) { + return geduldh.NewLocal(s.DocumentRoot) } type DocumentReference struct { @@ -205,191 +247,57 @@ func (s *Store) DocumentReferencesInDomain(domainUUID, hash string) ([]DocumentR return references, nil } -func (s *Store) persistDocument( - expectedHash string, - body string, -) error { +func (s *Store) persistDocument(expectedHash, body string) error { + if s.ReadOnly { + return ErrReadOnly + } if s.DocumentRoot == "" { return nil } - - actualHash := - revisionHash( - body, - ) - - if !strings.EqualFold(expectedHash, actualHash) { - return fmt.Errorf( - "%w: expected %s, got %s", - ErrDocumentHashMismatch, - expectedHash, - actualHash, - ) - } - - path, err := - s.documentPath( - actualHash, - ) - + backend, err := s.documentStorage() if err != nil { return err } - - if existing, err := os.ReadFile(path); err == nil { - if revisionHash(string(existing)) != actualHash { - return fmt.Errorf( - "%w: corrupt existing file %s", - ErrDocumentHashMismatch, - path, - ) - } - - return nil - } else if !errors.Is(err, os.ErrNotExist) { - return err - } - - dir := - filepath.Dir( - path, - ) - - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - - temp, err := - os.CreateTemp( - dir, - ".document-*", - ) - - if err != nil { - return err - } - - tempPath := - temp.Name() - - defer os.Remove(tempPath) - - if err := temp.Chmod(0644); err != nil { - temp.Close() - return err - } - - if _, err := temp.WriteString(body); err != nil { - temp.Close() - return err - } - - if err := temp.Sync(); err != nil { - temp.Close() - return err - } - - if err := temp.Close(); err != nil { - return err - } - - if err := os.Rename(tempPath, path); err != nil { - return err - } - - return nil + return backend.PutIfAbsent(context.Background(), expectedHash, int64(len(body)), strings.NewReader(body)) } -func (s *Store) readDocument( - hash string, -) (string, error) { +func (s *Store) readDocument(hash string) (string, error) { if s.DocumentRoot == "" { return "", os.ErrNotExist } - - path, err := - s.documentPath( - hash, - ) - + backend, err := s.documentStorage() if err != nil { return "", err } - - payload, err := - os.ReadFile( - path, - ) - + reader, err := backend.Open(context.Background(), hash) if err != nil { return "", err } - - body := - string( - payload, - ) - - if revisionHash(body) != strings.ToLower(hash) { - return "", fmt.Errorf( - "%w: corrupt file %s", - ErrDocumentHashMismatch, - path, - ) + defer reader.Close() + payload, err := io.ReadAll(reader) + if err != nil { + return "", err } - - return body, nil + if revisionHash(string(payload)) != strings.ToLower(strings.TrimSpace(hash)) { + return "", ErrDocumentHashMismatch + } + return string(payload), nil } func (s *Store) EnsureDocumentStore() error { - if s.DocumentRoot == "" { - return nil - } + return s.EnsureDocumentStoreContext(context.Background()) +} - if err := os.MkdirAll( - filepath.Join( - s.DocumentRoot, - "sha256", - ), - 0755, - ); err != nil { - return err - } - - rows, err := s.DB.Query(` - SELECT - document_hash, - COALESCE(body, '') - FROM revisions - WHERE document_hash IS NOT NULL - AND document_hash != '' - `) - - if err != nil { - return err - } - - for rows.Next() { - var hash string - var body string - - if err := rows.Scan(&hash, &body); err != nil { - rows.Close() +func (s *Store) EnsureDocumentStoreContext(ctx context.Context) error { + for { + progress, err := s.BackfillDocuments(ctx, 128) + if err != nil { return err } - - if err := s.persistDocument(hash, body); err != nil { - rows.Close() - return err + if progress.Completed { + return nil } } - - if err := rows.Err(); err != nil { - rows.Close() - return err - } - - return rows.Close() } func (s *Store) hydrateRevision( diff --git a/internal/store/document_window.go b/internal/store/document_window.go index 36aaf2b..f98932a 100644 --- a/internal/store/document_window.go +++ b/internal/store/document_window.go @@ -1,9 +1,9 @@ package store import ( + "context" "errors" "io" - "os" ) const MaxDocumentWindowBytes int64 = 256 << 10 @@ -17,6 +17,13 @@ type DocumentWindow struct { } func (s *Store) ReadDocumentWindow(documentHash string, offset, limit int64) (DocumentWindow, error) { + return s.ReadDocumentWindowContext(context.Background(), documentHash, offset, limit) +} + +func (s *Store) ReadDocumentWindowContext(ctx context.Context, documentHash string, offset, limit int64) (DocumentWindow, error) { + if err := ctx.Err(); err != nil { + return DocumentWindow{}, err + } if offset < 0 { return DocumentWindow{}, errors.New("negative Document offset") } @@ -27,32 +34,32 @@ func (s *Store) ReadDocumentWindow(documentHash string, offset, limit int64) (Do limit = MaxDocumentWindowBytes } if s.DocumentRoot != "" { - path, err := s.DocumentPath(documentHash) + backend, err := s.documentStorage() if err != nil { return DocumentWindow{}, err } - file, err := os.Open(path) + file, err := backend.Open(ctx, documentHash) if err != nil { return DocumentWindow{}, err } defer file.Close() - info, err := file.Stat() + size, err := file.Seek(0, io.SeekEnd) if err != nil { return DocumentWindow{}, err } - if offset > info.Size() { - offset = info.Size() + if offset > size { + offset = size } - data := make([]byte, minInt64(limit, info.Size()-offset)) + data := make([]byte, minInt64(limit, size-offset)) n, err := file.ReadAt(data, offset) if err != nil && !errors.Is(err, io.EOF) { return DocumentWindow{}, err } data = data[:n] - return DocumentWindow{Data: data, Offset: offset, NextOffset: offset + int64(n), TotalBytes: info.Size(), Truncated: offset+int64(n) < info.Size()}, nil + return DocumentWindow{Data: data, Offset: offset, NextOffset: offset + int64(n), TotalBytes: size, Truncated: offset+int64(n) < size}, nil } var body string - if err := s.DB.QueryRow(`SELECT COALESCE(body,'') FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, documentHash).Scan(&body); err != nil { + if err := s.DB.QueryRowContext(ctx, `SELECT COALESCE(body,'') FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, documentHash).Scan(&body); err != nil { return DocumentWindow{}, err } total := int64(len(body)) @@ -79,18 +86,25 @@ func (s *Store) GetRevisionMetadata(revisionUUID string) (*Revision, error) { } func (s *Store) DocumentSize(documentHash string) (int64, error) { + return s.DocumentSizeContext(context.Background(), documentHash) +} + +func (s *Store) DocumentSizeContext(ctx context.Context, documentHash string) (int64, error) { + if err := ctx.Err(); err != nil { + return 0, err + } if s.DocumentRoot != "" { - path, err := s.DocumentPath(documentHash) + backend, err := s.documentStorage() if err != nil { return 0, err } - info, err := os.Stat(path) + info, err := backend.Stat(ctx, documentHash) if err != nil { return 0, err } - return info.Size(), nil + return info.Size, nil } var size int64 - err := s.DB.QueryRow(`SELECT length(COALESCE(body,'')) FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, documentHash).Scan(&size) + err := s.DB.QueryRowContext(ctx, `SELECT length(CAST(COALESCE(body,'') AS BLOB)) FROM revisions WHERE document_hash=? ORDER BY rowid DESC LIMIT 1`, documentHash).Scan(&size) return size, err } diff --git a/internal/store/dregarnuhr_test.go b/internal/store/dregarnuhr_test.go new file mode 100644 index 0000000..438554f --- /dev/null +++ b/internal/store/dregarnuhr_test.go @@ -0,0 +1,104 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "testing" + "time" + + "pageswiki/internal/dregarnuhr" +) + +func TestDregarnuhrPersistedLifecycle(t *testing.T) { + for _, terminal := range []string{"succeeded", "failed", "skipped", "cancelled"} { + t.Run(terminal, func(t *testing.T) { + s := newSchemaTestStore(t) + if err := s.EnsureEventJobsSchema(); err != nil { + t.Fatal(err) + } + settings, err := s.GetQueueSettings() + if err != nil { + t.Fatal(err) + } + settings.RetryBaseSeconds = 30 + if err := s.SetQueueSettings(settings); err != nil { + t.Fatal(err) + } + at := time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC) + if _, err := s.EnqueueEventWithOptions("", "test.lifecycle", terminal, "{}", at, 7, 3); err != nil { + t.Fatal(err) + } + ctx := context.Background() + if _, err := s.ClaimDueEventAt(ctx, at.Add(-time.Millisecond)); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("early claim: %v", err) + } + for attempt := 1; attempt <= 3; attempt++ { + job, err := s.ClaimDueEventAt(ctx, at) + if err != nil { + t.Fatal(err) + } + if job.Attempts != attempt || job.Status != "running" || job.StartedAtMS != at.UnixMilli() { + t.Fatalf("claim: %+v", job) + } + finished := at.Add(time.Second) + runErr := errors.New("synthetic failure") + if attempt == 3 { + switch terminal { + case "succeeded": + runErr = nil + case "skipped": + runErr = fmt.Errorf("inapplicable: %w", ErrEventSkipped) + case "cancelled": + if err := s.CancelEventJob(job.UUID); err != nil { + t.Fatal(err) + } + } + } + if err := s.FinishEventAt(ctx, job.UUID, runErr, finished); err != nil { + t.Fatal(err) + } + persisted, err := scanEventJob(s.DB.QueryRow(`SELECT `+eventJobColumns+` FROM event_jobs WHERE uuid=?`, job.UUID)) + if err != nil { + t.Fatal(err) + } + if persisted.UpdatedAt != finished.UnixMilli() { + t.Fatalf("completion time: %+v", persisted) + } + if attempt < 3 { + at = finished.Add(time.Duration(30*(1<<(attempt-1))) * time.Second) + if persisted.Status != "queued" || persisted.FinishedAtMS != 0 || persisted.RunAtMS != at.UnixMilli() { + t.Fatalf("retry: %+v", persisted) + } + if _, err := s.ClaimDueEventAt(ctx, at.Add(-time.Millisecond)); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("early retry: %v", err) + } + } else if persisted.Status != terminal || persisted.FinishedAtMS != finished.UnixMilli() { + t.Fatalf("terminal: %+v", persisted) + } + } + if _, err := s.ClaimDueEventAt(ctx, at.Add(48*time.Hour)); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("terminal claimed again: %v", err) + } + }) + } + if ErrEventSkipped != dregarnuhr.ErrSkipped { + t.Fatal("legacy skip identity changed") + } +} + +func TestDregarnuhrCancellationErrorsPropagate(t *testing.T) { + s := newSchemaTestStore(t) + if err := s.EnsureEventJobsSchema(); err != nil { + t.Fatal(err) + } + if _, err := s.EventCancellationState(context.Background(), "missing"); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("missing job: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := s.ClaimDueEventAt(ctx, time.Now()); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled claim: %v", err) + } +} diff --git a/internal/store/editor_revision_test.go b/internal/store/editor_revision_test.go new file mode 100644 index 0000000..576502c --- /dev/null +++ b/internal/store/editor_revision_test.go @@ -0,0 +1,84 @@ +package store_test + +import ( + "errors" + "pageswiki/internal/db" + "pageswiki/internal/store" + "sync" + "testing" +) + +func TestEditorRevisionCAS(t *testing.T) { + file := t.TempDir() + "/editor.db" + database, err := db.Open(file) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if err = db.Migrate(database); err != nil { + t.Fatal(err) + } + root := t.TempDir() + "/documents" + s := store.NewWithDocumentRoot(database, root) + object, err := s.CreatePageAtPath("/editor") + if err != nil { + t.Fatal(err) + } + base, err := s.SaveRevisionContentIfHead(object, []byte("base"), "", "und", "text/html", "", "") + if err != nil { + t.Fatal(err) + } + if _, err = s.SaveRevisionContentIfHead(object, []byte("stale"), "", "und", "text/html", "", ""); !errors.Is(err, store.ErrRevisionConflict) { + t.Fatalf("stale empty head: %v", err) + } + en, err := s.GetRevisionForExactLanguage(object, "en") + if err != nil || en.UUID != "" || en.Body != "" { + t.Fatalf("language fallback: %+v %v", en, err) + } + if _, err = s.SaveRevisionContentIfHead(object, []byte("english"), "", "en", "text/html", "", ""); err != nil { + t.Fatal(err) + } + // Independent Store and connection: mutex alone cannot enforce this CAS. + database2, err := db.Open(file) + if err != nil { + t.Fatal(err) + } + defer database2.Close() + s2 := store.NewWithDocumentRoot(database2, root) + start := make(chan struct{}) + results := make(chan error, 2) + var wg sync.WaitGroup + for _, st := range []*store.Store{s, s2} { + wg.Add(1) + go func(st *store.Store) { + defer wg.Done() + <-start + _, e := st.SaveRevisionContentIfHead(object, []byte("next"), "", "und", "text/html", "", base.UUID) + results <- e + }(st) + } + close(start) + wg.Wait() + close(results) + won, lost := 0, 0 + for err := range results { + if err == nil { + won++ + } else if errors.Is(err, store.ErrRevisionConflict) { + lost++ + } else { + t.Fatal(err) + } + } + if won != 1 || lost != 1 { + t.Fatalf("won=%d lost=%d", won, lost) + } + old, err := s.GetRevision(base.UUID) + if err != nil || old.Body != "base" { + t.Fatal("immutable base overwritten", err) + } + head, err := s.GetRevisionMetadataForExactLanguage(object, "und") + if err != nil || head.UUID == base.UUID || head.Body != "" { + t.Fatalf("head metadata: %+v %v", head, err) + } +} diff --git a/internal/store/event_claim_test.go b/internal/store/event_claim_test.go new file mode 100644 index 0000000..e9b38d5 --- /dev/null +++ b/internal/store/event_claim_test.go @@ -0,0 +1,124 @@ +package store + +import ( + "database/sql" + "errors" + "fmt" + "path/filepath" + "sync" + "testing" + "time" + + "pageswiki/internal/db" +) + +func TestClaimDueEventPriorityAndReturnedState(t *testing.T) { + s := newSchemaTestStore(t) + if err := s.EnsureEventJobsSchema(); err != nil { + t.Fatal(err) + } + for _, item := range []struct { + key string + priority int + due time.Time + }{ + {"low", 1, time.Now().Add(-time.Hour)}, {"high", 9, time.Now().Add(-time.Minute)}, {"future", 99, time.Now().Add(time.Hour)}, + } { + if _, err := s.EnqueueEventWithOptions("", "test", item.key, "opaque", item.due, item.priority, 3); err != nil { + t.Fatal(err) + } + } + for _, key := range []string{"high", "low"} { + job, err := s.ClaimDueEvent() + if err != nil { + t.Fatal(err) + } + if job.Key != key || job.Status != "running" || job.Attempts != 1 || job.StartedAtMS == 0 || job.UpdatedAt != job.StartedAtMS { + t.Fatalf("wrong claimed state: %+v", job) + } + } + if _, err := s.ClaimDueEvent(); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("future/running claimed: %v", err) + } + s.ReadOnly = true + if _, err := s.ClaimDueEvent(); !errors.Is(err, ErrReadOnly) { + t.Fatalf("standby claim: %v", err) + } +} + +func TestClaimDueEventConcurrentWriters(t *testing.T) { + path := filepath.Join(t.TempDir(), "queue.db") + database, err := db.Open(path) + if err != nil { + t.Fatal(err) + } + defer database.Close() + s := &Store{DB: database} + if err := s.EnsureEventJobsSchema(); err != nil { + t.Fatal(err) + } + const total = 64 + for n := 0; n < total; n++ { + if _, err := s.EnqueueEvent("test", fmt.Sprint(n), "opaque", time.Now().Add(-time.Second)); err != nil { + t.Fatal(err) + } + } + var wg sync.WaitGroup + jobs := make(chan string, total) + errs := make(chan error, 9) + // Independent DB handles exercise file-level contention, not an in-process + // mutex or a single sql.DB connection serializing the test for us. + for range 8 { + other, err := db.Open(path) + if err != nil { + t.Fatal(err) + } + wg.Add(1) + go func() { + defer wg.Done() + defer other.Close() + worker := &Store{DB: other} + for { + job, err := worker.ClaimDueEvent() + if errors.Is(err, sql.ErrNoRows) { + return + } + if err != nil { + errs <- err + return + } + jobs <- job.UUID + } + }() + } + wg.Add(1) + go func() { + defer wg.Done() + for n := 0; n < 100; n++ { + if _, err := database.Exec(`UPDATE event_queue_settings SET max_depth=max_depth+1 WHERE id=1`); err != nil { + errs <- err + return + } + } + }() + wg.Wait() + close(jobs) + close(errs) + for err := range errs { + t.Error(err) + } + seen := map[string]bool{} + for id := range jobs { + if seen[id] { + t.Error("duplicate claim", id) + } + seen[id] = true + } + if len(seen) != total { + t.Fatalf("claimed=%d want=%d", len(seen), total) + } + var once int + if err := database.QueryRow(`SELECT count(*) FROM event_jobs WHERE status='running' AND attempts=1 AND started_at_ms=updated_at_ms`).Scan(&once); err != nil || once != total { + t.Fatalf("persisted claims=%d err=%v", once, err) + } +} diff --git a/internal/store/event_jobs.go b/internal/store/event_jobs.go index 66ac905..acd1eaf 100644 --- a/internal/store/event_jobs.go +++ b/internal/store/event_jobs.go @@ -9,26 +9,11 @@ import ( "time" "github.com/google/uuid" + "pageswiki/internal/dregarnuhr" ) -type EventJob struct { - UUID string `json:"uuid"` - EventUUID string `json:"event_uuid,omitempty"` - Type string `json:"event_type"` - Key string `json:"event_key"` - Payload string `json:"payload,omitempty"` - Status string `json:"status"` - Priority int `json:"priority"` - Attempts int `json:"attempts"` - MaxAttempts int `json:"max_attempts"` - RunAtMS int64 `json:"run_at_ms"` - StartedAtMS int64 `json:"started_at_ms,omitempty"` - FinishedAtMS int64 `json:"finished_at_ms,omitempty"` - CancelRequested bool `json:"cancel_requested"` - Error string `json:"error,omitempty"` - CreatedAt int64 `json:"created_at_ms"` - UpdatedAt int64 `json:"updated_at_ms"` -} +// EventJob retains the existing public JSON shape and source compatibility. +type EventJob = dregarnuhr.Job type EventDefinition struct { UUID string `json:"uuid"` @@ -74,13 +59,7 @@ type EventLogTrend struct { SampleMessage string `json:"sample_message,omitempty"` } -type QueueSettings struct { - Paused bool `json:"paused"` - Concurrency int `json:"concurrency"` - MaxDepth int `json:"max_depth"` - RetryBaseSeconds int `json:"retry_base_seconds"` - RetentionHours int `json:"retention_hours"` -} +type QueueSettings = dregarnuhr.Settings type EventJobStats struct { Queued int `json:"queued"` @@ -91,97 +70,18 @@ type EventJobStats struct { Cancelled int `json:"cancelled"` } -var ErrEventSkipped = errors.New("event is not applicable") +var ErrEventSkipped = dregarnuhr.ErrSkipped func (s *Store) EnsureEventJobsSchema() error { - if _, err := s.DB.Exec(` - CREATE TABLE IF NOT EXISTS event_jobs ( - uuid TEXT PRIMARY KEY, - event_type TEXT NOT NULL, - event_key TEXT NOT NULL, - payload TEXT NOT NULL, - status TEXT NOT NULL, - attempts INTEGER NOT NULL DEFAULT 0, - run_at_ms INTEGER NOT NULL, - error TEXT NOT NULL DEFAULT '', - created_at_ms INTEGER NOT NULL, - updated_at_ms INTEGER NOT NULL - ) - `); err != nil { + if err := s.ensureSchemaComponent("event-jobs"); err != nil { return err } - for _, statement := range []string{ - `ALTER TABLE event_jobs ADD COLUMN event_uuid TEXT NOT NULL DEFAULT ''`, - `ALTER TABLE event_jobs ADD COLUMN priority INTEGER NOT NULL DEFAULT 0`, - `ALTER TABLE event_jobs ADD COLUMN max_attempts INTEGER NOT NULL DEFAULT 3`, - `ALTER TABLE event_jobs ADD COLUMN started_at_ms INTEGER NOT NULL DEFAULT 0`, - `ALTER TABLE event_jobs ADD COLUMN finished_at_ms INTEGER NOT NULL DEFAULT 0`, - `ALTER TABLE event_jobs ADD COLUMN cancel_requested INTEGER NOT NULL DEFAULT 0`, - } { - _, _ = s.DB.Exec(statement) - } - statements := []string{ - `CREATE INDEX IF NOT EXISTS idx_event_jobs_listing ON event_jobs (created_at_ms DESC, uuid DESC)`, - `CREATE INDEX IF NOT EXISTS idx_event_jobs_status_listing ON event_jobs (status, created_at_ms DESC, uuid DESC)`, - `CREATE INDEX IF NOT EXISTS idx_event_jobs_history ON event_jobs (created_at_ms DESC, uuid DESC) WHERE status IN ('succeeded','skipped','failed','cancelled')`, - `CREATE INDEX IF NOT EXISTS idx_event_jobs_due - ON event_jobs (status, priority DESC, run_at_ms, created_at_ms)`, - `CREATE TABLE IF NOT EXISTS event_definitions ( - uuid TEXT PRIMARY KEY, name TEXT NOT NULL, event_type TEXT NOT NULL, - event_key TEXT NOT NULL, payload TEXT NOT NULL DEFAULT '', - schedule_kind TEXT NOT NULL DEFAULT 'manual', - interval_seconds INTEGER NOT NULL DEFAULT 0, - next_run_at_ms INTEGER NOT NULL DEFAULT 0, - enabled INTEGER NOT NULL DEFAULT 1, priority INTEGER NOT NULL DEFAULT 0, - max_attempts INTEGER NOT NULL DEFAULT 3, - created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL - )`, - `CREATE INDEX IF NOT EXISTS idx_event_definitions_due - ON event_definitions (enabled, next_run_at_ms)`, - `CREATE TABLE IF NOT EXISTS event_logs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, job_uuid TEXT NOT NULL, - event_uuid TEXT NOT NULL DEFAULT '', level TEXT NOT NULL, - message TEXT NOT NULL, created_at_ms INTEGER NOT NULL - )`, - `CREATE INDEX IF NOT EXISTS idx_event_logs_job - ON event_logs (job_uuid, created_at_ms DESC)`, - `CREATE INDEX IF NOT EXISTS idx_event_logs_job_id ON event_logs (job_uuid, id DESC)`, - `CREATE TABLE IF NOT EXISTS event_log_policies ( - event_type TEXT PRIMARY KEY, - retention_hours INTEGER NOT NULL DEFAULT 168, - aggregate_after_hours INTEGER NOT NULL DEFAULT 24, - bucket_seconds INTEGER NOT NULL DEFAULT 60, - max_rows INTEGER NOT NULL DEFAULT 5000 - )`, - `INSERT OR IGNORE INTO event_log_policies(event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows) - VALUES('*',168,24,60,5000)`, - `CREATE TABLE IF NOT EXISTS event_log_buckets ( - event_type TEXT NOT NULL, - bucket_start_ms INTEGER NOT NULL, - level TEXT NOT NULL, - count INTEGER NOT NULL DEFAULT 0, - first_at_ms INTEGER NOT NULL, - last_at_ms INTEGER NOT NULL, - sample_message TEXT NOT NULL DEFAULT '', - PRIMARY KEY(event_type,bucket_start_ms,level) - )`, - `CREATE INDEX IF NOT EXISTS idx_event_log_buckets_time ON event_log_buckets(event_type,bucket_start_ms DESC)`, - `CREATE TABLE IF NOT EXISTS event_queue_settings ( - id INTEGER PRIMARY KEY CHECK (id = 1), paused INTEGER NOT NULL DEFAULT 0, - concurrency INTEGER NOT NULL DEFAULT 1, - max_depth INTEGER NOT NULL DEFAULT 10000, - retry_base_seconds INTEGER NOT NULL DEFAULT 30, - retention_hours INTEGER NOT NULL DEFAULT 168 - )`, - `INSERT OR IGNORE INTO event_queue_settings - (id, paused, concurrency, max_depth, retry_base_seconds, retention_hours) - VALUES (1, 0, 1, 10000, 30, 168)`, - } - for _, statement := range statements { - if _, err := s.DB.Exec(statement); err != nil { - return err - } - } + return s.recoverInterruptedEventJobs() +} + +// Startup recovery mutates runtime state, not schema. Never call it from +// status/preflight, read-only standby or a migration rehearsal. +func (s *Store) recoverInterruptedEventJobs() error { now := time.Now().UnixMilli() if _, err := s.DB.Exec(`UPDATE event_jobs SET status='queued', error=CASE WHEN error='' THEN 'execution interrupted by process restart' ELSE error END, @@ -250,86 +150,67 @@ const eventJobColumns = `uuid, event_uuid, event_type, event_key, payload, finished_at_ms, cancel_requested, error, created_at_ms, updated_at_ms` func (s *Store) ClaimDueEvent() (*EventJob, error) { - tx, err := s.DB.Begin() - if err != nil { - return nil, err + return s.ClaimDueEventAt(context.Background(), time.Now()) +} + +// ClaimDueEventAt keeps selection and claim in one statement; injected time +// allows scheduler tests without real sleeps or production data. +func (s *Store) ClaimDueEventAt(ctx context.Context, at time.Time) (*EventJob, error) { + if s.ReadOnly { + return nil, ErrReadOnly } - defer tx.Rollback() - job, err := scanEventJob(tx.QueryRow(` - SELECT `+eventJobColumns+` FROM event_jobs - WHERE status = 'queued' AND run_at_ms <= ? - ORDER BY priority DESC, run_at_ms, created_at_ms LIMIT 1`, - time.Now().UnixMilli())) - if err != nil { - return nil, err - } - now := time.Now().UnixMilli() - result, err := tx.Exec(` + now := at.UnixMilli() + // A deferred SELECT transaction cannot always upgrade its WAL snapshot to a + // writer (SQLITE_BUSY[_SNAPSHOT]), even with busy_timeout. Select and claim in + // one write statement instead. SQLite serializes concurrent claimers; RETURNING + // gives the committed attempt/status/timestamps, not the pre-claim snapshot. + return scanEventJob(s.DB.QueryRowContext(ctx, ` UPDATE event_jobs SET status='running', attempts=attempts+1, - started_at_ms=?, updated_at_ms=? WHERE uuid=? AND status='queued'`, - now, now, job.UUID) - if err != nil { - return nil, err - } - count, err := result.RowsAffected() - if err != nil || count != 1 { - return nil, sql.ErrNoRows - } - if err := tx.Commit(); err != nil { - return nil, err - } - job.Status, job.Attempts, job.StartedAtMS = "running", job.Attempts+1, now - return job, nil + started_at_ms=?, updated_at_ms=? + WHERE uuid=(SELECT uuid FROM event_jobs + WHERE status='queued' AND run_at_ms<=? + ORDER BY priority DESC,run_at_ms,created_at_ms,uuid LIMIT 1) + AND status='queued' + RETURNING `+eventJobColumns, now, now, now)) } func (s *Store) FinishEvent(uuid string, runErr error) error { - job, err := scanEventJob(s.DB.QueryRow(`SELECT `+eventJobColumns+` FROM event_jobs WHERE uuid=?`, uuid)) + return s.FinishEventAt(context.Background(), uuid, runErr, time.Now()) +} + +func (s *Store) FinishEventAt(ctx context.Context, uuid string, runErr error, at time.Time) error { + job, err := scanEventJob(s.DB.QueryRowContext(ctx, `SELECT `+eventJobColumns+` FROM event_jobs WHERE uuid=?`, uuid)) if err != nil { return err } - status, message := "succeeded", "" - if runErr != nil { - message = runErr.Error() - if errors.Is(runErr, ErrEventSkipped) { - status = "skipped" - } else if job.CancelRequested { - status = "cancelled" - } else if job.Attempts < job.MaxAttempts { - settings, settingsErr := s.GetQueueSettings() - if settingsErr != nil { - return settingsErr - } - delay := time.Duration(settings.RetryBaseSeconds) * time.Second - for i := 1; i < job.Attempts; i++ { - delay *= 2 - if delay >= 24*time.Hour { - delay = 24 * time.Hour - break - } - } - _, err = s.DB.Exec(`UPDATE event_jobs SET status='queued', error=?, - run_at_ms=?, finished_at_ms=0, updated_at_ms=? WHERE uuid=?`, - message, time.Now().Add(delay).UnixMilli(), time.Now().UnixMilli(), uuid) - _ = s.AddEventLog(uuid, job.EventUUID, "warning", "execution failed; retry scheduled: "+message) + base := 1 + if runErr != nil && !errors.Is(runErr, ErrEventSkipped) && !job.CancelRequested && job.Attempts < job.MaxAttempts { + settings, err := s.GetQueueSettings() + if err != nil { return err - } else { - status = "failed" } + base = settings.RetryBaseSeconds } - now := time.Now().UnixMilli() - _, err = s.DB.Exec(`UPDATE event_jobs SET status=?, error=?, + outcome := dregarnuhr.Complete(*job, runErr, base, at) + if outcome.Status == "queued" { + _, err = s.DB.ExecContext(ctx, `UPDATE event_jobs SET status='queued', error=?, + run_at_ms=?, finished_at_ms=0, updated_at_ms=? WHERE uuid=?`, + outcome.Message, outcome.RetryAt.UnixMilli(), at.UnixMilli(), uuid) + if err == nil { + _ = s.AddEventLog(uuid, job.EventUUID, outcome.Level, "execution failed; retry scheduled: "+outcome.Message) + } + return err + } + _, err = s.DB.ExecContext(ctx, `UPDATE event_jobs SET status=?, error=?, finished_at_ms=?, updated_at_ms=? WHERE uuid=?`, - status, message, now, now, uuid) - level := "info" - if status == "failed" { - level = "error" + outcome.Status, outcome.Message, at.UnixMilli(), at.UnixMilli(), uuid) + message := outcome.Status + if outcome.Message != "" { + message += ": " + outcome.Message + } + if err == nil { + _ = s.AddEventLog(uuid, job.EventUUID, outcome.Level, message) } - _ = s.AddEventLog(uuid, job.EventUUID, level, status+func() string { - if message != "" { - return ": " + message - } - return "" - }()) return err } @@ -442,9 +323,16 @@ func (s *Store) RetryEventJob(jobUUID string) error { } func (s *Store) EventCancellationRequested(jobUUID string) bool { + requested, err := s.EventCancellationState(context.Background(), jobUUID) + return err == nil && requested +} + +// EventCancellationState lets the runner distinguish a read failure from a +// confirmed absence of cancellation. The boolean compatibility API stays. +func (s *Store) EventCancellationState(ctx context.Context, jobUUID string) (bool, error) { var requested int - err := s.DB.QueryRow(`SELECT cancel_requested FROM event_jobs WHERE uuid=?`, jobUUID).Scan(&requested) - return err == nil && requested != 0 + err := s.DB.QueryRowContext(ctx, `SELECT cancel_requested FROM event_jobs WHERE uuid=?`, jobUUID).Scan(&requested) + return requested != 0, err } func (s *Store) PurgeEventHistory(retentionHours int) error { diff --git a/internal/store/geduldh_contract_test.go b/internal/store/geduldh_contract_test.go new file mode 100644 index 0000000..3926616 --- /dev/null +++ b/internal/store/geduldh_contract_test.go @@ -0,0 +1,99 @@ +package store_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "path/filepath" + "testing" + + "pageswiki/internal/db" + "pageswiki/internal/geduldh" + "pageswiki/internal/store" +) + +func TestGeduldhPublicationPrecedesRevisionCommit(t *testing.T) { + for _, mode := range []string{"missing-root", "corrupt-existing", "metadata-conflict"} { + t.Run(mode, func(t *testing.T) { + root := t.TempDir() + database, err := db.Open(filepath.Join(root, "pages.db")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if err := db.Migrate(database); err != nil { + t.Fatal(err) + } + documents := filepath.Join(root, "documents") + st := store.NewWithDocumentRoot(database, documents) + page, err := st.CreatePageAtPath("/geduldh-contract") + if err != nil { + t.Fatal(err) + } + base, err := st.SaveRevisionContent(page, []byte("base"), "base", "en", "text/plain", "base.txt") + if err != nil { + t.Fatal(err) + } + payload := []byte("new immutable payload") + sum := sha256.Sum256(payload) + hash := hex.EncodeToString(sum[:]) + filename := filepath.Join(documents, "sha256", hash[:2], hash) + switch mode { + case "missing-root": + st.DocumentRoot = filepath.Join(root, "unavailable") + case "corrupt-existing": + if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filename, []byte("damaged"), 0644); err != nil { + t.Fatal(err) + } + } + expected := base.UUID + if mode == "metadata-conflict" { + expected = "not-the-current-head" + } + _, err = st.SaveRevisionContentIfHead(page, payload, "must not commit", "en", "text/plain", "new.txt", expected) + if err == nil { + t.Fatal("unsafe revision committed") + } + if mode == "corrupt-existing" && (!errors.Is(err, geduldh.ErrHashMismatch) || !errors.Is(err, store.ErrDocumentHashMismatch)) { + t.Fatalf("storage error identity changed: %v", err) + } + if mode == "metadata-conflict" && !errors.Is(err, store.ErrRevisionConflict) { + t.Fatalf("conflict changed: %v", err) + } + var count int + if err := database.QueryRow(`SELECT count(*) FROM revisions WHERE object_uuid=?`, page).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 1 { + t.Fatalf("failed save added revision: count=%d", count) + } + st.DocumentRoot = documents + loaded, err := st.GetRevision(base.UUID) + if err != nil || loaded.Body != "base" { + t.Fatalf("previous revision lost: %v", err) + } + switch mode { + case "corrupt-existing": + data, err := os.ReadFile(filename) + if err != nil || string(data) != "damaged" { + t.Fatalf("corruption silently repaired: %v", err) + } + case "metadata-conflict": + // A failed metadata transaction may leave a valid unreferenced blob; + // the payload layer must not delete it without reference/retention proof. + backend, err := geduldh.NewLocal(documents) + if err != nil { + t.Fatal(err) + } + if err := backend.Verify(context.Background(), hash); err != nil { + t.Fatal(err) + } + } + }) + } +} diff --git a/internal/store/groups.go b/internal/store/groups.go index 7060a76..6d1bc8b 100644 --- a/internal/store/groups.go +++ b/internal/store/groups.go @@ -7,6 +7,8 @@ import ( "time" "github.com/google/uuid" + + "pageswiki/internal/schutzaria" ) var groupNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{1,63}$`) @@ -25,19 +27,8 @@ type GroupPermission struct { UUID, GroupUUID, ScopePath, Permission string } -var groupPermissions = map[string]map[string]bool{ - "read": {"read": true}, - "read_write_once": {"read": true, "create": true, "upload": true}, - "read_write": {"read": true, "create": true, "write": true, "upload": true, "rename": true, "link": true}, - "upload_only": {"upload": true}, -} - func NormalizeGroupPermission(value string) (string, error) { - value = strings.ToLower(strings.TrimSpace(value)) - if _, ok := groupPermissions[value]; !ok { - return "", errors.New("invalid group permission") - } - return value, nil + return schutzaria.NormalizeGroupPermission(value) } func normalizeGroupName(name string) (string, error) { @@ -375,8 +366,7 @@ func groupPermissionAllowsFrom(q pathQueryer, domainUUID, principalUUID, capabil if err := rows.Scan(&scope, &permission); err != nil { return false, err } - inside := targetPath == scope || scope == "/" || strings.HasPrefix(targetPath, strings.TrimRight(scope, "/")+"/") - if inside && groupPermissions[permission][capability] { + if schutzaria.GroupPermissionAllows(scope, permission, capability, targetPath) { return true, nil } } diff --git a/internal/store/imports.go b/internal/store/imports.go index 06562a1..a5be606 100644 --- a/internal/store/imports.go +++ b/internal/store/imports.go @@ -1,7 +1,6 @@ package store import ( - "database/sql" "errors" "strings" "time" @@ -30,47 +29,7 @@ type ImportItem struct { } func (s *Store) EnsureImportSchema() error { - statements := []string{ - `CREATE TABLE IF NOT EXISTS import_sessions(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,actor_uuid TEXT NOT NULL DEFAULT '',archive_name TEXT NOT NULL,archive_path TEXT NOT NULL,target_prefix TEXT NOT NULL DEFAULT '/',status TEXT NOT NULL DEFAULT 'queued',total_files INTEGER NOT NULL DEFAULT 0,markdown_files INTEGER NOT NULL DEFAULT 0,error TEXT NOT NULL DEFAULT '',created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`, - `CREATE TABLE IF NOT EXISTS import_items(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,source_path TEXT NOT NULL,target_path TEXT NOT NULL,size INTEGER NOT NULL DEFAULT 0,content_type TEXT NOT NULL DEFAULT '',title TEXT NOT NULL DEFAULT '',suggested_quartier TEXT NOT NULL DEFAULT '',suggested_street TEXT NOT NULL DEFAULT '',suggested_spot TEXT NOT NULL DEFAULT '',confidence REAL NOT NULL DEFAULT 0,selected INTEGER NOT NULL DEFAULT 1,PRIMARY KEY(session_uuid,sequence))`, - `CREATE TABLE IF NOT EXISTS import_journal(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,path TEXT NOT NULL,object_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,sequence))`, - `CREATE TABLE IF NOT EXISTS import_city_journal(session_uuid TEXT NOT NULL,kind TEXT NOT NULL,node_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,kind,node_uuid))`, - `CREATE TABLE IF NOT EXISTS import_city_edges(session_uuid TEXT NOT NULL,spot_uuid TEXT NOT NULL,page_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,spot_uuid,page_uuid))`, - `CREATE INDEX IF NOT EXISTS idx_import_sessions_domain ON import_sessions(domain_uuid,created_at_ms DESC)`, - } - for _, statement := range statements { - if _, err := s.DB.Exec(statement); err != nil { - return err - } - } - _ = ensureImportColumn(s.DB, "import_journal", "completed", "INTEGER NOT NULL DEFAULT 0") - return nil -} - -func ensureImportColumn(db *sql.DB, table, column, declaration string) error { - rows, err := db.Query(`PRAGMA table_info(` + table + `)`) - if err != nil { - return err - } - found := false - for rows.Next() { - var cid, notnull, pk int - var name, kind string - var defaultValue any - if err := rows.Scan(&cid, &name, &kind, ¬null, &defaultValue, &pk); err != nil { - rows.Close() - return err - } - if name == column { - found = true - } - } - rows.Close() - if found { - return nil - } - _, err = db.Exec(`ALTER TABLE ` + table + ` ADD COLUMN ` + column + ` ` + declaration) - return err + return s.ensureSchemaComponent("imports") } func (s *Store) CreateImportSession(domainUUID, actor, archiveName, archivePath, targetPrefix string) (string, error) { diff --git a/internal/store/list_children.go b/internal/store/list_children.go index 6dd9bd5..3a12729 100644 --- a/internal/store/list_children.go +++ b/internal/store/list_children.go @@ -2,7 +2,6 @@ package store import ( "fmt" - "os" "strings" ) @@ -246,10 +245,8 @@ func (s *Store) ListSelectedChildEntriesInDomain(domainUUID, parent string, page entry.Title = firstHTMLHeading(sourcePrefix) } if documentHash != "" && s.DocumentRoot != "" { - if path, pathErr := s.documentPath(documentHash); pathErr == nil { - if info, statErr := os.Stat(path); statErr == nil { - entry.Size = int(info.Size()) - } + if size, err := s.DocumentSize(documentHash); err == nil { + entry.Size = int(size) } } entries = append(entries, entry) diff --git a/internal/store/locality_access.go b/internal/store/locality_access.go index 96a69c3..4d0fd32 100644 --- a/internal/store/locality_access.go +++ b/internal/store/locality_access.go @@ -7,6 +7,8 @@ import ( "errors" "strings" "time" + + "pageswiki/internal/schutzaria" ) var ErrLocalityAccessEnforcing = errors.New("city.access.enforcing_error") @@ -41,52 +43,14 @@ func (s *Store) ensureLocalityAccessSchema() error { return nil } -type localityAccessLayer struct{ read, write, grant bool } -type localityAccessSnapshot struct { - admin bool - paths map[string][]localityAccessLayer -} - -func (p *localityAccessSnapshot) decide(path, capability string, basis bool) bool { - if !strings.HasPrefix(path, "/") { - return false - } - if p == nil || p.admin { - return basis - } - grant := false - for at := path; ; { - for _, layer := range p.paths[at] { - allowed := layer.write - if capability == "read" { - allowed = layer.read - } - if !allowed { - return false - } - if layer.grant && (capability == "read" || capability == "write" || capability == "create" || capability == "upload") { - grant = true - } - } - if at == "/" { - break - } - i := strings.LastIndex(at, "/") - if i <= 0 { - at = "/" - } else { - at = at[:i] - } - } - return basis || grant -} +type localityAccessSnapshot = schutzaria.LocalityAccess func (s *Store) CheckLocalityAccessCaps(domain, principal, capability, path string) error { p, err := loadLocalityAccess(s.DB, domain, principal) if err != nil { return err } - if !p.decide(path, capability, true) { + if !p.Decide(path, capability, true) { return ErrAccessDenied } return nil @@ -112,7 +76,7 @@ func (s *Store) CheckObjectLocalityAccessCaps(domain, principal, capability, obj if err != nil { return err } - if p.admin || len(p.paths) == 0 { + if !p.RequiresPathChecks() { return nil } rows, err := s.DB.Query(`SELECT path FROM paths WHERE domain_uuid=? AND object_uuid=? LIMIT 10001`, domain, object) @@ -130,7 +94,7 @@ func (s *Store) CheckObjectLocalityAccessCaps(domain, principal, capability, obj if err = rows.Scan(&path); err != nil { return err } - if !p.decide(path, capability, true) { + if !p.Decide(path, capability, true) { return ErrAccessDenied } } @@ -176,7 +140,8 @@ func installLocalityAccessGuards(ctx context.Context, tx *sql.Tx) error { } func loadLocalityAccess(q pathQueryer, domain, principal string) (*localityAccessSnapshot, error) { - p := &localityAccessSnapshot{paths: map[string][]localityAccessLayer{}} + var layers []schutzaria.AccessLayer + var admin bool rows, err := q.Query(`SELECT a.scope_path,a.grants_access, EXISTS(SELECT 1 FROM locality_access_entries e JOIN domain_memberships m ON m.domain_uuid=e.domain_uuid AND m.principal_uuid=? AND m.state='active' JOIN principals u ON u.uuid=m.principal_uuid AND u.state IN ('active','readonly') WHERE e.domain_uuid=a.domain_uuid AND e.node_kind=a.node_kind AND e.node_uuid=a.node_uuid AND ((e.subject_kind='user' AND e.subject_uuid=m.uuid) OR (e.subject_kind='group' AND EXISTS(SELECT 1 FROM groups g JOIN group_members gm ON gm.group_uuid=g.uuid AND gm.membership_uuid=m.uuid WHERE g.uuid=e.subject_uuid AND g.domain_uuid=m.domain_uuid AND g.state='active')))), EXISTS(SELECT 1 FROM locality_access_entries e JOIN domain_memberships m ON m.domain_uuid=e.domain_uuid AND m.principal_uuid=? AND m.state='active' JOIN principals u ON u.uuid=m.principal_uuid AND u.state='active' WHERE e.domain_uuid=a.domain_uuid AND e.node_kind=a.node_kind AND e.node_uuid=a.node_uuid AND e.permission='read_write' AND ((e.subject_kind='user' AND e.subject_uuid=m.uuid) OR (e.subject_kind='group' AND EXISTS(SELECT 1 FROM groups g JOIN group_members gm ON gm.group_uuid=g.uuid AND gm.membership_uuid=m.uuid WHERE g.uuid=e.subject_uuid AND g.domain_uuid=m.domain_uuid AND g.state='active')))), @@ -193,15 +158,17 @@ func loadLocalityAccess(q pathQueryer, domain, principal string) (*localityAcces return nil, ErrGovernanceInvalid } var path string - var layer localityAccessLayer - var admin bool - if err = rows.Scan(&path, &layer.grant, &layer.read, &layer.write, &admin); err != nil { + var layer schutzaria.AccessLayer + if err = rows.Scan(&path, &layer.Grant, &layer.Read, &layer.Write, &admin); err != nil { return nil, err } - p.admin = admin - p.paths[path] = append(p.paths[path], layer) + layer.Path = path + layers = append(layers, layer) } - return p, rows.Err() + if err := rows.Err(); err != nil { + return nil, err + } + return schutzaria.NewLocalityAccess(admin, layers), nil } func readLocalityAccess(ctx context.Context, tx *sql.Tx, domain, kind, id string, g CityGovernance) (LocalityAccess, error) { diff --git a/internal/store/locality_publication.go b/internal/store/locality_publication.go index 62bf9b7..9768f3a 100644 --- a/internal/store/locality_publication.go +++ b/internal/store/locality_publication.go @@ -11,6 +11,8 @@ import ( "sort" "strings" "time" + + "pageswiki/internal/schutzaria" ) var ErrPublicationImpact = errors.New("city.publication.impact_error") @@ -313,20 +315,7 @@ func (s *Store) LocalityPublication(ctx context.Context, domain, actor, kind, id // Explicitly exclude every application/user/group/editor namespace. Public // projections must never turn an arbitrary reserved handler into a document. func PublicSemanticPath(path string) bool { - if !strings.HasPrefix(path, "/") || strings.ContainsAny(path, "\\\x00\r\n") || strings.Contains(path, "//") { - return false - } - for _, segment := range strings.Split(path, "/") { - if segment == "." || segment == ".." { - return false - } - } - first := strings.Split(strings.TrimPrefix(path, "/"), "/")[0] - switch first { - case "_", "-", ":", "o", "r", "v", "l", "e", "s", "u", "g", "static", "health", "new": - return false - } - return true + return schutzaria.PublicSemanticPath(path) } // Lookup is metadata-only and Path-indexed. The manifest is fenced by current diff --git a/internal/store/mestionora_compatibility_test.go b/internal/store/mestionora_compatibility_test.go new file mode 100644 index 0000000..e044805 --- /dev/null +++ b/internal/store/mestionora_compatibility_test.go @@ -0,0 +1,61 @@ +package store_test + +import ( + "context" + "reflect" + "strings" + "testing" + + "pageswiki/internal/mestionora" + "pageswiki/internal/store" +) + +func TestMestionoraStoreCompatibilityAndRequestIsolation(t *testing.T) { + st := searchStoreFixture(t) + domain := defaultDomainUUID(t, st) + object := insertSearchFilterPage(t, st, domain, "/docs/needle", "text/html", "needle.html", "

    Needle Guide

    needle body

    ", 1, 2000) + insertSearchFilterPage(t, st, domain, "/private/needle", "text/html", "secret.html", "

    Secret Needle

    ", 1, 2000) + if _, err := st.DB.Exec(`INSERT INTO paths(domain_uuid,path,object_uuid,created_at) VALUES(?,'/private/alias',?,1)`, domain, object); err != nil { + t.Fatal(err) + } + engine := mestionora.NewSQLite(st.DB) + allow := func(p string) bool { return strings.HasPrefix(p, "/docs/") } + ctx := context.Background() + for _, options := range []store.SearchOptions{{}, {FileType: "html"}, {Extension: "html"}, {FromMS: searchFilterTime(1000), BeforeMS: searchFilterTime(3000)}} { + legacy, err := st.SearchPathsInDomainWithOptions(ctx, domain, "/", "needle", 20, allow, options) + if err != nil { + t.Fatal(err) + } + current, err := engine.SearchPathsInDomainWithOptions(ctx, domain, "/", "needle", 20, allow, options) + if err != nil || !reflect.DeepEqual(legacy, current) || len(current) != 1 || current[0].Path != "/docs/needle" { + t.Fatalf("incompatible/exposed results: %v %v", current, err) + } + } + legacy, err := st.SearchPathSuggestionsInDomain(ctx, domain, "/", "needle", 8, allow) + if err != nil { + t.Fatal(err) + } + current, err := engine.SearchPathSuggestionsInDomain(ctx, domain, "/", "needle", 8, allow) + if err != nil || !reflect.DeepEqual(legacy, current) || len(current) != 1 || current[0].Title != "Needle Guide" { + t.Fatalf("suggestion compatibility: %v %v", current, err) + } + for _, request := range []struct { + domain string + allowed func(string) bool + }{ + {domain, nil}, {domain, func(string) bool { return false }}, {"other-domain", allow}, + } { + results, err := engine.SearchPathsInDomainFiltered(ctx, request.domain, "/", "needle", 20, request.allowed) + if err != nil || len(results) != 0 { + t.Fatalf("prior request leaked: %v %v", results, err) + } + } + if store.ErrInvalidSearchOptions != mestionora.ErrInvalidSearchOptions || store.SearchCandidateLimit != mestionora.SearchCandidateLimit { + t.Fatal("compatibility identity changed") + } + // Alias method sets preserve City display/navigation projections too. + var city store.CitySearchResult = mestionora.CitySearchResult{Kind: "spot", Name: "home-undefined", Title: "Undefined", StreetTitle: "Home", PathPrefix: "/life/home/_undefined", NavigationPath: "/life/home"} + if city.DisplayTitle() != "Home" || city.ContentPath() != "/life/home" { + t.Fatal("City projection changed") + } +} diff --git a/internal/store/notification_push.go b/internal/store/notification_push.go index d66d871..2a375bd 100644 --- a/internal/store/notification_push.go +++ b/internal/store/notification_push.go @@ -8,9 +8,11 @@ import ( "errors" "strings" "time" + + "pageswiki/internal/ordoschnellic" ) -const NotificationPushEventType = "notification.push.deliver" +const NotificationPushEventType = ordoschnellic.PushEventType func (s *Store) ensureNotificationPushSchema() error { _, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS notification_push_keys(id INTEGER PRIMARY KEY CHECK(id=1),private_key TEXT NOT NULL,public_key TEXT NOT NULL); @@ -81,11 +83,7 @@ func (s *Store) DeleteNotificationPush(ctx context.Context, domain, owner, endpo return err } -type NotificationPushDelivery struct { - ID, Notification, Domain, Owner, Subscription string - Focused bool - Attempt int -} +type NotificationPushDelivery = ordoschnellic.PushDelivery func (s *Store) AcknowledgeNotificationPush(ctx context.Context, domain, owner, endpoint string, ids []string) error { if len(ids) == 0 { @@ -104,12 +102,20 @@ func (s *Store) AcknowledgeNotificationPush(ctx context.Context, domain, owner, return err } func (s *Store) DeferFocusedNotificationPush(ctx context.Context, d *NotificationPushDelivery) error { - _, err := s.DB.ExecContext(ctx, `UPDATE notification_push_queue SET attempts=attempts-1,lease_until_ms=0,next_at_ms=? WHERE subscription_id=? AND notification_uuid=? AND attempts=?`, time.Now().Add(time.Minute).UnixMilli(), d.ID, d.Notification, d.Attempt) + return s.DeferFocusedNotificationPushAt(ctx, d, time.Now()) +} + +func (s *Store) DeferFocusedNotificationPushAt(ctx context.Context, d *NotificationPushDelivery, at time.Time) error { + _, err := s.DB.ExecContext(ctx, `UPDATE notification_push_queue SET attempts=attempts-1,lease_until_ms=0,next_at_ms=? WHERE subscription_id=? AND notification_uuid=? AND attempts=?`, at.Add(ordoschnellic.PushFocusDelay).UnixMilli(), d.ID, d.Notification, d.Attempt) return err } func (s *Store) ClaimNotificationPush(ctx context.Context) (*NotificationPushDelivery, error) { - now := time.Now().UnixMilli() + return s.ClaimNotificationPushAt(ctx, time.Now()) +} + +func (s *Store) ClaimNotificationPushAt(ctx context.Context, at time.Time) (*NotificationPushDelivery, error) { + now := at.UnixMilli() tx, err := s.DB.BeginTx(ctx, nil) if err != nil { return nil, err @@ -117,7 +123,7 @@ func (s *Store) ClaimNotificationPush(ctx context.Context) (*NotificationPushDel defer tx.Rollback() // Atomic claim across scheduler workers/processes. Outbound I/O is never inside this transaction. var d NotificationPushDelivery - err = tx.QueryRowContext(ctx, `UPDATE notification_push_queue SET lease_until_ms=?,attempts=attempts+1 WHERE (subscription_id,notification_uuid)=(SELECT subscription_id,notification_uuid FROM notification_push_queue WHERE next_at_ms<=? AND lease_until_ms<=? ORDER BY next_at_ms LIMIT 1) RETURNING subscription_id,notification_uuid,attempts`, now+60000, now, now).Scan(&d.ID, &d.Notification, &d.Attempt) + err = tx.QueryRowContext(ctx, `UPDATE notification_push_queue SET lease_until_ms=?,attempts=attempts+1 WHERE (subscription_id,notification_uuid)=(SELECT subscription_id,notification_uuid FROM notification_push_queue WHERE next_at_ms<=? AND lease_until_ms<=? ORDER BY next_at_ms LIMIT 1) RETURNING subscription_id,notification_uuid,attempts`, at.Add(ordoschnellic.PushLeaseDuration).UnixMilli(), now, now).Scan(&d.ID, &d.Notification, &d.Attempt) if errors.Is(err, sql.ErrNoRows) { return nil, nil } @@ -141,15 +147,32 @@ func (s *Store) ClaimNotificationPush(ctx context.Context) (*NotificationPushDel } func (s *Store) FinishNotificationPush(ctx context.Context, d *NotificationPushDelivery, retry, expired bool) error { + return s.FinishNotificationPushAt(ctx, d, retry, expired, time.Now()) +} + +func (s *Store) FinishNotificationPushAt(ctx context.Context, d *NotificationPushDelivery, retry, expired bool, at time.Time) error { if expired { if _, err := s.DB.ExecContext(ctx, `DELETE FROM notification_push_subscriptions WHERE id=?`, d.ID); err != nil { return err } } - if retry && d.Attempt < 5 { - _, err := s.DB.ExecContext(ctx, `UPDATE notification_push_queue SET next_at_ms=?,lease_until_ms=0 WHERE subscription_id=? AND notification_uuid=? AND attempts=?`, time.Now().Add(time.Duration(1<=? AND path0) AS is_dir FROM relative WHERE rest!='' GROUP BY 1 + ) SELECT name,is_dir FROM children + WHERE substr(lower(name),1,length(?))=lower(?) ORDER BY name LIMIT 1001`, base, domain, base, strings.TrimSuffix(base, "/")+"0", parent, prefix, prefix) + if err != nil { + return nil, false, err + } + defer rows.Close() + entries := make([]BrowserEntry, 0) + for rows.Next() { + var entry BrowserEntry + if err := rows.Scan(&entry.Name, &entry.IsDir); err != nil { + return nil, false, err + } + entry.Path = base + entry.Name + entries = append(entries, entry) + } + if err := rows.Err(); err != nil { + return nil, false, err + } + truncated := len(entries) > 1000 + if truncated { + entries = entries[:1000] + } + return entries, truncated, nil +} diff --git a/internal/store/path_completion_test.go b/internal/store/path_completion_test.go new file mode 100644 index 0000000..b583c9d --- /dev/null +++ b/internal/store/path_completion_test.go @@ -0,0 +1,38 @@ +package store + +import ( + "context" + "testing" +) + +func TestPathCompletionNamesNoRevisionsAndBounded(t *testing.T) { + s := newSchemaTestStore(t) + // Deliberately omit revisions and documents: completion must never hydrate them. + _, err := s.DB.Exec(`CREATE TABLE paths(domain_uuid TEXT,path TEXT,object_uuid TEXT); + INSERT INTO paths VALUES('d','/docs/日本語 #?/child','o'),('d','/docs/a%b','o'),('d','/docs/a_b','o'),('other','/docs/secret','o'),('d','/docs-other/no','o');`) + if err != nil { + t.Fatal(err) + } + for _, prefix := range []string{"日本", "a%", "a_"} { + entries, more, err := s.ListPathCompletionNames(context.Background(), "d", "/docs", prefix) + if err != nil || more || len(entries) != 1 { + t.Fatalf("%q: %+v more=%v err=%v", prefix, entries, more, err) + } + if prefix == "日本" && (!entries[0].IsDir || entries[0].Path != "/docs/日本語 #?") { + t.Fatal(entries) + } + } + _, err = s.DB.Exec(`WITH RECURSIVE n(v) AS (VALUES(1) UNION ALL SELECT v+1 FROM n WHERE v<1200) INSERT INTO paths SELECT 'd','/many/'||printf('%04d',v),'o' FROM n`) + if err != nil { + t.Fatal(err) + } + entries, more, err := s.ListPathCompletionNames(context.Background(), "d", "/many", "") + if err != nil || !more || len(entries) != 1000 { + t.Fatalf("budget: %d %v %v", len(entries), more, err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, _, err = s.ListPathCompletionNames(ctx, "d", "/many", ""); err == nil { + t.Fatal("ignored cancellation") + } +} diff --git a/internal/store/periodicals.go b/internal/store/periodicals.go index bc2b4b8..a8e4f2f 100644 --- a/internal/store/periodicals.go +++ b/internal/store/periodicals.go @@ -252,7 +252,7 @@ func (s *Store) changePeriodical(ctx context.Context, domain, actor, kind string if err != nil { return err } - if !caps.decide(e.Path, capability, basis == nil) { + if !caps.Decide(e.Path, capability, basis == nil) { return ErrAccessDenied } } diff --git a/internal/store/profile_contact.go b/internal/store/profile_contact.go new file mode 100644 index 0000000..0d67c8c --- /dev/null +++ b/internal/store/profile_contact.go @@ -0,0 +1,185 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "strings" + "time" + + "pageswiki/internal/db" +) + +var profileContactMigrations = []db.SQLMigration{{Version: 1, Name: "private_profile_phone", Statements: []string{ + `CREATE TABLE profile_contacts ( + domain_uuid TEXT NOT NULL, principal_uuid TEXT NOT NULL, + phone_display TEXT NOT NULL DEFAULT '', phone_number TEXT NOT NULL DEFAULT '', + visibility TEXT NOT NULL DEFAULT 'private' CHECK(visibility IN ('private','domain','public')), + version INTEGER NOT NULL DEFAULT 1, updated_at_ms INTEGER NOT NULL, + PRIMARY KEY(domain_uuid,principal_uuid))`, +}, Assertions: []string{ + schemaColumns("profile_contacts", "domain_uuid,principal_uuid,phone_display,phone_number,visibility,version,updated_at_ms"), + schemaUnique("profile_contacts", "domain_uuid,principal_uuid"), +}}} + +// Kept outside Principal: account lists, identity federation and auth responses +// must not accidentally export personal contact details. +type ProfileContact struct { + PhoneDisplay string + PhoneNumber string + Visibility string + Version int64 +} + +var ErrContactPhone = errors.New("contact.invalid_phone") + +// Initial scope: ordinary local/international numbers only. No service codes, +// extensions, pauses, URI parameters, or control characters are dialled. +func NormalizeContactPhone(value string) (string, string, error) { + if len(value) > 64 { + return "", "", ErrContactPhone + } + for _, c := range value { + if !(c >= '0' && c <= '9') && c != '+' && c != ' ' && c != '-' && c != '(' && c != ')' { + return "", "", ErrContactPhone + } + } + display := strings.TrimSpace(value) + if display == "" { + return "", "", nil + } + var digits strings.Builder + for i, c := range display { + if c >= '0' && c <= '9' || c == '+' && i == 0 { + digits.WriteRune(c) + } else if c == '+' { + return "", "", ErrContactPhone + } + } + number := digits.String() + n := len(strings.TrimPrefix(number, "+")) + if n < 3 || n > 15 { + return "", "", ErrContactPhone + } + return display, number, nil +} + +func loadProfileContact(ctx context.Context, q interface { + QueryRowContext(context.Context, string, ...any) *sql.Row +}, domain, owner string) (ProfileContact, error) { + c := ProfileContact{Visibility: "private"} + err := q.QueryRowContext(ctx, `SELECT phone_display,phone_number,visibility,version FROM profile_contacts WHERE domain_uuid=? AND principal_uuid=?`, domain, owner).Scan(&c.PhoneDisplay, &c.PhoneNumber, &c.Visibility, &c.Version) + if errors.Is(err, sql.ErrNoRows) { + err = nil + } + return c, err +} + +// ProfileContactForViewer returns no private data, including visibility/version, +// to unauthorized readers. Administrator status is not a contact consent grant. +func (s *Store) ProfileContactForViewer(ctx context.Context, domain, owner, viewer string) (ProfileContact, error) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + tx, err := s.DB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return ProfileContact{}, err + } + defer tx.Rollback() + var username string + err = tx.QueryRowContext(ctx, `SELECT i.subject FROM principals p JOIN domain_memberships m ON m.principal_uuid=p.uuid AND m.domain_uuid=? AND m.state='active' JOIN principal_identities i ON i.principal_uuid=p.uuid AND i.provider_type='password' WHERE p.uuid=? AND p.state IN ('active','readonly')`, domain, owner).Scan(&username) + if err != nil { + return ProfileContact{}, err + } + if viewer != "" { + var member bool + if err = tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM domain_memberships m JOIN principals p ON p.uuid=m.principal_uuid WHERE m.domain_uuid=? AND m.principal_uuid=? AND m.state='active' AND p.state IN ('active','readonly'))`, domain, viewer).Scan(&member); err != nil { + return ProfileContact{}, err + } + if !member { + return ProfileContact{}, nil + } + } + c, err := loadProfileContact(ctx, tx, domain, owner) + if err != nil { + return ProfileContact{}, err + } + if viewer != owner { + policy, err := loadUserSpacePolicy(ctx, tx, domain, owner, username) + if err != nil { + return ProfileContact{}, err + } + if viewer != "" { + if userSpaceVisibilityRank(c.Visibility) < 1 || userSpaceVisibilityRank(policy.ProfileVisibility) < 1 { + return ProfileContact{}, nil + } + } else { + allowed, err := s.domainAllowsUserSpacePublic(ctx, tx, domain) + if err != nil { + return ProfileContact{}, err + } + if !allowed || c.Visibility != "public" || policy.ProfileVisibility != "public" { + return ProfileContact{}, nil + } + } + } + if err = tx.Commit(); err != nil { + return ProfileContact{}, err + } + return c, nil +} + +// The actor is the owner; callers cannot pass a separate target principal. +func (s *Store) SaveProfileContact(ctx context.Context, domain, actor string, proposed ProfileContact, confirmPublic bool) error { + if s.ReadOnly { + return ErrReadOnly + } + display, number, err := NormalizeContactPhone(proposed.PhoneDisplay) + if err != nil { + return err + } + if userSpaceVisibilityRank(proposed.Visibility) < 0 || proposed.Version < 0 { + return ErrGovernanceInvalid + } + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + tx, err := s.DB.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + var username string + if err = tx.QueryRowContext(ctx, `SELECT i.subject FROM principals p JOIN domain_memberships m ON m.principal_uuid=p.uuid AND m.domain_uuid=? AND m.state='active' JOIN principal_identities i ON i.principal_uuid=p.uuid AND i.provider_type='password' WHERE p.uuid=? AND p.state='active'`, domain, actor).Scan(&username); err != nil { + return err + } + current, err := loadProfileContact(ctx, tx, domain, actor) + if err != nil { + return err + } + if current.Version != proposed.Version { + return ErrGovernanceConflict + } + if number == "" { + proposed.Visibility = "private" + } + if proposed.Visibility == "public" { + policy, err := loadUserSpacePolicy(ctx, tx, domain, actor, username) + if err != nil { + return err + } + allowed, err := s.domainAllowsUserSpacePublic(ctx, tx, domain) + if err != nil { + return err + } + if !allowed || policy.ProfileVisibility != "public" { + return ErrGovernanceCeiling + } + if (current.Visibility != "public" || current.PhoneNumber != number) && !confirmPublic { + return ErrGovernanceInvalid + } + } + _, err = tx.ExecContext(ctx, `INSERT INTO profile_contacts(domain_uuid,principal_uuid,phone_display,phone_number,visibility,version,updated_at_ms) VALUES(?,?,?,?,?,1,?) ON CONFLICT(domain_uuid,principal_uuid) DO UPDATE SET phone_display=excluded.phone_display,phone_number=excluded.phone_number,visibility=excluded.visibility,version=profile_contacts.version+1,updated_at_ms=excluded.updated_at_ms`, domain, actor, display, number, proposed.Visibility, time.Now().UnixMilli()) + if err != nil { + return err + } + return tx.Commit() +} diff --git a/internal/store/profile_contact_test.go b/internal/store/profile_contact_test.go new file mode 100644 index 0000000..e143962 --- /dev/null +++ b/internal/store/profile_contact_test.go @@ -0,0 +1,19 @@ +package store + +import "testing" + +func TestNormalizeContactPhone(t *testing.T) { + for _, tc := range []struct{ input, want string }{ + {"", ""}, {" 03-1234-5678 ", "0312345678"}, {"+81 (90) 1234-5678", "+819012345678"}, + } { + _, got, err := NormalizeContactPhone(tc.input) + if err != nil || got != tc.want { + t.Fatalf("%q: %q %v", tc.input, got, err) + } + } + for _, input := range []string{"*21*123#", "tel:1234", "javascript:alert(1)", "123;ext=4", "123,456", "123\r\n456", "\t123", "123", "12+34", "+", "12", "1234567890123456", "(+)123", "123\x1b"} { + if _, _, err := NormalizeContactPhone(input); err == nil { + t.Errorf("accepted %q", input) + } + } +} diff --git a/internal/store/projection_metadata.go b/internal/store/projection_metadata.go index 546a9b2..c6a46de 100644 --- a/internal/store/projection_metadata.go +++ b/internal/store/projection_metadata.go @@ -27,20 +27,7 @@ type ProjectionMetadata struct { } func (s *Store) EnsureProjectionMetadataSchema() error { - for _, statement := range []string{ - `CREATE TABLE IF NOT EXISTS projection_metadata ( - document_hash TEXT NOT NULL,kind TEXT NOT NULL,version INTEGER NOT NULL, - payload TEXT NOT NULL DEFAULT '{}',error TEXT NOT NULL DEFAULT '', - updated_at_ms INTEGER NOT NULL,retry_at_ms INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY(document_hash,kind,version))`, - `CREATE INDEX IF NOT EXISTS projection_metadata_eviction ON projection_metadata(updated_at_ms DESC)`, - `CREATE INDEX IF NOT EXISTS idx_event_jobs_identity_status ON event_jobs(event_type,event_key,status)`, - } { - if _, err := s.DB.Exec(statement); err != nil { - return err - } - } - return nil + return s.ensureSchemaComponent("projection-metadata") } func (s *Store) GetProjectionMetadata(hash, kind string, version int) (*ProjectionMetadata, error) { diff --git a/internal/store/publication_approval.go b/internal/store/publication_approval.go index d699a61..1530f01 100644 --- a/internal/store/publication_approval.go +++ b/internal/store/publication_approval.go @@ -11,9 +11,10 @@ import ( "time" "github.com/google/uuid" + "pageswiki/internal/schutzaria" ) -var ErrApprovalTransition = errors.New("invalid approval request transition") +var ErrApprovalTransition = schutzaria.ErrApprovalTransition type ApprovalRequest struct { UUID string `json:"uuid"` @@ -127,7 +128,7 @@ func (s *Store) DecideApprovalRequestScoped(ctx context.Context, domainUUID, req } func (s *Store) decideApprovalRequest(ctx context.Context, domainUUID, requestUUID, reviewerUUID, decision, justification string, generation int64) error { - if requestUUID == "" || reviewerUUID == "" || generation < 1 || (decision != "approved" && decision != "rejected" && decision != "changes_requested") { + if requestUUID == "" || reviewerUUID == "" || generation < 1 || !schutzaria.ValidApprovalDecision(decision) { return ErrApprovalTransition } tx, err := s.DB.BeginTx(ctx, nil) @@ -146,17 +147,14 @@ func (s *Store) decideApprovalRequest(ctx context.Context, domainUUID, requestUU if err := tx.QueryRowContext(ctx, query, args...).Scan(&state, ¤tGeneration); err != nil { return err } - if state != "pending" || currentGeneration != generation { - return ErrApprovalTransition + newState, err := schutzaria.ApprovalNextState(state, currentGeneration, generation, decision) + if err != nil { + return err } now := time.Now().UnixMilli() if _, err := tx.ExecContext(ctx, `INSERT INTO publication_decisions(uuid,request_uuid,reviewer_uuid,generation,decision,justification,created_at_ms) VALUES(?,?,?,?,?,?,?)`, uuid.NewString(), requestUUID, reviewerUUID, generation, decision, justification, now); err != nil { return err } - newState := decision - if decision == "changes_requested" { - newState = "rejected" - } if _, err := tx.ExecContext(ctx, `UPDATE publication_requests SET state=?,updated_at_ms=? WHERE uuid=? AND state='pending' AND generation=?`, newState, now, requestUUID, generation); err != nil { return err } diff --git a/internal/store/read_policy.go b/internal/store/read_policy.go index 00b7d40..a556205 100644 --- a/internal/store/read_policy.go +++ b/internal/store/read_policy.go @@ -3,38 +3,12 @@ package store import ( "errors" "strings" + + "pageswiki/internal/schutzaria" ) -// ReadPathPolicy is an immutable, request-scoped snapshot of positive Path -// read grants. Never share it across requests or use it to authorize writes. -type ReadPathPolicy struct { - access *localityAccessSnapshot - all bool - exact map[string]bool - subtrees map[string]bool -} - -func (p *ReadPathPolicy) Allows(path string) bool { - if p == nil { - return false - } - return p.access.decide(path, "read", p.allowsBase(path)) -} - -func (p *ReadPathPolicy) allowsBase(path string) bool { - if p == nil || !strings.HasPrefix(path, "/") { - return false - } - if p.all || p.exact[path] { - return true - } - for at := strings.LastIndex(path, "/"); at > 0; at = strings.LastIndex(path[:at], "/") { - if p.subtrees[path[:at]] { - return true - } - } - return false -} +// ReadPathPolicy is a request-scoped, read-only policy snapshot. +type ReadPathPolicy = schutzaria.ReadPathPolicy // LoadReadPathPolicy uses one bounded query, independent of the number of // directory entries. It deliberately excludes Object grants: an Object grant @@ -44,9 +18,10 @@ func (s *Store) LoadReadPathPolicy(domain, principal string) (*ReadPathPolicy, e } func loadReadPathPolicy(q pathQueryer, domain, principal string) (*ReadPathPolicy, error) { - p := &ReadPathPolicy{exact: map[string]bool{}, subtrees: map[string]bool{}} + var all bool + var exact, subtrees []string if domain == "" || principal == "" { - return p, ErrAccessDenied + return &ReadPathPolicy{}, ErrAccessDenied } rows, err := q.Query(` SELECT 'role',g.target_type,g.target_id FROM principals p @@ -79,15 +54,15 @@ func loadReadPathPolicy(q pathQueryer, domain, principal string) (*ReadPathPolic } if source == "role" { if kind == "domain" && target == domain || kind == "path" && target == "*" { - p.all = true + all = true } else if kind == "path" { - p.exact[target] = true + exact = append(exact, target) } } else if target == "/" { - p.all = true + all = true } else if strings.HasPrefix(target, "/") { - p.exact[target] = true - p.subtrees[strings.TrimRight(target, "/")] = true + exact = append(exact, target) + subtrees = append(subtrees, strings.TrimRight(target, "/")) } } if err = rows.Err(); err != nil { @@ -96,6 +71,9 @@ func loadReadPathPolicy(q pathQueryer, domain, principal string) (*ReadPathPolic if err = rows.Close(); err != nil { return nil, err } - p.access, err = loadLocalityAccess(q, domain, principal) - return p, err + access, err := loadLocalityAccess(q, domain, principal) + if err != nil { + return nil, err + } + return schutzaria.NewReadPathPolicy(all, exact, subtrees, access), nil } diff --git a/internal/store/readonly.go b/internal/store/readonly.go new file mode 100644 index 0000000..304a4ad --- /dev/null +++ b/internal/store/readonly.go @@ -0,0 +1,38 @@ +package store + +import ( + "crypto/sha256" + "database/sql" + "errors" + "os" + "path/filepath" + "strings" +) + +var ErrReadOnly = errors.New("DR standby is read-only") + +// No migrations, schema repair, key generation or file creation on standby. +func NewReadOnlyWithDocumentRoot(database *sql.DB, root string) (*Store, error) { + var readOnly int + if err := database.QueryRow("PRAGMA query_only").Scan(&readOnly); err != nil { + return nil, err + } + if readOnly != 1 { + return nil, errors.New("standby requires a read-only database connection") + } + st := &Store{DB: database, DocumentRoot: root, childCache: map[string][]string{}, ReadOnly: true} + if encoded := strings.TrimSpace(os.Getenv("PAGESWIKI_AUTH_KEY")); encoded != "" { + sum := sha256.Sum256([]byte(encoded)) + st.authKey = sum[:] + } else { + key, err := os.ReadFile(filepath.Join(filepath.Dir(root), "auth.key")) + if err != nil { + return nil, err + } + if len(key) != 32 { + return nil, errors.New("standby requires the original 32-byte auth.key") + } + st.authKey = key + } + return st, nil +} diff --git a/internal/store/recent_objects.go b/internal/store/recent_objects.go index e40db86..f195c36 100644 --- a/internal/store/recent_objects.go +++ b/internal/store/recent_objects.go @@ -1,7 +1,5 @@ package store -import "os" - type RecentObject struct { UUID string Path string @@ -117,9 +115,9 @@ func (s *Store) ListRecentObjectsInDomain( if out[i].documentHash == "" { continue } - if path, err := s.documentPath(out[i].documentHash); err == nil && path != "" { - if info, err := os.Stat(path); err == nil { - out[i].Size = int(info.Size()) + if s.DocumentRoot != "" { + if size, err := s.DocumentSize(out[i].documentHash); err == nil { + out[i].Size = int(size) } } } diff --git a/internal/store/resource_history.go b/internal/store/resource_history.go index dec62df..b206f92 100644 --- a/internal/store/resource_history.go +++ b/internal/store/resource_history.go @@ -4,17 +4,15 @@ import ( "context" "encoding/json" "time" + + "pageswiki/internal/seheweit" ) -const ResourceEventID = "builtin.resource.sample" -const ResourceEventType = "resource.sample" +const ResourceEventID = seheweit.ResourceEventID +const ResourceEventType = seheweit.ResourceEventType func (s *Store) EnsureResourceHistorySchema() error { - _, err := s.DB.Exec(`CREATE TABLE IF NOT EXISTS resource_samples ( - id INTEGER PRIMARY KEY, job_uuid TEXT NOT NULL UNIQUE, - timestamp_ms INTEGER NOT NULL, payload TEXT NOT NULL - ); CREATE INDEX IF NOT EXISTS idx_resource_samples_time ON resource_samples(timestamp_ms)`) - return err + return s.ensureSchemaComponent("resource-history") } func (s *Store) EnsureBuiltinResourceEvent() error { @@ -26,37 +24,34 @@ func (s *Store) EnsureBuiltinResourceEvent() error { } func (s *Store) SaveResourceSample(job string, at int64, payload []byte) error { - tx, err := s.DB.Begin() + return s.SaveResourceSampleContext(context.Background(), job, at, payload) +} + +func (s *Store) SaveResourceSampleContext(ctx context.Context, job string, at int64, payload []byte) error { + if s.ReadOnly { + return ErrReadOnly + } + tx, err := s.DB.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() - if _, err = tx.Exec(`INSERT OR IGNORE INTO resource_samples(job_uuid,timestamp_ms,payload) VALUES(?,?,?)`, job, at, string(payload)); err != nil { + if _, err = tx.ExecContext(ctx, `INSERT OR IGNORE INTO resource_samples(job_uuid,timestamp_ms,payload) VALUES(?,?,?)`, job, at, string(payload)); err != nil { return err } - if _, err = tx.Exec(`DELETE FROM resource_samples WHERE timestamp_ms < ?`, at-int64(7*24*time.Hour/time.Millisecond)); err != nil { + if _, err = tx.ExecContext(ctx, `DELETE FROM resource_samples WHERE timestamp_ms < ?`, seheweit.RetentionCutoff(at)); err != nil { return err } return tx.Commit() } -// Return the last sample in each time bucket, not averages disguised as peaks. +// Return the last inserted sample in each time bucket, not averages or peaks. // At most 361 points and an indexed seven-day scan; no request-side collection. func (s *Store) ResourceHistory(ctx context.Context, hours int, now time.Time) ([]json.RawMessage, error) { - if hours < 1 { - hours = 1 - } - if hours > 168 { - hours = 168 - } - span := int64(hours) * 3600000 - bucket := span / 360 - if bucket < 60000 { - bucket = 60000 - } + window := seheweit.Window(hours, now) rows, err := s.DB.QueryContext(ctx, `SELECT payload FROM resource_samples WHERE id IN (SELECT MAX(id) FROM resource_samples WHERE timestamp_ms>=? AND timestamp_ms<=? GROUP BY timestamp_ms / ?) - ORDER BY timestamp_ms LIMIT 361`, now.UnixMilli()-span, now.UnixMilli(), bucket) + ORDER BY timestamp_ms LIMIT ?`, window.FromMS, window.ToMS, window.BucketMS, seheweit.MaxHistoryPoints) if err != nil { return nil, err } diff --git a/internal/store/revision_store.go b/internal/store/revision_store.go index 0ee555e..5e82095 100644 --- a/internal/store/revision_store.go +++ b/internal/store/revision_store.go @@ -107,6 +107,19 @@ func (s *Store) saveRevisionContentUnlocked( contentType string, filename string, ) (*Revision, error) { + return s.saveRevisionContentExpected(objectUUID, payload, message, rawLanguage, contentType, filename, nil) +} + +// SaveRevisionContentIfHead compares the exact language head and appends within +// one SQLite write transaction, including writers outside this Store process. +// An empty expected UUID asserts that this language has no revision yet. +func (s *Store) SaveRevisionContentIfHead(objectUUID string, payload []byte, message, lang, contentType, filename, expected string) (*Revision, error) { + s.revisionWriteMu.Lock() + defer s.revisionWriteMu.Unlock() + return s.saveRevisionContentExpected(objectUUID, payload, message, lang, contentType, filename, &expected) +} + +func (s *Store) saveRevisionContentExpected(objectUUID string, payload []byte, message, rawLanguage, contentType, filename string, expected *string) (*Revision, error) { lang, err := normalizeRevisionLanguage( rawLanguage, @@ -165,6 +178,29 @@ func (s *Store) saveRevisionContentUnlocked( return nil, err } defer tx.Rollback() + if expected != nil { + // Acquire a write lock before reading the head. Avoid a read-to-write + // upgrade race with CLI/another process writing this same database. + if _, err = tx.Exec(`UPDATE objects SET uuid=uuid WHERE uuid=?`, objectUUID); err != nil { + return nil, err + } + var current string + var currentMS int64 + err = tx.QueryRow(`SELECT uuid,COALESCE(created_at_ms,created_at*1000) FROM revisions WHERE object_uuid=? AND COALESCE(NULLIF(lang,''),'und')=? ORDER BY COALESCE(created_at_ms,created_at*1000) DESC,rowid DESC LIMIT 1`, objectUUID, lang).Scan(¤t, ¤tMS) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return nil, err + } + if current != *expected { + return nil, ErrRevisionConflict + } + // Hashing/persisting may wait behind another process. The revision we + // accepted must sort after its base even if the wall clock moved back. + revision.CreatedAtMS = time.Now().UnixMilli() + if revision.CreatedAtMS <= currentMS { + revision.CreatedAtMS = currentMS + 1 + } + revision.CreatedAt = revision.CreatedAtMS / 1000 + } if err = insertRevision(tx, revision, embeddedHashes); err != nil { return nil, err } @@ -178,6 +214,36 @@ func (s *Store) saveRevisionContentUnlocked( return revision, nil } +// GetRevisionForExactLanguage deliberately has no viewer-language fallback. +func (s *Store) GetRevisionForExactLanguage(objectUUID, rawLanguage string) (*Revision, error) { + return s.revisionForExactLanguage(objectUUID, rawLanguage, false) +} + +// GetRevisionMetadataForExactLanguage does not read the document file. Polling +// an editor head must not hydrate a potentially large body on every request. +func (s *Store) GetRevisionMetadataForExactLanguage(objectUUID, rawLanguage string) (*Revision, error) { + return s.revisionForExactLanguage(objectUUID, rawLanguage, true) +} + +func (s *Store) revisionForExactLanguage(objectUUID, rawLanguage string, metadata bool) (*Revision, error) { + lang, err := normalizeRevisionLanguage(rawLanguage) + if err != nil { + return nil, err + } + var id string + err = s.DB.QueryRow(`SELECT uuid FROM revisions WHERE object_uuid=? AND COALESCE(NULLIF(lang,''),'und')=? ORDER BY COALESCE(created_at_ms,created_at*1000) DESC,rowid DESC LIMIT 1`, objectUUID, lang).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return &Revision{ObjectUUID: objectUUID, Language: lang, ContentType: "text/html; charset=utf-8"}, nil + } + if err != nil { + return nil, err + } + if metadata { + return s.GetRevisionMetadata(id) + } + return s.GetRevision(id) +} + func insertRevision(tx *sql.Tx, revision *Revision, embeddedHashes []string) error { result, err := tx.Exec(` INSERT INTO revisions ( diff --git a/internal/store/schema_inspect.go b/internal/store/schema_inspect.go new file mode 100644 index 0000000..21a20ce --- /dev/null +++ b/internal/store/schema_inspect.go @@ -0,0 +1,29 @@ +package store + +import ( + "context" + "database/sql" + + "pageswiki/internal/db" +) + +type SchemaInspection struct { + Coverage string `json:"coverage"` + Components []db.MigrationInspection `json:"components"` + LegacyAreas []string `json:"legacy_areas"` +} + +// InspectSchema deliberately does not construct a Store: constructors can +// migrate, backfill, generate keys, seed defaults and requeue interrupted jobs. +func InspectSchema(ctx context.Context, database *sql.DB) (SchemaInspection, error) { + plans := make(map[string][]db.SQLMigration, len(managedStoreSchemas)+1) + for name, steps := range managedStoreSchemas { + plans[name] = steps + } + plans["document-store"] = documentBackfillMigrations + components, err := db.InspectSQLiteMigrations(ctx, database, plans) + return SchemaInspection{Coverage: "managed-components-only", Components: components, LegacyAreas: []string{ + "core-bootstrap", "campuses/domains/federation", "auth/ACL/user-space", "outbox/notifications/publication-approval", + "City", "tasks/calendars", "periodicals", "search/object-listing/assets/GC", + }}, err +} diff --git a/internal/store/schema_inspect_test.go b/internal/store/schema_inspect_test.go new file mode 100644 index 0000000..1304a38 --- /dev/null +++ b/internal/store/schema_inspect_test.go @@ -0,0 +1,83 @@ +package store + +import ( + "context" + "testing" + + "pageswiki/internal/db" +) + +func TestSchemaInspectionDoesNotRecoverJobsOrBackfill(t *testing.T) { + st := newSchemaTestStore(t) + for name := range managedStoreSchemas { + if err := st.ensureSchemaComponent(name); err != nil { + t.Fatal(err) + } + } + if err := db.ApplySQLiteMigrations(context.Background(), st.DB, "document-store", documentBackfillMigrations); err != nil { + t.Fatal(err) + } + if _, err := st.DB.Exec(`INSERT INTO event_jobs(uuid,event_type,event_key,payload,status,run_at_ms,created_at_ms,updated_at_ms) VALUES('running','test','key','','running',1,1,1); INSERT INTO document_backfills VALUES('store','cursor',7,0,1)`); err != nil { + t.Fatal(err) + } + report, err := InspectSchema(context.Background(), st.DB) + if err != nil { + t.Fatal(err) + } + for _, component := range report.Components { + if component.State != "current" { + t.Fatalf("component=%+v", component) + } + } + var status string + if err := st.DB.QueryRow(`SELECT status FROM event_jobs WHERE uuid='running'`).Scan(&status); err != nil || status != "running" { + t.Fatalf("job modified: %q %v", status, err) + } + var processed, complete int + if err := st.DB.QueryRow(`SELECT processed,completed FROM document_backfills WHERE store_id='store'`).Scan(&processed, &complete); err != nil || processed != 7 || complete != 0 { + t.Fatalf("backfill modified: %d %d %v", processed, complete, err) + } + if err := st.EnsureEventJobsSchema(); err != nil { + t.Fatal(err) + } + if err := st.DB.QueryRow(`SELECT status FROM event_jobs WHERE uuid='running'`).Scan(&status); err != nil || status != "queued" { + t.Fatalf("startup recovery broken: %q %v", status, err) + } +} + +func TestEventJobMigrationAdoptsLegacyAndPreservesSettings(t *testing.T) { + st := newSchemaTestStore(t) + if _, err := st.DB.Exec(eventJobMigrations[0].Statements[0]); err != nil { + t.Fatal(err) + } + if _, err := st.DB.Exec(`INSERT INTO event_jobs(uuid,event_type,event_key,payload,status,run_at_ms,created_at_ms,updated_at_ms) VALUES('old','test','key','keep','running',1,1,1)`); err != nil { + t.Fatal(err) + } + if err := st.ensureSchemaComponent("event-jobs"); err != nil { + t.Fatal(err) + } + var payload, status string + var attempts int + if err := st.DB.QueryRow(`SELECT payload,status,max_attempts FROM event_jobs WHERE uuid='old'`).Scan(&payload, &status, &attempts); err != nil || payload != "keep" || status != "running" || attempts != 3 { + t.Fatalf("legacy job=%q %q %d %v", payload, status, attempts, err) + } + if _, err := st.DB.Exec(`UPDATE event_queue_settings SET paused=1,concurrency=3; UPDATE event_log_policies SET max_rows=42 WHERE event_type='*'`); err != nil { + t.Fatal(err) + } + if err := st.ensureSchemaComponent("event-jobs"); err != nil { + t.Fatal(err) + } + var paused, workers, rows int + if err := st.DB.QueryRow(`SELECT paused,concurrency FROM event_queue_settings WHERE id=1`).Scan(&paused, &workers); err != nil || paused != 1 || workers != 3 { + t.Fatalf("settings overwritten %d %d %v", paused, workers, err) + } + if err := st.DB.QueryRow(`SELECT max_rows FROM event_log_policies WHERE event_type='*'`).Scan(&rows); err != nil || rows != 42 { + t.Fatalf("policy overwritten %d %v", rows, err) + } + if _, err := st.DB.Exec(`DROP INDEX idx_event_jobs_due`); err != nil { + t.Fatal(err) + } + if err := st.ensureSchemaComponent("event-jobs"); err == nil { + t.Fatal("missing queue index accepted") + } +} diff --git a/internal/store/schema_jobs.go b/internal/store/schema_jobs.go new file mode 100644 index 0000000..f71fad8 --- /dev/null +++ b/internal/store/schema_jobs.go @@ -0,0 +1,120 @@ +package store + +import "pageswiki/internal/db" + +// Version 1 repairs only the known legacy columns; version 2 can then safely +// create indexes using them. Recovery of running jobs is intentionally separate. +var eventJobMigrations = []db.SQLMigration{ + {Version: 1, Name: "adopt_event_job_columns", Statements: []string{` + CREATE TABLE IF NOT EXISTS event_jobs ( + uuid TEXT PRIMARY KEY, + event_type TEXT NOT NULL, + event_key TEXT NOT NULL, + payload TEXT NOT NULL, + status TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + run_at_ms INTEGER NOT NULL, + error TEXT NOT NULL DEFAULT '', + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL + ) + `}, AddColumns: []db.SQLiteColumn{ + {Table: "event_jobs", Name: "event_uuid", Declaration: "TEXT NOT NULL DEFAULT ''"}, + {Table: "event_jobs", Name: "priority", Declaration: "INTEGER NOT NULL DEFAULT 0"}, + {Table: "event_jobs", Name: "max_attempts", Declaration: "INTEGER NOT NULL DEFAULT 3"}, + {Table: "event_jobs", Name: "started_at_ms", Declaration: "INTEGER NOT NULL DEFAULT 0"}, + {Table: "event_jobs", Name: "finished_at_ms", Declaration: "INTEGER NOT NULL DEFAULT 0"}, + {Table: "event_jobs", Name: "cancel_requested", Declaration: "INTEGER NOT NULL DEFAULT 0"}, + }}, + {Version: 2, Name: "adopt_event_queue_and_logs", Statements: []string{ + `CREATE INDEX IF NOT EXISTS idx_event_jobs_identity_status ON event_jobs(event_type,event_key,status)`, + `CREATE INDEX IF NOT EXISTS idx_event_jobs_listing ON event_jobs (created_at_ms DESC, uuid DESC)`, + `CREATE INDEX IF NOT EXISTS idx_event_jobs_status_listing ON event_jobs (status, created_at_ms DESC, uuid DESC)`, + `CREATE INDEX IF NOT EXISTS idx_event_jobs_history ON event_jobs (created_at_ms DESC, uuid DESC) WHERE status IN ('succeeded','skipped','failed','cancelled')`, + // Legacy installations already have this name without priority. IF NOT + // EXISTS would retain that definition and fail the contract below. This + // non-unique scheduling index is rebuilt once, in the adoption transaction; + // neither jobs nor their status/attempts are changed. + `DROP INDEX IF EXISTS idx_event_jobs_due`, + `CREATE INDEX IF NOT EXISTS idx_event_jobs_due + ON event_jobs (status, priority DESC, run_at_ms, created_at_ms)`, + `CREATE TABLE IF NOT EXISTS event_definitions ( + uuid TEXT PRIMARY KEY, name TEXT NOT NULL, event_type TEXT NOT NULL, + event_key TEXT NOT NULL, payload TEXT NOT NULL DEFAULT '', + schedule_kind TEXT NOT NULL DEFAULT 'manual', + interval_seconds INTEGER NOT NULL DEFAULT 0, + next_run_at_ms INTEGER NOT NULL DEFAULT 0, + enabled INTEGER NOT NULL DEFAULT 1, priority INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 3, + created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS idx_event_definitions_due + ON event_definitions (enabled, next_run_at_ms)`, + `CREATE TABLE IF NOT EXISTS event_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, job_uuid TEXT NOT NULL, + event_uuid TEXT NOT NULL DEFAULT '', level TEXT NOT NULL, + message TEXT NOT NULL, created_at_ms INTEGER NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS idx_event_logs_job + ON event_logs (job_uuid, created_at_ms DESC)`, + `CREATE INDEX IF NOT EXISTS idx_event_logs_job_id ON event_logs (job_uuid, id DESC)`, + `CREATE TABLE IF NOT EXISTS event_log_policies ( + event_type TEXT PRIMARY KEY, + retention_hours INTEGER NOT NULL DEFAULT 168, + aggregate_after_hours INTEGER NOT NULL DEFAULT 24, + bucket_seconds INTEGER NOT NULL DEFAULT 60, + max_rows INTEGER NOT NULL DEFAULT 5000 + )`, + `INSERT OR IGNORE INTO event_log_policies(event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows) + VALUES('*',168,24,60,5000)`, + `CREATE TABLE IF NOT EXISTS event_log_buckets ( + event_type TEXT NOT NULL, + bucket_start_ms INTEGER NOT NULL, + level TEXT NOT NULL, + count INTEGER NOT NULL DEFAULT 0, + first_at_ms INTEGER NOT NULL, + last_at_ms INTEGER NOT NULL, + sample_message TEXT NOT NULL DEFAULT '', + PRIMARY KEY(event_type,bucket_start_ms,level) + )`, + `CREATE INDEX IF NOT EXISTS idx_event_log_buckets_time ON event_log_buckets(event_type,bucket_start_ms DESC)`, + `CREATE TABLE IF NOT EXISTS event_queue_settings ( + id INTEGER PRIMARY KEY CHECK (id = 1), paused INTEGER NOT NULL DEFAULT 0, + concurrency INTEGER NOT NULL DEFAULT 1, + max_depth INTEGER NOT NULL DEFAULT 10000, + retry_base_seconds INTEGER NOT NULL DEFAULT 30, + retention_hours INTEGER NOT NULL DEFAULT 168 + )`, + `INSERT OR IGNORE INTO event_queue_settings + (id, paused, concurrency, max_depth, retry_base_seconds, retention_hours) + VALUES (1, 0, 1, 10000, 30, 168)`, + }, Assertions: []string{ + schemaColumns("event_jobs", "uuid,event_type,event_key,payload,status,attempts,run_at_ms,error,created_at_ms,updated_at_ms,event_uuid,priority,max_attempts,started_at_ms,finished_at_ms,cancel_requested"), + schemaColumns("event_definitions", "uuid,name,event_type,event_key,payload,schedule_kind,interval_seconds,next_run_at_ms,enabled,priority,max_attempts,created_at_ms,updated_at_ms"), + schemaColumns("event_logs", "id,job_uuid,event_uuid,level,message,created_at_ms"), + schemaColumns("event_log_policies", "event_type,retention_hours,aggregate_after_hours,bucket_seconds,max_rows"), + schemaColumns("event_log_buckets", "event_type,bucket_start_ms,level,count,first_at_ms,last_at_ms,sample_message"), + schemaColumns("event_queue_settings", "id,paused,concurrency,max_depth,retry_base_seconds,retention_hours"), + schemaUnique("event_jobs", "uuid"), schemaUnique("event_definitions", "uuid"), + schemaUnique("event_log_policies", "event_type"), schemaUnique("event_log_buckets", "event_type,bucket_start_ms,level"), + schemaIndex("event_jobs", "idx_event_jobs_identity_status", "event_type,event_key,status"), + schemaIndex("event_jobs", "idx_event_jobs_due", "status,priority,run_at_ms,created_at_ms"), + schemaIndex("event_jobs", "idx_event_jobs_listing", "created_at_ms,uuid"), + schemaIndex("event_jobs", "idx_event_jobs_status_listing", "status,created_at_ms,uuid"), + schemaIndex("event_jobs", "idx_event_jobs_history", "created_at_ms,uuid"), + schemaIndexPredicate("idx_event_jobs_history", "status IN ('succeeded','skipped','failed','cancelled')"), + schemaIndex("event_definitions", "idx_event_definitions_due", "enabled,next_run_at_ms"), + schemaIndex("event_logs", "idx_event_logs_job", "job_uuid,created_at_ms"), + schemaIndex("event_logs", "idx_event_logs_job_id", "job_uuid,id"), + schemaIndex("event_log_buckets", "idx_event_log_buckets_time", "event_type,bucket_start_ms"), + schemaAddedColumn("event_jobs", "priority", "INTEGER", "0"), + schemaAddedColumn("event_jobs", "max_attempts", "INTEGER", "3"), + schemaAddedColumn("event_jobs", "event_uuid", "TEXT", "''"), + schemaAddedColumn("event_jobs", "started_at_ms", "INTEGER", "0"), + schemaAddedColumn("event_jobs", "finished_at_ms", "INTEGER", "0"), + schemaAddedColumn("event_jobs", "cancel_requested", "INTEGER", "0"), + `SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_table_info('event_logs') WHERE name='id' AND type='INTEGER' AND pk=1)`, + `SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_table_info('event_queue_settings') WHERE name='id' AND type='INTEGER' AND pk=1)`, + `SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM event_queue_settings WHERE id=1)`, + }}, +} diff --git a/internal/store/schema_modules.go b/internal/store/schema_modules.go new file mode 100644 index 0000000..ed8ff83 --- /dev/null +++ b/internal/store/schema_modules.go @@ -0,0 +1,141 @@ +package store + +import ( + "context" + "fmt" + "strings" + + "pageswiki/internal/db" +) + +// These definitions adopt known legacy tables without replacing their rows. +// Once released, add a new version; never edit an applied definition. +// Auth, City, tasks and calendars remain in their existing migration paths. +var managedStoreSchemas = map[string][]db.SQLMigration{ + "profile-contacts": profileContactMigrations, + "event-jobs": eventJobMigrations, + "projection-metadata": {{Version: 1, Name: "adopt_projection_metadata", Statements: []string{ + `CREATE TABLE IF NOT EXISTS projection_metadata ( + document_hash TEXT NOT NULL,kind TEXT NOT NULL,version INTEGER NOT NULL, + payload TEXT NOT NULL DEFAULT '{}',error TEXT NOT NULL DEFAULT '', + updated_at_ms INTEGER NOT NULL,retry_at_ms INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(document_hash,kind,version))`, + `CREATE INDEX IF NOT EXISTS projection_metadata_eviction ON projection_metadata(updated_at_ms DESC)`, + }, Assertions: []string{ + schemaColumns("projection_metadata", "document_hash,kind,version,payload,error,updated_at_ms,retry_at_ms"), + schemaUnique("projection_metadata", "document_hash,kind,version"), + schemaIndex("projection_metadata", "projection_metadata_eviction", "updated_at_ms"), + }}}, + "resource-history": {{Version: 1, Name: "adopt_resource_history", Statements: []string{ + `CREATE TABLE IF NOT EXISTS resource_samples ( + id INTEGER PRIMARY KEY, job_uuid TEXT NOT NULL UNIQUE, + timestamp_ms INTEGER NOT NULL, payload TEXT NOT NULL)`, + `CREATE INDEX IF NOT EXISTS idx_resource_samples_time ON resource_samples(timestamp_ms)`, + }, Assertions: []string{ + schemaColumns("resource_samples", "id,job_uuid,timestamp_ms,payload"), + schemaUnique("resource_samples", "job_uuid"), + `SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_table_info('resource_samples') WHERE name='id' AND type='INTEGER' AND pk=1)`, + schemaIndex("resource_samples", "idx_resource_samples_time", "timestamp_ms"), + }}}, + "broadcasts": {{Version: 1, Name: "adopt_broadcasts", Statements: []string{ + `CREATE TABLE IF NOT EXISTS broadcasts (uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,title TEXT NOT NULL,body TEXT NOT NULL,severity TEXT NOT NULL DEFAULT 'info' CHECK(severity IN ('info','success','warning','critical')),starts_at_ms INTEGER NOT NULL DEFAULT 0,ends_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`, + `CREATE INDEX IF NOT EXISTS idx_broadcast_active ON broadcasts(domain_uuid,starts_at_ms,ends_at_ms,updated_at_ms DESC)`, + }, Assertions: []string{ + schemaColumns("broadcasts", "uuid,domain_uuid,title,body,severity,starts_at_ms,ends_at_ms,created_at_ms,updated_at_ms"), + schemaUnique("broadcasts", "uuid"), + schemaIndex("broadcasts", "idx_broadcast_active", "domain_uuid,starts_at_ms,ends_at_ms,updated_at_ms"), + }}}, + "browser-navigation": {{Version: 1, Name: "adopt_browser_navigation", Statements: []string{ + `CREATE TABLE IF NOT EXISTS browser_navigation_links ( + uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,principal_uuid TEXT NOT NULL, + label TEXT NOT NULL,path TEXT NOT NULL,is_directory INTEGER NOT NULL DEFAULT 0, + position INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL, + UNIQUE(domain_uuid, principal_uuid, path))`, + `CREATE INDEX IF NOT EXISTS idx_browser_navigation_owner ON browser_navigation_links(domain_uuid, principal_uuid, position, created_at_ms)`, + }, AddColumns: []db.SQLiteColumn{{Table: "browser_navigation_links", Name: "is_directory", Declaration: "INTEGER NOT NULL DEFAULT 0"}}, Assertions: []string{ + schemaColumns("browser_navigation_links", "uuid,domain_uuid,principal_uuid,label,path,is_directory,position,created_at_ms,updated_at_ms"), + schemaUnique("browser_navigation_links", "uuid"), + schemaUnique("browser_navigation_links", "domain_uuid,principal_uuid,path"), + schemaAddedColumn("browser_navigation_links", "is_directory", "INTEGER", "0"), + schemaIndex("browser_navigation_links", "idx_browser_navigation_owner", "domain_uuid,principal_uuid,position,created_at_ms"), + }}}, + "adapter-sessions": {{Version: 1, Name: "adopt_adapter_sessions", Statements: []string{ + `CREATE TABLE IF NOT EXISTS adapter_sessions ( + ticket_hash TEXT PRIMARY KEY,session_hash TEXT NOT NULL DEFAULT '', + domain_uuid TEXT NOT NULL,plugin_id TEXT NOT NULL,object_uuid TEXT NOT NULL, + principal_uuid TEXT NOT NULL DEFAULT '',base_revision_uuid TEXT NOT NULL, + ticket_expires_at_ms INTEGER NOT NULL,session_expires_at_ms INTEGER NOT NULL DEFAULT 0, + exchanged_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL)`, + `CREATE UNIQUE INDEX IF NOT EXISTS idx_adapter_session_hash ON adapter_sessions(session_hash) WHERE session_hash != ''`, + }, AddColumns: []db.SQLiteColumn{ + {Table: "adapter_sessions", Name: "root_path", Declaration: "TEXT NOT NULL DEFAULT ''"}, + {Table: "adapter_sessions", Name: "principal_uuid", Declaration: "TEXT NOT NULL DEFAULT ''"}, + }, Assertions: []string{ + schemaColumns("adapter_sessions", "ticket_hash,session_hash,domain_uuid,plugin_id,object_uuid,principal_uuid,base_revision_uuid,ticket_expires_at_ms,session_expires_at_ms,exchanged_at_ms,created_at_ms,root_path"), + schemaUnique("adapter_sessions", "ticket_hash"), + schemaAddedColumn("adapter_sessions", "root_path", "TEXT", "''"), + schemaAddedColumn("adapter_sessions", "principal_uuid", "TEXT", "''"), + schemaIndex("adapter_sessions", "idx_adapter_session_hash", "session_hash"), + `SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_index_list('adapter_sessions') WHERE name='idx_adapter_session_hash' AND "unique"=1 AND partial=1)`, + schemaIndexPredicate("idx_adapter_session_hash", "session_hash != ''"), + }}}, + "imports": {{Version: 1, Name: "adopt_imports", Statements: []string{ + `CREATE TABLE IF NOT EXISTS import_sessions(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,actor_uuid TEXT NOT NULL DEFAULT '',archive_name TEXT NOT NULL,archive_path TEXT NOT NULL,target_prefix TEXT NOT NULL DEFAULT '/',status TEXT NOT NULL DEFAULT 'queued',total_files INTEGER NOT NULL DEFAULT 0,markdown_files INTEGER NOT NULL DEFAULT 0,error TEXT NOT NULL DEFAULT '',created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`, + `CREATE TABLE IF NOT EXISTS import_items(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,source_path TEXT NOT NULL,target_path TEXT NOT NULL,size INTEGER NOT NULL DEFAULT 0,content_type TEXT NOT NULL DEFAULT '',title TEXT NOT NULL DEFAULT '',suggested_quartier TEXT NOT NULL DEFAULT '',suggested_street TEXT NOT NULL DEFAULT '',suggested_spot TEXT NOT NULL DEFAULT '',confidence REAL NOT NULL DEFAULT 0,selected INTEGER NOT NULL DEFAULT 1,PRIMARY KEY(session_uuid,sequence))`, + `CREATE TABLE IF NOT EXISTS import_journal(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,path TEXT NOT NULL,object_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,sequence))`, + `CREATE TABLE IF NOT EXISTS import_city_journal(session_uuid TEXT NOT NULL,kind TEXT NOT NULL,node_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,kind,node_uuid))`, + `CREATE TABLE IF NOT EXISTS import_city_edges(session_uuid TEXT NOT NULL,spot_uuid TEXT NOT NULL,page_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,spot_uuid,page_uuid))`, + `CREATE INDEX IF NOT EXISTS idx_import_sessions_domain ON import_sessions(domain_uuid,created_at_ms DESC)`, + }, AddColumns: []db.SQLiteColumn{{Table: "import_journal", Name: "completed", Declaration: "INTEGER NOT NULL DEFAULT 0"}}, Assertions: []string{ + schemaColumns("import_sessions", "uuid,domain_uuid,actor_uuid,archive_name,archive_path,target_prefix,status,total_files,markdown_files,error,created_at_ms,updated_at_ms"), + schemaColumns("import_items", "session_uuid,sequence,source_path,target_path,size,content_type,title,suggested_quartier,suggested_street,suggested_spot,confidence,selected"), + schemaColumns("import_journal", "session_uuid,sequence,path,object_uuid,created_at_ms,completed"), + schemaColumns("import_city_journal", "session_uuid,kind,node_uuid,created_at_ms"), + schemaColumns("import_city_edges", "session_uuid,spot_uuid,page_uuid,created_at_ms"), + schemaUnique("import_sessions", "uuid"), + schemaUnique("import_items", "session_uuid,sequence"), + schemaUnique("import_journal", "session_uuid,sequence"), + schemaUnique("import_city_journal", "session_uuid,kind,node_uuid"), + schemaUnique("import_city_edges", "session_uuid,spot_uuid,page_uuid"), + schemaAddedColumn("import_journal", "completed", "INTEGER", "0"), + schemaIndex("import_sessions", "idx_import_sessions_domain", "domain_uuid,created_at_ms"), + }}}, +} + +func (s *Store) ensureSchemaComponent(component string) error { + if s.ReadOnly { + return ErrReadOnly + } + steps, ok := managedStoreSchemas[component] + if !ok { + return fmt.Errorf("unknown schema component %q", component) + } + return db.ApplySQLiteMigrations(context.Background(), s.DB, component, steps) +} + +// These helpers consume only static definitions above, never request input. +func schemaColumns(table, columns string) string { + return fmt.Sprintf("SELECT %s FROM %q LIMIT 0", columns, table) +} + +func schemaUnique(table, columns string) string { + return fmt.Sprintf(`SELECT 1 WHERE NOT EXISTS ( + SELECT 1 FROM pragma_index_list('%s') AS i WHERE i."unique"=1 AND i.partial=0 + AND (SELECT group_concat(name,',') FROM (SELECT name FROM pragma_index_info(i.name) ORDER BY seqno))='%s')`, table, columns) +} + +func schemaIndex(table, name, columns string) string { + return fmt.Sprintf(`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_index_list('%s') WHERE name='%s') + OR (SELECT group_concat(name,',') FROM (SELECT name FROM pragma_index_info('%s') ORDER BY seqno)) IS NOT '%s'`, table, name, name, strings.TrimSpace(columns)) +} + +func schemaAddedColumn(table, column, kind, defaultSQL string) string { + return fmt.Sprintf(`SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM pragma_table_info('%s') WHERE name='%s' AND upper(type)='%s' AND "notnull"=1 AND dflt_value='%s')`, table, column, kind, strings.ReplaceAll(defaultSQL, "'", "''")) +} + +func schemaIndexPredicate(name, predicate string) string { + normalized := strings.ToLower(strings.Join(strings.Fields(predicate), "")) + return fmt.Sprintf(`SELECT 1 WHERE NOT EXISTS ( + SELECT 1 FROM (SELECT lower(replace(replace(replace(replace(sql,' ',''),char(9),''),char(10),''),char(13),'')) AS normalized FROM sqlite_master WHERE type='index' AND name='%s') + WHERE substr(normalized,instr(normalized,'where')+5)='%s')`, name, strings.ReplaceAll(normalized, "'", "''")) +} diff --git a/internal/store/schema_modules_test.go b/internal/store/schema_modules_test.go new file mode 100644 index 0000000..6512a2c --- /dev/null +++ b/internal/store/schema_modules_test.go @@ -0,0 +1,143 @@ +package store + +import ( + "errors" + "path/filepath" + "testing" + + "pageswiki/internal/db" +) + +func newSchemaTestStore(t *testing.T) *Store { + t.Helper() + database, err := db.Open(filepath.Join(t.TempDir(), "schema.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { database.Close() }) + return &Store{DB: database} +} + +func TestManagedSchemasFreshRepeatAndReadOnly(t *testing.T) { + for component := range managedStoreSchemas { + t.Run(component, func(t *testing.T) { + st := newSchemaTestStore(t) + if err := st.ensureSchemaComponent(component); err != nil { + t.Fatal(err) + } + if err := st.ensureSchemaComponent(component); err != nil { + t.Fatal(err) + } + var count int + if err := st.DB.QueryRow(`SELECT count(*) FROM schema_migration_steps WHERE component=?`, component).Scan(&count); err != nil || count != len(managedStoreSchemas[component]) { + t.Fatalf("ledger=%d err=%v", count, err) + } + st.ReadOnly = true + if err := st.ensureSchemaComponent(component); !errors.Is(err, ErrReadOnly) { + t.Fatalf("standby migration: %v", err) + } + }) + } +} + +func TestManagedSchemasAdoptLegacyRowsWithoutRecreatingTables(t *testing.T) { + for _, tc := range []struct{ component, ddl, insert, query, want string }{ + {"browser-navigation", `CREATE TABLE browser_navigation_links(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,principal_uuid TEXT NOT NULL,label TEXT NOT NULL,path TEXT NOT NULL,position INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL,UNIQUE(domain_uuid,principal_uuid,path))`, + `INSERT INTO browser_navigation_links VALUES('u','d','p','keep','/page',3,10,20)`, `SELECT label||':'||position||':'||is_directory FROM browser_navigation_links WHERE uuid='u'`, "keep:3:0"}, + {"adapter-sessions", `CREATE TABLE adapter_sessions(ticket_hash TEXT PRIMARY KEY,session_hash TEXT NOT NULL DEFAULT '',domain_uuid TEXT NOT NULL,plugin_id TEXT NOT NULL,object_uuid TEXT NOT NULL,base_revision_uuid TEXT NOT NULL,ticket_expires_at_ms INTEGER NOT NULL,session_expires_at_ms INTEGER NOT NULL DEFAULT 0,exchanged_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL)`, + `INSERT INTO adapter_sessions VALUES('ticket','session','d','code','o','r',100,200,50,10)`, `SELECT session_hash||':'||root_path||':'||principal_uuid FROM adapter_sessions WHERE ticket_hash='ticket'`, "session::"}, + {"imports", `CREATE TABLE import_journal(session_uuid TEXT NOT NULL,sequence INTEGER NOT NULL,path TEXT NOT NULL,object_uuid TEXT NOT NULL,created_at_ms INTEGER NOT NULL,PRIMARY KEY(session_uuid,sequence))`, + `INSERT INTO import_journal VALUES('s',2,'/keep','o',20)`, `SELECT path||':'||completed FROM import_journal WHERE session_uuid='s' AND sequence=2`, "/keep:0"}, + {"resource-history", `CREATE TABLE resource_samples(id INTEGER PRIMARY KEY,job_uuid TEXT NOT NULL UNIQUE,timestamp_ms INTEGER NOT NULL,payload TEXT NOT NULL)`, + `INSERT INTO resource_samples VALUES(9,'j',100,'keep')`, `SELECT id||':'||payload FROM resource_samples WHERE job_uuid='j'`, "9:keep"}, + {"projection-metadata", `CREATE TABLE projection_metadata(document_hash TEXT NOT NULL,kind TEXT NOT NULL,version INTEGER NOT NULL,payload TEXT NOT NULL DEFAULT '{}',error TEXT NOT NULL DEFAULT '',updated_at_ms INTEGER NOT NULL,retry_at_ms INTEGER NOT NULL DEFAULT 0,PRIMARY KEY(document_hash,kind,version))`, + `INSERT INTO projection_metadata VALUES('h','pdf',1,'keep','',10,0)`, `SELECT payload FROM projection_metadata WHERE document_hash='h'`, "keep"}, + {"broadcasts", `CREATE TABLE broadcasts(uuid TEXT PRIMARY KEY,domain_uuid TEXT NOT NULL,title TEXT NOT NULL,body TEXT NOT NULL,severity TEXT NOT NULL DEFAULT 'info' CHECK(severity IN ('info','success','warning','critical')),starts_at_ms INTEGER NOT NULL DEFAULT 0,ends_at_ms INTEGER NOT NULL DEFAULT 0,created_at_ms INTEGER NOT NULL,updated_at_ms INTEGER NOT NULL)`, + `INSERT INTO broadcasts VALUES('u','d','title','keep','info',0,0,10,20)`, `SELECT body FROM broadcasts WHERE uuid='u'`, "keep"}, + } { + t.Run(tc.component, func(t *testing.T) { + st := newSchemaTestStore(t) + for _, statement := range []string{tc.ddl, tc.insert} { + if _, err := st.DB.Exec(statement); err != nil { + t.Fatal(err) + } + } + if err := st.ensureSchemaComponent(tc.component); err != nil { + t.Fatal(err) + } + var value string + if err := st.DB.QueryRow(tc.query).Scan(&value); err != nil || value != tc.want { + t.Fatalf("row=%q want=%q err=%v", value, tc.want, err) + } + if err := st.ensureSchemaComponent(tc.component); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestManagedSchemasRejectBrokenAdoptionAndLaterDrift(t *testing.T) { + st := newSchemaTestStore(t) + // Matching columns without the conflict-target uniqueness is not sufficient. + if _, err := st.DB.Exec(`CREATE TABLE resource_samples(id INTEGER PRIMARY KEY,job_uuid TEXT NOT NULL,timestamp_ms INTEGER NOT NULL,payload TEXT NOT NULL); INSERT INTO resource_samples VALUES(1,'j',10,'keep')`); err != nil { + t.Fatal(err) + } + if err := st.EnsureResourceHistorySchema(); err == nil { + t.Fatal("missing unique constraint accepted") + } + var payload string + if err := st.DB.QueryRow(`SELECT payload FROM resource_samples WHERE id=1`).Scan(&payload); err != nil || payload != "keep" { + t.Fatal("failed adoption modified data", err) + } + var count int + if err := st.DB.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='idx_resource_samples_time'`).Scan(&count); err != nil || count != 0 { + t.Fatal("failed adoption left partial DDL", err) + } + if _, err := st.DB.Exec(`CREATE UNIQUE INDEX restored_resource_unique ON resource_samples(job_uuid)`); err != nil { + t.Fatal(err) + } + if err := st.EnsureResourceHistorySchema(); err != nil { + t.Fatal(err) + } + if _, err := st.DB.Exec(`DROP INDEX idx_resource_samples_time`); err != nil { + t.Fatal(err) + } + if err := st.EnsureResourceHistorySchema(); err == nil { + t.Fatal("ledger masked missing index") + } + if err := st.EnsureAdapterSessionsSchema(); err != nil { + t.Fatal(err) + } + if _, err := st.DB.Exec(`DROP INDEX idx_adapter_session_hash; CREATE UNIQUE INDEX idx_adapter_session_hash ON adapter_sessions(session_hash) WHERE session_hash = ''`); err != nil { + t.Fatal(err) + } + if err := st.EnsureAdapterSessionsSchema(); err == nil { + t.Fatal("incorrect partial index predicate accepted") + } +} + +func TestEventJobsAdoptLegacyDueIndex(t *testing.T) { + st := newSchemaTestStore(t) + // The deployed pre-ledger scheduler used the same index name without priority. + for _, statement := range []string{ + eventJobMigrations[0].Statements[0], + `CREATE INDEX idx_event_jobs_due ON event_jobs(status,run_at_ms,created_at_ms)`, + `INSERT INTO event_jobs(uuid,event_type,event_key,payload,status,attempts,run_at_ms,created_at_ms,updated_at_ms) VALUES('keep','test','key','unchanged','running',2,10,20,30)`, + } { + if _, err := st.DB.Exec(statement); err != nil { + t.Fatal(err) + } + } + for range 2 { + if err := st.ensureSchemaComponent("event-jobs"); err != nil { + t.Fatal(err) + } + } + var columns, row string + if err := st.DB.QueryRow(`SELECT group_concat(name,',') FROM (SELECT name FROM pragma_index_info('idx_event_jobs_due') ORDER BY seqno)`).Scan(&columns); err != nil || columns != "status,priority,run_at_ms,created_at_ms" { + t.Fatalf("index=%q err=%v", columns, err) + } + if err := st.DB.QueryRow(`SELECT payload||':'||status||':'||attempts||':'||run_at_ms||':'||created_at_ms||':'||updated_at_ms FROM event_jobs WHERE uuid='keep'`).Scan(&row); err != nil || row != "unchanged:running:2:10:20:30" { + t.Fatalf("adoption changed job: %q err=%v", row, err) + } +} diff --git a/internal/store/schutzaria_test.go b/internal/store/schutzaria_test.go new file mode 100644 index 0000000..376c8b9 --- /dev/null +++ b/internal/store/schutzaria_test.go @@ -0,0 +1,91 @@ +package store + +import ( + "database/sql" + "encoding/json" + "testing" + + "pageswiki/internal/schutzaria" +) + +func schutzariaStoreFixture(t *testing.T) *Store { + t.Helper() + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { db.Close() }) + s := &Store{DB: db} + if err = s.EnsureAuthSchema(); err != nil { + t.Fatal(err) + } + for _, stmt := range []string{ + `INSERT INTO principals(uuid,state,created_at_ms,updated_at_ms) VALUES('p','active',1,1)`, + `INSERT INTO domain_memberships(uuid,domain_uuid,principal_uuid,state,created_at_ms) VALUES('m','d','p','active',1)`, + `INSERT INTO auth_roles(uuid,domain_uuid,name,created_at_ms) VALUES('r','d','reader',1)`, + `INSERT INTO auth_role_bindings(membership_uuid,role_uuid,created_at_ms) VALUES('m','r',1)`, + } { + if _, err = db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + return s +} + +func TestSchutzariaReadLoaderBudgetsAndErrors(t *testing.T) { + for _, kind := range []string{"grant", "locality", "db-error"} { + t.Run(kind, func(t *testing.T) { + s := schutzariaStoreFixture(t) + if _, err := s.DB.Exec(`INSERT INTO auth_capability_grants(uuid,role_uuid,capability,target_type,target_id,created_at_ms) VALUES('all','r','read','domain','d',1)`); err != nil { + t.Fatal(err) + } + switch kind { + case "grant": + _, err := s.DB.Exec(`WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<4096) INSERT INTO auth_capability_grants(uuid,role_uuid,capability,target_type,target_id,created_at_ms) SELECT 'grant-'||x,'r','read','path','/page/'||x,1 FROM n`) + if err != nil { + t.Fatal(err) + } + case "locality": + _, err := s.DB.Exec(`WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<4097) INSERT INTO locality_access_policies(domain_uuid,node_kind,node_uuid,scope_path,grants_access,actor_uuid,authority_rank) SELECT 'd','spot','spot-'||x,'/scope/'||x,0,'p',3 FROM n`) + if err != nil { + t.Fatal(err) + } + case "db-error": + if _, err := s.DB.Exec(`DROP TABLE locality_access_policies`); err != nil { + t.Fatal(err) + } + } + policy, err := s.LoadReadPathPolicy("d", "p") + if err == nil || policy != nil || policy.Allows("/would-be-allowed") { + t.Fatal("incomplete ACL snapshot escaped", policy, err) + } + }) + } +} + +func TestSchutzariaStoreCompatibility(t *testing.T) { + var policy *ReadPathPolicy = schutzaria.NewReadPathPolicy(false, []string{"/exact"}, nil, nil) + if !policy.Allows("/exact") || policy.Allows("/exact/child") { + t.Fatal("alias mismatch") + } + var locality LocalityPolicy = schutzaria.LocalityPolicy{Kind: "spot", UUID: "id", Policy: "public"} + data, err := json.Marshal(locality) + if err != nil || string(data) != `{"kind":"spot","uuid":"id","policy":"public"}` { + t.Fatal(string(data), err) + } + var user UserSpacePolicy = schutzaria.UserSpacePolicy{Username: "alice", RootVisibility: "private", DescendantVisibility: "domain"} + for _, p := range []string{"/u/alice", "/u/alice/page", "/u/bob"} { + if EffectiveUserSpaceVisibility(user, p) != schutzaria.EffectiveUserSpaceVisibility(user, p) { + t.Fatal("user policy wrapper mismatch") + } + } + if ErrApprovalTransition != schutzaria.ErrApprovalTransition { + t.Fatal("error identity changed") + } + for _, p := range []string{"/public", "/u/private", "/v/tree"} { + if PublicSemanticPath(p) != schutzaria.PublicSemanticPath(p) { + t.Fatal("publication wrapper mismatch") + } + } +} diff --git a/internal/store/search.go b/internal/store/search.go index 26e1054..6a2d2b3 100644 --- a/internal/store/search.go +++ b/internal/store/search.go @@ -2,378 +2,34 @@ package store import ( "context" - "database/sql" - "errors" - "sort" - "strings" - "time" + + "pageswiki/internal/mestionora" ) -const SearchCandidateLimit = 1000 -const SearchExcerptDocumentBytes = 64 << 10 -const SearchExcerptTotalBytes = 1 << 20 -const searchExcerptTimeout = 400 * time.Millisecond +type SearchResult = mestionora.SearchResult +type PathSuggestion = mestionora.PathSuggestion -const searchResultColumns = `p.path,COALESCE(r.content_type,''),COALESCE(r.filename,''),COALESCE(r.document_hash,''),COALESCE(r.revision_uuid,''),COALESCE(r.created_at_ms/1000,0)` -const searchPathBoundary = `(p.path=? OR (p.path>=? AND p.path0 OR instr(lower(COALESCE(r.filename,'')),lower(?))>0)` -const searchPathCandidatesSQL = searchPathCandidatesBaseSQL + searchPathQueryPredicate + ` ORDER BY p.path LIMIT ?` +const ( + SearchCandidateLimit = mestionora.SearchCandidateLimit + SearchExcerptDocumentBytes = mestionora.SearchExcerptDocumentBytes + SearchExcerptTotalBytes = mestionora.SearchExcerptTotalBytes +) -// CROSS JOIN is intentional: revision_uuid is an UNINDEXED FTS payload column. -// A Path-first plan re-runs the complete MATCH scan for every Path. Start with -// MATCH exactly once, then seek current Revision metadata and Domain bindings -// through their B-tree indexes. Keep scope filtering before the candidate cap. -const searchFTSCandidatesBaseSQL = `SELECT ` + searchResultColumns + `,revision_search.rowid - FROM revision_search - CROSS JOIN latest_revision_index r ON r.revision_uuid=revision_search.revision_uuid - CROSS JOIN paths p ON p.object_uuid=r.object_uuid - WHERE revision_search MATCH ? AND p.domain_uuid=? AND ` + searchPathBoundary -const searchFTSCandidatesSQL = searchFTSCandidatesBaseSQL + ` LIMIT ?` - -func searchBodyExcerptsSQL(count int) string { - // UUID is not indexed by FTS5. Restrict the MATCH/snippet work using the - // integer rowids collected from the same read snapshot, not payload UUIDs. - return `SELECT revision_uuid,substr(snippet(revision_search,3,'','',' … ',36),1,4096) - FROM revision_search WHERE revision_search MATCH ? AND rowid IN (` + strings.TrimRight(strings.Repeat("?,", count), ",") + `)` -} - -// Excerpts are optional enrichment. Even small documents can have expensive -// prefix hit-position scans, so their time budget is separate from finding -// results. A spent excerpt budget omits snippets, not already-authorized hits. -func loadSearchBodyExcerpts(parent context.Context, tx *sql.Tx, expression string, ids []any, budget time.Duration) (excerpts map[string]string, err error) { - ctx, cancel := context.WithTimeout(parent, budget) - defer cancel() - defer func() { - if err != nil && ctx.Err() != nil { - if parent.Err() != nil { - err = parent.Err() - } else { - excerpts, err = nil, nil - } - } - }() - args := append([]any{"body : (" + expression + ")"}, ids...) - rows, err := tx.QueryContext(ctx, searchBodyExcerptsSQL(len(ids)), args...) - if err != nil { - return nil, err - } - defer rows.Close() - excerpts = make(map[string]string) - for rows.Next() { - var id, excerpt string - if err := rows.Scan(&id, &excerpt); err != nil { - return nil, err - } - excerpts[id] = excerpt - } - return excerpts, rows.Err() -} - -type SearchResult struct { - Path string - ContentType string - Filename string - DocumentHash string - RevisionUUID string - Size int64 // Kept for API compatibility; search does not stat Document files. - UpdatedAt int64 - PathMatch bool - FilenameMatch bool - BodyMatch bool - Excerpt string // Untrusted source text, never HTML-safe markup. - ExcerptOmitted bool // A body match remains visible when snippet work exceeds its budget. -} - -type PathSuggestion struct { - SearchResult - Title string -} - -// SearchPathsInDomain preserves the store API for trusted callers. HTTP callers -// must use SearchPathsInDomainFiltered with a request-scoped authorization filter. +// SearchPathsInDomain is the legacy trusted-caller entry point. HTTP callers +// must pass their request policy to the filtered APIs; there is no implicit +// allow-all entry point in Mestionora itself. func (s *Store) SearchPathsInDomain(domainUUID, scope, query string, limit int) ([]SearchResult, error) { return s.SearchPathsInDomainFiltered(context.Background(), domainUUID, scope, query, limit, func(string) bool { return true }) } -// SearchPathsInDomainFiltered bounds SQL candidates before materialization and -// authorizes each Path before loading an excerpt. It never computes a global hit -// count or stats Document files. A narrow scope improves both recall and cost. func (s *Store) SearchPathsInDomainFiltered(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]SearchResult, error) { - return s.SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}) + return mestionora.NewSQLite(s.DB).SearchPathsInDomainFiltered(ctx, domainUUID, scope, query, limit, allowed) } -// SearchPathsInDomainWithOptions also permits filter-only searches. File/date -// predicates apply before SQL candidate caps, while ACL checks stay ahead of all -// snippet enrichment. Zero options preserve the original indexed query plans. -func (s *Store) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) (found []SearchResult, searchErr error) { - return s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options, true) +func (s *Store) SearchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions) ([]SearchResult, error) { + return mestionora.NewSQLite(s.DB).SearchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, options) } -// SearchPathSuggestionsInDomain shares the current-Revision FTS and ACL path -// with full search, but deliberately skips snippet generation. Typeahead calls -// are frequent and only need a bounded display label, never a body excerpt. func (s *Store) SearchPathSuggestionsInDomain(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool) ([]PathSuggestion, error) { - results, err := s.searchPathsInDomainWithOptions(ctx, domainUUID, scope, query, limit, allowed, SearchOptions{}, false) - if err != nil || len(results) == 0 { - return nil, err - } - ids := make([]any, 0, len(results)) - seen := make(map[string]bool, len(results)) - for _, result := range results { - if result.RevisionUUID != "" && strings.HasPrefix(strings.ToLower(result.ContentType), "text/html") && !seen[result.RevisionUUID] { - ids = append(ids, result.RevisionUUID) - seen[result.RevisionUUID] = true - } - } - headings := make(map[string]string, len(ids)) - if len(ids) > 0 { - rows, queryErr := s.DB.QueryContext(ctx, `SELECT uuid,substr(CAST(body AS TEXT),1,65536) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...) - if queryErr != nil { - return nil, queryErr - } - for rows.Next() { - var revision, source string - if queryErr = rows.Scan(&revision, &source); queryErr != nil { - rows.Close() - return nil, queryErr - } - headings[revision] = firstHTMLHeading(source) - } - queryErr = rows.Err() - rows.Close() - if queryErr != nil { - return nil, queryErr - } - } - out := make([]PathSuggestion, 0, len(results)) - for _, result := range results { - out = append(out, PathSuggestion{SearchResult: result, Title: headings[result.RevisionUUID]}) - } - return out, nil -} - -func (s *Store) searchPathsInDomainWithOptions(ctx context.Context, domainUUID, scope, query string, limit int, allowed func(string) bool, options SearchOptions, loadExcerpts bool) (found []SearchResult, searchErr error) { - if err := options.Validate(); err != nil { - return nil, err - } - scope, query = normalizePath(scope), strings.TrimSpace(query) - if query == "" && !options.HasFilters() || allowed == nil { - return nil, nil - } - if len(query) > 512 || len(strings.Fields(query)) > 16 { - return nil, errors.New("search query is too long") - } - if limit <= 0 || limit > 200 { - limit = 100 - } - ctx, cancel := context.WithTimeout(ctx, 2*time.Second) - defer cancel() - defer func() { - // SQLite can report an interrupted statement instead of wrapping the - // context error. Preserve cancellation/deadline semantics for HTTP. - if searchErr != nil && ctx.Err() != nil { - searchErr = ctx.Err() - } - }() - tx, err := s.DB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) - if err != nil { - return nil, err - } - defer tx.Rollback() - prefix := strings.TrimRight(scope, "/") + "/" - upper := strings.TrimRight(scope, "/") + "0" - results := make(map[string]SearchResult) - ftsRows := make(map[string]int64) - collect := func(statement string, body, filename bool, args ...any) error { - rows, err := tx.QueryContext(ctx, statement, args...) - if err != nil { - return err - } - defer rows.Close() - for rows.Next() { - var result SearchResult - var rowID int64 - if err := rows.Scan(&result.Path, &result.ContentType, &result.Filename, &result.DocumentHash, &result.RevisionUUID, &result.UpdatedAt, &rowID); err != nil { - return err - } - if !allowed(result.Path) { - continue - } - if rowID != 0 { - ftsRows[result.RevisionUUID] = rowID - } - result.PathMatch = query != "" && strings.Contains(strings.ToLower(result.Path), strings.ToLower(query)) - result.FilenameMatch = filename || results[result.Path].FilenameMatch || query != "" && strings.Contains(strings.ToLower(result.Filename), strings.ToLower(query)) - result.BodyMatch = body || results[result.Path].BodyMatch - results[result.Path] = result - } - return rows.Err() - } - filterSQL, filterArgs := searchOptionsPredicate(options) - pathSQL := searchPathCandidatesBaseSQL - pathArgs := []any{domainUUID, scope, prefix, upper} - if query != "" { - pathSQL += searchPathQueryPredicate - pathArgs = append(pathArgs, query, query) - } - pathSQL += filterSQL + ` ORDER BY p.path LIMIT ?` - pathArgs = append(pathArgs, filterArgs...) - pathArgs = append(pathArgs, SearchCandidateLimit) - if err := collect(pathSQL, false, false, pathArgs...); err != nil { - return nil, err - } - expression := ftsPrefixExpression(query) - if expression != "" { - // No global ORDER BY or unbounded IN materialization: stop the FTS scan - // after at most this many current, in-Domain, in-scope Path candidates. - for _, field := range []string{"body", "filename"} { - args := []any{field + " : (" + expression + ")", domainUUID, scope, prefix, upper} - args = append(args, filterArgs...) - args = append(args, SearchCandidateLimit) - if err := collect(searchFTSCandidatesBaseSQL+filterSQL+` LIMIT ?`, field == "body", field == "filename", args...); err != nil { - return nil, err - } - } - } - out := make([]SearchResult, 0, len(results)) - for _, result := range results { - out = append(out, result) - } - sort.Slice(out, func(i, j int) bool { - // Direct name/Path matches precede body-only hits; all ties are stable. - iName, jName := out[i].PathMatch || out[i].FilenameMatch, out[j].PathMatch || out[j].FilenameMatch - if iName != jName { - return iName - } - if len(out[i].Path) != len(out[j].Path) { - return len(out[i].Path) < len(out[j].Path) - } - return out[i].Path < out[j].Path - }) - if len(out) > limit { - out = out[:limit] - } - if !loadExcerpts { - return out, nil - } - // Only authorized, displayed results receive text excerpts. Batch the FTS - // lookup so a broad query is not re-scanned once for every visible result. - ids := make([]any, 0, len(out)) - seen := make(map[string]bool) - for _, result := range out { - if result.BodyMatch && !seen[result.RevisionUUID] { - ids = append(ids, ftsRows[result.RevisionUUID]) - seen[result.RevisionUUID] = true - } - } - if len(ids) != 0 { - // snippet's output token/character limit does not bound its input work: - // FTS5 may tokenize the entire matched document. Read byte lengths only - // for authorized displayed rows before spending the snippet budget. - rows, err := tx.QueryContext(ctx, `SELECT rowid,octet_length(body) - FROM revision_search WHERE rowid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...) - if err != nil { - return nil, err - } - sizes := make(map[int64]int64) - for rows.Next() { - var rowID, size int64 - if err := rows.Scan(&rowID, &size); err != nil { - rows.Close() - return nil, err - } - sizes[rowID] = size - } - err = rows.Err() - rows.Close() - if err != nil { - return nil, err - } - ids = nil - selected := make(map[string]bool) - remaining := int64(SearchExcerptTotalBytes) - for i := range out { - if !out[i].BodyMatch { - continue - } - revision := out[i].RevisionUUID - include, decided := selected[revision] - if !decided { - rowID := ftsRows[revision] - size, exists := sizes[rowID] - include = exists && size >= 0 && size <= SearchExcerptDocumentBytes && size <= remaining - if include { - ids = append(ids, rowID) - remaining -= size - } - selected[revision] = include - } - out[i].ExcerptOmitted = !include - } - } - if len(ids) != 0 { - excerpts, err := loadSearchBodyExcerpts(ctx, tx, expression, ids, searchExcerptTimeout) - if err != nil { - return nil, err - } - for i := range out { - if out[i].BodyMatch { - var present bool - out[i].Excerpt, present = excerpts[out[i].RevisionUUID] - out[i].ExcerptOmitted = !present - } - } - } - // A filename/Path-only hit still benefits from a short Page summary. Read - // only a bounded prefix from displayed textual revisions, in one batch. - ids, seen = nil, make(map[string]bool) - for _, result := range out { - media := strings.ToLower(result.ContentType) - textual := strings.HasPrefix(media, "text/") || strings.HasPrefix(media, "application/json") || strings.HasPrefix(media, "application/xml") || strings.HasPrefix(media, "application/javascript") || strings.HasPrefix(media, "application/yaml") || strings.HasPrefix(media, "application/x-yaml") - if !result.BodyMatch && textual && result.RevisionUUID != "" && !seen[result.RevisionUUID] { - ids = append(ids, result.RevisionUUID) - seen[result.RevisionUUID] = true - } - } - if len(ids) != 0 { - rows, err := tx.QueryContext(ctx, `SELECT uuid,substr(body,1,1024) FROM revisions WHERE uuid IN (`+strings.TrimRight(strings.Repeat("?,", len(ids)), ",")+`)`, ids...) - if err != nil { - return nil, err - } - excerpts := make(map[string]string) - for rows.Next() { - var id, excerpt string - if err := rows.Scan(&id, &excerpt); err != nil { - rows.Close() - return nil, err - } - excerpts[id] = excerpt - } - err = rows.Err() - rows.Close() - if err != nil { - return nil, err - } - for i := range out { - if !out[i].BodyMatch { - out[i].Excerpt = excerpts[out[i].RevisionUUID] - } - } - } - return out, nil -} - -func ftsPrefixExpression(query string) string { - words := strings.Fields(query) - terms := make([]string, 0, len(words)) - for _, word := range words { - word = strings.ReplaceAll(word, `"`, `""`) - if word != "" { - terms = append(terms, `"`+word+`"*`) - } - } - return strings.Join(terms, " AND ") + return mestionora.NewSQLite(s.DB).SearchPathSuggestionsInDomain(ctx, domainUUID, scope, query, limit, allowed) } diff --git a/internal/store/search_filters.go b/internal/store/search_filters.go index 5c98020..c213847 100644 --- a/internal/store/search_filters.go +++ b/internal/store/search_filters.go @@ -1,135 +1,7 @@ package store -import ( - "errors" - "fmt" - "strings" -) +import "pageswiki/internal/mestionora" -var ErrInvalidSearchOptions = errors.New("invalid search options") +type SearchOptions = mestionora.SearchOptions -// SearchOptions applies to current Page metadata, not historical Revisions. -// Dates are Unix milliseconds: FromMS is inclusive and BeforeMS is exclusive. -// The created field refers to Object creation, not creation of a Path alias. -// FileType and Extension are normalized lower-case values; Extension has no dot -// prefix and may contain a compound suffix such as tar.gz. -type SearchOptions struct { - FileType string - Extension string - DateField string - FromMS *int64 - BeforeMS *int64 -} - -func (o SearchOptions) HasFilters() bool { - return o.FileType != "" || o.Extension != "" || o.FromMS != nil || o.BeforeMS != nil -} - -func (o SearchOptions) Validate() error { - switch o.FileType { - case "", "html", "text", "image", "audio", "video", "pdf", "archive", "csv", "json", "code", "office", "other": - default: - return fmt.Errorf("%w: unknown file type", ErrInvalidSearchOptions) - } - switch o.DateField { - case "", "updated", "created": - default: - return fmt.Errorf("%w: unknown date field", ErrInvalidSearchOptions) - } - if len(o.Extension) > 32 { - return fmt.Errorf("%w: extension is too long", ErrInvalidSearchOptions) - } - for i, c := range o.Extension { - alnum := c >= 'a' && c <= 'z' || c >= '0' && c <= '9' - if !alnum && (i == 0 || !strings.ContainsRune("._+-", c)) { - return fmt.Errorf("%w: invalid extension", ErrInvalidSearchOptions) - } - } - if o.FromMS != nil && o.BeforeMS != nil && *o.FromMS >= *o.BeforeMS { - return fmt.Errorf("%w: empty or reversed date range", ErrInvalidSearchOptions) - } - return nil -} - -const searchMIMESQL = `lower(trim(substr(COALESCE(r.content_type,''),1,instr(COALESCE(r.content_type,'')||';',';')-1)))` - -// The original filename wins over the Path suffix when present. Names remain -// data: even extension patterns are bound arguments and do not become SQL. -const searchFilenameSQL = `lower(CASE WHEN COALESCE(r.filename,'')<>'' THEN r.filename ELSE p.path END)` - -func searchSuffixesSQL(suffixes ...string) string { - clauses := make([]string, 0, len(suffixes)) - for _, suffix := range suffixes { - // Only package-owned ASCII constants reach this helper. GLOB has no - // escaping ambiguity for underscores, unlike LIKE. - clauses = append(clauses, searchFilenameSQL+` GLOB '*.`+suffix+`'`) - } - return "(" + strings.Join(clauses, " OR ") + ")" -} - -// Specific MIME types are authoritative. Generic text/binary types may use a -// known suffix (e.g. a Python source uploaded as text/plain); unrecognized, -// specific MIME types stay Other instead of pretending to be a different type. -// This uses metadata only and never hydrates a Document to sniff its bytes. -var searchFileTypeSQL = `CASE - WHEN ` + searchMIMESQL + ` IN ('text/html','application/xhtml+xml') THEN 'html' - WHEN ` + searchMIMESQL + `='application/pdf' THEN 'pdf' - WHEN ` + searchMIMESQL + ` LIKE 'image/%' THEN 'image' - WHEN ` + searchMIMESQL + ` LIKE 'audio/%' THEN 'audio' - WHEN ` + searchMIMESQL + ` LIKE 'video/%' THEN 'video' - WHEN ` + searchMIMESQL + ` IN ('text/csv','application/csv','text/tab-separated-values') THEN 'csv' - WHEN ` + searchMIMESQL + ` IN ('application/json','text/json') OR ` + searchMIMESQL + ` GLOB '*+json' THEN 'json' - WHEN ` + searchMIMESQL + ` IN ('application/zip','application/x-zip-compressed','application/x-7z-compressed','application/vnd.rar','application/x-rar','application/x-rar-compressed','application/x-tar','application/gzip','application/x-gzip','application/x-xz','application/x-bzip2','application/zstd','application/x-zstd','application/x-compress','application/x-cpio') THEN 'archive' - WHEN ` + searchMIMESQL + ` IN ('application/msword','application/vnd.ms-excel','application/vnd.ms-powerpoint','application/rtf','text/rtf') OR ` + searchMIMESQL + ` GLOB 'application/vnd.openxmlformats-officedocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.oasis.opendocument.*' OR ` + searchMIMESQL + ` GLOB 'application/vnd.ms-*.macroenabled.*' THEN 'office' - WHEN ` + searchMIMESQL + ` IN ('text/markdown','text/x-markdown') THEN 'text' - WHEN ` + searchMIMESQL + ` IN ('application/javascript','application/x-javascript','text/javascript','application/typescript','text/typescript','application/xml','text/xml','application/yaml','application/x-yaml','text/yaml','application/toml','application/x-sh','application/x-shellscript','application/x-python','application/x-python-code','application/x-httpd-php','application/sql','text/css') OR ` + searchMIMESQL + ` GLOB 'text/x-*' OR ` + searchMIMESQL + ` GLOB '*+xml' THEN 'code' - WHEN ` + searchMIMESQL + ` NOT IN ('','text/plain','application/octet-stream','binary/octet-stream','application/x-empty') THEN CASE WHEN ` + searchMIMESQL + ` LIKE 'text/%' THEN 'text' ELSE 'other' END - WHEN ` + searchSuffixesSQL("html", "htm", "xhtml") + ` THEN 'html' - WHEN ` + searchSuffixesSQL("pdf") + ` THEN 'pdf' - WHEN ` + searchSuffixesSQL("png", "jpg", "jpeg", "gif", "webp", "avif", "bmp", "svg", "tif", "tiff", "ico", "heic", "heif") + ` THEN 'image' - WHEN ` + searchSuffixesSQL("mp3", "flac", "wav", "ogg", "oga", "opus", "m4a", "aac", "aiff", "alac", "mid", "midi") + ` THEN 'audio' - WHEN ` + searchSuffixesSQL("mp4", "webm", "mkv", "mov", "avi", "m4v", "ogv", "mpeg", "mpg") + ` THEN 'video' - WHEN ` + searchSuffixesSQL("zip", "7z", "rar", "tar", "gz", "tgz", "xz", "txz", "bz2", "tbz", "tbz2", "zst", "tzst", "cbz", "cbr", "cpio") + ` THEN 'archive' - WHEN ` + searchSuffixesSQL("csv", "tsv") + ` THEN 'csv' - WHEN ` + searchSuffixesSQL("json", "jsonl", "ndjson", "excalidraw", "ipynb") + ` THEN 'json' - WHEN ` + searchSuffixesSQL("doc", "docx", "docm", "xls", "xlsx", "xlsm", "ppt", "pptx", "pptm", "odt", "ods", "odp", "rtf") + ` THEN 'office' - WHEN ` + searchSuffixesSQL("py", "pyw", "sh", "bash", "zsh", "fish", "ps1", "bat", "cmd", "js", "mjs", "cjs", "jsx", "ts", "tsx", "go", "rs", "c", "h", "cpp", "hpp", "cc", "cs", "java", "kt", "kts", "swift", "rb", "php", "pl", "pm", "lua", "r", "sql", "css", "scss", "sass", "less", "xml", "xsl", "yaml", "yml", "toml", "ini", "conf", "vue", "svelte", "dart", "ex", "exs", "erl", "hs", "scala", "clj", "lisp", "el", "vim", "wasm") + ` THEN 'code' - WHEN ` + searchMIMESQL + `='text/plain' OR ` + searchSuffixesSQL("txt", "text", "log", "md", "markdown", "rst", "adoc", "org") + ` THEN 'text' - ELSE 'other' END` - -// The returned predicate is inserted before each candidate stream's LIMIT, -// including FTS-first streams. Filtering after their cap silently loses hits. -func searchOptionsPredicate(options SearchOptions) (string, []any) { - var predicate strings.Builder - var args []any - if options.FromMS != nil || options.BeforeMS != nil { - dateSQL := "r.created_at_ms" - if options.DateField == "created" { - // Objects.uuid is a primary-key lookup. Avoid joining Objects in the - // normal search plan when Object creation is not being filtered. - predicate.WriteString(" AND EXISTS (SELECT 1 FROM objects o WHERE o.uuid=p.object_uuid") - dateSQL = "o.created_at*1000" - } - if options.FromMS != nil { - predicate.WriteString(" AND " + dateSQL + ">=?") - args = append(args, *options.FromMS) - } - if options.BeforeMS != nil { - predicate.WriteString(" AND " + dateSQL + "= latest_revision_index.created_at_ms; - END - `); err != nil { - return err - } - if _, err := s.DB.Exec(` - CREATE TRIGGER IF NOT EXISTS revision_search_delete - AFTER DELETE ON revisions BEGIN - DELETE FROM revision_search WHERE revision_uuid = old.uuid; - DELETE FROM revision_search_state WHERE revision_uuid = old.uuid; - DELETE FROM latest_revision_index - WHERE object_uuid = old.object_uuid - AND revision_uuid = old.uuid; - INSERT OR REPLACE INTO latest_revision_index ( - object_uuid, revision_uuid, content_type, filename, - document_hash, created_at_ms - ) - SELECT - object_uuid, uuid, COALESCE(content_type, ''), - COALESCE(filename, ''), COALESCE(document_hash, ''), - COALESCE(created_at_ms, created_at * 1000, 0) - FROM revisions - WHERE object_uuid = old.object_uuid - ORDER BY COALESCE(created_at_ms, created_at * 1000) DESC, rowid DESC - LIMIT 1; - END - `); err != nil { - return err - } - _, err := s.DB.Exec(` - INSERT INTO revision_search ( - revision_uuid, object_uuid, filename, body - ) - SELECT - r.uuid, - r.object_uuid, - COALESCE(r.filename, ''), - CASE - WHEN lower(COALESCE(r.content_type, '')) LIKE 'text/%' - OR lower(COALESCE(r.content_type, '')) LIKE 'application/json%' - OR lower(COALESCE(r.content_type, '')) LIKE 'application/xml%' - OR lower(COALESCE(r.content_type, '')) LIKE 'application/javascript%' - OR lower(COALESCE(r.content_type, '')) LIKE 'application/x-yaml%' - OR lower(COALESCE(r.content_type, '')) LIKE 'application/yaml%' - THEN COALESCE(r.body, '') - ELSE '' - END - FROM revisions r - WHERE NOT EXISTS ( - SELECT 1 FROM revision_search_state state - WHERE state.revision_uuid = r.uuid - ) - `) - if err != nil { - return err - } - _, err = s.DB.Exec(` - INSERT OR IGNORE INTO revision_search_state (revision_uuid) - SELECT uuid FROM revisions - `) - return err + return mestionora.NewSQLite(s.DB).EnsureSearchIndexSchema() } diff --git a/internal/store/seheweit_history_test.go b/internal/store/seheweit_history_test.go new file mode 100644 index 0000000..c845c07 --- /dev/null +++ b/internal/store/seheweit_history_test.go @@ -0,0 +1,124 @@ +package store + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "pageswiki/internal/seheweit" +) + +func resourceTestStore(t *testing.T) *Store { + t.Helper() + s := newSchemaTestStore(t) + if err := s.EnsureResourceHistorySchema(); err != nil { + t.Fatal(err) + } + return s +} + +func TestSeheweitHistoryBucketCompatibility(t *testing.T) { + s := resourceTestStore(t) + now := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC) + // Boundary-inclusive query. Later insertion, not later timestamp, wins a + // bucket, matching the old MAX(id) query even with out-of-order samples. + for i, tc := range []struct { + offset time.Duration + payload string + }{ + {-time.Hour - time.Millisecond, `{"label":"too-old"}`}, + {-time.Hour, `{"label":"start"}`}, + {-10 * time.Second, `{"label":"later-time"}`}, + {-20 * time.Second, `{"label":"last-insert"}`}, + {0, `{"label":"end"}`}, + {time.Millisecond, `{"label":"future"}`}, + } { + if err := s.SaveResourceSample(fmt.Sprint(i), now.Add(tc.offset).UnixMilli(), []byte(tc.payload)); err != nil { + t.Fatal(err) + } + } + got, err := s.ResourceHistory(t.Context(), 1, now) + if err != nil { + t.Fatal(err) + } + want := []string{`{"label":"start"}`, `{"label":"last-insert"}`, `{"label":"end"}`} + if len(got) != len(want) { + t.Fatalf("samples %s", got) + } + for i := range want { + if string(got[i]) != want[i] { + t.Fatalf("sample %d: %s", i, got[i]) + } + } +} + +func TestSeheweitHistoryRetentionAndIdempotence(t *testing.T) { + s := resourceTestStore(t) + at := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC).UnixMilli() + cutoff := seheweit.RetentionCutoff(at) + for _, tc := range []struct { + job string + at int64 + }{{"expired", cutoff - 1}, {"boundary", cutoff}, {"current", at}} { + if err := s.SaveResourceSample(tc.job, tc.at, []byte(`{"original":true}`)); err != nil { + t.Fatal(err) + } + } + if err := s.SaveResourceSample("current", at+1, []byte(`{"replacement":true}`)); err != nil { + t.Fatal(err) + } + var payload string + if err := s.DB.QueryRow(`SELECT payload FROM resource_samples WHERE job_uuid='current'`).Scan(&payload); err != nil || payload != `{"original":true}` { + t.Fatalf("retry replaced sample: %s %v", payload, err) + } + // The retry uses its collection time for pruning, so the old cutoff is now + // expired too. A distinct test covers exact retention-boundary preservation. + var count int + if err := s.DB.QueryRow(`SELECT COUNT(*) FROM resource_samples`).Scan(&count); err != nil || count != 1 { + t.Fatalf("retention count=%d %v", count, err) + } + s2 := resourceTestStore(t) + if err := s2.SaveResourceSample("boundary", cutoff, []byte(`{}`)); err != nil { + t.Fatal(err) + } + if err := s2.SaveResourceSample("current", at, []byte(`{}`)); err != nil { + t.Fatal(err) + } + if err := s2.DB.QueryRow(`SELECT COUNT(*) FROM resource_samples`).Scan(&count); err != nil || count != 2 { + t.Fatalf("inclusive cutoff count=%d %v", count, err) + } +} + +func TestSeheweitHistoryCancelledReadOnlyAndRollback(t *testing.T) { + s := resourceTestStore(t) + at := time.Now().UnixMilli() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := s.SaveResourceSampleContext(ctx, "cancelled", at, []byte(`{}`)); !errors.Is(err, context.Canceled) { + t.Fatalf("cancel: %v", err) + } + if _, err := s.ResourceHistory(ctx, 24, time.Now()); !errors.Is(err, context.Canceled) { + t.Fatalf("read cancel: %v", err) + } + s.ReadOnly = true + if err := s.SaveResourceSample("readonly", at, []byte(`{}`)); !errors.Is(err, ErrReadOnly) { + t.Fatalf("readonly: %v", err) + } + s.ReadOnly = false + if err := s.SaveResourceSample("old", seheweit.RetentionCutoff(at)-1, []byte(`{}`)); err != nil { + t.Fatal(err) + } + // Failure during expiration must roll back the insertion as well. + if _, err := s.DB.Exec(`CREATE TRIGGER prevent_resource_expire BEFORE DELETE ON resource_samples BEGIN SELECT RAISE(ABORT,'synthetic failure'); END`); err != nil { + t.Fatal(err) + } + if err := s.SaveResourceSample("rollback", at, []byte(`{}`)); err == nil { + t.Fatal("expiration failure ignored") + } + var count int + if err := s.DB.QueryRow(`SELECT COUNT(*) FROM resource_samples WHERE job_uuid<>'old'`).Scan(&count); err != nil || count != 0 { + t.Fatalf("partial transaction or cancelled write count=%d %v", count, err) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 653e91f..68acc91 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -8,15 +8,24 @@ import ( "path/filepath" "strings" "sync" + "time" + + "pageswiki/internal/geduldh" ) type Store struct { - DB *sql.DB - DocumentRoot string - pathCacheMu sync.RWMutex - revisionWriteMu sync.Mutex - childCache map[string][]string - authKey []byte + // Standby is immutable for the lifetime of this process. Role changes require restart. + ReadOnly bool + StandbyReadUntil time.Time + BackgroundJobsDisabled bool + DB *sql.DB + DocumentRoot string + documentMaterializerOnce sync.Once + documentMaterializer *geduldh.Materializer + pathCacheMu sync.RWMutex + revisionWriteMu sync.Mutex + childCache map[string][]string + authKey []byte } func New( @@ -108,6 +117,9 @@ func NewWithDocumentRoot( if err := st.EnsureUserSpaceSchema(); err != nil { panic(err) } + if err := st.ensureSchemaComponent("profile-contacts"); err != nil { + panic(err) + } if err := st.EnsureBuiltinAdminGroups(); err != nil { panic(err) } diff --git a/internal/store/user_space.go b/internal/store/user_space.go index 7d01c7e..6b7f48f 100644 --- a/internal/store/user_space.go +++ b/internal/store/user_space.go @@ -8,26 +8,14 @@ import ( "sort" "strings" "time" + + "pageswiki/internal/schutzaria" ) const MaxUserSpaceOverrides = 256 -type UserSpacePathPolicy struct { - Path string `json:"path"` - Visibility string `json:"visibility"` -} - -type UserSpacePolicy struct { - PrincipalUUID string `json:"principal_uuid"` - Username string `json:"username"` - ProfileVisibility string `json:"profile_visibility"` - AvatarVisibility string `json:"avatar_visibility"` - RootVisibility string `json:"root_visibility"` - DescendantVisibility string `json:"descendant_visibility"` - Version int64 `json:"version"` - DomainAllowsPublic bool `json:"domain_allows_public"` - Overrides []UserSpacePathPolicy `json:"overrides"` -} +type UserSpacePathPolicy = schutzaria.UserSpacePathPolicy +type UserSpacePolicy = schutzaria.UserSpacePolicy type UserSpacePage struct { Path string @@ -76,18 +64,7 @@ func defaultUserSpacePolicy(principal, username string) UserSpacePolicy { return UserSpacePolicy{PrincipalUUID: principal, Username: username, ProfileVisibility: "domain", AvatarVisibility: "domain", RootVisibility: "domain", DescendantVisibility: "domain", Overrides: []UserSpacePathPolicy{}} } -func userSpaceVisibilityRank(value string) int { - switch value { - case "private": - return 0 - case "domain": - return 1 - case "public": - return 2 - default: - return -1 - } -} +func userSpaceVisibilityRank(value string) int { return schutzaria.UserSpaceVisibilityRank(value) } func userSpaceRoot(username string) string { return "/u/" + username } @@ -98,18 +75,7 @@ func escapeUserSpaceLike(value string) string { } func userSpaceDocumentPath(username, value string, descendant bool) bool { - root := userSpaceRoot(username) - if value != path.Clean(value) || strings.ContainsAny(value, "\\\x00\r\n") { - return false - } - if descendant { - if !strings.HasPrefix(value, root+"/") { - return false - } - rest := strings.TrimPrefix(value, root+"/") - return rest != "" && !strings.HasPrefix(rest, "_/") && rest != "_" - } - return value == root + return schutzaria.UserSpaceDocumentPath(username, value, descendant) } func (s *Store) domainAllowsUserSpacePublic(ctx context.Context, q interface { @@ -170,22 +136,7 @@ func (s *Store) UserSpacePolicy(ctx context.Context, domain, principal string) ( } func userSpaceEffectiveVisibility(p UserSpacePolicy, target string) string { - root := userSpaceRoot(p.Username) - if target == root { - return p.RootVisibility - } - if !userSpaceDocumentPath(p.Username, target, true) { - return "private" - } - visibility := p.DescendantVisibility - for _, item := range p.Overrides { - if target == item.Path || strings.HasPrefix(target, strings.TrimRight(item.Path, "/")+"/") { - if userSpaceVisibilityRank(item.Visibility) < userSpaceVisibilityRank(visibility) { - visibility = item.Visibility - } - } - } - return visibility + return schutzaria.EffectiveUserSpaceVisibility(p, target) } func EffectiveUserSpaceVisibility(p UserSpacePolicy, target string) string { diff --git a/pages-cli/README.md b/pages-cli/README.md new file mode 100644 index 0000000..ac99bc2 --- /dev/null +++ b/pages-cli/README.md @@ -0,0 +1,108 @@ +# pages-cli + +**`pages` — PagesWikiをターミナルから読む、たどる、編集する。** + +プロファイルや言語は初回に設定し、普段のコマンドは短く保ちます。 +DBに直接アクセスする `data/pageswiki-cli` とは独立したHTTPクライアントです。 + +## はじめる + +Python 3.9以降、Linux/macOS等のUnix系環境を使用します。外部Pythonパッケージは不要です。 +リポジトリ内では `./pages-cli/pages` で実行できます。PATHへ配置する場合: + +```sh +mkdir -p ~/.local/bin +install -m 755 pages-cli/pages ~/.local/bin/pages +# ~/.local/bin をPATHに含めてください。 +``` + +```sh +pages configure --origin https://pages.example.test --username anya +pages login +pages +``` + +パスワードと必要なTOTPは非表示プロンプトへ入力します。`pages` 単独で対話シェル、 +`pages ls /manual` のように続ければワンショット操作です。 + +```text +pages[default]:/> cd /manual +pages[default]:/manual> ls +pages[default]:/manual> edit start +pages[default]:/manual> exit +``` + +`exit` はシェルを閉じるだけです。セッションを失効させる場合は `pages logout`。 + +## 設定する + +```sh +pages configure # 設定を表示(トークンは表示しない) +pages configure --lang ja # 現在の既定接続の編集言語 +pages configure list # 接続先一覧、★が既定 +pages configure set --profile work --origin https://work.example.test --username anya --lang en +pages configure use work # 以後はworkを指定不要 +pages status +pages edit /guide +``` + +既定プロファイル名は `default`、既定言語は `und`(言語未指定)です。OSや表示言語から +書き込み先の言語を推測しません。最初に名前付きプロファイルを作った場合はそれを既定とします。 +言語は接続先ごとに保持し、編集時はその言語の既存Revisionを使用します。 + +一度だけ変更したいときは `pages --profile personal --lang ja edit /guide` も使えます。 +プロファイル自体は先にconfigureで登録します。引数の上書きは設定ファイルを書き換えません。 + +設定ファイルは `~/.config/pages-cli/config.json`(`XDG_CONFIG_HOME`対応)。 + +```json +{ + "version": 1, + "default_profile": "default", + "profiles": { + "default": { + "origin": "https://pages.example.test", + "username": "anya", + "lang": "und" + } + } +} +``` + +セッションは別の `sessions/PROFILE.json` に保存します。設定にパスワードやBearerを入れないでください。 +ディレクトリ0700/ファイル0600を要求します。セッション保持中のorigin・username変更には +事前logoutが必要です。同じプロファイルのシェルが開いている間の設定変更はロックで拒否します。 +手編集で接続先を変えても、旧セッションを新接続先に送信しません。 + +旧試作名 `data/pageswiki-remote` は新コマンドへの案内用ランチャーです。 +旧 `~/.config/pageswiki-remote/` の資格情報は自動移行・削除しません。 +新しいconfigureで接続先を登録して再ログインし、不要な旧セッションはサーバのセッション管理で失効してください。 + +## 操作 + +| コマンド | 用途 | +| --- | --- | +| `ls` / `cd` / `pwd` | Semantic Pathをたどる。仮想フォルダにも移動可能 | +| `cat` / `view` | 正本の出力/`$PAGER`で閲覧 | +| `get ADDRESS LOCAL` | 正本をダウンロード。既存のローカルファイルは上書きしない | +| `edit ADDRESS` | `$EDITOR`で編集。Wikiはauthoring Markdown、他のテキストは原文 | +| `put LOCAL ADDRESS --base REVISION_UUID` | 明示した基点RevisionとのCASで更新 | +| `search QUERY` | 権限付きの検索候補 | +| `open ADDRESS` | ブラウザで閲覧。ブラウザ側ログインを使用 | +| `status` / `logout` | 接続状態表示/セッション失効 | + +`pages --help`、`pages configure --help`、`pages edit --help` で引数を確認できます。 +Tab補完、上下キーのプロセス内履歴に対応。複雑な引用位置の補完、Ctrl+Pのfuzzy UIは後続です。 + +初期版は既存Page・既存言語の更新まで。新規作成、OIDC引き継ぎ、バイナリ更新は未実装です。 +競合や通信断で自動上書きせず、ローカル原稿を保持します。HTTPS必須(開発用loopbackのみHTTP可)。 +詳しいURI・保存・安全性の契約は[設計書](../docs/design/ja/remote-cli.md)を参照してください。 + +## テスト + +```sh +python3 pages-cli/test_pages.py +go test ./internal/server -run TestRemoteCLIHTTPIntegration -count=1 +``` + +偽サーバと一時DBのみを使用します。本番資格情報や外部カレンダーは不要です。 diff --git a/pages-cli/pages b/pages-cli/pages new file mode 100755 index 0000000..71b06a4 --- /dev/null +++ b/pages-cli/pages @@ -0,0 +1,528 @@ +#!/usr/bin/env python3 +"""pages — navigate and edit PagesWiki from your terminal (pages-cli).""" +import argparse +import contextlib +import difflib +import fcntl +import getpass +import hashlib +import json +import os +from pathlib import Path +import posixpath +import re +import shlex +import stat +import subprocess +import sys +import tempfile +import time +import unicodedata +import urllib.error +import urllib.parse as url +import urllib.request as http +import webbrowser + + +class Failure(Exception): + def __init__(self, message, status=0, payload=None): + super().__init__(message) + self.status, self.payload = status, payload or {} + + +def safe_text(value): + return ''.join(c for c in str(value) if c in '\n\t' or not unicodedata.category(c).startswith('C')) + + +def origin(value): + if not isinstance(value, str): raise Failure('Origin must be an HTTPS URL string.') + p = url.urlsplit(value) + if (p.scheme not in ('https', 'http') or not p.hostname or p.username or p.password + or p.path not in ('', '/') or p.query or p.fragment or any(c.isspace() for c in value)): + raise Failure('Use an HTTPS origin without credentials, path, query or fragment.') + if p.scheme == 'http' and p.hostname not in ('127.0.0.1', '::1', 'localhost'): + raise Failure('HTTP is permitted only for loopback development; use HTTPS remotely.') + return url.urlunsplit((p.scheme, p.netloc.lower(), '', '', '')) + + +def semantic(value, cwd='/'): + value = value.removeprefix('path:') + if '://' in value or '\\' in value or any(unicodedata.category(c).startswith('C') for c in value): + raise Failure('Expected a semantic Path, not a URL or control characters.') + return '/' + posixpath.normpath('/' + posixpath.join(cwd, value)).lstrip('/') + + +def address(value, cwd='/'): + if value.startswith(('uuid:', 'rev:')): + kind, ident = value.split(':', 1) + if not re.fullmatch(r'[0-9a-fA-F-]{36}', ident): + raise Failure('Invalid UUID address.') + return kind + ':' + ident + if re.match(r'^[a-zA-Z]+:', value) and not value.startswith('path:'): + raise Failure('This client supports path:, uuid: and read-only rev: addresses.') + return 'path:' + semantic(value, cwd) + + +class NoRedirect(http.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None # Never forward credentials or turn a login HTML page into success. + + +class Client: + def __init__(self, base, token='', lang='und'): + self.base, self.token, self.lang = origin(base), token, lang + self.opener = http.build_opener(NoRedirect()) + + def request(self, route, method='GET', data=None, form=None, limit=4 << 20): + if not route.startswith('/') or route.startswith('//'): + raise Failure('Invalid API route.') + headers = {'Accept': 'application/json', 'User-Agent': 'pages-cli/0.1'} + if self.token: + headers['Authorization'] = 'Bearer ' + self.token + body = None + if data is not None: + body = json.dumps(data, ensure_ascii=False).encode() + headers['Content-Type'] = 'application/json' + if form is not None: + body = url.urlencode(form).encode() + headers['Content-Type'] = 'application/x-www-form-urlencoded' + if body is not None and len(body) > 2 << 20: + raise Failure('Request exceeds the 2 MiB client limit.') + req = http.Request(self.base + route, body, headers, method=method) + try: + with self.opener.open(req, timeout=20) as response: + raw = response.read(limit + 1) + if len(raw) > limit: + raise Failure('Response exceeds client limit; use the browser for large files.') + return raw, response.headers + except urllib.error.HTTPError as e: + raw = e.read(65536) + e.close() + try: + payload = json.loads(raw) + except (ValueError, UnicodeError): + payload = {} + if not isinstance(payload, dict): + payload = {} + messages = {303: 'Login redirect refused; log in again.', 401: 'Session expired or authentication required.', 403: 'Access denied.', + 404: 'Not found or not readable.', 409: 'Conflict: remote revision changed.'} + raise Failure(messages.get(e.code, 'HTTP request failed (%d); redirects are not followed.' % e.code), e.code, payload) from None + except (urllib.error.URLError, TimeoutError, OSError): + raise Failure('Connection/TLS failure; no automatic write retry was attempted.') from None + + def json(self, route, **kwargs): + raw, headers = self.request(route, **kwargs) + if headers.get_content_type() != 'application/json': + raise Failure('Expected JSON, not a login page or other projection.') + try: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError() + return value + except (ValueError, UnicodeError): + raise Failure('Invalid JSON response.') from None + + def login(self, username, password, otp=getpass.getpass): + try: + result = self.json('/_/auth/login', method='POST', form={'username': username, 'password': password}) + except Failure as e: + if e.status != 401 or not e.payload.get('mfa_required') or not e.payload.get('challenge_token'): + raise + result = self.json('/_/auth/totp', method='POST', form={ + 'challenge_token': e.payload['challenge_token'], 'code': otp('TOTP: ')}) + token = result.get('access_token') + if not isinstance(token, str) or not re.fullmatch(r'[A-Za-z0-9_.~-]{16,1024}', token) or result.get('token_type') != 'Bearer': + raise Failure('Server did not issue a Bearer session.') + self.token = token + return time.time() + int(result['expires_in']) + + def listing(self, path, prefix=''): + return self.json('/_/api/v1/path-picker?' + url.urlencode({'path': path, 'mode': 'children', 'prefix': prefix})) + + def raw(self, addr, method='GET'): + raw, headers = self.request('/v/raw/' + url.quote(addr, safe='/:') + '?' + url.urlencode({'lang': self.lang}), method=method, limit=64 << 20) + digest = headers.get('X-PagesWiki-Document-SHA256') + if method == 'GET' and (not digest or hashlib.sha256(raw).hexdigest() != digest): + raise Failure('Document SHA-256 mismatch.') + return raw, headers + + def snapshot(self, addr): + if addr.startswith('rev:'): + raise Failure('Revision addresses are immutable; edit a path: or uuid: address.') + _, headers = self.raw(addr, 'HEAD') + ident = headers.get('X-PagesWiki-Object-UUID') + if not ident: + raise Failure('Missing Object UUID metadata.') + state = self.json('/_/api/v1/editor/state?' + url.urlencode({'uuid': ident, 'lang': self.lang})) + if not state.get('editable') or not state.get('principal_uuid'): + raise Failure('Not an authenticated editable text document; use the browser.') + if not state.get('head_revision_uuid') or not state.get('content_type'): + raise Failure('No saved revision in this exact language; creation/translation is not supported yet.') + state['object_uuid'] = ident + return state + + def save(self, state, text, markdown=False): + if len(text.encode()) > 1 << 20: + raise Failure('Text edit exceeds the 1 MiB client limit.') + if markdown: + if not state['content_type'].startswith('text/html'): + raise Failure('Markdown authoring is supported only for HTML Wiki documents.') + text = self.json('/_/api/v1/render/markdown', method='POST', data={ + 'markdown': text, 'authoring': True, 'projection': False})['html'] + if text == state['body']: + return {'status': 'unchanged', 'revision_uuid': state['head_revision_uuid']} + return self.json('/_/api/v1/revisions/save', method='POST', data={ + 'object_uuid': state['object_uuid'], 'body': text, 'lang': state['lang'], + 'content_type': state['content_type'], 'filename': state['filename'], + 'base_revision_uuid': state['head_revision_uuid'], + 'editor_principal_uuid': state['principal_uuid'], 'message': 'Remote CLI edit'}) + + +def private_file(path): + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077: + raise Failure('Credential file must be a user-owned regular file with mode 0600.') + + +class PrivateJSON: + def __init__(self, root, filename): + self.root = Path(root) + self.root.mkdir(mode=0o700, parents=True, exist_ok=True) + info = self.root.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077: + raise Failure('Profile directory must be owned by you with mode 0700; symlinks are not accepted.') + self.path = self.root / filename + self.data = {} + + @contextlib.contextmanager + def locked(self): + fd = os.open(str(self.path) + '.lock', os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, 'w') as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise Failure('This profile is already in use by another CLI process.') from None + if self.path.exists() or self.path.is_symlink(): + private_file(self.path) + self.data = json.loads(self.path.read_text()) + if not isinstance(self.data, dict): + raise Failure('Invalid configuration object.') + yield self + + def save(self): + fd, temp = tempfile.mkstemp(dir=self.root, prefix='.profile-') + try: + with os.fdopen(fd, 'w') as out: + json.dump(self.data, out) + out.flush() + os.fsync(out.fileno()) + os.replace(temp, self.path) + finally: + if os.path.exists(temp): + os.unlink(temp) + + +def config_root(root=None): + return Path(root or os.environ.get('XDG_CONFIG_HOME', str(Path.home() / '.config'))) / 'pages-cli' + + +def profile_name(name): + if not isinstance(name, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,64}', name): + raise Failure('Profile names use 1–64 letters, digits, underscores or hyphens.') + return name + + +def language(value): + if not isinstance(value, str) or not re.fullmatch(r'[A-Za-z]{1,8}(?:-[A-Za-z0-9]{1,8})*', value): + raise Failure('Use a language tag such as und, ja or en-US; the server validates the exact tag.') + return value + + +class Profile(PrivateJSON): + def __init__(self, name, root=None): + self.name = profile_name(name) + # Validate the parent before creating sessions, including on first use. + PrivateJSON(config_root(root), 'config.json') + super().__init__(config_root(root) / 'sessions', profile_name(name) + '.json') + + +class Configuration(PrivateJSON): + def __init__(self, root=None): + super().__init__(config_root(root), 'config.json') + + def validated(self): + if not self.data: + return {'version': 1, 'default_profile': 'default', 'profiles': {}} + if set(self.data) - {'version', 'default_profile', 'profiles'} or self.data.get('version') != 1: + raise Failure('Unsupported configuration format; credentials belong in sessions/, not config.json.') + default = profile_name(self.data.get('default_profile', 'default')) + profiles = self.data.get('profiles', {}) + if not isinstance(profiles, dict): raise Failure('Invalid profiles configuration.') + normalized = {} + for name, spec in profiles.items(): + profile_name(name) + if not isinstance(spec, dict) or set(spec) - {'origin', 'username', 'lang'}: + raise Failure('Profiles accept only origin, username and lang; never store a password here.') + if not isinstance(spec.get('username', ''), str) or any(unicodedata.category(c).startswith('C') for c in spec.get('username', '')): + raise Failure('Invalid configured username.') + normalized[name] = dict(spec, origin=origin(spec.get('origin', '')), lang=language(spec.get('lang', 'und'))) + if profiles and default not in profiles: raise Failure('Default profile does not exist.') + return {'version': 1, 'default_profile': default, 'profiles': normalized} + + +def configure(words, root=None): + parser = argparse.ArgumentParser(prog='pages configure', description='Manage connections and defaults. No network requests are made.') + sub = parser.add_subparsers(dest='action', required=True) + p = sub.add_parser('set', help='Create or update a connection profile') + p.add_argument('--profile'); p.add_argument('--origin'); p.add_argument('--username'); p.add_argument('--lang') + p = sub.add_parser('use', help='Select the default profile'); p.add_argument('profile') + sub.add_parser('show', help='Show settings, never session tokens') + sub.add_parser('list', help='List configured connections') + if not words: words = ['show'] + elif words[0].startswith('--') and words[0] != '--help': words = ['set'] + words + args = parser.parse_args(words) + with Configuration(root).locked() as config: + data = config.validated() + if args.action == 'show': + print(json.dumps(data, ensure_ascii=False, indent=2)); return + if args.action == 'list': + for name, spec in data['profiles'].items(): + print(safe_text(('★ ' if name == data['default_profile'] else ' ') + name + ' ' + spec['origin'] + ' ' + spec.get('username', '') + ' [' + spec.get('lang', 'und') + ']')) + if not data['profiles']: print('No connections. Use: pages configure --origin https://wiki.example --username NAME') + return + if args.action == 'use': + name = profile_name(args.profile) + if name not in data['profiles']: raise Failure('Configure this profile before selecting it.') + data['default_profile'] = name + else: + name = profile_name(args.profile or data['default_profile']) + spec = dict(data['profiles'].get(name, {'lang': 'und'})) + if args.origin is not None: spec['origin'] = origin(args.origin) + if args.username is not None: spec['username'] = args.username.strip() + if args.lang is not None: spec['lang'] = language(args.lang) + if not spec.get('origin'): raise Failure('A new connection requires --origin https://HOST.') + # Configuration and session locks use this order. Never retarget an + # active credential to a different Domain or account. + with Profile(name, root).locked() as session: + if session.data.get('token') and (spec['origin'] != session.data.get('origin') or spec.get('username', '') != session.data.get('username')): + raise Failure('Log out this profile before changing its origin or username.') + data['profiles'][name] = spec + if len(data['profiles']) == 1: data['default_profile'] = name + config.data = data + config.validated() + config.save() + print('Configured:', name); return + config.data = data; config.save() + print('Default profile:', data['default_profile']) + + +def editor(client, addr, mode): + state = client.snapshot(addr) + markdown = mode == 'markdown' or (mode == 'auto' and state['content_type'].startswith('text/html')) + initial = state['markdown'] if markdown else state['body'] + directory = Path(tempfile.mkdtemp(prefix='pageswiki-edit-')) + file = directory / ('document.md' if markdown else 'document.txt') + file.touch(mode=0o600) + file.write_text(initial, encoding='utf-8') + success = False + try: + command = shlex.split(os.environ.get('EDITOR', 'vi')) + if not command or subprocess.run(command + [str(file)], check=False).returncode: + raise Failure('Editor exited unsuccessfully.') + text = file.read_text(encoding='utf-8') + if text == initial: + print('Unchanged.') + else: + try: + result = client.save(state, text, markdown) + except Failure as error: + if error.status == 409: + latest = client.snapshot('uuid:' + state['object_uuid']) + remote = latest['markdown'] if markdown else latest['body'] + print(safe_text(''.join(difflib.unified_diff(remote.splitlines(True), text.splitlines(True), fromfile='remote', tofile='local')))) + raise + print('Saved revision:', safe_text(result['revision_uuid'])) + success = True + finally: + if success: + file.unlink() + directory.rmdir() + else: + metadata = directory / 'recovery.json' + metadata.touch(mode=0o600) + metadata.write_text(json.dumps({'origin': client.base, 'address': 'uuid:' + state['object_uuid'], + 'base_revision_uuid': state['head_revision_uuid'], 'lang': state['lang'], 'format': 'markdown' if markdown else 'raw'})) + print('Local edit retained:', safe_text(file), file=sys.stderr) + + +def execute(client, profile, words): + parser = argparse.ArgumentParser(prog='pages') + sub = parser.add_subparsers(dest='command', required=True) + for command in ('ls', 'cd', 'cat', 'view', 'edit', 'open'): + p = sub.add_parser(command) + p.add_argument('path', nargs='?', default='.') + if command == 'edit': + p.add_argument('--format', choices=['auto', 'raw', 'markdown'], default='auto') + p = sub.add_parser('get'); p.add_argument('path'); p.add_argument('local') + p = sub.add_parser('put'); p.add_argument('local'); p.add_argument('path'); p.add_argument('--base', required=True) + p.add_argument('--format', choices=['raw', 'markdown'], default='raw') + p = sub.add_parser('search'); p.add_argument('query') + sub.add_parser('pwd'); sub.add_parser('logout'); sub.add_parser('status') + args = parser.parse_args(words) + cwd = profile.data.get('cwd', '/') + cmd = args.command + if cmd == 'logout': + client.json('/_/auth/logout', method='POST') + profile.data.pop('token', None); profile.data.pop('expires', None); profile.save() + print('Session revoked.'); return + if cmd == 'pwd': + print(safe_text(cwd)); return + if cmd == 'status': + print(safe_text(client.base + '\n' + profile.data.get('username', '') + '\nPath: ' + cwd)) + print('Language:', safe_text(client.lang)) + print('Expires:', time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(profile.data.get('expires', 0)))) + return + if cmd == 'search': + result = client.json('/_/api/v1/search/suggest?' + url.urlencode({'q': args.query})) + for item in result.get('items', []): + print(safe_text(item['path'] + '\t' + item['label'])) + return + addr = address(args.path, cwd) + if cmd in ('ls', 'cd'): + if not addr.startswith('path:'): raise Failure('ls/cd require a Path.') + path = addr[5:] + result = client.listing(path) + if cmd == 'cd': + profile.data['cwd'] = path; profile.save() + else: + for item in result['entries']: + print(safe_text(item['name'] + ('/' if item['kind'] == 'folder' else ''))) + if result.get('truncated'): print('Listing truncated; narrow the Path.', file=sys.stderr) + elif cmd == 'open': + route = addr[5:] if addr.startswith('path:') else '/o/' + addr if addr.startswith('uuid:') else '/r/view/' + addr[4:] + webbrowser.open(client.base + url.quote(route, safe='/:'), new=2) + elif cmd == 'edit': + editor(client, addr, args.format) + elif cmd == 'put': + state = client.snapshot(addr) + if state['head_revision_uuid'] != args.base: raise Failure('Conflict: supplied base is not the current language head.', 409) + text = Path(args.local).read_text(encoding='utf-8') + print('Revision:', safe_text(client.save(state, text, args.format == 'markdown')['revision_uuid'])) + else: + raw, headers = client.raw(addr) + if cmd == 'get': + fd = os.open(args.local, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, 'wb') as out: out.write(raw) + print('Revision:', safe_text(headers.get('X-PagesWiki-Revision-UUID', 'unknown'))) + elif cmd == 'cat' and not sys.stdout.isatty(): + sys.stdout.buffer.write(raw) + else: + if not (headers.get_content_type().startswith('text/') or headers.get_content_type() in ('application/json', 'application/xml')): + raise Failure('Binary document: use get or open.') + text = safe_text(raw.decode('utf-8')) + if cmd == 'view' and sys.stdout.isatty(): + subprocess.run(shlex.split(os.environ.get('PAGER', 'less')) , input=text.encode(), check=False) + else: print(text, end='') + + +def shell(client, profile): + import readline + print(safe_text('pages · ' + client.base + ' · ' + profile.data.get('username', '') + ' · ' + client.lang)) + # History remains in memory, never on disk; login/password commands are not supported here. + readline.set_history_length(200) + readline.set_completer_delims(' \t\n') + matches = [] + def complete(text, index): + if index == 0: + matches.clear() + try: + p = semantic(text, profile.data.get('cwd', '/')) + parent = p if text.endswith('/') else posixpath.dirname(p) + prefix = '' if text.endswith('/') else posixpath.basename(p) + for item in client.listing(parent, prefix)['entries']: + if any(unicodedata.category(c).startswith('C') for c in item['path']): + continue + candidate = item['path'] + ('/' if item['kind'] == 'folder' else '') + if not text.startswith('/'): candidate = posixpath.relpath(candidate, profile.data.get('cwd', '/')) + ('/' if item['kind'] == 'folder' else '') + matches.append(shlex.quote(candidate)) + except Failure: + pass + return matches[index] if index < len(matches) else None + readline.set_completer(complete) + readline.parse_and_bind('tab: complete') + while True: + try: + line = input('pages[' + profile.name + ']:' + safe_text(profile.data.get('cwd', '/')) + '> ') + words = shlex.split(line) + if not words: continue + if words[0] in ('exit', 'quit'): break + if words[0] == 'login': raise Failure('Exit shell and use the login command.') + if words[0] != 'logout' and profile.data.get('expires', 0) <= time.time(): raise Failure('Session expired; log in again.') + execute(client, profile, words) + if words[0] == 'logout': break + except (Failure, ValueError, OSError) as e: print(safe_text(str(e)), file=sys.stderr) + except SystemExit: pass + except KeyboardInterrupt: print() + except EOFError: break + + +def main(argv=None): + parser = argparse.ArgumentParser(prog='pages', description=__doc__, epilog='Commands: configure, login, logout, status, shell, ls, cd, pwd, cat, view, get, put, edit, search, open. Use pages COMMAND --help.') + parser.add_argument('--profile', help='Use another configured connection for this invocation') + parser.add_argument('--lang', help='Override the configured exact language for this invocation') + parser.add_argument('command', nargs='?', default='shell') + parser.add_argument('args', nargs=argparse.REMAINDER) + args = parser.parse_args(argv) + if args.command == 'configure': + if args.profile or args.lang: parser.error('Use pages configure set --profile NAME --lang TAG.') + configure(args.args); return + if args.command == 'help': + parser.print_help(); return + if args.command == 'login': + login = argparse.ArgumentParser(prog='pages login', description='Log into the configured connection. Password and TOTP are prompted privately.') + login.parse_args(args.args) + elif '--help' in args.args: + execute(None, None, [args.command] + args.args); return + with contextlib.ExitStack() as stack: + with Configuration().locked() as config: + settings = config.validated() + name = profile_name(args.profile or settings['default_profile']) + spec = settings['profiles'].get(name) + if spec is None: + raise Failure('No connection configured. Run: pages configure --origin https://HOST --username NAME') + lang = language(args.lang or spec.get('lang', 'und')) + # Hold the profile lock before releasing the configuration lock. + profile = stack.enter_context(Profile(name).locked()) + if args.command == 'login': + if not spec.get('username'): raise Failure('Set your account with pages configure --username NAME.') + if profile.data.get('token'): raise Failure('Log out this profile before replacing its session.') + client = Client(spec['origin'], lang=lang) + expires = client.login(spec['username'], getpass.getpass('Password: ')) + same_identity = profile.data.get('origin') == client.base and profile.data.get('username') == spec['username'] + cwd = profile.data.get('cwd', '/') if same_identity else '/' + profile.data = {'origin': client.base, 'username': spec['username'], 'token': client.token, 'expires': expires, 'cwd': cwd} + try: + profile.save() + except OSError: + # Do not intentionally leave a usable session behind when its + # local credential cannot be persisted. Expiry is the fallback + # if the revocation request itself cannot reach the server. + try: client.json('/_/auth/logout', method='POST') + except Failure: pass + raise Failure('Could not save credentials; attempted to revoke the new session.') from None + print('Logged in. Session saved privately.'); return + if not profile.data.get('token'): raise Failure('Log in first.') + if args.command != 'logout' and (profile.data.get('origin') != spec['origin'] or profile.data.get('username') != spec.get('username')): + raise Failure('Session and configured identity differ. Log out before reconnecting.') + if args.command not in ('logout', 'status') and profile.data.get('expires', 0) <= time.time(): raise Failure('Session expired; log out and log in again.') + client = Client(profile.data['origin'], profile.data['token'], lang) + if args.command == 'shell': shell(client, profile) + else: execute(client, profile, [args.command] + args.args) + + +if __name__ == '__main__': + try: + main() + except (Failure, OSError, ValueError, KeyError, KeyboardInterrupt) as e: + print(safe_text(str(e)) or 'Interrupted.', file=sys.stderr) + sys.exit(1) diff --git a/pages-cli/test_pages.py b/pages-cli/test_pages.py new file mode 100644 index 0000000..6575648 --- /dev/null +++ b/pages-cli/test_pages.py @@ -0,0 +1,264 @@ +import contextlib +from email.message import Message +import hashlib +import io +import json +import os +from pathlib import Path +import runpy +import stat +import sys +import tempfile +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest.mock import patch + +m = runpy.run_path(str(Path(__file__).with_name('pages'))) +Client, Failure, Profile = m['Client'], m['Failure'], m['Profile'] + + +@contextlib.contextmanager +def server(callback): + class Handler(BaseHTTPRequestHandler): + def do_GET(self): callback(self) + def do_POST(self): callback(self) + def log_message(self, *args): pass + instance = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread(target=instance.serve_forever, daemon=True) + thread.start() + try: yield 'http://127.0.0.1:%d' % instance.server_port + finally: instance.shutdown(); instance.server_close(); thread.join() + + +def reply(h, status, body, mime='application/json'): + raw = json.dumps(body).encode() if isinstance(body, dict) else body + h.send_response(status); h.send_header('Content-Type', mime); h.end_headers(); h.wfile.write(raw) + + +class RemoteTests(unittest.TestCase): + def configure(self, root, *args): + with contextlib.redirect_stdout(io.StringIO()) as output: + m['configure'](list(args), root) + return output.getvalue() + + def test_configuration_defaults_and_switch(self): + with tempfile.TemporaryDirectory() as root: + self.configure(root,'--origin','https://one.example','--username','anya') + self.configure(root,'set','--profile','work','--origin','https://two.example','--username','worker','--lang','ja') + self.configure(root,'use','work') + settings=json.loads(self.configure(root,'show')) + self.assertEqual(settings['default_profile'],'work') + self.assertEqual(settings['profiles']['default']['lang'],'und') + self.assertEqual(settings['profiles']['work']['lang'],'ja') + self.configure(root,'--lang','en-US') + settings=json.loads(self.configure(root,'show')) + self.assertEqual(settings['profiles']['work']['lang'],'en-US') + with m['Configuration'](root).locked() as cfg: + self.assertEqual(stat.S_IMODE(cfg.path.stat().st_mode),0o600) + with self.assertRaises(Failure): self.configure(root,'use','missing') + with self.assertRaises(Failure): self.configure(root,'--lang','ja_JP') + + def test_active_credentials_cannot_be_retargeted_or_shown(self): + with tempfile.TemporaryDirectory() as root: + self.configure(root,'--origin','https://one.example','--username','anya') + with Profile('default',root).locked() as profile: + profile.data={'origin':'https://one.example','username':'anya','token':'private-token'}; profile.save() + for args in [('--origin','https://other.example'),('--username','other')]: + with self.assertRaises(Failure): self.configure(root,*args) + self.assertNotIn('private-token',self.configure(root,'show')) + self.assertNotIn('private-token',self.configure(root,'list')) + self.configure(root,'--lang','ja') + with Profile('default',root).locked(): + with self.assertRaises(Failure): self.configure(root,'--lang','en') + with Profile('default',root).locked() as profile: + profile.data.pop('token'); profile.save() + self.configure(root,'--origin','https://other.example') + + def test_main_uses_saved_defaults_and_rejects_manual_retarget(self): + with tempfile.TemporaryDirectory() as root, patch.dict(os.environ,{'XDG_CONFIG_HOME':root}): + self.configure(root,'set','--profile','work','--origin','https://work.example','--username','worker','--lang','ja') + with Profile('work',root).locked() as profile: + profile.data={'origin':'https://work.example','username':'worker','token':'session-token','expires':time.time()+100,'cwd':'/notes'}; profile.save() + # Patch the globals used by main rather than running a real request. + globals=m['main'].__globals__ + with patch.dict(globals,{'execute':lambda client,profile,args: self.assertEqual((client.base,client.lang,args),('https://work.example','ja',['ls']))}): + m['main'](['ls']) + with patch.dict(globals,{'execute':lambda client,profile,args: self.assertEqual(client.lang,'en')}): + m['main'](['--lang','en','ls']) + with m['Configuration'](root).locked() as cfg: + cfg.data['profiles']['work']['origin']='https://evil.example'; cfg.save() + with self.assertRaises(Failure): m['main'](['ls']) + + def test_bad_configuration_is_not_printed(self): + with tempfile.TemporaryDirectory() as root: + with m['Configuration'](root).locked() as cfg: + cfg.data={'version':1,'default_profile':'default','profiles':{'default':{'origin':'https://example.test','token':'secret'}}}; cfg.save() + with self.assertRaises(Failure): self.configure(root,'show') + for invalid in [None,42,[],{}]: + with self.assertRaises(Failure): m['profile_name'](invalid) + with self.assertRaises(Failure): m['language'](invalid) + with self.assertRaises(Failure): m['origin'](invalid) + + def test_origin_and_paths(self): + for value in ['http://example.com', 'https://user:pass@example.com', 'https://example.com/a', 'https://example.com?token=x']: + with self.assertRaises(Failure): m['origin'](value) + self.assertEqual(m['address']('../日本語 #?%', '/docs/deep'), 'path:/docs/日本語 #?%') + self.assertEqual(m['semantic']('../../..', '/'), '/') + for value in ['https://evil.test', 'asset:x:y', 'rev:bad', '/bad\x1b[31m']: + with self.assertRaises(Failure): m['address'](value) + self.assertNotIn('\x1b', m['safe_text']('\x1b]52;clipboard\x07')) + self.assertNotIn('\u202e', m['safe_text']('abc\u202e')) + + def test_redirect_never_receives_bearer(self): + seen = [] + with server(lambda h: (seen.append(h.headers.get('Authorization')), reply(h, 200, {}))) as sink: + def redirect(h): + h.send_response(302); h.send_header('Location', sink); h.end_headers() + with server(redirect) as base: + with self.assertRaises(Failure) as error: Client(base, 'secret').json('/test') + self.assertEqual(error.exception.status, 302) + self.assertEqual(seen, []) + + def test_protocol_failures_and_limits(self): + for status, body, mime in [(200,b'login','text/html'), (200,b'not-json','application/json'), (403,b'private internal error','text/plain')]: + with server(lambda h: reply(h,status,body,mime)) as base: + with self.assertRaises(Failure) as error: Client(base).json('/test') + self.assertNotIn('private internal', str(error.exception)) + with server(lambda h: reply(h,200,b'x'*100)) as base: + with self.assertRaises(Failure): Client(base).request('/large',limit=10) + + def test_password_totp_contract(self): + requests = [] + def handle(h): + requests.append((h.path, h.rfile.read(int(h.headers['Content-Length'])).decode())) + if h.path.endswith('/login'): reply(h,401,{'mfa_required':True,'challenge_token':'short-lived'}) + else: reply(h,200,{'access_token':'a'*32,'token_type':'Bearer','expires_in':3600}) + with server(handle) as base: + client = Client(base) + client.login('someone','secret',lambda prompt:'123456') + self.assertEqual(client.token,'a'*32) + self.assertIn('challenge_token=short-lived',requests[1][1]) + self.assertNotIn('password',requests[1][1]) + + def test_private_profile_lock_and_symlinks(self): + with tempfile.TemporaryDirectory() as root: + with Profile('work',root).locked() as profile: + profile.data={'token':'secret'}; profile.save() + self.assertEqual(stat.S_IMODE(profile.path.stat().st_mode),0o600) + self.assertEqual(stat.S_IMODE(profile.root.stat().st_mode),0o700) + with self.assertRaises(Failure): + with Profile('work',root).locked(): pass + with Profile('work',root).locked() as loaded: self.assertEqual(loaded.data['token'],'secret') + profile.path.chmod(0o644) + with self.assertRaises(Failure): + with Profile('work',root).locked(): pass + profile.path.unlink(); profile.path.symlink_to(Path(root)/'missing') + with self.assertRaises(Failure): + with Profile('work',root).locked(): pass + + def test_cas_and_authoring_contract(self): + state={'object_uuid':'object','head_revision_uuid':'base','principal_uuid':'person','lang':'ja', + 'content_type':'text/html','filename':'a.html','body':'old'} + client=Client('https://example.test') + calls=[] + def api(route, **kwargs): + calls.append((route,kwargs['data'])) + return {'html':'

    new

    '} if route.endswith('/markdown') else {'revision_uuid':'new'} + with patch.object(client,'json',side_effect=api): client.save(state,'new',True) + self.assertTrue(calls[0][1]['authoring']); self.assertFalse(calls[0][1]['projection']) + self.assertEqual(calls[1][1]['base_revision_uuid'],'base') + self.assertEqual(calls[1][1]['editor_principal_uuid'],'person') + self.assertEqual(calls[1][1]['lang'],'ja') + with patch.object(client,'json',side_effect=Failure('conflict',409)): + with self.assertRaises(Failure): client.save(state,'new') + + def test_download_private_and_no_clobber(self): + headers=Message(); headers['X-PagesWiki-Revision-UUID']='rev' + with tempfile.TemporaryDirectory() as root: + profile=Profile('test',root) + client=Client('https://example.test') + file=str(Path(root)/'download') + with patch.object(client,'raw',return_value=(b'data',headers)), contextlib.redirect_stdout(io.StringIO()): + m['execute'](client,profile,['get','/path',file]) + self.assertEqual(stat.S_IMODE(Path(file).stat().st_mode),0o600) + with self.assertRaises(FileExistsError): m['execute'](client,profile,['get','/path',file]) + + def test_editor_conflict_retains_private_recovery(self): + state={'object_uuid':'object','head_revision_uuid':'base','principal_uuid':'person','lang':'und', + 'content_type':'text/plain','filename':'x.txt','body':'old','markdown':'old'} + client=Client('https://example.test','secret-token') + with tempfile.TemporaryDirectory() as root: + directory=Path(root)/'working'; directory.mkdir(mode=0o700) + def edit(args, **kwargs): + Path(args[-1]).write_text('my edit') + class Result: returncode=0 + return Result() + with patch('tempfile.mkdtemp',return_value=str(directory)), patch('subprocess.run',side_effect=edit), patch.object(client,'snapshot',return_value=state), patch.object(client,'save',side_effect=Failure('conflict',409)), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(Failure): m['editor'](client,'path:/x','auto') + self.assertEqual((directory/'document.txt').read_text(),'my edit') + recovery=(directory/'recovery.json').read_text() + self.assertNotIn('secret-token',recovery) + self.assertIn('base',recovery) + self.assertEqual(stat.S_IMODE((directory/'recovery.json').stat().st_mode),0o600) + + +def integration(): + fixture=json.load(sys.stdin) + # Exercise the public entry point with persisted defaults, not only Client. + with tempfile.TemporaryDirectory() as root, patch.dict(os.environ,{'XDG_CONFIG_HOME':root}), contextlib.redirect_stdout(io.StringIO()) as output: + m['main'](['configure','--origin',fixture['origin'],'--username','admin']) + with patch('getpass.getpass',return_value='a long test password'): m['main'](['login']) + with Profile('default',root).locked() as session: session_token=session.data['token'] + m['main'](['ls']) + m['main'](['status']) + def local_editor(args, **kwargs): + file=Path(args[-1]); file.write_text(file.read_text()+'\nConfigured CLI edit\n') + class Result: returncode=0 + return Result() + with patch('subprocess.run',side_effect=local_editor): m['main'](['edit','/editor-fixture']) + check=Client(fixture['origin'],session_token) + assert 'Configured CLI edit' in check.snapshot('path:/editor-fixture')['body'] + m['main'](['logout']) + assert session_token not in output.getvalue() + with Profile('default',root).locked() as session: assert 'token' not in session.data + client=Client(fixture['origin']) + client.login('admin','a long test password') + addr='path:/editor-fixture' + state=client.snapshot(addr) + assert state['lang']=='und' and state['principal_uuid'] + missing=Client(fixture['origin'], client.token, 'ja') + try: missing.snapshot(addr) + except Failure: pass + else: raise AssertionError('missing language silently fell back') + assert client.listing('/')['entries'] + assert client.save(state,'# Remote\n\n{{children}}\n',True)['revision_uuid'] != state['head_revision_uuid'] + latest=client.snapshot(addr) + assert '{{children}}' in latest['markdown'] and '.brand{flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis} + .mode-search .topbar>.topbar-search{order:2;flex:1 0 100%;max-width:none} +} .mode-search .topbar-search{display:flex;align-items:center;gap:.4rem;flex:1;max-width:46rem;margin:0}.mode-search .topbar-search input{flex:1;min-width:0;width:auto;margin:0}.mode-search .topbar-search button{flex-shrink:0;margin:0;line-height:1;display:grid;place-items:center;padding:.55rem}.mode-search .topbar-search button svg{width:1rem;height:1rem;fill:none;stroke:currentColor;stroke-width:1.8} .mode-search .breadcrumb-bar{gap:1rem;margin-bottom:.4rem} .mode-search .breadcrumb-bar h1{font-size:1.2rem;margin:0} diff --git a/static/css/theme-picker.css b/static/css/theme-picker.css new file mode 100644 index 0000000..a2a0273 --- /dev/null +++ b/static/css/theme-picker.css @@ -0,0 +1,9 @@ +.theme-picker { position:relative; flex:none; margin-inline:.3rem; color:var(--fg); } +.theme-picker summary { display:grid; place-items:center; width:2rem; height:2rem; border-radius:50%; cursor:pointer; list-style:none; font-size:1.4rem; } +.theme-picker summary::-webkit-details-marker { display:none; } +.theme-picker summary:hover, .theme-picker summary:focus-visible { background:var(--button-hover-bg); outline:1px solid var(--border); } +.theme-picker-panel { position:absolute; z-index:100; right:0; top:calc(100% + .5rem); width:min(17rem,calc(100vw - 2rem)); box-sizing:border-box; padding:.8rem; background:var(--input-bg); color:var(--fg); border:1px solid var(--border); border-radius:.5rem; box-shadow:0 .4rem 1.2rem #0005; } +.theme-picker-panel label { display:grid; gap:.4rem; } +.theme-picker-panel select { width:100%; min-height:2.4rem; background:var(--input-bg); color:var(--input-fg); border:1px solid var(--input-border); } +.theme-picker-panel small { display:block; color:var(--muted); margin-top:.4rem; } +.theme-picker-panel [role="status"] { font-size:.85rem; color:var(--danger); } diff --git a/static/css/theme/base.css b/static/css/theme/base.css index a219ed7..4670330 100644 --- a/static/css/theme/base.css +++ b/static/css/theme/base.css @@ -16,17 +16,77 @@ --button-hover-bg: #222; --danger: #ff6d6d; --accent: #b077ff; + --editor-heading: #f2d278; + --editor-keyword: color-mix(in srgb, var(--accent) 55%, var(--fg)); + --editor-type: #91d2f4; + --editor-string: #ade0b1; + --editor-number: #edc58c; + --editor-comment: #99a6b8; + --editor-selection: color-mix(in srgb, var(--accent) 26%, transparent); + --editor-caret: var(--link); --scrollbar-track: color-mix(in srgb, var(--bg) 88%, var(--fg) 12%); --scrollbar-thumb: color-mix(in srgb, var(--muted) 72%, var(--accent) 28%); --scrollbar-thumb-hover: color-mix(in srgb, var(--muted) 38%, var(--accent) 62%); } +/* Shared semantic tokens: CodeMirror editors, read-only file viewers and + projected HTML code blocks use the same palette, including live switches. */ +.pw-syntax-heading{color:var(--editor-heading);font-weight:bold} +.pw-syntax-link{color:var(--link);text-decoration:underline} +.pw-syntax-emphasis{font-style:italic}.pw-syntax-strong{font-weight:bold} +.pw-syntax-keyword{color:var(--editor-keyword)}.pw-syntax-type{color:var(--editor-type)} +.pw-syntax-string{color:var(--editor-string)}.pw-syntax-number{color:var(--editor-number)} +.pw-syntax-comment{color:var(--editor-comment);font-style:italic} +.pw-syntax-variable{color:var(--editor-variable,var(--fg))} +.pw-syntax-function{color:var(--editor-function,var(--link))} +.pw-syntax-operator{color:var(--editor-operator,var(--editor-keyword))} +.pw-syntax-invalid{text-decoration:underline wavy var(--danger)} + html, body { background: var(--bg); color: var(--fg); } +/* Document code blocks share theme tokens across viewers and editor previews. + Keep overflow local: a long command must not widen the whole document. */ +:where(.page-html, .periodical-body, .preview, #preview) pre { + box-sizing:border-box; + max-width:100%; + overflow:auto; + padding:.85rem 1rem; + border:1px solid var(--border); + border-inline-start:3px solid var(--accent); + border-radius:.35rem; + background:var(--code-bg,var(--input-bg)); + color:var(--code-fg,var(--fg)); + font: .9em/1.6 "JetBrains Mono", "Hack Nerd Font", monospace; + tab-size:4; + white-space:pre; + overscroll-behavior-inline:contain; +} +:where(.page-html, .periodical-body, .preview, #preview) pre code { + background:transparent; + color:inherit; + padding:0; + border:0; + font:inherit; +} +:where(.page-html, .periodical-body, .preview, #preview) :not(pre) > code { + padding:.1em .3em; + border-radius:.2em; + background:var(--code-bg,var(--input-bg)); + color:var(--code-fg,var(--fg)); + font-family:"JetBrains Mono", "Hack Nerd Font", monospace; + overflow-wrap:anywhere; +} +@media print { + :where(.page-html, .periodical-body, .preview, #preview) pre { + overflow:visible; white-space:pre-wrap; overflow-wrap:anywhere; + color:#000; background:#fff; border-color:#999; + } +} + /* Theme-aware global scrollbars. Firefox uses the standard properties; Chromium/WebKit use the pseudo-elements below. */ * { diff --git a/static/css/theme/index-white.css b/static/css/theme/index-white.css index 7a62df4..6adf532 100644 --- a/static/css/theme/index-white.css +++ b/static/css/theme/index-white.css @@ -1,4 +1,10 @@ :root { + --editor-heading: #805600; + --editor-keyword: #8d2868; + --editor-type: #075b8b; + --editor-string: #27662f; + --editor-number: #854d0b; + --editor-comment: #586574; --bg: #f8fbff; --fg: #17253d; --muted: #687b98; diff --git a/static/css/theme/light.css b/static/css/theme/light.css index d31867e..f015283 100644 --- a/static/css/theme/light.css +++ b/static/css/theme/light.css @@ -1,4 +1,10 @@ :root { + --editor-heading: #805600; + --editor-keyword: #8d2868; + --editor-type: #075b8b; + --editor-string: #27662f; + --editor-number: #854d0b; + --editor-comment: #586574; --bg: #f7f7f7; --fg: #202020; --muted: #666; diff --git a/static/css/theme/railgun-white.css b/static/css/theme/railgun-white.css index 3e0a242..2765903 100644 --- a/static/css/theme/railgun-white.css +++ b/static/css/theme/railgun-white.css @@ -1,4 +1,10 @@ :root { + --editor-heading: #805600; + --editor-keyword: #8d2868; + --editor-type: #075b8b; + --editor-string: #27662f; + --editor-number: #854d0b; + --editor-comment: #586574; --bg: #fffaf4; --fg: #30241d; --muted: #806d61; diff --git a/static/css/theme/solarized-light.css b/static/css/theme/solarized-light.css index 2ba936a..e975b4a 100644 --- a/static/css/theme/solarized-light.css +++ b/static/css/theme/solarized-light.css @@ -1,4 +1,10 @@ :root { + --editor-heading: #805600; + --editor-keyword: #8d2868; + --editor-type: #075b8b; + --editor-string: #27662f; + --editor-number: #854d0b; + --editor-comment: #586574; --bg: #fdf6e3; --fg: #586e75; --muted: #839496; diff --git a/static/js/browser-address.js b/static/js/browser-address.js new file mode 100644 index 0000000..5d5034b --- /dev/null +++ b/static/js/browser-address.js @@ -0,0 +1,89 @@ +// These commands resolve Wiki paths only. They never run shell commands or open +// user-supplied URL schemes, and all writes remain in the authorized editor. +(() => { + function resolvePath(value, current = '/') { + if (/^[a-z][a-z0-9+.-]*:/i.test(value) || value.startsWith('//') || /[\\\u0000-\u001f\u007f]/.test(value)) { + throw new Error('invalid_path'); + } + const parts = (value.startsWith('/') ? [] : current.split('/').filter(Boolean)); + for (const part of value.split('/')) { + if (!part || part === '.') continue; + if (part === '..') parts.pop(); + else parts.push(part); + } + return '/' + parts.join('/'); + } + function destination(value, current = '/') { + value = value.trim(); + let command = 'cd'; + const match = /^(cd|vi|view)(?:\s+(.*))?$/s.exec(value); + if (match) { + command = match[1]; + value = match[2] || (command === 'cd' ? '/' : current); + } + const path = resolvePath(value, current); + // Each segment is literal. In particular #, ?, % and Unicode belong to the + // Path, not URL syntax; never decode user input a second time. + const encoded = path.split('/').map(encodeURIComponent).join('/'); + return {command, path, href: command === 'vi' ? '/e/-' + encoded : command === 'view' ? encoded : '/v/browser' + encoded}; + } + if (typeof module !== 'undefined' && module.exports) module.exports = {resolvePath, destination}; + if (typeof document === 'undefined') return; + window.PagesWikiPaths = {resolvePath, destination}; + const root = document.querySelector('[data-browser-address]'); + if (!root) return; + const current = root.dataset.path; + const input = root.querySelector('input'); + const error = document.getElementById('browser-address-error'); + const navigate = value => { + try { + const target = destination(value, current); + error.textContent = ''; + location.assign(target.href); + } catch (_) { + error.textContent = root.dataset.invalidLabel; + input.focus(); + } + }; + root.addEventListener('submit', event => { + event.preventDefault(); + navigate('cd ' + input.value); + }); + input.addEventListener('keydown', event => { + if (event.key === 'Escape') { input.value = current; error.textContent = ''; input.blur(); } + }); + input.addEventListener('input', () => { error.textContent = ''; }); + document.querySelector('[data-browser-up]')?.addEventListener('click', () => navigate('cd ..')); + const dialog = document.querySelector('[data-browser-create-dialog]'); + const name = dialog.querySelector('[name="path"]'); + const createError = dialog.querySelector('[role="alert"]'); + let mode = 'vi'; + document.querySelectorAll('[data-browser-create]').forEach(button => button.addEventListener('click', () => { + mode = button.dataset.browserCreate; + dialog.querySelector('h2').textContent = button.title; + dialog.querySelector('[type="submit"]').textContent = mode === 'vi' ? root.dataset.editLabel : root.dataset.openLabel; + name.value = ''; + name.setCustomValidity(''); + createError.textContent = ''; + dialog.querySelector('[data-browser-format]').hidden = mode !== 'vi'; + dialog.showModal(); + name.focus(); + })); + dialog.querySelector('[data-browser-create-cancel]').addEventListener('click', () => dialog.close()); + name.addEventListener('input', () => { name.setCustomValidity(''); createError.textContent = ''; }); + dialog.querySelector('form').addEventListener('submit', event => { + event.preventDefault(); + try { + if (!name.value.trim()) throw new Error('invalid_path'); + const path = resolvePath(name.value.trim(), current); + if (mode === 'vi' && path === '/') throw new Error('invalid_path'); + let href = destination(mode + ' ' + path, current).href; + if (mode === 'vi' && dialog.querySelector('[name="format"]').value === 'plain') href += '?edittype=plain'; + location.assign(href); + } catch (_) { + createError.textContent = root.dataset.invalidLabel; + name.setCustomValidity(root.dataset.invalidLabel); + name.reportValidity(); + } + }); +})(); diff --git a/static/js/city-estates.js b/static/js/city-estates.js index ff71663..67435b6 100644 --- a/static/js/city-estates.js +++ b/static/js/city-estates.js @@ -8,6 +8,7 @@ // API may suggest a descendant as a fallback portal (also in undefined // buckets); such a Page must keep its independently selectable building. const surroundingPages=spot=>uniquePages(spot).filter(p=>!(p.page_uuid===spot.portal_page_uuid&&p.path===spot.path_prefix)); + const displayTitle = spot => spot.display_title || spot.title || spot.name; const undefinedSpot=spot=>spot.name?.endsWith('-undefined')||spot.path_prefix?.endsWith('/_undefined'); const pageName=p=>p.label||(p.path||'').split('/').filter(Boolean).pop()||p.page_uuid; const pageURL=p=>p.path?p.path.split('/').map(encodeURIComponent).join('/'):`/o/uuid:${encodeURIComponent(p.page_uuid)}`; @@ -73,14 +74,14 @@ group.append(el('path',{d:`M${x-w/2+12} ${y-h/2+35}V${y+h/2-35}M${x+w/2-12} ${y-h/2+35}V${y+h/2-35}`,class:'estate-balustrade'})); } function entry(group,spot,x,y,onSpot,isUndefined,large=false) { - const g=el('g',{class:large?'city-spot-monument':'city-spot-entry',role:large?'img':'button',...(large?{}:{tabindex:0}),'aria-label':spot.title||spot.name}); + const g=el('g',{class:large?'city-spot-monument':'city-spot-entry',role:large?'img':'button',...(large?{}:{tabindex:0}),'aria-label':displayTitle(spot)}); if(!isUndefined)use(g,'city-palazzo',x-61,y-49,122,92); else {g.append(el('circle',{cx:x,cy:y-5,r:25,class:'estate-fountain'}),el('circle',{cx:x,cy:y-5,r:12,class:'estate-fountain-inner'}));} - label(g,'city-spot-point-label',x,y+59,isUndefined?(document.documentElement.lang.startsWith('ja')?'未分類のPage':'Unclassified Pages'):spot.title||spot.name,large?260:195,large?20:17); + label(g,'city-spot-point-label',x,y+59,displayTitle(spot),large?260:195,large?20:17); group.append(g); } function spotGroup(spot,onSpot,focused=false) { - const group=el('g',{class:'city-node city-spot-tile','data-city-type':'Spot','data-city-uuid':spot.uuid,role:'group','aria-label':spot.title||spot.name}); + const group=el('g',{class:'city-node city-spot-tile','data-city-type':'Spot','data-city-uuid':spot.uuid,role:'group','aria-label':displayTitle(spot)}); group.addEventListener('click',event=>{if(event.target.closest('a'))return;event.stopPropagation();if(!focused)onSpot(spot);});return group; } function streetScene(street,roads,page,onSpot) { diff --git a/static/js/city-map.js b/static/js/city-map.js index 3ce3a4c..3751246 100644 --- a/static/js/city-map.js +++ b/static/js/city-map.js @@ -22,6 +22,7 @@ let suspended = false; let navigation = null; let organizer = null; + const displayTitle = item => item?.display_title || item?.title || item?.name || ''; const label = key => svg.dataset[key] || key; const popupMode = new URLSearchParams(location.search).get('popup') === '1'; if (popupMode) { document.documentElement.classList.add('city-map-popup-mode'); document.body.classList.add('city-map-popup-mode'); } @@ -42,7 +43,7 @@ }; const show = (type, item, parent = '') => { inspector.dataset.cityType=type;inspector.dataset.cityUuid=item.uuid; - inspector.innerHTML = `

    ${escapeHTML(item.title || item.name)}

    Type
    ${type}
    Name
    ${escapeHTML(item.name)}
    Path
    ${escapeHTML(item.path_prefix || '—')}
    ${parent ? `
    Location
    ${escapeHTML(parent)}
    ` : ''}
    State
    ${escapeHTML(item.state)}
    ${item.description || item.summary ? `

    ${escapeHTML(item.description || item.summary)}

    ` : ''}

    Details →

    ${document.querySelector('.city-planner') ? `` : ''}`; + inspector.innerHTML = `

    ${escapeHTML(displayTitle(item))}

    Type
    ${type}
    Name
    ${escapeHTML(item.name)}
    Path
    ${escapeHTML(item.path_prefix || '—')}
    ${parent ? `
    Location
    ${escapeHTML(parent)}
    ` : ''}
    State
    ${escapeHTML(item.state)}
    ${item.description || item.summary ? `

    ${escapeHTML(item.description || item.summary)}

    ` : ''}

    Details →

    ${document.querySelector('.city-planner') ? `` : ''}`; inspector.querySelector('[data-delete-city-node]')?.addEventListener('click', event => deleteNode(event.currentTarget.dataset.cityType, item)); organizer?.decorateInspector(inspector,type,item); }; @@ -62,9 +63,9 @@ const pageList = pages.length ? `

    Pages

    ` : '

    No Pages

    '; inspector.insertAdjacentHTML('beforeend', pageList); const relations = city.relations.filter(relation => relation.source_spot_uuid === spot.uuid); - if (relations.length) inspector.insertAdjacentHTML('beforeend', `

    Relations

      ${relations.map(relation => { const target = city.quartiers.flatMap(q => q.streets.flatMap(s => s.spots)).find(item => item.uuid === relation.target_spot_uuid); return `
    • ${escapeHTML(relation.relation_type)} → ${escapeHTML(target?.title || relation.target_spot_uuid)}${canPlan ? ` ` : ''}
    • `; }).join('')}
    `); + if (relations.length) inspector.insertAdjacentHTML('beforeend', `

    Relations

      ${relations.map(relation => { const target = city.quartiers.flatMap(q => q.streets.flatMap(s => s.spots)).find(item => item.uuid === relation.target_spot_uuid); return `
    • ${escapeHTML(relation.relation_type)} → ${escapeHTML(displayTitle(target) || relation.target_spot_uuid)}${canPlan ? ` ` : ''}
    • `; }).join('')}
    `); if (canPlan) { - const options = city.quartiers.flatMap(q => q.streets.flatMap(s => s.spots)).filter(item => item.uuid !== spot.uuid).map(item => ``).join(''); + const options = city.quartiers.flatMap(q => q.streets.flatMap(s => s.spots)).filter(item => item.uuid !== spot.uuid).map(item => ``).join(''); inspector.insertAdjacentHTML('beforeend', `
    SpotGraph
    `); inspector.querySelector('[data-spot-page]')?.addEventListener('submit', async event => { event.preventDefault(); const response = await fetch(`/_/api/v1/city/spots/${spot.uuid}/pages`, {method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(Object.fromEntries(new FormData(event.target)))}); if (response.ok) await load(); else alert((await response.json()).error); }); inspector.querySelector('[data-spot-relation]')?.addEventListener('submit', async event => { event.preventDefault(); const response = await fetch(`/_/api/v1/city/spots/${spot.uuid}/relations`, {method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(Object.fromEntries(new FormData(event.target)))}); if (response.ok) await load(); else alert((await response.json()).error); }); @@ -85,7 +86,7 @@ const closeContext = () => { contextMenu?.remove(); contextMenu = null; }; const deleteNode = async (type, item) => { const ja = document.documentElement.lang.toLowerCase().startsWith('ja'); - if (!confirm(ja ? `${item.title || item.name} を削除しますか? 下位要素も案内図から削除されます。ドキュメント自体は削除されません。` : `Delete ${item.title || item.name}? Descendants are removed from the map; documents are preserved.`)) return; + if (!confirm(ja ? `${displayTitle(item)} を削除しますか? 下位要素も案内図から削除されます。ドキュメント自体は削除されません。` : `Delete ${displayTitle(item)}? Descendants are removed from the map; documents are preserved.`)) return; const plural = type === 'Spot' ? 'spots' : 'zones'; const options = {method:'DELETE', headers:{'Content-Type':'application/json'}}; if (type !== 'Spot') options.body = JSON.stringify({kind:type.toLowerCase()}); @@ -99,7 +100,7 @@ const url = event.target.closest?.('.city-map-page-link')?.getAttribute('href') || detailURL(type, item); contextMenu = document.createElement('div'); contextMenu.className = 'city-context-menu'; contextMenu.setAttribute('role', 'menu'); - contextMenu.innerHTML = `${escapeHTML(item.title || item.name)}${ja ? '詳細を開く' : 'Open details'}${ja ? '新しいタブで開く' : 'Open in new tab'}${document.querySelector('.city-planner') ? `` : ''}`; + contextMenu.innerHTML = `${escapeHTML(displayTitle(item))}${ja ? '詳細を開く' : 'Open details'}${ja ? '新しいタブで開く' : 'Open in new tab'}${document.querySelector('.city-planner') ? `` : ''}`; document.body.append(contextMenu); organizer?.context(contextMenu,type,item,event); const rect = contextMenu.getBoundingClientRect(); @@ -132,7 +133,7 @@ function updateCityTrail() { let trail=document.querySelector('[data-city-trail]'); if(!trail){trail=document.createElement('nav');trail.className='city-trail';trail.dataset.cityTrail='';document.querySelector('.city-toolbar').prepend(trail);} const q=city.quartiers.find(item=>item.uuid===cityView.quartier), s=q?.streets.find(item=>item.uuid===cityView.street),p=s?.spots.find(item=>item.uuid===cityView.spot); - trail.innerHTML=`${q?`›`:''}${s?`›`:''}${cityView.level==='spot'&&p?`›${escapeHTML(p.title)}`:''}`; + trail.innerHTML=`${q?`›`:''}${s?`›`:''}${cityView.level==='spot'&&p?`›${escapeHTML(displayTitle(p))}`:''}`; trail.querySelector('[data-city-level="domain"]')?.addEventListener('click',()=>{cityView={level:'domain',quartier:null,street:null};renderScene();}); trail.querySelector('[data-city-level="quartier"]')?.addEventListener('click',()=>{cityView={level:'quartier',quartier:q.uuid,street:null};renderScene();}); trail.querySelector('[data-city-level="street"]')?.addEventListener('click',()=>{cityView={level:'street',quartier:q.uuid,street:s.uuid};renderScene();}); @@ -251,7 +252,7 @@ } } function renderDirectory(data) { - const jump=(type,item)=>``; + const jump=(type,item)=>``; directory.innerHTML = `
    ${data.quartiers.map(q => `

    ${jump('quartier',q)}

    ${q.description ? `

    ${escapeHTML(q.description)}

    ` : ''}${q.streets.map(s => `

    ${jump('street',s)}

      ${s.spots.map(p => `
    • ${jump('spot',p)}
    • `).join('')}
    `).join('')}
    `).join('')}
    `; directory.querySelectorAll('[data-city-jump]').forEach(button=>button.addEventListener('click',()=>{ const type=button.dataset.cityJump,id=button.dataset.cityId; @@ -270,9 +271,9 @@ } function installCityAutoPlan() { const planner=document.querySelector('.city-planner');if(!planner||document.querySelector('[data-city-autoplan]'))return; - const ja=document.documentElement.lang.startsWith('ja'),panel=document.createElement('details');panel.className='city-autoplan';panel.dataset.cityAutoplan='';panel.innerHTML=`${ja?'URL Pathから都市を自動生成':'Generate City from URL Paths'}

    ${ja?'第1階層をQuartier、第2階層をStreet、第3階層をSpotとしてdry-runします。浅いPageは未定義Spotへ所属します。':'Dry-run: level 1 becomes Quartier, level 2 Street, and level 3 Spot. Shallow Pages use an undefined Spot.'}

    `;planner.before(panel); + const ja=document.documentElement.lang.startsWith('ja'),panel=document.createElement('details');panel.className='city-autoplan';panel.dataset.cityAutoplan='';panel.innerHTML=`${ja?'URL Pathから都市を自動生成':'Generate City from URL Paths'}

    ${ja?'第1階層をQuartier、第2階層をStreet、第3階層をSpotとしてdry-runします。浅いPageはStreetの入口にまとめます。':'Dry-run: level 1 becomes Quartier, level 2 Street, and level 3 Spot. Shallow Pages are grouped at their Street entrance.'}

    `;planner.before(panel); const result=panel.querySelector('[data-autoplan-result]'),results=panel.querySelector('[data-autoplan-results]'),apply=panel.querySelector('[data-autoplan-apply]'); - panel.querySelector('[data-autoplan-preview]').addEventListener('click',async()=>{result.textContent='…';const response=await fetch('/_/api/v1/city/autoplan');const data=await response.json();if(!response.ok){result.textContent=data.error||'Failed';return}result.textContent=`${data.count} Pages`;const rows=(data.items||[]).slice(0,300);results.innerHTML=`${rows.map(item=>``).join('')}
    PathQuartierStreetSpot${ja?'処理':'Action'}
    ${escapeHTML(item.path)}${escapeHTML(item.Quartier)}${escapeHTML(item.Street)}${escapeHTML(item.Spot)}${escapeHTML(item.action)}
    ${data.count>rows.length?`

    + ${data.count-rows.length}

    `:''}`;apply.hidden=!data.count;}); + panel.querySelector('[data-autoplan-preview]').addEventListener('click',async()=>{result.textContent='…';const response=await fetch('/_/api/v1/city/autoplan');const data=await response.json();if(!response.ok){result.textContent=data.error||'Failed';return}result.textContent=`${data.count} Pages`;const rows=(data.items||[]).slice(0,300);results.innerHTML=`${rows.map(item=>``).join('')}
    PathQuartierStreetSpot${ja?'処理':'Action'}
    ${escapeHTML(item.path)}${escapeHTML(item.Quartier)}${escapeHTML(item.Street)}${escapeHTML(item.undefined_spot ? item.Street : item.Spot)}${escapeHTML(item.action)}
    ${data.count>rows.length?`

    + ${data.count-rows.length}

    `:''}`;apply.hidden=!data.count;}); apply.addEventListener('click',async()=>{if(!confirm(ja?'この計画をCityへ適用しますか?':'Apply this plan to the City?'))return;apply.disabled=true;result.textContent='…';const response=await fetch('/_/api/v1/city/autoplan',{method:'POST'}),data=await response.json();apply.disabled=false;if(!response.ok){result.textContent=data.error||'Failed';return}result.textContent=`${data.applied} Pages`;await load();}); } function reset() { transform = { x: 0, y: 0, scale: 1 }; applyTransform(); } diff --git a/static/js/city-organizer.js b/static/js/city-organizer.js index 824a83e..52e243f 100644 --- a/static/js/city-organizer.js +++ b/static/js/city-organizer.js @@ -37,6 +37,7 @@ window.PagesWikiCityOrganizer = ({svg, getCity, reload, dismissMenus, canPlan}) forbidden:'Domain administrator access is required.', refresh:'Refresh map', hint:'Ctrl / ⌘ + click to select multiple items' }; const escape = value => String(value ?? '').replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); + const displayTitle = item => item.display_title || item.title || item.name; const selection = new Map(); const spots = () => (getCity()?.quartiers || []).flatMap(q => q.streets.flatMap(s => s.spots.map(p => ({...p, street:s, quartier:q})))); const streets = () => (getCity()?.quartiers || []).flatMap(q => q.streets.map(s => ({...s, quartier:q}))); @@ -48,7 +49,7 @@ window.PagesWikiCityOrganizer = ({svg, getCity, reload, dismissMenus, canPlan}) const item = spots().find(p => p.uuid === spot.dataset.cityUuid); if (!item) return null; if (page) return {key:pageKey(item.uuid,page.dataset.pageUuid),spot_uuid:item.uuid,page_uuid:page.dataset.pageUuid,title:page.getAttribute('aria-label')}; - return {key:`spot:${item.uuid}`,spot_uuid:item.uuid,title:item.title || item.name}; + return {key:`spot:${item.uuid}`,spot_uuid:item.uuid,title:displayTitle(item)}; }; const bar = document.createElement('div'); bar.className='city-selection-bar'; bar.hidden=true; @@ -112,7 +113,7 @@ window.PagesWikiCityOrganizer = ({svg, getCity, reload, dismissMenus, canPlan}) }; const rename = (type, item) => { const version=getCity().version; - const el=openDialog(words.rename,`

    ${escape(words.renameHelp)}

    `,data=>post(`/_/api/v1/city/${type==='Spot'?'spots':'zones'}/${encodeURIComponent(item.uuid)}/title`,{kind:type.toLowerCase(),title:data.get('title'),base_version:version})); + const el=openDialog(words.rename,`

    ${escape(words.renameHelp)}

    `,data=>post(`/_/api/v1/city/${type==='Spot'?'spots':'zones'}/${encodeURIComponent(item.uuid)}/title`,{kind:type.toLowerCase(),title:data.get('title'),base_version:version})); el.querySelector('input').select(); }; const organize = action => { @@ -125,7 +126,7 @@ window.PagesWikiCityOrganizer = ({svg, getCity, reload, dismissMenus, canPlan}) const name=`spot-${crypto.randomUUID().slice(0,8)}`; const choices=spots().filter(p=>!entries.some(e=>e.spot_uuid===p.uuid)&&!p.name.endsWith('-undefined')&&!p.path_prefix.endsWith('/_undefined')); let fields=`
      ${entries.map(e=>`
    • ${escape(e.title)}
    • `).join('')}

    ${escape(action==='move'?words.moveHelp:words.mergeHelp)}

    `; - if(action==='merge')fields+=`${choices.length?'':`

    ${escape(words.empty)}

    `}`; + if(action==='merge')fields+=`${choices.length?'':`

    ${escape(words.empty)}

    `}`; else { fields+=``; if(action==='create')fields+=`
    ${escape(words.advanced)}
    `; @@ -148,7 +149,7 @@ window.PagesWikiCityOrganizer = ({svg, getCity, reload, dismissMenus, canPlan}) refresh(){const current=spots();for(const [key,item] of selection)if(!current.some(s=>s.uuid===item.spot_uuid&&(!item.page_uuid||s.pages.some(p=>p.page_uuid===item.page_uuid))))selection.delete(key);syncSelection();}, decorateInspector(container,type,item){ const heading=container.querySelector('h2');if(heading)button(heading,words.rename,'✎',()=>rename(type,item),'data-city-rename'); - if(heading&&type==='Spot')button(heading,words.move,'⇄',()=>{selection.clear();add({key:`spot:${item.uuid}`,spot_uuid:item.uuid,title:item.title||item.name});syncSelection();organize('move');},'data-city-move-selection'); + if(heading&&type==='Spot')button(heading,words.move,'⇄',()=>{selection.clear();add({key:`spot:${item.uuid}`,spot_uuid:item.uuid,title:displayTitle(item)});syncSelection();organize('move');},'data-city-move-selection'); }, context(menu,type,item,event){ const hit=pick(event.target); diff --git a/static/js/code-highlight.js b/static/js/code-highlight.js new file mode 100644 index 0000000..9656866 --- /dev/null +++ b/static/js/code-highlight.js @@ -0,0 +1,54 @@ +// Lazy, bounded projection. No CDN, HTML injection, or document writes. +const selector = '.page-html pre code, .periodical-body pre code, #preview pre code, .preview pre code, .preview-content pre code, .public-document pre code'; +const seen = new WeakMap(), queue = new Set(); +let scheduled = false, bundle; +const idle = callback => window.requestIdleCallback ? requestIdleCallback(callback, {timeout:1000}) : setTimeout(callback, 0); +function language(code) { + return [...code.classList, ...code.parentElement.classList].find(c => /^(language|lang)-/.test(c))?.replace(/^(language|lang)-/, '') || ''; +} +function schedule() { + if (scheduled || !queue.size) return; + scheduled = true; + idle(async () => { + try { + const code = queue.values().next().value; queue.delete(code); + if (!code?.isConnected) return; + const text = code.textContent, mode = language(code); + const signature = mode + '\0' + text; + if (!mode || text.length > 32768 || seen.get(code) === signature) return; + bundle ||= import('/static/vendor/codemirror/editor.bundle.js?v=command-ux-1'); + await bundle; + if (!code.isConnected || code.textContent !== text || language(code) !== mode) { watch(code); return; } + seen.set(code, signature); + window.PagesWikiEditor.highlightCodeBlock(code, mode); + } catch (_) { /* Keep the readable plain-text fallback. */ } + finally { scheduled = false; schedule(); } + }); +} +const visible = 'IntersectionObserver' in window ? new IntersectionObserver(entries => { + for (const entry of entries) if (entry.isIntersecting) { visible.unobserve(entry.target); queue.add(entry.target); } + schedule(); +}, {rootMargin:'100px'}) : null; +function watch(code) { + if (!language(code) || code.textContent.length > 32768 || seen.get(code) === language(code) + '\0' + code.textContent) return; + if (visible) visible.observe(code); else { queue.add(code); schedule(); } +} +function scan(root) { + if (root.matches?.(selector)) watch(root); + root.querySelectorAll?.(selector).forEach(watch); +} +scan(document); +new MutationObserver(records => { + for (const record of records) { + // Preview replacements and fetched projections are processed incrementally. + for (const node of record.addedNodes) if (node.nodeType === 1) scan(node); + for (const node of record.removedNodes) if (node.nodeType === 1) { + const release = code => { visible?.unobserve(code); queue.delete(code); }; + if (node.matches('code')) release(node); + node.querySelectorAll('code').forEach(release); + } + if (record.type === 'attributes' && record.target.matches('pre,code')) scan(record.target); + const code = record.target.nodeType === 1 ? record.target.closest?.('pre code') : record.target.parentElement?.closest('pre code'); + if (code?.matches(selector)) watch(code); + } +}).observe(document.body, {childList:true, subtree:true, characterData:true, attributes:true, attributeFilter:['class']}); diff --git a/static/js/command-bar.js b/static/js/command-bar.js new file mode 100644 index 0000000..8e7af15 --- /dev/null +++ b/static/js/command-bar.js @@ -0,0 +1,84 @@ +(() => { + const bar = document.querySelector('[data-command-bar]'), paths = window.PagesWikiPaths; + if (!bar || !paths) return; + document.body.append(bar); // Escape transformed/clipped headers. + const input = bar.querySelector('input'), list = bar.querySelector('[role=listbox]'), status = bar.querySelector('[role=status]'); + const browser = bar.dataset.browser === 'true', current = bar.dataset.commandPath || '/'; + let previous, request, generation = 0, options = [], index = -1, seed = '', completing = false, composing = false; + const interactive = target => target?.closest?.('input,textarea,select,button,a,summary,[contenteditable]:not([contenteditable=false]),[role=textbox],[role=button],.cm-editor'); + const reset = () => { + generation++; request?.abort(); completing = false; options = []; index = -1; + list.replaceChildren(); list.hidden = true; status.textContent = ''; input.setAttribute('aria-expanded', 'false'); input.removeAttribute('aria-activedescendant'); + }; + const open = text => { + previous = document.activeElement; bar.hidden = false; reset(); input.value = text; + input.focus(); input.setSelectionRange(input.value.length, input.value.length); + }; + const close = () => { + reset(); input.value = ''; if (!browser) bar.hidden = true; input.blur(); + if (previous?.isConnected && !bar.contains(previous)) previous.focus(); + }; + const choose = i => { + index = (i + options.length) % options.length; + input.value = options[index].value; + input.setSelectionRange(input.value.length, input.value.length); + [...list.children].forEach((el,n) => el.setAttribute('aria-selected', String(n === index))); + input.setAttribute('aria-activedescendant', list.children[index].id); + list.children[index].scrollIntoView({block:'nearest'}); + }; + async function complete(backwards) { + if (options.length && !(options.length === 1 && input.value.endsWith('/')) && (input.value === seed || input.value === options[index]?.value)) { choose(index + (backwards ? -1 : 1)); return; } + if (completing) return; + reset(); seed = input.value; + const match = /^(cd|vi|view)\s+(.*)$/s.exec(seed); + if (!match || input.selectionStart !== input.value.length || input.selectionEnd !== input.value.length) return; + const raw = match[2], slash = raw.lastIndexOf('/'); + const directory = slash < 0 ? '' : raw.slice(0, slash + 1), prefix = raw.slice(slash + 1); + let parent; + try { parent = paths.resolvePath(directory || '.', current); } catch { status.textContent = bar.dataset.invalid; return; } + const ticket = generation; request = new AbortController(); completing = true; + try { + const url = '/_/api/v1/path-picker?' + new URLSearchParams({path:parent, mode:'children', prefix}); + const response = await fetch(url, {signal:request.signal, headers:{Accept:'application/json'}, cache:'no-store'}); + if (!response.ok || !response.headers.get('content-type')?.includes('application/json')) throw Error('unavailable'); + const data = await response.json(); + if (ticket !== generation || input.value !== seed) return; + options = (data.entries || []).filter(e => typeof e.name === 'string' && e.name.toLowerCase().startsWith(prefix.toLowerCase())).slice(0,100).map(e => ({value:match[1] + ' ' + directory + e.name + (e.kind === 'folder' ? '/' : ''), label:e.name + (e.kind === 'folder' ? '/' : '')})); + status.textContent = !options.length ? bar.dataset.empty : data.truncated || data.entries.length > 100 ? bar.dataset.more : ''; + for (const [i,item] of options.entries()) { + const option = document.createElement('div'); option.role = 'option'; option.id = 'wiki-command-option-' + i; option.textContent = item.label; + option.addEventListener('mousedown', event => event.preventDefault()); + option.addEventListener('click', () => { choose(i); input.focus(); }); list.append(option); + } + list.hidden = !options.length; input.setAttribute('aria-expanded', String(!!options.length)); + if (options.length) choose(backwards ? options.length - 1 : 0); + } catch (error) { if (ticket === generation && error.name !== 'AbortError') status.textContent = bar.dataset.unavailable; } + finally { if (ticket === generation) completing = false; } + } + bar.addEventListener('submit', event => { + event.preventDefault(); + if (composing || !input.value.trim()) return; + try { + if (!/^(cd|vi|view)(?:\s|$)/.test(input.value.trim())) throw Error('invalid'); + location.assign(paths.destination(input.value, current).href); + } catch { status.textContent = bar.dataset.invalid; } + }); + input.addEventListener('input', reset); + input.addEventListener('compositionstart', () => { composing = true; reset(); }); + input.addEventListener('compositionend', () => { composing = false; }); + bar.addEventListener('focusout', event => { if (!bar.contains(event.relatedTarget)) reset(); }); + input.addEventListener('keydown', event => { + if (event.isComposing || event.keyCode === 229 || event.ctrlKey || event.metaKey || event.altKey) return; + if (event.key === 'Escape') { event.preventDefault(); close(); } + if (event.key === 'Tab' && /^(cd|vi|view)\s/.test(input.value) && input.selectionStart === input.value.length && input.selectionEnd === input.value.length) { event.preventDefault(); complete(event.shiftKey); } + if (options.length && ['ArrowDown','ArrowUp'].includes(event.key)) { event.preventDefault(); choose(index + (event.key === 'ArrowDown' ? 1 : -1)); } + }); + bar.querySelector('[data-command-close]').addEventListener('click', close); + document.addEventListener('keydown', event => { + if (event.defaultPrevented || event.ctrlKey || event.metaKey || event.altKey || event.repeat || interactive(event.target) || document.querySelector('dialog[open],.auth-menu details[open],.theme-picker[open]') || String(window.getSelection() || '')) return; + if (browser && (event.key === 'Process' || event.keyCode === 229)) { open(''); return; } + if (event.isComposing) return; + if (!browser && event.key === 'g') { event.preventDefault(); open(''); } + else if (browser && event.key.length === 1) { event.preventDefault(); open(event.key); } + }, true); +})(); diff --git a/static/js/editor-core.js b/static/js/editor-core.js new file mode 100644 index 0000000..62f220b --- /dev/null +++ b/static/js/editor-core.js @@ -0,0 +1,25 @@ +(function(root) { + // Conservative three-way merge: each side is treated as one changed range. + // Multiple scattered changes may conflict, but are never silently discarded. + function merge(base, local, remote) { + if (local === remote || remote === base) return local + if (local === base) return remote + const change = text => { + let from = 0, end = 0 + while (from < base.length && from < text.length && base[from] === text[from]) from++ + while (end < base.length-from && end < text.length-from && base[base.length-1-end] === text[text.length-1-end]) end++ + return {from, to:base.length-end, insert:text.slice(from,text.length-end)} + } + const a = change(local), b = change(remote) + // Adjacent/identical insertion points require a decision too. + if (!(a.to < b.from || b.to < a.from)) return null + const [first,last] = [a,b].sort((x,y) => x.from-y.from) + return base.slice(0,first.from)+first.insert+base.slice(first.to,last.from)+last.insert+base.slice(last.to) + } + function validDraft(draft, now = Date.now()) { + return draft?.version === 1 && typeof draft.text === 'string' && typeof draft.baseBody === 'string' + && typeof draft.baseRevision === 'string' && ['html','markdown','code','plain'].includes(draft.format) + && Number.isFinite(draft.savedAt) && now-draft.savedAt < 7*86400000 && draft.savedAt <= now+60000 + } + root.PagesEditorCore = {merge,validDraft} +})(typeof window === 'undefined' ? globalThis : window) diff --git a/static/js/editor.js b/static/js/editor.js index da41c6e..7543eb9 100644 --- a/static/js/editor.js +++ b/static/js/editor.js @@ -1,406 +1,445 @@ +/* Editing state is independent of projection HTML. No save follows a failed conversion. */ window.PagesEditor = { - - autosaveTimer: null, - init() { - - const editor = - document.getElementById( - "editor", - ) - - const preview = - document.getElementById( - "preview", - ) - - const editType = - document.getElementById( - "edit-type", - ) - - if ( - !editor - || - !preview - ) { - return - } - - const renderPreview = - async () => { - this.previewController?.abort() - const controller = new AbortController() - this.previewController = controller - const sequence = (this.previewSequence || 0) + 1 - this.previewSequence = sequence - const source = editor.value - - if (editType.value === "code") { - preview.classList.add("code-preview") - preview.textContent = - editor.value - - return editor.value - } - - try { - const res = - await fetch( - "/_/api/v1/render/markdown", - { - method: "POST", - signal: controller.signal, - - headers: { - "Content-Type": - "application/json", - }, - - body: JSON.stringify({ - ...(editType.value === "html" ? {html: source} : {markdown: source}), - path: window.PagesWikiCurrentPath || location.pathname, - projection: true, - }), - }, - ) - - if (controller.signal.aborted || sequence !== this.previewSequence) return - if (!res.ok) { - preview.classList.add("code-preview") - - preview.innerText = - source - - return editor.value - } - - const data = - await res.json() - if (controller.signal.aborted || sequence !== this.previewSequence) return - - preview.classList.remove("code-preview") - preview.innerHTML = - data.html - - if (window.PagesWikiLoadChildren) { - window.PagesWikiLoadChildren(preview) - } - - return data.html - } catch (error) { - if (controller.signal.aborted || sequence !== this.previewSequence) return - preview.classList.add("code-preview") - preview.textContent = source - return source - } - } - - this.renderPreview = - renderPreview - + this.editor = document.getElementById('editor') + this.preview = document.getElementById('preview') + this.mount = document.getElementById('editor-mount') + if (!this.editor || !this.preview || !this.mount) return + this.baseRevision = this.mount.dataset.baseRevision || '' + this.baseBody = document.getElementById('editor-base')?.value || '' + this.initialText = this.editor.value + this.owner = this.mount.dataset.owner || '' + this.serial = 0 + this.paused = false + // The editor also runs on explicitly configured HTTP instances. + // This is a draft namespace, not an authentication credential. + this.tab = globalThis.crypto?.randomUUID?.() || Date.now().toString(36)+'-'+Math.random().toString(36).slice(2) + this.ja = document.documentElement.lang.startsWith('ja') + this.expireDrafts() + this.editor.addEventListener('input', () => this.changed()) document.querySelectorAll('[data-editor-action]').forEach(button => button.addEventListener('click', event => { event.preventDefault() - switch (button.dataset.editorAction) { - case 'save': this.saveNow(); break - case 'rename': window.PagesPathActions.renameCurrentPath(); break - case 'delete': window.PagesPathActions.removeCurrentPath(); break - case 'html': window.switchEditorFormat('html'); break - case 'markdown': window.switchEditorFormat('markdown'); break - case 'preview': window.toggleEditorPreview(); break - } + const action = button.dataset.editorAction + if (action === 'save') this.saveNow() + if (action === 'rename') window.PagesPathActions?.renameCurrentPath() + if (action === 'delete') window.PagesPathActions?.removeCurrentPath() + if (action === 'html' || action === 'markdown') this.switchFormat(action) + if (action === 'preview') window.toggleEditorPreview() + if (action === 'tools') this.tools() + if (action === 'language') this.languageDialog() + if (action === 'restore') this.historyDialog() })) - - editor.addEventListener( - "input", - async () => { - - clearTimeout(this.previewTimer) - this.previewController?.abort() - this.previewTimer = setTimeout(renderPreview, 150) - - clearTimeout( - this.autosaveTimer, - ) - - this.autosaveTimer = - setTimeout( - () => { - this.autosave() - }, - 3000, - ) - }, - ) - - document.addEventListener( - "keydown", - (ev) => { - - if ( - (ev.ctrlKey || ev.metaKey) - && - ev.key === "s" - ) { - - ev.preventDefault() - - this.saveNow() - } - }, - ) - - renderPreview() + document.addEventListener('keydown', event => { + if ((event.ctrlKey || event.metaKey) && event.key.toLowerCase() === 's') {event.preventDefault(); this.saveNow()} + }) + window.addEventListener('beforeunload', event => { + if (this.dirty) { this.persist(); event.preventDefault(); event.returnValue = '' } + }) + window.addEventListener('pagehide', () => { if (this.dirty) this.persist() }) + window.addEventListener('offline', () => this.fail({kind:'offline'})) + window.addEventListener('online', () => { this.paused = true; this.status('reconnected'); this.checkState() }) + document.addEventListener('visibilitychange', () => { + if (document.hidden) {if (this.dirty) this.persist()} else this.checkState() + }) + this.stateTimer = setInterval(() => { if (!document.hidden) this.checkState() },30000) + this.renderPreview() + // CodeMirror is mounted by the next DOMContentLoaded listener. + setTimeout(() => this.offerRecovery(), 0) }, - - async getCanonicalHTML() { - - const editor = - document.getElementById( - "editor", - ) - - const editType = - document.getElementById( - "edit-type", - ) - - if ( - editType.value - === - "html" - ) { - - return editor.value + t(key) { + const text = { + dirty:['未保存','Unsaved'], draft:['端末に下書きを保存済み','Draft saved on this device'], saved:['保存済み','Saved'], saving:['保存中…','Saving…'], + auth:['認証切れ:再ログイン後に保存してください','Session expired: sign in again, then save'], + forbidden:['権限がありません。下書きをダウンロードできます','Permission denied. You can download your draft'], + offline:['通信できません。保存を中断しました','Connection unavailable. Saving paused'], + failed:['保存・変換に失敗しました。本文は保持しています','Save/conversion failed. Your text is retained'], + storage:['端末保存に失敗しました。下書きをダウンロードしてください','Device storage failed. Download your draft'], + reconnected:['接続が戻りました。内容を確認して保存してください','Reconnected. Review your work and save'], + remote:['別の編集による更新があります。保存時に照合します','Another edit was saved. Changes will be checked when you save'], + conflict:['競合の確認(保存するまで反映されません)','Resolve conflict (not applied until saved)'], + base:['編集開始時','Base'], local:['自分の編集','Your changes'], latest:['最新の内容','Latest content'], result:['保存する内容(HTML/コード)','Result to save (HTML / code)'], + apply:['作業バッファへ反映','Apply to working buffer'], cancel:['閉じる','Close'], recovery:['未保存の下書きがあります','An unsaved draft is available'], + recover:['復元','Restore draft'], discard:['破棄','Discard'], export:['下書きダウンロード','Download draft'], tools:['挿入・下書き','Insert / drafts'], + privacy:['下書きはこの端末に平文で保存し、復元対象は7日以内です。期限切れはエディタ起動時に削除します。共有端末では作業後に削除してください。','Drafts are stored as plaintext on this device and recoverable for 7 days. Expired drafts are removed when the editor opens. Remove them after using a shared device.'], + language:['言語の編集・分岐(BCP 47)','Edit / fork language (BCP 47)'], open:['指定言語を開く','Open language'], fork:['現在の本文を指定言語へ分岐','Fork current text into language'], + exists:['指定言語には既に本文があります。編集として開いてください','This language already exists. Open it for editing'], history:['過去のリビジョンを復元','Restore a previous revision'], + signin:['再ログイン(別タブ)','Sign in again (new tab)'], insert:['挿入','Insert'], invalid:['言語タグを確認してください','Check the language tag'], identity:['別アカウントに切り替わっています。元のアカウントで開き直してください','Account changed. Reopen using the original account'], + incompatible:['ファイル形式が変更されています。下書きを退避して開き直してください','File type changed. Download your draft and reopen'], older:['さらに古い履歴','Older revisions'], + children:['子ページ一覧','Children'], table:['表','Table'], figure:['画像とキャプション','Image + caption'], note:['補足','Note'], code:['コードブロック','Code block'], quote:['引用','Quote'], + protected:['Markdown で表せない HTML は {=html} ブロックに保持します。ブロック内も編集でき、保存・プレビュー時に HTML として扱います。','HTML not representable in Markdown is retained in editable {=html} blocks and rendered as HTML on save / preview.'], } - - if (editType.value === "code") { - return editor.value + return text[key]?.[this.ja ? 0 : 1] || key + }, + value(id) { return document.getElementById(id)?.value || '' }, + format() {return this.value('edit-type')}, + text() {return window.PagesWikiCodeMirror?.state.doc.toString() ?? this.editor.value}, + status(key) { + const status = document.getElementById('editor-save-status') + if (status) {status.textContent = this.t(key); status.dataset.state = key} + }, + keyPrefix() {return 'pageswiki.draft.v1:'+JSON.stringify([this.mount.dataset.domain,this.owner,this.value('object-uuid'),this.value('page-language')||'und'])+':'}, + expireDrafts() { + try { + for (const key of Object.keys(localStorage)) { + if (!key.startsWith('pageswiki.draft.v1:')) continue + let draft + try {draft=JSON.parse(localStorage.getItem(key))} catch (_) {} + if (!PagesEditorCore.validDraft(draft)) localStorage.removeItem(key) + } + } catch (_) {this.status('storage')} + }, + persist() { + if (!this.dirty) return + try { + const draft = {version:1, text:this.text(), format:this.format(), baseRevision:this.baseRevision, baseBody:this.baseBody, savedAt:Date.now()} + const data = JSON.stringify(draft) + if (new TextEncoder().encode(data).length > 2*1024*1024) throw Error('draft too large') + localStorage.setItem(this.keyPrefix()+this.tab,data) + this.draftStored = true + if (!this.paused && !this.saving) this.status('draft') + } catch (_) {this.draftStored = false; this.status('storage')} + }, + clearDraft() { + clearTimeout(this.draftTimer) + try {localStorage.removeItem(this.keyPrefix()+this.tab)} catch (_) {} + }, + offerRecovery() { + let candidate + try { + for (const key of Object.keys(localStorage)) { + if (!key.startsWith(this.keyPrefix())) continue + let draft + try {draft = JSON.parse(localStorage.getItem(key))} catch (_) {} + if (!PagesEditorCore.validDraft(draft)) {localStorage.removeItem(key); continue} + if (!candidate || draft.savedAt > candidate.draft.savedAt) candidate = {key,draft} + } + } catch (_) {this.status('storage'); return} + if (!candidate) return + this.paused = true + const {dialog,content,button} = this.dialog('recovery') + this.paragraph(content,this.t('privacy')) + this.paragraph(content,new Date(candidate.draft.savedAt).toLocaleString()) + button('recover',() => { + // A stale draft must not change a plain file into HTML (or vice + // versa) merely because the editor was reopened in another mode. + const family = format => ['plain','code'].includes(format) ? format : 'html' + if (family(candidate.draft.format) !== family(this.format())) {this.fail({kind:'incompatible'}); return} + this.baseRevision = candidate.draft.baseRevision + this.baseBody = candidate.draft.baseBody + this.initialText = null + this.setText(candidate.draft.text,candidate.draft.format) + this.persist() + // Move this recovered snapshot, never remove another tab's newer + // draft. A failed local write must leave the recovery source intact. + if (this.draftStored && localStorage.getItem(candidate.key) === JSON.stringify(candidate.draft)) localStorage.removeItem(candidate.key) + dialog.close() + }) + button('discard',() => {localStorage.removeItem(candidate.key); this.paused=false; dialog.close()}) + button('export',() => this.download(candidate.draft.text,candidate.draft.format)) + }, + changed() { + this.serial++ + this.dirty = true + this.status('dirty') + clearTimeout(this.draftTimer) + this.draftTimer = setTimeout(() => this.persist(),250) + clearTimeout(this.previewTimer) + this.previewController?.abort() + this.previewTimer = setTimeout(() => this.renderPreview(),300) + clearTimeout(this.autosaveTimer) + if (!this.paused) this.autosaveTimer = setTimeout(() => this.save(false),3000) + }, + async request(url,options = {}) { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(),15000) + const abort = () => controller.abort() + options.signal?.addEventListener('abort',abort,{once:true}) + if (options.signal?.aborted) controller.abort() + try { + const response = await fetch(url,{...options,signal:controller.signal,credentials:'same-origin',cache:'no-store'}) + if (response.status === 401 || response.redirected) throw {kind:'auth'} + if (response.status === 403) throw {kind:'forbidden'} + if (response.status === 409) throw {kind:'conflict'} + if (!response.ok) throw {kind:'failed'} + if (!response.headers.get('Content-Type')?.includes('application/json')) throw {kind:'auth'} + return await response.json() + } catch (error) { + if (error.kind) throw error + throw {kind:'offline'} + } finally {clearTimeout(timer); options.signal?.removeEventListener('abort',abort)} + }, + post(url,body,options={}) {return this.request(url,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body),...options})}, + fail(error) { + this.paused = true + clearTimeout(this.autosaveTimer) + this.persist() + this.status(error.kind || 'failed') + if (this.dirty && this.draftStored===false) { + const status=document.getElementById('editor-save-status') + if (status) status.textContent+=' / '+this.t('storage') } - - const res = - await fetch( - "/_/api/v1/render/markdown", - { - method: "POST", - - headers: { - "Content-Type": - "application/json", - }, - - body: JSON.stringify({ - markdown: - editor.value, - }), - }, - ) - - if (!res.ok) { - return editor.value - } - - const data = - await res.json() - + }, + async canonical(text = this.text(), format = this.format()) { + if (text === this.initialText && format === this.initialFormat) return this.baseBody + if (format !== 'markdown') return text + const data = await this.post('/_/api/v1/render/markdown',{markdown:text,authoring:true}) + if (typeof data.html !== 'string') throw {kind:'failed'} return data.html }, - - async autosave() { - - const objectUUID = - document.getElementById( - "object-uuid", - ) - - if ( - !objectUUID - || - !objectUUID.value - ) { - return + getCanonicalHTML() {return this.canonical()}, + async renderPreview() { + this.previewController?.abort() + const controller = new AbortController() + this.previewController = controller + const text = this.text() + if (this.format() === 'code' || this.format() === 'plain') {this.preview.classList.add('code-preview'); this.preview.textContent=text; return} + try { + const data = await this.post('/_/api/v1/render/markdown',{ + ...(this.format()==='html' ? {html:text} : {markdown:text}),path:window.PagesWikiCurrentPath || '/',projection:true,authoring:true, + },{signal:controller.signal}) + if (controller.signal.aborted || text !== this.text()) return + this.preview.classList.remove('code-preview') + this.preview.innerHTML = data.html + window.PagesWikiLoadChildren?.(this.preview) + } catch (error) { + if (controller.signal.aborted) return + this.preview.classList.add('code-preview'); this.preview.textContent=text + this.fail(error) } - - const canonicalHTML = - await this.getCanonicalHTML() - - const pageLanguage = - document.getElementById( - "page-language", - ) - const pageContentType = - document.getElementById("page-content-type") - const pageFilename = - document.getElementById("page-filename") - - await fetch( - "/_/api/v1/revisions/save", - { - method: "POST", - - headers: { - "Content-Type": - "application/json", - }, - - body: JSON.stringify({ - object_uuid: - objectUUID.value, - - body: - canonicalHTML, - - message: - "autosave", - - lang: - pageLanguage - ? pageLanguage.value - : "und", - - content_type: - pageContentType - ? pageContentType.value - : "", - - filename: - pageFilename - ? pageFilename.value - : "", - }), - }, - ) }, - - async saveNow() { - - const objectUUID = - document.getElementById( - "object-uuid", - ) - - if ( - !objectUUID - || - !objectUUID.value - ) { + stateURL(extra={}) { + const query = new URLSearchParams({uuid:this.value('object-uuid'),lang:this.value('page-language')||'und',...extra}) + return '/_/api/v1/editor/state?'+query + }, + async state(extra={}) { + const data = await this.request(this.stateURL(extra)) + if (data.principal_uuid !== this.owner) throw {kind:'identity'} + return data + }, + async checkState() { + if (this.checking || this.saving) return + this.checking = true + try { + const latest = await this.state({metadata:'1'}) + if (latest.revision_uuid !== this.baseRevision) this.status('remote') + } catch (error) {this.fail(error)} finally {this.checking=false} + }, + autosave() {return this.save(false)}, + saveNow() {return this.save(true)}, + async save(manual) { + if (this.saving) return false + if (!this.dirty) return !this.paused + if (!manual && this.paused) return false + if (manual) this.mergeAttempts=0 + this.persist() + this.saving = true + this.status('saving') + const text = this.text(), format = this.format(), serial = this.serial + let body + try { + body = await this.canonical(text,format) + const data = await this.post('/_/api/v1/revisions/save',{ + object_uuid:this.value('object-uuid'),body,message:manual?'manual-save':'autosave', + lang:this.value('page-language')||'und',content_type:this.value('page-content-type'),filename:this.value('page-filename'), + base_revision_uuid:this.baseRevision,editor_principal_uuid:this.owner, + }) + if (!data.revision_uuid) throw {kind:'failed'} + this.baseRevision = data.revision_uuid + this.baseBody = typeof data.body==='string' ? data.body : body + this.initialText = text; this.initialFormat = format + this.paused = false + this.mergeAttempts=0 + if (serial === this.serial) {this.dirty=false; this.clearDraft(); this.status('saved')} + else {this.persist(); this.autosaveTimer=setTimeout(() => this.save(false),3000)} + return serial === this.serial + } catch (error) { + if (error.kind === 'conflict' && body !== undefined) { + try {await this.resolveConflict(body,serial)} catch (failure) {this.fail(failure)} + } else this.fail(error) + return false + } finally {this.saving=false} + }, + async resolveConflict(local,serial) { + this.paused = true + const remote = await this.state() + if ((remote.content_type||'').split(';')[0]!==this.value('page-content-type').split(';')[0]) {this.fail({kind:'incompatible'}); return} + // If a save succeeded but its response was lost, acknowledge it safely. + if (serial === this.serial && local === remote.body) { + this.baseRevision=remote.revision_uuid; this.baseBody=remote.body + this.initialText=this.text(); this.initialFormat=this.format() + this.dirty=false; this.paused=false; this.clearDraft(); this.status('saved'); return + } + if (serial !== this.serial) {this.status('remote'); this.persist(); return} + const merged = PagesEditorCore.merge(this.baseBody,local,remote.body) + const apply = result => { + this.baseRevision=remote.revision_uuid; this.baseBody=remote.body + document.getElementById('page-filename').value=remote.filename||'' + this.initialText=null + this.setText(result,['code','plain'].includes(this.format())?this.format():'html') + this.persist() + } + if (merged !== null && (this.mergeAttempts || 0)<3) { + this.mergeAttempts=(this.mergeAttempts || 0)+1 + apply(merged) + // A second CAS still guards a writer racing this automatic merge. + this.paused=false + this.autosaveTimer=setTimeout(() => this.save(false),500) return } - - const canonicalHTML = - await this.getCanonicalHTML() - - const pageLanguage = - document.getElementById( - "page-language", - ) - const pageContentType = - document.getElementById("page-content-type") - const pageFilename = - document.getElementById("page-filename") - - const res = - await fetch( - "/_/api/v1/revisions/save", - { - method: "POST", - - headers: { - "Content-Type": - "application/json", - }, - - body: JSON.stringify({ - object_uuid: - objectUUID.value, - - body: - canonicalHTML, - - message: - "manual-save", - - lang: - pageLanguage - ? pageLanguage.value - : "und", - - content_type: - pageContentType - ? pageContentType.value - : "", - - filename: - pageFilename - ? pageFilename.value - : "", - }), - }, - ) - - if (!res.ok) { - - alert( - "save failed", - ) - - return + this.persist(); this.status('conflict') + const {dialog,content,button} = this.dialog('conflict') + this.area(content,'base',this.baseBody,true) + this.area(content,'local',local,true) + this.area(content,'latest',remote.body,true) + const result = this.area(content,'result',local,false) + button('apply',() => { + if (serial !== this.serial) {this.status('remote'); dialog.close(); return} + apply(result.value); dialog.close() + }) + }, + setText(text,format=this.format()) { + const oldFormat = this.format() + document.getElementById('edit-type').value=format + const status=document.getElementById('editor-format-status') + if (status) status.textContent=format + this.editor.value=text + if (oldFormat !== format && window.PagesWikiCodeMirror) { + window.PagesWikiCodeMirror.destroy() + window.PagesWikiCodeMirror = window.PagesWikiEditor.create(this.mount,text,format,{ + keymap:this.mount.dataset.keymap,syntax:this.mount.dataset.syntax==='true',autocomplete:this.mount.dataset.autocomplete==='true', + }) + } else if (window.PagesWikiCodeMirror) { + const cm=window.PagesWikiCodeMirror + cm.dispatch({changes:{from:0,to:cm.state.doc.length,insert:text}}) } - - alert( - "saved", - ) + document.querySelectorAll('[data-editor-action="html"],[data-editor-action="markdown"]').forEach(button => { + const selected=button.dataset.editorAction===format + button.classList.toggle('active',selected); button.setAttribute('aria-pressed',String(selected)) + }) + this.changed() + }, + async switchFormat(format) { + // Representation switching is only lossless inside the HTML canonical + // family. Plain text/code must never silently turn into executable HTML. + if (!['html','markdown'].includes(format) || ['plain','code'].includes(this.format())) return + if (format===this.format() || this.saving) return + this.paused=true; this.persist() + const serial=this.serial + try { + let body=await this.canonical() + if (serial!==this.serial) return + if (format==='markdown') { + const data=await this.post('/_/api/v1/editor/convert',{html:body}) + if (serial!==this.serial) return + if (typeof data.markdown!=='string') throw {kind:'failed'} + body=data.markdown + } + this.setText(body,format) + this.persist() + } catch (error) {this.fail(error)} + }, + dialog(title) { + document.querySelector('#editor-dialog')?.remove() + const dialog=document.createElement('dialog'); dialog.id='editor-dialog'; dialog.className='editor-dialog' + const heading=document.createElement('h2'); heading.textContent=this.t(title); dialog.append(heading) + const content=document.createElement('div'); content.className='editor-dialog-content'; dialog.append(content) + const footer=document.createElement('footer'); dialog.append(footer) + const button=(label,action) => { + const b=document.createElement('button'); b.type='button'; b.textContent=this.t(label) + b.addEventListener('click',() => Promise.resolve().then(action).catch(error => this.fail(error))) + footer.append(b); return b + } + button('cancel',() => dialog.close()) + dialog.addEventListener('close',() => dialog.remove()) + document.body.append(dialog); dialog.showModal() + return {dialog,content,button} + }, + paragraph(parent,text) {const p=document.createElement('p'); p.textContent=text; parent.append(p)}, + area(parent,label,text,readOnly) { + const wrapper=document.createElement('label'); wrapper.textContent=this.t(label) + const area=document.createElement('textarea'); area.value=text; area.readOnly=readOnly; area.rows=6; wrapper.append(area); parent.append(wrapper); return area + }, + download(text=this.text(),format=this.format()) { + const url=URL.createObjectURL(new Blob([text],{type:'text/plain;charset=utf-8'})) + const a=document.createElement('a'); a.href=url; a.download='pageswiki-draft.'+(format==='markdown'?'md':format==='html'?'html':'txt'); a.click() + setTimeout(() => URL.revokeObjectURL(url),1000) + }, + tools() { + const {dialog,content,button}=this.dialog('tools') + this.paragraph(content,this.t('privacy')) + const select=document.createElement('select'); select.setAttribute('aria-label',this.t('insert')) + this.paragraph(content,this.t('protected')) + const parts=[['children','{{children}}'],['table','\n\n\n
    Column
    Value
    '],['figure','
    Description
    Caption
    '],['note',''],['code','
    code
    '],['quote','

    Quote

    ']] + parts.forEach(([label],i) => {const option=document.createElement('option'); option.value=i; option.textContent=this.t(label); select.append(option)}) + content.append(select) + const login=document.createElement('a'); login.href='/_/auth/login?return='+encodeURIComponent(location.pathname+location.search); login.target='_blank'; login.rel='noopener'; login.textContent=this.t('signin'); content.append(login) + button('insert',() => { + const part=parts[Number(select.value)] + const source=this.format()==='markdown' && part[0]!=='children' ? '```{=html}\n'+part[1]+'\n```' : part[1] + const text='\n\n'+source+'\n\n' + const cm=window.PagesWikiCodeMirror + if (cm) cm.dispatch(cm.state.replaceSelection(text)); else this.setText(this.text()+text) + dialog.close(); cm?.focus() + }) + button('export',() => this.download()) + button('discard',() => { + this.clearDraft() + this.status('dirty'); dialog.close() + }) + }, + languageDialog() { + if (this.saving) return + this.paused=true; this.persist() + const {dialog,content,button}=this.dialog('language') + const input=document.createElement('input'); input.value=this.value('page-language')||'und'; input.placeholder='ja / en / ja-JP'; input.setAttribute('aria-label',this.t('language')); content.append(input) + const lang=() => { + try {return input.value==='und'?'und':Intl.getCanonicalLocales(input.value.trim())[0] || (()=>{throw Error()})()} catch (_) {throw {kind:'invalid'}} + } + button('open',() => {const url=new URL(location.href); url.searchParams.set('lang',lang()); location.href=url.toString()}) + button('fork',async () => { + if (this.saving) return + const target=lang(), serial=this.serial + const state=await this.state({lang:target}) + if (state.revision_uuid) {this.paragraph(content,this.t('exists')); return} + if (serial!==this.serial || this.saving) return + document.getElementById('page-language').value=state.lang + this.baseRevision=''; this.baseBody=''; this.initialText=''; this.initialFormat=this.format() + const url=new URL(location.href); url.searchParams.set('lang',state.lang); history.replaceState(null,'',url) + this.changed(); this.persist(); dialog.close() + }) + }, + async historyDialog() { + if (this.saving) return + this.paused=true; this.persist() + try { + const {dialog,content,button}=this.dialog('history') + const select=document.createElement('select'); select.setAttribute('aria-label',this.t('history')) + content.append(select) + let before='' + const load=async () => { + older.disabled=true + const rows=await this.request('/_/api/v1/revisions/list?'+new URLSearchParams({uuid:this.value('object-uuid'),limit:'50',before})) + rows.filter(row => row.lang===(this.value('page-language')||'und')).forEach(row => {const option=document.createElement('option'); option.value=row.uuid; option.textContent=new Date(row.created_at*1000).toLocaleString()+' '+row.message+' '+row.uuid.slice(0,8); select.append(option)}) + before=rows.at(-1)?.uuid||before + older.disabled=rows.length<50 + } + const older=button('older',load) + button('recover',async () => { + if (!select.value || this.saving) return + const serial=this.serial + const old=await this.state({revision:select.value}) + if (serial!==this.serial || this.saving) return + if (!old.editable) throw {kind:'incompatible'} + this.setText(this.format()==='markdown'?old.markdown:old.body) + this.persist(); dialog.close() + }) + await load() + } catch (error) {this.fail(error)} }, } -document.addEventListener( - "DOMContentLoaded", - () => { - - if ( - document.body.classList.contains( - "mode-editor", - ) - ) { - - PagesEditor.init() - } - }, -) - -document.addEventListener( - "DOMContentLoaded", - () => { - - const sel = - document.getElementById( - "editor-type", - ) - - if (!sel) { - return - } - - sel.addEventListener( - "change", - () => { - - const u = - new URL( - location.href, - ) - - u.searchParams.set( - "edittype", - sel.value, - ) - - location.href = - u.toString() - }, - ) - }, -) - -// HTML/Markdown previews and diagram projections are produced by the common -// server boundary. Do not reinterpret authoring source in a second client pass. +document.addEventListener('DOMContentLoaded',() => { + if (!document.body.classList.contains('mode-editor')) return + PagesEditor.init() + PagesEditor.initialFormat=PagesEditor.format() +}) diff --git a/static/js/home-pages.js b/static/js/home-pages.js index ef130e1..3c4f450 100644 --- a/static/js/home-pages.js +++ b/static/js/home-pages.js @@ -43,6 +43,7 @@ }; buttons.forEach(button => button.addEventListener('click', () => show(button.dataset.homePageTarget, true))); + document.addEventListener('home:search', () => show('search')); pager.addEventListener('wheel', event => { if (modalOpen()) return; if (page === 1 && (event.deltaY > 0 || bookmarks.scrollTop > 0)) return; diff --git a/static/js/home-type-search.js b/static/js/home-type-search.js new file mode 100644 index 0000000..c7f4156 --- /dev/null +++ b/static/js/home-type-search.js @@ -0,0 +1,24 @@ +// Type-to-search is intentionally limited to the home page. Never capture typing +// in another editor, a modal, an open menu, or an OS/browser keyboard shortcut. +(() => { + const input = document.querySelector('[data-home-type-search]'); + if (!input) return; + document.addEventListener('keydown', event => { + if (event.defaultPrevented || event.ctrlKey || event.metaKey || event.altKey || event.isComposing) return; + if (event.target.closest?.('input,textarea,select,button,a,summary,[contenteditable]:not([contenteditable="false"]),[role="textbox"]')) return; + if ([...document.querySelectorAll('dialog[open],details[open],[role="dialog"]:not(dialog),[aria-modal="true"]:not(dialog)')] + .some(surface => !surface.hidden && surface.getClientRects().length)) return; + if (window.getSelection()?.toString()) return; + const composing = event.key === 'Process' || event.keyCode === 229; + if (!composing && ([...event.key].length !== 1 || event.key.trim() === '')) return; + document.dispatchEvent(new Event('home:search')); + if (input.closest('[inert]') || !input.getClientRects().length) return; + input.focus({preventScroll: true}); + // Redirect IME's native composition to the input without synthesizing text. + if (composing) return; + event.preventDefault(); + // Search inputs do not support setRangeText/selectionStart in all browsers. + input.value += event.key; + input.dispatchEvent(new Event('input', {bubbles: true})); + }); +})(); diff --git a/static/js/theme-picker.js b/static/js/theme-picker.js new file mode 100644 index 0000000..0d2b389 --- /dev/null +++ b/static/js/theme-picker.js @@ -0,0 +1,37 @@ +(() => { + const picker = document.querySelector('[data-theme-picker]'); + if (!picker) return; + const select = picker.querySelector('[data-theme-select]'); + const status = picker.querySelector('[data-theme-status]'); + const themeLink = [...document.querySelectorAll('link[rel="stylesheet"]')] + .find(link => link.getAttribute('href') === picker.dataset.currentCss); + if (!themeLink) { picker.hidden = true; return; } + let current = picker.dataset.currentId; + select.addEventListener('change', () => { + const option = select.selectedOptions[0]; + const candidate = document.createElement('link'); + candidate.rel = 'stylesheet'; + candidate.href = option.dataset.css; + select.disabled = true; + status.textContent = ''; + // Load before replacing, so failure cannot leave an unstyled page. Keeping + // the link's position preserves editor/component overrides and avoids reload. + candidate.media = 'not all'; + candidate.onload = () => { + themeLink.href = candidate.href; + candidate.remove(); + current = option.value; + const secure = location.protocol === 'https:' ? '; Secure' : ''; + document.cookie = 'pageswiki_theme=' + encodeURIComponent(current) + '; Path=/; SameSite=Lax; Max-Age=' + (current ? 31536000 : 0) + secure; + select.disabled = false; + document.dispatchEvent(new Event('pageswiki:theme')); + }; + candidate.onerror = () => { + candidate.remove(); select.value = current; select.disabled = false; + status.textContent = picker.dataset.failed; + }; + document.head.append(candidate); + }); + document.addEventListener('pointerdown', event => { if (!picker.contains(event.target)) picker.open = false; }); + picker.addEventListener('keydown', event => { if (event.key === 'Escape') { picker.open = false; picker.querySelector('summary').focus(); } }); +})(); diff --git a/static/js/viewer-shortcuts.js b/static/js/viewer-shortcuts.js index 608ff7a..4413c79 100644 --- a/static/js/viewer-shortcuts.js +++ b/static/js/viewer-shortcuts.js @@ -22,7 +22,7 @@ }; const shortcutSheet = () => { const table=document.createElement('div'); table.className='viewer-shortcut-grid'; - [['e','Edit'],['/','Find'],['v','Change view'],['c','Create child'],['u','Upload'],['s','Share'],['p','Permissions'],['Ctrl+↑','Parent'],['Ctrl+↓','Children'],['Ctrl+→','Previous'],['Ctrl+←','Next'],['?','This cheat sheet']].forEach(([key,label])=>{const k=document.createElement('kbd');k.textContent=key;const d=document.createElement('span');d.textContent=label;table.append(k,d);}); + [['e','Edit'],['/','Find'],['g','Command: cd / vi / view · Tab completion'],['v','Change view'],['c','Create child'],['u','Upload'],['s','Share'],['p','Permissions'],['Ctrl+↑','Parent'],['Ctrl+↓','Children'],['Ctrl+→','Previous'],['Ctrl+←','Next'],['?','This cheat sheet']].forEach(([key,label])=>{const k=document.createElement('kbd');k.textContent=key;const d=document.createElement('span');d.textContent=label;table.append(k,d);}); openDialog('Keyboard shortcuts',table); }; const childrenDialog = async () => { diff --git a/static/vendor/codemirror/editor.bundle.js b/static/vendor/codemirror/editor.bundle.js index 725b44b..3a0171e 100644 --- a/static/vendor/codemirror/editor.bundle.js +++ b/static/vendor/codemirror/editor.bundle.js @@ -11996,6 +11996,32 @@ return state.facet(selectionConfig); } var UnicodeRegexpSupport = /x/.unicode != null ? "gu" : "g"; + function highlightActiveLine() { + return activeLineHighlighter; + } + var lineDeco = /* @__PURE__ */ Decoration.line({ class: "cm-activeLine" }); + var activeLineHighlighter = /* @__PURE__ */ ViewPlugin.fromClass(class { + constructor(view) { + this.decorations = this.getDeco(view); + } + update(update) { + if (update.docChanged || update.selectionSet) + this.decorations = this.getDeco(update.view); + } + getDeco(view) { + let lastLineStart = -1, deco = []; + for (let r of view.state.selection.ranges) { + let line = view.lineBlockAt(r.head); + if (line.from > lastLineStart) { + deco.push(lineDeco.range(line.from)); + lastLineStart = line.from; + } + } + return Decoration.set(deco); + } + }, { + decorations: (v) => v.decorations + }); var Outside = "-10000px"; var TooltipViewManager = class { constructor(view, facet, createTooltipView, removeTooltipView) { @@ -12969,6 +12995,26 @@ last = last * 10 + 9; return last; } + var activeLineGutterMarker = /* @__PURE__ */ new class extends GutterMarker { + constructor() { + super(...arguments); + this.elementClass = "cm-activeLineGutter"; + } + }(); + var activeLineGutterHighlighter = /* @__PURE__ */ gutterLineClass.compute(["selection"], (state) => { + let marks2 = [], last = -1; + for (let range of state.selection.ranges) { + let linePos = state.doc.lineAt(range.head).from; + if (linePos > last) { + last = linePos; + marks2.push(activeLineGutterMarker.range(linePos)); + } + } + return RangeSet.of(marks2); + }); + function highlightActiveLineGutter() { + return activeLineGutterHighlighter; + } // node_modules/@lezer/common/dist/index.js var DefaultBufferLength = 1024; @@ -38654,28 +38700,50 @@ } // src/editor.js + var pagesWikiHighlightStyle = HighlightStyle.define([ + { tag: tags.heading, class: "pw-syntax-heading" }, + { tag: [tags.link, tags.url], class: "pw-syntax-link" }, + { tag: tags.emphasis, class: "pw-syntax-emphasis" }, + { tag: tags.strong, class: "pw-syntax-strong" }, + { tag: [tags.keyword, tags.operatorKeyword], class: "pw-syntax-keyword" }, + { tag: [tags.tagName, tags.typeName, tags.monospace], class: "pw-syntax-type" }, + { tag: [tags.string, tags.special(tags.string)], class: "pw-syntax-string" }, + { tag: [tags.number, tags.bool, tags.null, tags.attributeName], class: "pw-syntax-number" }, + { tag: tags.comment, class: "pw-syntax-comment" }, + { tag: [tags.variableName, tags.propertyName], class: "pw-syntax-variable" }, + { tag: tags.function(tags.variableName), class: "pw-syntax-function" }, + { tag: tags.operator, class: "pw-syntax-operator" }, + { tag: tags.invalid, class: "pw-syntax-invalid" } + ]); var pagesWikiTheme = EditorView.theme({ ".cm-selectionBackground": { - backgroundColor: "rgba(255, 216, 102, 0.48) !important" + backgroundColor: "var(--editor-selection) !important" }, "&.cm-focused .cm-selectionBackground": { - backgroundColor: "rgba(255, 216, 102, 0.58) !important" + backgroundColor: "var(--editor-selection) !important" }, ".cm-selectionLayer .cm-selectionBackground": { - backgroundColor: "rgba(255, 216, 102, 0.58) !important" + backgroundColor: "var(--editor-selection) !important" }, ".cm-content ::selection": { - backgroundColor: "rgba(255, 216, 102, 0.48) !important", - color: "#ffffff !important" + backgroundColor: "var(--editor-selection) !important", + color: "var(--fg) !important" } }); var vimMode = "INSERT"; + var languageAliases = { js: "javascript", ts: "typescript", py: "python", sh: "shell", bash: "shell", zsh: "shell", yml: "yaml", md: "markdown", c: "cpp", "c++": "cpp", xml: "html", htm: "html" }; function languageExtension(mode) { - switch (String(mode || "").toLowerCase()) { + mode = String(mode || "").toLowerCase(); + switch (languageAliases[mode] || mode) { case "html": return html(); case "markdown": - return markdown(); + return markdown({ codeLanguages: (info) => { + const mode2 = info.trim().split(/\s+/)[0].toLowerCase(); + if (["md", "markdown"].includes(mode2)) return null; + const support = languageExtension(mode2); + return support?.language || support; + } }); case "javascript": return javascript({ jsx: true }); case "typescript": @@ -38744,6 +38812,7 @@ } function setEditorMode(mode) { vimMode = mode; + document.body.dataset.editorMode = mode.toLowerCase(); const el = document.getElementById("editor-vim-mode"); if (!el) { return; @@ -38756,6 +38825,7 @@ if (!word || word.from === word.to && !context.explicit) { return null; } + if (!context.explicit && !word.text.startsWith("{{")) return null; return { from: word.from, options: [ @@ -38767,10 +38837,13 @@ { label: "```go", type: "snippet", apply: "```go\n\n```" }, { label: "```bash", type: "snippet", apply: "```bash\n\n```" }, { label: "[link](url)", type: "snippet", apply: "[text](url)" }, - { label: "{{children}}", type: "function", apply: "{{children}}" }, + { label: "{{children}}", type: "function", apply: "{{children}}", boost: 5 }, { label: "{{children:/}}", type: "function", apply: "{{children:/}}" }, { label: "{{children:/ exclude=/draft}}", type: "function", apply: "{{children:/ exclude=/draft}}" } - ] + ].map((option) => option.label.startsWith("{{") ? { ...option, apply(view, completion, from, to) { + const tail = view.state.sliceDoc(to, to + 2).match(/^\}{1,2}/)?.[0] || ""; + view.dispatch({ changes: { from, to: to + tail.length, insert: option.label }, selection: { anchor: from + option.label.length } }); + } } : option) }; } function installVimCommands() { @@ -38786,9 +38859,8 @@ const quit = () => { location.href = window.PagesWikiCurrentPath || "/"; }; - const saveQuit = () => { - save(); - setTimeout(quit, 120); + const saveQuit = async () => { + if (await window.PagesEditor?.saveNow()) quit(); }; Vim.defineEx("write", "w", save); Vim.defineEx("w", "w", save); @@ -38798,41 +38870,11 @@ Vim.defineEx("x", "x", saveQuit); } function attachVimModeTracking(view) { - view.dom.addEventListener( - "keydown", - (ev) => { - if (ev.key === "Escape" || ev.ctrlKey && ev.key === "[") { - setEditorMode("NORMAL"); - return; - } - if (vimMode === "NORMAL") { - if (ev.key === "i" || ev.key === "a" || ev.key === "o" || ev.key === "O") { - setEditorMode("INSERT"); - return; - } - if (ev.key === "R") { - setEditorMode("REPLACE"); - return; - } - if (ev.key === "v" || ev.key === "V") { - setEditorMode("VISUAL"); - return; - } - } - if (vimMode === "VISUAL" && ev.key === "Escape") { - setEditorMode("NORMAL"); - } - }, - true - ); - view.dom.addEventListener( - "focusin", - () => { - if (!vimMode) { - setEditorMode("INSERT"); - } - } - ); + getCM(view)?.on("vim-mode-change", (event) => { + const mode = String(event.mode || "normal").toUpperCase(); + setEditorMode(mode); + view.dom.classList.toggle("pageswiki-vim-visual", mode === "VISUAL"); + }); } function attachStatusBar(view) { const cursorEl = document.getElementById("editor-cursor"); @@ -38851,10 +38893,30 @@ view.dom.addEventListener("focusin", updateCursor); } window.PagesWikiEditor = { + highlightCodeBlock(code, mode) { + mode = String(mode || "").toLowerCase(); + const language2 = languageExtension(mode); + const text = code.textContent; + if (!language2 || text.length > 32768) return false; + const parser14 = (language2.language || language2).parser; + const fragment = document.createDocumentFragment(); + let offset = 0; + highlightTree(parser14.parse(text), pagesWikiHighlightStyle, (from, to, classes) => { + if (from > offset) fragment.append(document.createTextNode(text.slice(offset, from))); + const span = document.createElement("span"); + span.className = classes; + span.textContent = text.slice(from, to); + fragment.append(span); + offset = to; + }); + fragment.append(document.createTextNode(text.slice(offset))); + code.replaceChildren(fragment); + return true; + }, createViewer(target, text, mode, minimap) { const extensions = [ lineNumbers(), - syntaxHighlighting(defaultHighlightStyle), + syntaxHighlighting(pagesWikiHighlightStyle), pagesWikiTheme, EditorState.readOnly.of(true), EditorView.editable.of(false) @@ -38874,9 +38936,12 @@ return view; }, create(target, initialText, mode, options = {}) { - const keymapName = String(options.keymap || "default").toLowerCase(); + const mobile = window.matchMedia("(pointer: coarse) and (hover: none)").matches; + const keymapName = mobile ? "default" : String(options.keymap || "default").toLowerCase(); const extensions = [ lineNumbers(), + highlightActiveLine(), + highlightActiveLineGutter(), history(), bracketMatching(), closeBrackets(), @@ -38904,7 +38969,7 @@ ]) ); if (options.syntax !== false) { - extensions.push(syntaxHighlighting(defaultHighlightStyle)); + extensions.push(syntaxHighlighting(pagesWikiHighlightStyle)); } if (options.autocomplete !== false) { extensions.push( @@ -38918,6 +38983,9 @@ if (language2) { extensions.push(language2); } + if (mode === "html" || mode === "markdown") { + extensions.push(EditorState.languageData.of(() => [{ closeBrackets: { brackets: ["(", "[", "'", '"', "`"] } }])); + } const view = new EditorView({ state: EditorState.create({ doc: initialText, @@ -38928,87 +38996,16 @@ attachStatusBar(view); if (keymapName === "vim") { attachVimModeTracking(view); - attachPagesWikiExFallback(view); } return view; } }; - function attachPagesWikiExFallback(view) { - let active = false; - let buffer = ""; - function save() { - if (window.PagesEditor && window.PagesEditor.saveNow) { - window.PagesEditor.saveNow(); - } - } - function quit() { - location.href = window.PagesWikiCurrentPath || "/"; - } - view.dom.addEventListener( - "keydown", - (ev) => { - if (!active && vimMode === "NORMAL" && ev.key === ":") { - active = true; - buffer = ""; - return; - } - if (!active) { - return; - } - if (ev.key === "Escape" || ev.ctrlKey && ev.key === "[") { - active = false; - buffer = ""; - return; - } - if (ev.key === "Enter") { - const cmd2 = buffer.trim(); - active = false; - buffer = ""; - if (cmd2 === "w") { - ev.preventDefault(); - save(); - return; - } - if (cmd2 === "q") { - ev.preventDefault(); - quit(); - return; - } - if (cmd2 === "wq" || cmd2 === "x") { - ev.preventDefault(); - save(); - setTimeout(quit, 120); - return; - } - return; - } - if (ev.key.length === 1) { - buffer += ev.key; - } - if (ev.key === "Backspace") { - buffer = buffer.slice(0, -1); - } - }, - true - ); - } - var pagesWikiPreviewTimer = null; function schedulePagesWikiPreview(view) { const textarea = document.getElementById("editor"); if (textarea) { textarea.value = view.state.doc.toString(); + textarea.dispatchEvent(new Event("input", { bubbles: true })); } - if (pagesWikiPreviewTimer) { - clearTimeout(pagesWikiPreviewTimer); - } - pagesWikiPreviewTimer = setTimeout( - () => { - if (window.PagesEditor && window.PagesEditor.renderPreview) { - window.PagesEditor.renderPreview(); - } - }, - 350 - ); } })(); //# sourceMappingURL=editor.bundle.js.map diff --git a/static/vendor/codemirror/editor.bundle.js.map b/static/vendor/codemirror/editor.bundle.js.map index 98c49fa..32a5bb5 100644 --- a/static/vendor/codemirror/editor.bundle.js.map +++ b/static/vendor/codemirror/editor.bundle.js.map @@ -1,7 +1,7 @@ { "version": 3, "sources": ["../../../assets/editor/node_modules/@marijn/find-cluster-break/src/index.js", "../../../assets/editor/node_modules/@codemirror/state/dist/index.js", "../../../assets/editor/node_modules/style-mod/src/style-mod.js", "../../../assets/editor/node_modules/w3c-keyname/index.js", "../../../assets/editor/node_modules/@codemirror/view/dist/index.js", "../../../assets/editor/node_modules/@lezer/common/dist/index.js", "../../../assets/editor/node_modules/@lezer/highlight/dist/index.js", "../../../assets/editor/node_modules/@codemirror/language/dist/index.js", "../../../assets/editor/node_modules/@codemirror/commands/dist/index.js", "../../../assets/editor/node_modules/@codemirror/autocomplete/dist/index.js", "../../../assets/editor/node_modules/@lezer/lr/dist/index.js", "../../../assets/editor/node_modules/@lezer/html/dist/index.js", "../../../assets/editor/node_modules/@lezer/css/dist/index.js", "../../../assets/editor/node_modules/@codemirror/lang-css/dist/index.js", "../../../assets/editor/node_modules/@lezer/javascript/dist/index.js", "../../../assets/editor/node_modules/@codemirror/lang-javascript/dist/index.js", "../../../assets/editor/node_modules/@codemirror/lang-html/dist/index.js", "../../../assets/editor/node_modules/@lezer/markdown/dist/index.js", "../../../assets/editor/node_modules/@codemirror/lang-markdown/dist/index.js", "../../../assets/editor/node_modules/@lezer/python/dist/index.js", "../../../assets/editor/node_modules/@codemirror/lang-python/dist/index.js", "../../../assets/editor/node_modules/@lezer/json/dist/index.js", "../../../assets/editor/node_modules/@codemirror/lang-json/dist/index.js", "../../../assets/editor/node_modules/@codemirror/lang-sql/dist/index.js", "../../../assets/editor/node_modules/@lezer/yaml/dist/index.js", "../../../assets/editor/node_modules/@codemirror/lang-yaml/dist/index.js", "../../../assets/editor/node_modules/@lezer/go/dist/index.js", "../../../assets/editor/node_modules/@codemirror/lang-go/dist/index.js", "../../../assets/editor/node_modules/@lezer/rust/dist/index.js", "../../../assets/editor/node_modules/@codemirror/lang-rust/dist/index.js", "../../../assets/editor/node_modules/@lezer/java/dist/index.js", "../../../assets/editor/node_modules/@codemirror/lang-java/dist/index.js", "../../../assets/editor/node_modules/@lezer/cpp/dist/index.js", "../../../assets/editor/node_modules/@codemirror/lang-cpp/dist/index.js", "../../../assets/editor/node_modules/@codemirror/legacy-modes/mode/shell.js", "../../../assets/editor/node_modules/@codemirror/legacy-modes/mode/toml.js", "../../../assets/editor/node_modules/@codemirror/search/dist/index.js", "../../../assets/editor/node_modules/@replit/codemirror-vim/dist/index.js", "../../../assets/editor/src/editor.js"], - "sourcesContent": ["// These are filled with ranges (rangeFrom[i] up to but not including\n// rangeTo[i]) of code points that count as extending characters.\nlet rangeFrom = [], rangeTo = []\n\n;(() => {\n // Compressed representation of the Grapheme_Cluster_Break=Extend\n // information from\n // http://www.unicode.org/Public/16.0.0/ucd/auxiliary/GraphemeBreakProperty.txt.\n // Each pair of elements represents a range, as an offet from the\n // previous range and a length. Numbers are in base-36, with the empty\n // string being a shorthand for 1.\n let numbers = \"lc,34,7n,7,7b,19,,,,2,,2,,,20,b,1c,l,g,,2t,7,2,6,2,2,,4,z,,u,r,2j,b,1m,9,9,,o,4,,9,,3,,5,17,3,3b,f,,w,1j,,,,4,8,4,,3,7,a,2,t,,1m,,,,2,4,8,,9,,a,2,q,,2,2,1l,,4,2,4,2,2,3,3,,u,2,3,,b,2,1l,,4,5,,2,4,,k,2,m,6,,,1m,,,2,,4,8,,7,3,a,2,u,,1n,,,,c,,9,,14,,3,,1l,3,5,3,,4,7,2,b,2,t,,1m,,2,,2,,3,,5,2,7,2,b,2,s,2,1l,2,,,2,4,8,,9,,a,2,t,,20,,4,,2,3,,,8,,29,,2,7,c,8,2q,,2,9,b,6,22,2,r,,,,,,1j,e,,5,,2,5,b,,10,9,,2u,4,,6,,2,2,2,p,2,4,3,g,4,d,,2,2,6,,f,,jj,3,qa,3,t,3,t,2,u,2,1s,2,,7,8,,2,b,9,,19,3,3b,2,y,,3a,3,4,2,9,,6,3,63,2,2,,1m,,,7,,,,,2,8,6,a,2,,1c,h,1r,4,1c,7,,,5,,14,9,c,2,w,4,2,2,,3,1k,,,2,3,,,3,1m,8,2,2,48,3,,d,,7,4,,6,,3,2,5i,1m,,5,ek,,5f,x,2da,3,3x,,2o,w,fe,6,2x,2,n9w,4,,a,w,2,28,2,7k,,3,,4,,p,2,5,,47,2,q,i,d,,12,8,p,b,1a,3,1c,,2,4,2,2,13,,1v,6,2,2,2,2,c,,8,,1b,,1f,,,3,2,2,5,2,,,16,2,8,,6m,,2,,4,,fn4,,kh,g,g,g,a6,2,gt,,6a,,45,5,1ae,3,,2,5,4,14,3,4,,4l,2,fx,4,ar,2,49,b,4w,,1i,f,1k,3,1d,4,2,2,1x,3,10,5,,8,1q,,c,2,1g,9,a,4,2,,2n,3,2,,,2,6,,4g,,3,8,l,2,1l,2,,,,,m,,e,7,3,5,5f,8,2,3,,,n,,29,,2,6,,,2,,,2,,2,6j,,2,4,6,2,,2,r,2,2d,8,2,,,2,2y,,,,2,6,,,2t,3,2,4,,5,77,9,,2,6t,,a,2,,,4,,40,4,2,2,4,,w,a,14,6,2,4,8,,9,6,2,3,1a,d,,2,ba,7,,6,,,2a,m,2,7,,2,,2,3e,6,3,,,2,,7,,,20,2,3,,,,9n,2,f0b,5,1n,7,t4,,1r,4,29,,f5k,2,43q,,,3,4,5,8,8,2,7,u,4,44,3,1iz,1j,4,1e,8,,e,,m,5,,f,11s,7,,h,2,7,,2,,5,79,7,c5,4,15s,7,31,7,240,5,gx7k,2o,3k,6o\".split(\",\").map(s => s ? parseInt(s, 36) : 1)\n for (let i = 0, n = 0; i < numbers.length; i++)\n (i % 2 ? rangeTo : rangeFrom).push(n = n + numbers[i])\n})()\n\nexport function isExtendingChar(code) {\n if (code < 768) return false\n for (let from = 0, to = rangeFrom.length;;) {\n let mid = (from + to) >> 1\n if (code < rangeFrom[mid]) to = mid\n else if (code >= rangeTo[mid]) from = mid + 1\n else return true\n if (from == to) return false\n }\n}\n\nfunction isRegionalIndicator(code) {\n return code >= 0x1F1E6 && code <= 0x1F1FF\n}\n\nfunction check(code) {\n for (let i = 0; i < rangeFrom.length; i++) {\n if (rangeTo[i] > code) return rangeFrom[i] <= code\n }\n return false\n}\n\nconst ZWJ = 0x200d\n\nexport function findClusterBreak(str, pos, forward = true, includeExtending = true) {\n return (forward ? nextClusterBreak : prevClusterBreak)(str, pos, includeExtending)\n}\n\nfunction nextClusterBreak(str, pos, includeExtending) {\n if (pos == str.length) return pos\n // If pos is in the middle of a surrogate pair, move to its start\n if (pos && surrogateLow(str.charCodeAt(pos)) && surrogateHigh(str.charCodeAt(pos - 1))) pos--\n let prev = codePointAt(str, pos)\n pos += codePointSize(prev)\n while (pos < str.length) {\n let next = codePointAt(str, pos)\n if (prev == ZWJ || next == ZWJ || includeExtending && isExtendingChar(next)) {\n pos += codePointSize(next)\n prev = next\n } else if (isRegionalIndicator(next)) {\n let countBefore = 0, i = pos - 2\n while (i >= 0 && isRegionalIndicator(codePointAt(str, i))) { countBefore++; i -= 2 }\n if (countBefore % 2 == 0) break\n else pos += 2\n } else {\n break\n }\n }\n return pos\n}\n\nfunction prevClusterBreak(str, pos, includeExtending) {\n while (pos > 0) {\n let found = nextClusterBreak(str, pos - 2, includeExtending)\n if (found < pos) return found\n pos--\n }\n return 0\n}\n\nfunction codePointAt(str, pos) {\n let code0 = str.charCodeAt(pos)\n if (!surrogateHigh(code0) || pos + 1 == str.length) return code0\n let code1 = str.charCodeAt(pos + 1)\n if (!surrogateLow(code1)) return code0\n return ((code0 - 0xd800) << 10) + (code1 - 0xdc00) + 0x10000\n}\n\nfunction surrogateLow(ch) { return ch >= 0xDC00 && ch < 0xE000 }\nfunction surrogateHigh(ch) { return ch >= 0xD800 && ch < 0xDC00 }\nfunction codePointSize(code) { return code < 0x10000 ? 1 : 2 }\n", "import { findClusterBreak as findClusterBreak$1 } from '@marijn/find-cluster-break';\n\n/**\nThe data structure for documents. @nonabstract\n*/\nclass Text {\n /**\n Get the line description around the given position.\n */\n lineAt(pos) {\n if (pos < 0 || pos > this.length)\n throw new RangeError(`Invalid position ${pos} in document of length ${this.length}`);\n return this.lineInner(pos, false, 1, 0);\n }\n /**\n Get the description for the given (1-based) line number.\n */\n line(n) {\n if (n < 1 || n > this.lines)\n throw new RangeError(`Invalid line number ${n} in ${this.lines}-line document`);\n return this.lineInner(n, true, 1, 0);\n }\n /**\n Replace a range of the text with the given content.\n */\n replace(from, to, text) {\n [from, to] = clip(this, from, to);\n let parts = [];\n this.decompose(0, from, parts, 2 /* Open.To */);\n if (text.length)\n text.decompose(0, text.length, parts, 1 /* Open.From */ | 2 /* Open.To */);\n this.decompose(to, this.length, parts, 1 /* Open.From */);\n return TextNode.from(parts, this.length - (to - from) + text.length);\n }\n /**\n Append another document to this one.\n */\n append(other) {\n return this.replace(this.length, this.length, other);\n }\n /**\n Retrieve the text between the given points.\n */\n slice(from, to = this.length) {\n [from, to] = clip(this, from, to);\n let parts = [];\n this.decompose(from, to, parts, 0);\n return TextNode.from(parts, to - from);\n }\n /**\n Test whether this text is equal to another instance.\n */\n eq(other) {\n if (other == this)\n return true;\n if (other.length != this.length || other.lines != this.lines)\n return false;\n let start = this.scanIdentical(other, 1), end = this.length - this.scanIdentical(other, -1);\n let a = new RawTextCursor(this), b = new RawTextCursor(other);\n for (let skip = start, pos = start;;) {\n a.next(skip);\n b.next(skip);\n skip = 0;\n if (a.lineBreak != b.lineBreak || a.done != b.done || a.value != b.value)\n return false;\n pos += a.value.length;\n if (a.done || pos >= end)\n return true;\n }\n }\n /**\n Iterate over the text. When `dir` is `-1`, iteration happens\n from end to start. This will return lines and the breaks between\n them as separate strings.\n */\n iter(dir = 1) { return new RawTextCursor(this, dir); }\n /**\n Iterate over a range of the text. When `from` > `to`, the\n iterator will run in reverse.\n */\n iterRange(from, to = this.length) { return new PartialTextCursor(this, from, to); }\n /**\n Return a cursor that iterates over the given range of lines,\n _without_ returning the line breaks between, and yielding empty\n strings for empty lines.\n \n When `from` and `to` are given, they should be 1-based line numbers.\n */\n iterLines(from, to) {\n let inner;\n if (from == null) {\n inner = this.iter();\n }\n else {\n if (to == null)\n to = this.lines + 1;\n let start = this.line(from).from;\n inner = this.iterRange(start, Math.max(start, to == this.lines + 1 ? this.length : to <= 1 ? 0 : this.line(to - 1).to));\n }\n return new LineCursor(inner);\n }\n /**\n Return the document as a string, using newline characters to\n separate lines.\n */\n toString() { return this.sliceString(0); }\n /**\n Convert the document to an array of lines (which can be\n deserialized again via [`Text.of`](https://codemirror.net/6/docs/ref/#state.Text^of)).\n */\n toJSON() {\n let lines = [];\n this.flatten(lines);\n return lines;\n }\n /**\n @internal\n */\n constructor() { }\n /**\n Create a `Text` instance for the given array of lines.\n */\n static of(text) {\n if (text.length == 0)\n throw new RangeError(\"A document must have at least one line\");\n if (text.length == 1 && !text[0])\n return Text.empty;\n return text.length <= 32 /* Tree.Branch */ ? new TextLeaf(text) : TextNode.from(TextLeaf.split(text, []));\n }\n}\n// Leaves store an array of line strings. There are always line breaks\n// between these strings. Leaves are limited in size and have to be\n// contained in TextNode instances for bigger documents.\nclass TextLeaf extends Text {\n constructor(text, length = textLength(text)) {\n super();\n this.text = text;\n this.length = length;\n }\n get lines() { return this.text.length; }\n get children() { return null; }\n lineInner(target, isLine, line, offset) {\n for (let i = 0;; i++) {\n let string = this.text[i], end = offset + string.length;\n if ((isLine ? line : end) >= target)\n return new Line(offset, end, line, string);\n offset = end + 1;\n line++;\n }\n }\n decompose(from, to, target, open) {\n let text = from <= 0 && to >= this.length ? this\n : new TextLeaf(sliceText(this.text, from, to), Math.min(to, this.length) - Math.max(0, from));\n if (open & 1 /* Open.From */) {\n let prev = target.pop();\n let joined = appendText(text.text, prev.text.slice(), 0, text.length);\n if (joined.length <= 32 /* Tree.Branch */) {\n target.push(new TextLeaf(joined, prev.length + text.length));\n }\n else {\n let mid = joined.length >> 1;\n target.push(new TextLeaf(joined.slice(0, mid)), new TextLeaf(joined.slice(mid)));\n }\n }\n else {\n target.push(text);\n }\n }\n replace(from, to, text) {\n if (!(text instanceof TextLeaf))\n return super.replace(from, to, text);\n [from, to] = clip(this, from, to);\n let lines = appendText(this.text, appendText(text.text, sliceText(this.text, 0, from)), to);\n let newLen = this.length + text.length - (to - from);\n if (lines.length <= 32 /* Tree.Branch */)\n return new TextLeaf(lines, newLen);\n return TextNode.from(TextLeaf.split(lines, []), newLen);\n }\n sliceString(from, to = this.length, lineSep = \"\\n\") {\n [from, to] = clip(this, from, to);\n let result = \"\";\n for (let pos = 0, i = 0; pos <= to && i < this.text.length; i++) {\n let line = this.text[i], end = pos + line.length;\n if (pos > from && i)\n result += lineSep;\n if (from < end && to > pos)\n result += line.slice(Math.max(0, from - pos), to - pos);\n pos = end + 1;\n }\n return result;\n }\n flatten(target) {\n for (let line of this.text)\n target.push(line);\n }\n scanIdentical() { return 0; }\n static split(text, target) {\n let part = [], len = -1;\n for (let line of text) {\n part.push(line);\n len += line.length + 1;\n if (part.length == 32 /* Tree.Branch */) {\n target.push(new TextLeaf(part, len));\n part = [];\n len = -1;\n }\n }\n if (len > -1)\n target.push(new TextLeaf(part, len));\n return target;\n }\n}\n// Nodes provide the tree structure of the `Text` type. They store a\n// number of other nodes or leaves, taking care to balance themselves\n// on changes. There are implied line breaks _between_ the children of\n// a node (but not before the first or after the last child).\nclass TextNode extends Text {\n constructor(children, length) {\n super();\n this.children = children;\n this.length = length;\n this.lines = 0;\n for (let child of children)\n this.lines += child.lines;\n }\n lineInner(target, isLine, line, offset) {\n for (let i = 0;; i++) {\n let child = this.children[i], end = offset + child.length, endLine = line + child.lines - 1;\n if ((isLine ? endLine : end) >= target)\n return child.lineInner(target, isLine, line, offset);\n offset = end + 1;\n line = endLine + 1;\n }\n }\n decompose(from, to, target, open) {\n for (let i = 0, pos = 0; pos <= to && i < this.children.length; i++) {\n let child = this.children[i], end = pos + child.length;\n if (from <= end && to >= pos) {\n let childOpen = open & ((pos <= from ? 1 /* Open.From */ : 0) | (end >= to ? 2 /* Open.To */ : 0));\n if (pos >= from && end <= to && !childOpen)\n target.push(child);\n else\n child.decompose(from - pos, to - pos, target, childOpen);\n }\n pos = end + 1;\n }\n }\n replace(from, to, text) {\n [from, to] = clip(this, from, to);\n if (text.lines < this.lines)\n for (let i = 0, pos = 0; i < this.children.length; i++) {\n let child = this.children[i], end = pos + child.length;\n // Fast path: if the change only affects one child and the\n // child's size remains in the acceptable range, only update\n // that child\n if (from >= pos && to <= end) {\n let updated = child.replace(from - pos, to - pos, text);\n let totalLines = this.lines - child.lines + updated.lines;\n if (updated.lines < (totalLines >> (5 /* Tree.BranchShift */ - 1)) &&\n updated.lines > (totalLines >> (5 /* Tree.BranchShift */ + 1))) {\n let copy = this.children.slice();\n copy[i] = updated;\n return new TextNode(copy, this.length - (to - from) + text.length);\n }\n return super.replace(pos, end, updated);\n }\n pos = end + 1;\n }\n return super.replace(from, to, text);\n }\n sliceString(from, to = this.length, lineSep = \"\\n\") {\n [from, to] = clip(this, from, to);\n let result = \"\";\n for (let i = 0, pos = 0; i < this.children.length && pos <= to; i++) {\n let child = this.children[i], end = pos + child.length;\n if (pos > from && i)\n result += lineSep;\n if (from < end && to > pos)\n result += child.sliceString(from - pos, to - pos, lineSep);\n pos = end + 1;\n }\n return result;\n }\n flatten(target) {\n for (let child of this.children)\n child.flatten(target);\n }\n scanIdentical(other, dir) {\n if (!(other instanceof TextNode))\n return 0;\n let length = 0;\n let [iA, iB, eA, eB] = dir > 0 ? [0, 0, this.children.length, other.children.length]\n : [this.children.length - 1, other.children.length - 1, -1, -1];\n for (;; iA += dir, iB += dir) {\n if (iA == eA || iB == eB)\n return length;\n let chA = this.children[iA], chB = other.children[iB];\n if (chA != chB)\n return length + chA.scanIdentical(chB, dir);\n length += chA.length + 1;\n }\n }\n static from(children, length = children.reduce((l, ch) => l + ch.length + 1, -1)) {\n let lines = 0;\n for (let ch of children)\n lines += ch.lines;\n if (lines < 32 /* Tree.Branch */) {\n let flat = [];\n for (let ch of children)\n ch.flatten(flat);\n return new TextLeaf(flat, length);\n }\n let chunk = Math.max(32 /* Tree.Branch */, lines >> 5 /* Tree.BranchShift */), maxChunk = chunk << 1, minChunk = chunk >> 1;\n let chunked = [], currentLines = 0, currentLen = -1, currentChunk = [];\n function add(child) {\n let last;\n if (child.lines > maxChunk && child instanceof TextNode) {\n for (let node of child.children)\n add(node);\n }\n else if (child.lines > minChunk && (currentLines > minChunk || !currentLines)) {\n flush();\n chunked.push(child);\n }\n else if (child instanceof TextLeaf && currentLines &&\n (last = currentChunk[currentChunk.length - 1]) instanceof TextLeaf &&\n child.lines + last.lines <= 32 /* Tree.Branch */) {\n currentLines += child.lines;\n currentLen += child.length + 1;\n currentChunk[currentChunk.length - 1] = new TextLeaf(last.text.concat(child.text), last.length + 1 + child.length);\n }\n else {\n if (currentLines + child.lines > chunk)\n flush();\n currentLines += child.lines;\n currentLen += child.length + 1;\n currentChunk.push(child);\n }\n }\n function flush() {\n if (currentLines == 0)\n return;\n chunked.push(currentChunk.length == 1 ? currentChunk[0] : TextNode.from(currentChunk, currentLen));\n currentLen = -1;\n currentLines = currentChunk.length = 0;\n }\n for (let child of children)\n add(child);\n flush();\n return chunked.length == 1 ? chunked[0] : new TextNode(chunked, length);\n }\n}\nText.empty = /*@__PURE__*/new TextLeaf([\"\"], 0);\nfunction textLength(text) {\n let length = -1;\n for (let line of text)\n length += line.length + 1;\n return length;\n}\nfunction appendText(text, target, from = 0, to = 1e9) {\n for (let pos = 0, i = 0, first = true; i < text.length && pos <= to; i++) {\n let line = text[i], end = pos + line.length;\n if (end >= from) {\n if (end > to)\n line = line.slice(0, to - pos);\n if (pos < from)\n line = line.slice(from - pos);\n if (first) {\n target[target.length - 1] += line;\n first = false;\n }\n else\n target.push(line);\n }\n pos = end + 1;\n }\n return target;\n}\nfunction sliceText(text, from, to) {\n return appendText(text, [\"\"], from, to);\n}\nclass RawTextCursor {\n constructor(text, dir = 1) {\n this.dir = dir;\n this.done = false;\n this.lineBreak = false;\n this.value = \"\";\n this.nodes = [text];\n this.offsets = [dir > 0 ? 1 : (text instanceof TextLeaf ? text.text.length : text.children.length) << 1];\n }\n nextInner(skip, dir) {\n this.done = this.lineBreak = false;\n for (;;) {\n let last = this.nodes.length - 1;\n let top = this.nodes[last], offsetValue = this.offsets[last], offset = offsetValue >> 1;\n let size = top instanceof TextLeaf ? top.text.length : top.children.length;\n if (offset == (dir > 0 ? size : 0)) {\n if (last == 0) {\n this.done = true;\n this.value = \"\";\n return this;\n }\n if (dir > 0)\n this.offsets[last - 1]++;\n this.nodes.pop();\n this.offsets.pop();\n }\n else if ((offsetValue & 1) == (dir > 0 ? 0 : 1)) {\n this.offsets[last] += dir;\n if (skip == 0) {\n this.lineBreak = true;\n this.value = \"\\n\";\n return this;\n }\n skip--;\n }\n else if (top instanceof TextLeaf) {\n // Move to the next string\n let next = top.text[offset + (dir < 0 ? -1 : 0)];\n this.offsets[last] += dir;\n if (next.length > Math.max(0, skip)) {\n this.value = skip == 0 ? next : dir > 0 ? next.slice(skip) : next.slice(0, next.length - skip);\n return this;\n }\n skip -= next.length;\n }\n else {\n let next = top.children[offset + (dir < 0 ? -1 : 0)];\n if (skip > next.length) {\n skip -= next.length;\n this.offsets[last] += dir;\n }\n else {\n if (dir < 0)\n this.offsets[last]--;\n this.nodes.push(next);\n this.offsets.push(dir > 0 ? 1 : (next instanceof TextLeaf ? next.text.length : next.children.length) << 1);\n }\n }\n }\n }\n next(skip = 0) {\n if (skip < 0) {\n this.nextInner(-skip, (-this.dir));\n skip = this.value.length;\n }\n return this.nextInner(skip, this.dir);\n }\n}\nclass PartialTextCursor {\n constructor(text, start, end) {\n this.value = \"\";\n this.done = false;\n this.cursor = new RawTextCursor(text, start > end ? -1 : 1);\n this.pos = start > end ? text.length : 0;\n this.from = Math.min(start, end);\n this.to = Math.max(start, end);\n }\n nextInner(skip, dir) {\n if (dir < 0 ? this.pos <= this.from : this.pos >= this.to) {\n this.value = \"\";\n this.done = true;\n return this;\n }\n skip += Math.max(0, dir < 0 ? this.pos - this.to : this.from - this.pos);\n let limit = dir < 0 ? this.pos - this.from : this.to - this.pos;\n if (skip > limit)\n skip = limit;\n limit -= skip;\n let { value } = this.cursor.next(skip);\n this.pos += (value.length + skip) * dir;\n this.value = value.length <= limit ? value : dir < 0 ? value.slice(value.length - limit) : value.slice(0, limit);\n this.done = !this.value;\n return this;\n }\n next(skip = 0) {\n if (skip < 0)\n skip = Math.max(skip, this.from - this.pos);\n else if (skip > 0)\n skip = Math.min(skip, this.to - this.pos);\n return this.nextInner(skip, this.cursor.dir);\n }\n get lineBreak() { return this.cursor.lineBreak && this.value != \"\"; }\n}\nclass LineCursor {\n constructor(inner) {\n this.inner = inner;\n this.afterBreak = true;\n this.value = \"\";\n this.done = false;\n }\n next(skip = 0) {\n let { done, lineBreak, value } = this.inner.next(skip);\n if (done && this.afterBreak) {\n this.value = \"\";\n this.afterBreak = false;\n }\n else if (done) {\n this.done = true;\n this.value = \"\";\n }\n else if (lineBreak) {\n if (this.afterBreak) {\n this.value = \"\";\n }\n else {\n this.afterBreak = true;\n this.next();\n }\n }\n else {\n this.value = value;\n this.afterBreak = false;\n }\n return this;\n }\n get lineBreak() { return false; }\n}\nif (typeof Symbol != \"undefined\") {\n Text.prototype[Symbol.iterator] = function () { return this.iter(); };\n RawTextCursor.prototype[Symbol.iterator] = PartialTextCursor.prototype[Symbol.iterator] =\n LineCursor.prototype[Symbol.iterator] = function () { return this; };\n}\n/**\nThis type describes a line in the document. It is created\non-demand when lines are [queried](https://codemirror.net/6/docs/ref/#state.Text.lineAt).\n*/\nclass Line {\n /**\n @internal\n */\n constructor(\n /**\n The position of the start of the line.\n */\n from, \n /**\n The position at the end of the line (_before_ the line break,\n or at the end of document for the last line).\n */\n to, \n /**\n This line's line number (1-based).\n */\n number, \n /**\n The line's content.\n */\n text) {\n this.from = from;\n this.to = to;\n this.number = number;\n this.text = text;\n }\n /**\n The length of the line (not including any line break after it).\n */\n get length() { return this.to - this.from; }\n}\nfunction clip(text, from, to) {\n from = Math.max(0, Math.min(text.length, from));\n return [from, Math.max(from, Math.min(text.length, to))];\n}\n\n/**\nReturns a next grapheme cluster break _after_ (not equal to)\n`pos`, if `forward` is true, or before otherwise. Returns `pos`\nitself if no further cluster break is available in the string.\nMoves across surrogate pairs, extending characters (when\n`includeExtending` is true), characters joined with zero-width\njoiners, and flag emoji.\n*/\nfunction findClusterBreak(str, pos, forward = true, includeExtending = true) {\n return findClusterBreak$1(str, pos, forward, includeExtending);\n}\nfunction surrogateLow(ch) { return ch >= 0xDC00 && ch < 0xE000; }\nfunction surrogateHigh(ch) { return ch >= 0xD800 && ch < 0xDC00; }\n/**\nFind the code point at the given position in a string (like the\n[`codePointAt`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/codePointAt)\nstring method).\n*/\nfunction codePointAt(str, pos) {\n let code0 = str.charCodeAt(pos);\n if (!surrogateHigh(code0) || pos + 1 == str.length)\n return code0;\n let code1 = str.charCodeAt(pos + 1);\n if (!surrogateLow(code1))\n return code0;\n return ((code0 - 0xd800) << 10) + (code1 - 0xdc00) + 0x10000;\n}\n/**\nGiven a Unicode codepoint, return the JavaScript string that\nrespresents it (like\n[`String.fromCodePoint`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/fromCodePoint)).\n*/\nfunction fromCodePoint(code) {\n if (code <= 0xffff)\n return String.fromCharCode(code);\n code -= 0x10000;\n return String.fromCharCode((code >> 10) + 0xd800, (code & 1023) + 0xdc00);\n}\n/**\nThe amount of positions a character takes up in a JavaScript string.\n*/\nfunction codePointSize(code) { return code < 0x10000 ? 1 : 2; }\n\nconst DefaultSplit = /\\r\\n?|\\n/;\n/**\nDistinguishes different ways in which positions can be mapped.\n*/\nvar MapMode = /*@__PURE__*/(function (MapMode) {\n /**\n Map a position to a valid new position, even when its context\n was deleted.\n */\n MapMode[MapMode[\"Simple\"] = 0] = \"Simple\";\n /**\n Return null if deletion happens across the position.\n */\n MapMode[MapMode[\"TrackDel\"] = 1] = \"TrackDel\";\n /**\n Return null if the character _before_ the position is deleted.\n */\n MapMode[MapMode[\"TrackBefore\"] = 2] = \"TrackBefore\";\n /**\n Return null if the character _after_ the position is deleted.\n */\n MapMode[MapMode[\"TrackAfter\"] = 3] = \"TrackAfter\";\nreturn MapMode})(MapMode || (MapMode = {}));\n/**\nA change description is a variant of [change set](https://codemirror.net/6/docs/ref/#state.ChangeSet)\nthat doesn't store the inserted text. As such, it can't be\napplied, but is cheaper to store and manipulate.\n*/\nclass ChangeDesc {\n // Sections are encoded as pairs of integers. The first is the\n // length in the current document, and the second is -1 for\n // unaffected sections, and the length of the replacement content\n // otherwise. So an insertion would be (0, n>0), a deletion (n>0,\n // 0), and a replacement two positive numbers.\n /**\n @internal\n */\n constructor(\n /**\n @internal\n */\n sections) {\n this.sections = sections;\n }\n /**\n The length of the document before the change.\n */\n get length() {\n let result = 0;\n for (let i = 0; i < this.sections.length; i += 2)\n result += this.sections[i];\n return result;\n }\n /**\n The length of the document after the change.\n */\n get newLength() {\n let result = 0;\n for (let i = 0; i < this.sections.length; i += 2) {\n let ins = this.sections[i + 1];\n result += ins < 0 ? this.sections[i] : ins;\n }\n return result;\n }\n /**\n False when there are actual changes in this set.\n */\n get empty() { return this.sections.length == 0 || this.sections.length == 2 && this.sections[1] < 0; }\n /**\n Iterate over the unchanged parts left by these changes. `posA`\n provides the position of the range in the old document, `posB`\n the new position in the changed document.\n */\n iterGaps(f) {\n for (let i = 0, posA = 0, posB = 0; i < this.sections.length;) {\n let len = this.sections[i++], ins = this.sections[i++];\n if (ins < 0) {\n f(posA, posB, len);\n posB += len;\n }\n else {\n posB += ins;\n }\n posA += len;\n }\n }\n /**\n Iterate over the ranges changed by these changes. (See\n [`ChangeSet.iterChanges`](https://codemirror.net/6/docs/ref/#state.ChangeSet.iterChanges) for a\n variant that also provides you with the inserted text.)\n `fromA`/`toA` provides the extent of the change in the starting\n document, `fromB`/`toB` the extent of the replacement in the\n changed document.\n \n When `individual` is true, adjacent changes (which are kept\n separate for [position mapping](https://codemirror.net/6/docs/ref/#state.ChangeDesc.mapPos)) are\n reported separately.\n */\n iterChangedRanges(f, individual = false) {\n iterChanges(this, f, individual);\n }\n /**\n Get a description of the inverted form of these changes.\n */\n get invertedDesc() {\n let sections = [];\n for (let i = 0; i < this.sections.length;) {\n let len = this.sections[i++], ins = this.sections[i++];\n if (ins < 0)\n sections.push(len, ins);\n else\n sections.push(ins, len);\n }\n return new ChangeDesc(sections);\n }\n /**\n Compute the combined effect of applying another set of changes\n after this one. The length of the document after this set should\n match the length before `other`.\n */\n composeDesc(other) { return this.empty ? other : other.empty ? this : composeSets(this, other); }\n /**\n Map this description, which should start with the same document\n as `other`, over another set of changes, so that it can be\n applied after it. When `before` is true, map as if the changes\n in `this` happened before the ones in `other`.\n */\n mapDesc(other, before = false) { return other.empty ? this : mapSet(this, other, before); }\n mapPos(pos, assoc = -1, mode = MapMode.Simple) {\n let posA = 0, posB = 0;\n for (let i = 0; i < this.sections.length;) {\n let len = this.sections[i++], ins = this.sections[i++], endA = posA + len;\n if (ins < 0) {\n if (endA > pos)\n return posB + (pos - posA);\n posB += len;\n }\n else {\n if (mode != MapMode.Simple && endA >= pos &&\n (mode == MapMode.TrackDel && posA < pos && endA > pos ||\n mode == MapMode.TrackBefore && posA < pos ||\n mode == MapMode.TrackAfter && endA > pos))\n return null;\n if (endA > pos || endA == pos && assoc < 0 && !len)\n return pos == posA || assoc < 0 ? posB : posB + ins;\n posB += ins;\n }\n posA = endA;\n }\n if (pos > posA)\n throw new RangeError(`Position ${pos} is out of range for changeset of length ${posA}`);\n return posB;\n }\n /**\n Check whether these changes touch a given range. When one of the\n changes entirely covers the range, the string `\"cover\"` is\n returned.\n */\n touchesRange(from, to = from) {\n for (let i = 0, pos = 0; i < this.sections.length && pos <= to;) {\n let len = this.sections[i++], ins = this.sections[i++], end = pos + len;\n if (ins >= 0 && pos <= to && end >= from)\n return pos < from && end > to ? \"cover\" : true;\n pos = end;\n }\n return false;\n }\n /**\n @internal\n */\n toString() {\n let result = \"\";\n for (let i = 0; i < this.sections.length;) {\n let len = this.sections[i++], ins = this.sections[i++];\n result += (result ? \" \" : \"\") + len + (ins >= 0 ? \":\" + ins : \"\");\n }\n return result;\n }\n /**\n Serialize this change desc to a JSON-representable value.\n */\n toJSON() { return this.sections; }\n /**\n Create a change desc from its JSON representation (as produced\n by [`toJSON`](https://codemirror.net/6/docs/ref/#state.ChangeDesc.toJSON).\n */\n static fromJSON(json) {\n if (!Array.isArray(json) || json.length % 2 || json.some(a => typeof a != \"number\"))\n throw new RangeError(\"Invalid JSON representation of ChangeDesc\");\n return new ChangeDesc(json);\n }\n /**\n @internal\n */\n static create(sections) { return new ChangeDesc(sections); }\n}\n/**\nA change set represents a group of modifications to a document. It\nstores the document length, and can only be applied to documents\nwith exactly that length.\n*/\nclass ChangeSet extends ChangeDesc {\n constructor(sections, \n /**\n @internal\n */\n inserted) {\n super(sections);\n this.inserted = inserted;\n }\n /**\n Apply the changes to a document, returning the modified\n document.\n */\n apply(doc) {\n if (this.length != doc.length)\n throw new RangeError(\"Applying change set to a document with the wrong length\");\n iterChanges(this, (fromA, toA, fromB, _toB, text) => doc = doc.replace(fromB, fromB + (toA - fromA), text), false);\n return doc;\n }\n mapDesc(other, before = false) { return mapSet(this, other, before, true); }\n /**\n Given the document as it existed _before_ the changes, return a\n change set that represents the inverse of this set, which could\n be used to go from the document created by the changes back to\n the document as it existed before the changes.\n */\n invert(doc) {\n let sections = this.sections.slice(), inserted = [];\n for (let i = 0, pos = 0; i < sections.length; i += 2) {\n let len = sections[i], ins = sections[i + 1];\n if (ins >= 0) {\n sections[i] = ins;\n sections[i + 1] = len;\n let index = i >> 1;\n while (inserted.length < index)\n inserted.push(Text.empty);\n inserted.push(len ? doc.slice(pos, pos + len) : Text.empty);\n }\n pos += len;\n }\n return new ChangeSet(sections, inserted);\n }\n /**\n Combine two subsequent change sets into a single set. `other`\n must start in the document produced by `this`. If `this` goes\n `docA` \u2192 `docB` and `other` represents `docB` \u2192 `docC`, the\n returned value will represent the change `docA` \u2192 `docC`.\n */\n compose(other) { return this.empty ? other : other.empty ? this : composeSets(this, other, true); }\n /**\n Given another change set starting in the same document, maps this\n change set over the other, producing a new change set that can be\n applied to the document produced by applying `other`. When\n `before` is `true`, order changes as if `this` comes before\n `other`, otherwise (the default) treat `other` as coming first.\n \n Given two changes `A` and `B`, `A.compose(B.map(A))` and\n `B.compose(A.map(B, true))` will produce the same document. This\n provides a basic form of [operational\n transformation](https://en.wikipedia.org/wiki/Operational_transformation),\n and can be used for collaborative editing.\n */\n map(other, before = false) { return other.empty ? this : mapSet(this, other, before, true); }\n /**\n Iterate over the changed ranges in the document, calling `f` for\n each, with the range in the original document (`fromA`-`toA`)\n and the range that replaces it in the new document\n (`fromB`-`toB`).\n \n When `individual` is true, adjacent changes are reported\n separately.\n */\n iterChanges(f, individual = false) {\n iterChanges(this, f, individual);\n }\n /**\n Get a [change description](https://codemirror.net/6/docs/ref/#state.ChangeDesc) for this change\n set.\n */\n get desc() { return ChangeDesc.create(this.sections); }\n /**\n @internal\n */\n filter(ranges) {\n let resultSections = [], resultInserted = [], filteredSections = [];\n let iter = new SectionIter(this);\n done: for (let i = 0, pos = 0;;) {\n let next = i == ranges.length ? 1e9 : ranges[i++];\n while (pos < next || pos == next && iter.len == 0) {\n if (iter.done)\n break done;\n let len = Math.min(iter.len, next - pos);\n addSection(filteredSections, len, -1);\n let ins = iter.ins == -1 ? -1 : iter.off == 0 ? iter.ins : 0;\n addSection(resultSections, len, ins);\n if (ins > 0)\n addInsert(resultInserted, resultSections, iter.text);\n iter.forward(len);\n pos += len;\n }\n let end = ranges[i++];\n while (pos < end) {\n if (iter.done)\n break done;\n let len = Math.min(iter.len, end - pos);\n addSection(resultSections, len, -1);\n addSection(filteredSections, len, iter.ins == -1 ? -1 : iter.off == 0 ? iter.ins : 0);\n iter.forward(len);\n pos += len;\n }\n }\n return { changes: new ChangeSet(resultSections, resultInserted),\n filtered: ChangeDesc.create(filteredSections) };\n }\n /**\n Serialize this change set to a JSON-representable value.\n */\n toJSON() {\n let parts = [];\n for (let i = 0; i < this.sections.length; i += 2) {\n let len = this.sections[i], ins = this.sections[i + 1];\n if (ins < 0)\n parts.push(len);\n else if (ins == 0)\n parts.push([len]);\n else\n parts.push([len].concat(this.inserted[i >> 1].toJSON()));\n }\n return parts;\n }\n /**\n Create a change set for the given changes, for a document of the\n given length, using `lineSep` as line separator.\n */\n static of(changes, length, lineSep) {\n let sections = [], inserted = [], pos = 0;\n let total = null;\n function flush(force = false) {\n if (!force && !sections.length)\n return;\n if (pos < length)\n addSection(sections, length - pos, -1);\n let set = new ChangeSet(sections, inserted);\n total = total ? total.compose(set.map(total)) : set;\n sections = [];\n inserted = [];\n pos = 0;\n }\n function process(spec) {\n if (Array.isArray(spec)) {\n for (let sub of spec)\n process(sub);\n }\n else if (spec instanceof ChangeSet) {\n if (spec.length != length)\n throw new RangeError(`Mismatched change set length (got ${spec.length}, expected ${length})`);\n flush();\n total = total ? total.compose(spec.map(total)) : spec;\n }\n else {\n let { from, to = from, insert } = spec;\n if (from > to || from < 0 || to > length)\n throw new RangeError(`Invalid change range ${from} to ${to} (in doc of length ${length})`);\n let insText = !insert ? Text.empty : typeof insert == \"string\" ? Text.of(insert.split(lineSep || DefaultSplit)) : insert;\n let insLen = insText.length;\n if (from == to && insLen == 0)\n return;\n if (from < pos)\n flush();\n if (from > pos)\n addSection(sections, from - pos, -1);\n addSection(sections, to - from, insLen);\n addInsert(inserted, sections, insText);\n pos = to;\n }\n }\n process(changes);\n flush(!total);\n return total;\n }\n /**\n Create an empty changeset of the given length.\n */\n static empty(length) {\n return new ChangeSet(length ? [length, -1] : [], []);\n }\n /**\n Create a changeset from its JSON representation (as produced by\n [`toJSON`](https://codemirror.net/6/docs/ref/#state.ChangeSet.toJSON).\n */\n static fromJSON(json) {\n if (!Array.isArray(json))\n throw new RangeError(\"Invalid JSON representation of ChangeSet\");\n let sections = [], inserted = [];\n for (let i = 0; i < json.length; i++) {\n let part = json[i];\n if (typeof part == \"number\") {\n sections.push(part, -1);\n }\n else if (!Array.isArray(part) || typeof part[0] != \"number\" || part.some((e, i) => i && typeof e != \"string\")) {\n throw new RangeError(\"Invalid JSON representation of ChangeSet\");\n }\n else if (part.length == 1) {\n sections.push(part[0], 0);\n }\n else {\n while (inserted.length < i)\n inserted.push(Text.empty);\n inserted[i] = Text.of(part.slice(1));\n sections.push(part[0], inserted[i].length);\n }\n }\n return new ChangeSet(sections, inserted);\n }\n /**\n @internal\n */\n static createSet(sections, inserted) {\n return new ChangeSet(sections, inserted);\n }\n}\nfunction addSection(sections, len, ins, forceJoin = false) {\n if (len == 0 && ins <= 0)\n return;\n let last = sections.length - 2;\n if (last >= 0 && ins <= 0 && ins == sections[last + 1])\n sections[last] += len;\n else if (last >= 0 && len == 0 && sections[last] == 0)\n sections[last + 1] += ins;\n else if (forceJoin) {\n sections[last] += len;\n sections[last + 1] += ins;\n }\n else\n sections.push(len, ins);\n}\nfunction addInsert(values, sections, value) {\n if (value.length == 0)\n return;\n let index = (sections.length - 2) >> 1;\n if (index < values.length) {\n values[values.length - 1] = values[values.length - 1].append(value);\n }\n else {\n while (values.length < index)\n values.push(Text.empty);\n values.push(value);\n }\n}\nfunction iterChanges(desc, f, individual) {\n let inserted = desc.inserted;\n for (let posA = 0, posB = 0, i = 0; i < desc.sections.length;) {\n let len = desc.sections[i++], ins = desc.sections[i++];\n if (ins < 0) {\n posA += len;\n posB += len;\n }\n else {\n let endA = posA, endB = posB, text = Text.empty;\n for (;;) {\n endA += len;\n endB += ins;\n if (ins && inserted)\n text = text.append(inserted[(i - 2) >> 1]);\n if (individual || i == desc.sections.length || desc.sections[i + 1] < 0)\n break;\n len = desc.sections[i++];\n ins = desc.sections[i++];\n }\n f(posA, endA, posB, endB, text);\n posA = endA;\n posB = endB;\n }\n }\n}\nfunction mapSet(setA, setB, before, mkSet = false) {\n // Produce a copy of setA that applies to the document after setB\n // has been applied (assuming both start at the same document).\n let sections = [], insert = mkSet ? [] : null;\n let a = new SectionIter(setA), b = new SectionIter(setB);\n // Iterate over both sets in parallel. inserted tracks, for changes\n // in A that have to be processed piece-by-piece, whether their\n // content has been inserted already, and refers to the section\n // index.\n for (let inserted = -1;;) {\n if (a.done && b.len || b.done && a.len) {\n throw new Error(\"Mismatched change set lengths\");\n }\n else if (a.ins == -1 && b.ins == -1) {\n // Move across ranges skipped by both sets.\n let len = Math.min(a.len, b.len);\n addSection(sections, len, -1);\n a.forward(len);\n b.forward(len);\n }\n else if (b.ins >= 0 && (a.ins < 0 || inserted == a.i || a.off == 0 && (b.len < a.len || b.len == a.len && !before))) {\n // If there's a change in B that comes before the next change in\n // A (ordered by start pos, then len, then before flag), skip\n // that (and process any changes in A it covers).\n let len = b.len;\n addSection(sections, b.ins, -1);\n while (len) {\n let piece = Math.min(a.len, len);\n if (a.ins >= 0 && inserted < a.i && a.len <= piece) {\n addSection(sections, 0, a.ins);\n if (insert)\n addInsert(insert, sections, a.text);\n inserted = a.i;\n }\n a.forward(piece);\n len -= piece;\n }\n b.next();\n }\n else if (a.ins >= 0) {\n // Process the part of a change in A up to the start of the next\n // non-deletion change in B (if overlapping).\n let len = 0, left = a.len;\n while (left) {\n if (b.ins == -1) {\n let piece = Math.min(left, b.len);\n len += piece;\n left -= piece;\n b.forward(piece);\n }\n else if (b.ins == 0 && b.len < left) {\n left -= b.len;\n b.next();\n }\n else {\n break;\n }\n }\n addSection(sections, len, inserted < a.i ? a.ins : 0);\n if (insert && inserted < a.i)\n addInsert(insert, sections, a.text);\n inserted = a.i;\n a.forward(a.len - left);\n }\n else if (a.done && b.done) {\n return insert ? ChangeSet.createSet(sections, insert) : ChangeDesc.create(sections);\n }\n else {\n throw new Error(\"Mismatched change set lengths\");\n }\n }\n}\nfunction composeSets(setA, setB, mkSet = false) {\n let sections = [];\n let insert = mkSet ? [] : null;\n let a = new SectionIter(setA), b = new SectionIter(setB);\n for (let open = false;;) {\n if (a.done && b.done) {\n return insert ? ChangeSet.createSet(sections, insert) : ChangeDesc.create(sections);\n }\n else if (a.ins == 0) { // Deletion in A\n addSection(sections, a.len, 0, open);\n a.next();\n }\n else if (b.len == 0 && !b.done) { // Insertion in B\n addSection(sections, 0, b.ins, open);\n if (insert)\n addInsert(insert, sections, b.text);\n b.next();\n }\n else if (a.done || b.done) {\n throw new Error(\"Mismatched change set lengths\");\n }\n else {\n let len = Math.min(a.len2, b.len), sectionLen = sections.length;\n if (a.ins == -1) {\n let insB = b.ins == -1 ? -1 : b.off ? 0 : b.ins;\n addSection(sections, len, insB, open);\n if (insert && insB)\n addInsert(insert, sections, b.text);\n }\n else if (b.ins == -1) {\n addSection(sections, a.off ? 0 : a.len, len, open);\n if (insert)\n addInsert(insert, sections, a.textBit(len));\n }\n else {\n addSection(sections, a.off ? 0 : a.len, b.off ? 0 : b.ins, open);\n if (insert && !b.off)\n addInsert(insert, sections, b.text);\n }\n open = (a.ins > len || b.ins >= 0 && b.len > len) && (open || sections.length > sectionLen);\n a.forward2(len);\n b.forward(len);\n }\n }\n}\nclass SectionIter {\n constructor(set) {\n this.set = set;\n this.i = 0;\n this.next();\n }\n next() {\n let { sections } = this.set;\n if (this.i < sections.length) {\n this.len = sections[this.i++];\n this.ins = sections[this.i++];\n }\n else {\n this.len = 0;\n this.ins = -2;\n }\n this.off = 0;\n }\n get done() { return this.ins == -2; }\n get len2() { return this.ins < 0 ? this.len : this.ins; }\n get text() {\n let { inserted } = this.set, index = (this.i - 2) >> 1;\n return index >= inserted.length ? Text.empty : inserted[index];\n }\n textBit(len) {\n let { inserted } = this.set, index = (this.i - 2) >> 1;\n return index >= inserted.length && !len ? Text.empty\n : inserted[index].slice(this.off, len == null ? undefined : this.off + len);\n }\n forward(len) {\n if (len == this.len)\n this.next();\n else {\n this.len -= len;\n this.off += len;\n }\n }\n forward2(len) {\n if (this.ins == -1)\n this.forward(len);\n else if (len == this.ins)\n this.next();\n else {\n this.ins -= len;\n this.off += len;\n }\n }\n}\n\n/**\nA single selection range. When\n[`allowMultipleSelections`](https://codemirror.net/6/docs/ref/#state.EditorState^allowMultipleSelections)\nis enabled, a [selection](https://codemirror.net/6/docs/ref/#state.EditorSelection) may hold\nmultiple ranges. By default, selections hold exactly one range.\n*/\nclass SelectionRange {\n constructor(\n /**\n The lower boundary of the range.\n */\n from, \n /**\n The upper boundary of the range.\n */\n to, flags) {\n this.from = from;\n this.to = to;\n this.flags = flags;\n }\n /**\n The anchor of the range\u2014the side that doesn't move when you\n extend it.\n */\n get anchor() { return this.flags & 32 /* RangeFlag.Inverted */ ? this.to : this.from; }\n /**\n The head of the range, which is moved when the range is\n [extended](https://codemirror.net/6/docs/ref/#state.SelectionRange.extend).\n */\n get head() { return this.flags & 32 /* RangeFlag.Inverted */ ? this.from : this.to; }\n /**\n True when `anchor` and `head` are at the same position.\n */\n get empty() { return this.from == this.to; }\n /**\n If this is a cursor that is explicitly associated with the\n character on one of its sides, this returns the side. -1 means\n the character before its position, 1 the character after, and 0\n means no association.\n */\n get assoc() { return this.flags & 8 /* RangeFlag.AssocBefore */ ? -1 : this.flags & 16 /* RangeFlag.AssocAfter */ ? 1 : 0; }\n /**\n The bidirectional text level associated with this cursor, if\n any.\n */\n get bidiLevel() {\n let level = this.flags & 7 /* RangeFlag.BidiLevelMask */;\n return level == 7 ? null : level;\n }\n /**\n The goal column (stored vertical offset) associated with a\n cursor. This is used to preserve the vertical position when\n [moving](https://codemirror.net/6/docs/ref/#view.EditorView.moveVertically) across\n lines of different length.\n */\n get goalColumn() {\n let value = this.flags >> 6 /* RangeFlag.GoalColumnOffset */;\n return value == 16777215 /* RangeFlag.NoGoalColumn */ ? undefined : value;\n }\n /**\n Map this range through a change, producing a valid range in the\n updated document.\n */\n map(change, assoc = -1) {\n let from, to;\n if (this.empty) {\n from = to = change.mapPos(this.from, assoc);\n }\n else {\n from = change.mapPos(this.from, 1);\n to = change.mapPos(this.to, -1);\n }\n return from == this.from && to == this.to ? this : new SelectionRange(from, to, this.flags);\n }\n /**\n Extend this range to cover at least `from` to `to`.\n */\n extend(from, to = from, assoc = 0) {\n if (from <= this.anchor && to >= this.anchor)\n return EditorSelection.range(from, to, undefined, undefined, assoc);\n let head = Math.abs(from - this.anchor) > Math.abs(to - this.anchor) ? from : to;\n return EditorSelection.range(this.anchor, head, undefined, undefined, assoc);\n }\n /**\n Compare this range to another range.\n */\n eq(other, includeAssoc = false) {\n return this.anchor == other.anchor && this.head == other.head && this.goalColumn == other.goalColumn &&\n (!includeAssoc || !this.empty || this.assoc == other.assoc);\n }\n /**\n Return a JSON-serializable object representing the range.\n */\n toJSON() { return { anchor: this.anchor, head: this.head }; }\n /**\n Convert a JSON representation of a range to a `SelectionRange`\n instance.\n */\n static fromJSON(json) {\n if (!json || typeof json.anchor != \"number\" || typeof json.head != \"number\")\n throw new RangeError(\"Invalid JSON representation for SelectionRange\");\n return EditorSelection.range(json.anchor, json.head);\n }\n /**\n @internal\n */\n static create(from, to, flags) {\n return new SelectionRange(from, to, flags);\n }\n}\n/**\nAn editor selection holds one or more selection ranges.\n*/\nclass EditorSelection {\n constructor(\n /**\n The ranges in the selection, sorted by position. Ranges cannot\n overlap (but they may touch, if they aren't empty).\n */\n ranges, \n /**\n The index of the _main_ range in the selection (which is\n usually the range that was added last).\n */\n mainIndex) {\n this.ranges = ranges;\n this.mainIndex = mainIndex;\n }\n /**\n Map a selection through a change. Used to adjust the selection\n position for changes.\n */\n map(change, assoc = -1) {\n if (change.empty)\n return this;\n return EditorSelection.create(this.ranges.map(r => r.map(change, assoc)), this.mainIndex);\n }\n /**\n Compare this selection to another selection. By default, ranges\n are compared only by position. When `includeAssoc` is true,\n cursor ranges must also have the same\n [`assoc`](https://codemirror.net/6/docs/ref/#state.SelectionRange.assoc) value.\n */\n eq(other, includeAssoc = false) {\n if (this.ranges.length != other.ranges.length ||\n this.mainIndex != other.mainIndex)\n return false;\n for (let i = 0; i < this.ranges.length; i++)\n if (!this.ranges[i].eq(other.ranges[i], includeAssoc))\n return false;\n return true;\n }\n /**\n Get the primary selection range. Usually, you should make sure\n your code applies to _all_ ranges, by using methods like\n [`changeByRange`](https://codemirror.net/6/docs/ref/#state.EditorState.changeByRange).\n */\n get main() { return this.ranges[this.mainIndex]; }\n /**\n Make sure the selection only has one range. Returns a selection\n holding only the main range from this selection.\n */\n asSingle() {\n return this.ranges.length == 1 ? this : new EditorSelection([this.main], 0);\n }\n /**\n Extend this selection with an extra range.\n */\n addRange(range, main = true) {\n return EditorSelection.create([range].concat(this.ranges), main ? 0 : this.mainIndex + 1);\n }\n /**\n Replace a given range with another range, and then normalize the\n selection to merge and sort ranges if necessary.\n */\n replaceRange(range, which = this.mainIndex) {\n let ranges = this.ranges.slice();\n ranges[which] = range;\n return EditorSelection.create(ranges, this.mainIndex);\n }\n /**\n Convert this selection to an object that can be serialized to\n JSON.\n */\n toJSON() {\n return { ranges: this.ranges.map(r => r.toJSON()), main: this.mainIndex };\n }\n /**\n Create a selection from a JSON representation.\n */\n static fromJSON(json) {\n if (!json || !Array.isArray(json.ranges) || typeof json.main != \"number\" || json.main >= json.ranges.length)\n throw new RangeError(\"Invalid JSON representation for EditorSelection\");\n return new EditorSelection(json.ranges.map((r) => SelectionRange.fromJSON(r)), json.main);\n }\n /**\n Create a selection holding a single range.\n */\n static single(anchor, head = anchor) {\n return new EditorSelection([EditorSelection.range(anchor, head)], 0);\n }\n /**\n Sort and merge the given set of ranges, creating a valid\n selection.\n */\n static create(ranges, mainIndex = 0) {\n if (ranges.length == 0)\n throw new RangeError(\"A selection needs at least one range\");\n for (let pos = 0, i = 0; i < ranges.length; i++) {\n let range = ranges[i];\n if (range.empty ? range.from <= pos : range.from < pos)\n return EditorSelection.normalized(ranges.slice(), mainIndex);\n pos = range.to;\n }\n return new EditorSelection(ranges, mainIndex);\n }\n /**\n Create a cursor selection range at the given position. You can\n safely ignore the optional arguments in most situations.\n */\n static cursor(pos, assoc = 0, bidiLevel, goalColumn) {\n return SelectionRange.create(pos, pos, (assoc == 0 ? 0 : assoc < 0 ? 8 /* RangeFlag.AssocBefore */ : 16 /* RangeFlag.AssocAfter */) |\n (bidiLevel == null ? 7 : Math.min(6, bidiLevel)) |\n ((goalColumn !== null && goalColumn !== void 0 ? goalColumn : 16777215 /* RangeFlag.NoGoalColumn */) << 6 /* RangeFlag.GoalColumnOffset */));\n }\n /**\n Create a selection range.\n */\n static range(anchor, head, goalColumn, bidiLevel, assoc) {\n let flags = ((goalColumn !== null && goalColumn !== void 0 ? goalColumn : 16777215 /* RangeFlag.NoGoalColumn */) << 6 /* RangeFlag.GoalColumnOffset */) |\n (bidiLevel == null ? 7 : Math.min(6, bidiLevel));\n if (!assoc && anchor != head)\n assoc = head < anchor ? 1 : -1;\n return head < anchor ? SelectionRange.create(head, anchor, 32 /* RangeFlag.Inverted */ | 16 /* RangeFlag.AssocAfter */ | flags)\n : SelectionRange.create(anchor, head, (!assoc ? 0 : assoc < 0 ? 8 /* RangeFlag.AssocBefore */ : 16 /* RangeFlag.AssocAfter */) | flags);\n }\n /**\n @internal\n */\n static normalized(ranges, mainIndex = 0) {\n let main = ranges[mainIndex];\n ranges.sort((a, b) => a.from - b.from);\n mainIndex = ranges.indexOf(main);\n for (let i = 1; i < ranges.length; i++) {\n let range = ranges[i], prev = ranges[i - 1];\n if (range.empty ? range.from <= prev.to : range.from < prev.to) {\n let from = prev.from, to = Math.max(range.to, prev.to);\n if (i <= mainIndex)\n mainIndex--;\n ranges.splice(--i, 2, range.anchor > range.head ? EditorSelection.range(to, from) : EditorSelection.range(from, to));\n }\n }\n return new EditorSelection(ranges, mainIndex);\n }\n}\nfunction checkSelection(selection, docLength) {\n for (let range of selection.ranges)\n if (range.to > docLength)\n throw new RangeError(\"Selection points outside of document\");\n}\n\nlet nextID = 0;\n/**\nA facet is a labeled value that is associated with an editor\nstate. It takes inputs from any number of extensions, and combines\nthose into a single output value.\n\nExamples of uses of facets are the [tab\nsize](https://codemirror.net/6/docs/ref/#state.EditorState^tabSize), [editor\nattributes](https://codemirror.net/6/docs/ref/#view.EditorView^editorAttributes), and [update\nlisteners](https://codemirror.net/6/docs/ref/#view.EditorView^updateListener).\n\nNote that `Facet` instances can be used anywhere where\n[`FacetReader`](https://codemirror.net/6/docs/ref/#state.FacetReader) is expected.\n*/\nclass Facet {\n constructor(\n /**\n @internal\n */\n combine, \n /**\n @internal\n */\n compareInput, \n /**\n @internal\n */\n compare, isStatic, enables) {\n this.combine = combine;\n this.compareInput = compareInput;\n this.compare = compare;\n this.isStatic = isStatic;\n /**\n @internal\n */\n this.id = nextID++;\n this.default = combine([]);\n this.extensions = typeof enables == \"function\" ? enables(this) : enables;\n }\n /**\n Returns a facet reader for this facet, which can be used to\n [read](https://codemirror.net/6/docs/ref/#state.EditorState.facet) it but not to define values for it.\n */\n get reader() { return this; }\n /**\n Define a new facet.\n */\n static define(config = {}) {\n return new Facet(config.combine || ((a) => a), config.compareInput || ((a, b) => a === b), config.compare || (!config.combine ? sameArray : (a, b) => a === b), !!config.static, config.enables);\n }\n /**\n Returns an extension that adds the given value to this facet.\n */\n of(value) {\n return new FacetProvider([], this, 0 /* Provider.Static */, value);\n }\n /**\n Create an extension that computes a value for the facet from a\n state. You must take care to declare the parts of the state that\n this value depends on, since your function is only called again\n for a new state when one of those parts changed.\n \n In cases where your value depends only on a single field, you'll\n want to use the [`from`](https://codemirror.net/6/docs/ref/#state.Facet.from) method instead.\n */\n compute(deps, get) {\n if (this.isStatic)\n throw new Error(\"Can't compute a static facet\");\n return new FacetProvider(deps, this, 1 /* Provider.Single */, get);\n }\n /**\n Create an extension that computes zero or more values for this\n facet from a state.\n */\n computeN(deps, get) {\n if (this.isStatic)\n throw new Error(\"Can't compute a static facet\");\n return new FacetProvider(deps, this, 2 /* Provider.Multi */, get);\n }\n from(field, get) {\n if (!get)\n get = x => x;\n return this.compute([field], state => get(state.field(field)));\n }\n}\nfunction sameArray(a, b) {\n return a == b || a.length == b.length && a.every((e, i) => e === b[i]);\n}\nclass FacetProvider {\n constructor(dependencies, facet, type, value) {\n this.dependencies = dependencies;\n this.facet = facet;\n this.type = type;\n this.value = value;\n this.id = nextID++;\n }\n dynamicSlot(addresses) {\n var _a;\n let getter = this.value;\n let compare = this.facet.compareInput;\n let id = this.id, idx = addresses[id] >> 1, multi = this.type == 2 /* Provider.Multi */;\n let depDoc = false, depSel = false, depAddrs = [];\n for (let dep of this.dependencies) {\n if (dep == \"doc\")\n depDoc = true;\n else if (dep == \"selection\")\n depSel = true;\n else if ((((_a = addresses[dep.id]) !== null && _a !== void 0 ? _a : 1) & 1) == 0)\n depAddrs.push(addresses[dep.id]);\n }\n return {\n create(state) {\n state.values[idx] = getter(state);\n return 1 /* SlotStatus.Changed */;\n },\n update(state, tr) {\n if ((depDoc && tr.docChanged) || (depSel && (tr.docChanged || tr.selection)) || ensureAll(state, depAddrs)) {\n let newVal = getter(state);\n if (multi ? !compareArray(newVal, state.values[idx], compare) : !compare(newVal, state.values[idx])) {\n state.values[idx] = newVal;\n return 1 /* SlotStatus.Changed */;\n }\n }\n return 0;\n },\n reconfigure: (state, oldState) => {\n let newVal, oldAddr = oldState.config.address[id];\n if (oldAddr != null) {\n let oldVal = getAddr(oldState, oldAddr);\n if (this.dependencies.every(dep => {\n return dep instanceof Facet ? oldState.facet(dep) === state.facet(dep) :\n dep instanceof StateField ? oldState.field(dep, false) == state.field(dep, false) : true;\n }) || (multi ? compareArray(newVal = getter(state), oldVal, compare) : compare(newVal = getter(state), oldVal))) {\n state.values[idx] = oldVal;\n return 0;\n }\n }\n else {\n newVal = getter(state);\n }\n state.values[idx] = newVal;\n return 1 /* SlotStatus.Changed */;\n }\n };\n }\n}\nfunction compareArray(a, b, compare) {\n if (a.length != b.length)\n return false;\n for (let i = 0; i < a.length; i++)\n if (!compare(a[i], b[i]))\n return false;\n return true;\n}\nfunction ensureAll(state, addrs) {\n let changed = false;\n for (let addr of addrs)\n if (ensureAddr(state, addr) & 1 /* SlotStatus.Changed */)\n changed = true;\n return changed;\n}\nfunction dynamicFacetSlot(addresses, facet, providers) {\n let providerAddrs = providers.map(p => addresses[p.id]);\n let providerTypes = providers.map(p => p.type);\n let dynamic = providerAddrs.filter(p => !(p & 1));\n let idx = addresses[facet.id] >> 1;\n function get(state) {\n let values = [];\n for (let i = 0; i < providerAddrs.length; i++) {\n let value = getAddr(state, providerAddrs[i]);\n if (providerTypes[i] == 2 /* Provider.Multi */)\n for (let val of value)\n values.push(val);\n else\n values.push(value);\n }\n return facet.combine(values);\n }\n return {\n create(state) {\n for (let addr of providerAddrs)\n ensureAddr(state, addr);\n state.values[idx] = get(state);\n return 1 /* SlotStatus.Changed */;\n },\n update(state, tr) {\n if (!ensureAll(state, dynamic))\n return 0;\n let value = get(state);\n if (facet.compare(value, state.values[idx]))\n return 0;\n state.values[idx] = value;\n return 1 /* SlotStatus.Changed */;\n },\n reconfigure(state, oldState) {\n let depChanged = ensureAll(state, providerAddrs);\n let oldProviders = oldState.config.facets[facet.id], oldValue = oldState.facet(facet);\n if (oldProviders && !depChanged && sameArray(providers, oldProviders)) {\n state.values[idx] = oldValue;\n return 0;\n }\n let value = get(state);\n if (facet.compare(value, oldValue)) {\n state.values[idx] = oldValue;\n return 0;\n }\n state.values[idx] = value;\n return 1 /* SlotStatus.Changed */;\n }\n };\n}\nconst initField = /*@__PURE__*/Facet.define({ static: true });\n/**\nFields can store additional information in an editor state, and\nkeep it in sync with the rest of the state.\n*/\nclass StateField {\n constructor(\n /**\n @internal\n */\n id, createF, updateF, compareF, \n /**\n @internal\n */\n spec) {\n this.id = id;\n this.createF = createF;\n this.updateF = updateF;\n this.compareF = compareF;\n this.spec = spec;\n /**\n @internal\n */\n this.provides = undefined;\n }\n /**\n Define a state field.\n */\n static define(config) {\n let field = new StateField(nextID++, config.create, config.update, config.compare || ((a, b) => a === b), config);\n if (config.provide)\n field.provides = config.provide(field);\n return field;\n }\n create(state) {\n let init = state.facet(initField).find(i => i.field == this);\n return ((init === null || init === void 0 ? void 0 : init.create) || this.createF)(state);\n }\n /**\n @internal\n */\n slot(addresses) {\n let idx = addresses[this.id] >> 1;\n return {\n create: (state) => {\n state.values[idx] = this.create(state);\n return 1 /* SlotStatus.Changed */;\n },\n update: (state, tr) => {\n let oldVal = state.values[idx];\n let value = this.updateF(oldVal, tr);\n if (this.compareF(oldVal, value))\n return 0;\n state.values[idx] = value;\n return 1 /* SlotStatus.Changed */;\n },\n reconfigure: (state, oldState) => {\n let init = state.facet(initField), oldInit = oldState.facet(initField), reInit;\n if ((reInit = init.find(i => i.field == this)) && reInit != oldInit.find(i => i.field == this)) {\n state.values[idx] = reInit.create(state);\n return 1 /* SlotStatus.Changed */;\n }\n if (oldState.config.address[this.id] != null) {\n state.values[idx] = oldState.field(this);\n return 0;\n }\n state.values[idx] = this.create(state);\n return 1 /* SlotStatus.Changed */;\n }\n };\n }\n /**\n Returns an extension that enables this field and overrides the\n way it is initialized. Can be useful when you need to provide a\n non-default starting value for the field.\n */\n init(create) {\n return [this, initField.of({ field: this, create })];\n }\n /**\n State field instances can be used as\n [`Extension`](https://codemirror.net/6/docs/ref/#state.Extension) values to enable the field in a\n given state.\n */\n get extension() { return this; }\n}\nconst Prec_ = { lowest: 4, low: 3, default: 2, high: 1, highest: 0 };\nfunction prec(value) {\n return (ext) => new PrecExtension(ext, value);\n}\n/**\nBy default extensions are registered in the order they are found\nin the flattened form of nested array that was provided.\nIndividual extension values can be assigned a precedence to\noverride this. Extensions that do not have a precedence set get\nthe precedence of the nearest parent with a precedence, or\n[`default`](https://codemirror.net/6/docs/ref/#state.Prec.default) if there is no such parent. The\nfinal ordering of extensions is determined by first sorting by\nprecedence and then by order within each precedence.\n*/\nconst Prec = {\n /**\n The highest precedence level, for extensions that should end up\n near the start of the precedence ordering.\n */\n highest: /*@__PURE__*/prec(Prec_.highest),\n /**\n A higher-than-default precedence, for extensions that should\n come before those with default precedence.\n */\n high: /*@__PURE__*/prec(Prec_.high),\n /**\n The default precedence, which is also used for extensions\n without an explicit precedence.\n */\n default: /*@__PURE__*/prec(Prec_.default),\n /**\n A lower-than-default precedence.\n */\n low: /*@__PURE__*/prec(Prec_.low),\n /**\n The lowest precedence level. Meant for things that should end up\n near the end of the extension order.\n */\n lowest: /*@__PURE__*/prec(Prec_.lowest)\n};\nclass PrecExtension {\n constructor(inner, prec) {\n this.inner = inner;\n this.prec = prec;\n }\n}\n/**\nExtension compartments can be used to make a configuration\ndynamic. By [wrapping](https://codemirror.net/6/docs/ref/#state.Compartment.of) part of your\nconfiguration in a compartment, you can later\n[replace](https://codemirror.net/6/docs/ref/#state.Compartment.reconfigure) that part through a\ntransaction.\n*/\nclass Compartment {\n /**\n Create an instance of this compartment to add to your [state\n configuration](https://codemirror.net/6/docs/ref/#state.EditorStateConfig.extensions).\n */\n of(ext) { return new CompartmentInstance(this, ext); }\n /**\n Create an [effect](https://codemirror.net/6/docs/ref/#state.TransactionSpec.effects) that\n reconfigures this compartment.\n */\n reconfigure(content) {\n return Compartment.reconfigure.of({ compartment: this, extension: content });\n }\n /**\n Get the current content of the compartment in the state, or\n `undefined` if it isn't present.\n */\n get(state) {\n return state.config.compartments.get(this);\n }\n}\nclass CompartmentInstance {\n constructor(compartment, inner) {\n this.compartment = compartment;\n this.inner = inner;\n }\n}\nclass Configuration {\n constructor(base, compartments, dynamicSlots, address, staticValues, facets) {\n this.base = base;\n this.compartments = compartments;\n this.dynamicSlots = dynamicSlots;\n this.address = address;\n this.staticValues = staticValues;\n this.facets = facets;\n this.statusTemplate = [];\n while (this.statusTemplate.length < dynamicSlots.length)\n this.statusTemplate.push(0 /* SlotStatus.Unresolved */);\n }\n staticFacet(facet) {\n let addr = this.address[facet.id];\n return addr == null ? facet.default : this.staticValues[addr >> 1];\n }\n static resolve(base, compartments, oldState) {\n let fields = [];\n let facets = Object.create(null);\n let newCompartments = new Map();\n for (let ext of flatten(base, compartments, newCompartments)) {\n if (ext instanceof StateField)\n fields.push(ext);\n else\n (facets[ext.facet.id] || (facets[ext.facet.id] = [])).push(ext);\n }\n let address = Object.create(null);\n let staticValues = [];\n let dynamicSlots = [];\n for (let field of fields) {\n address[field.id] = dynamicSlots.length << 1;\n dynamicSlots.push(a => field.slot(a));\n }\n let oldFacets = oldState === null || oldState === void 0 ? void 0 : oldState.config.facets;\n for (let id in facets) {\n let providers = facets[id], facet = providers[0].facet;\n let oldProviders = oldFacets && oldFacets[id] || [];\n if (providers.every(p => p.type == 0 /* Provider.Static */)) {\n address[facet.id] = (staticValues.length << 1) | 1;\n if (sameArray(oldProviders, providers)) {\n staticValues.push(oldState.facet(facet));\n }\n else {\n let value = facet.combine(providers.map(p => p.value));\n staticValues.push(oldState && facet.compare(value, oldState.facet(facet)) ? oldState.facet(facet) : value);\n }\n }\n else {\n for (let p of providers) {\n if (p.type == 0 /* Provider.Static */) {\n address[p.id] = (staticValues.length << 1) | 1;\n staticValues.push(p.value);\n }\n else {\n address[p.id] = dynamicSlots.length << 1;\n dynamicSlots.push(a => p.dynamicSlot(a));\n }\n }\n address[facet.id] = dynamicSlots.length << 1;\n dynamicSlots.push(a => dynamicFacetSlot(a, facet, providers));\n }\n }\n let dynamic = dynamicSlots.map(f => f(address));\n return new Configuration(base, newCompartments, dynamic, address, staticValues, facets);\n }\n}\nfunction flatten(extension, compartments, newCompartments) {\n let result = [[], [], [], [], []];\n let seen = new Map();\n function inner(ext, prec) {\n let known = seen.get(ext);\n if (known != null) {\n if (known <= prec)\n return;\n let found = result[known].indexOf(ext);\n if (found > -1)\n result[known].splice(found, 1);\n if (ext instanceof CompartmentInstance)\n newCompartments.delete(ext.compartment);\n }\n seen.set(ext, prec);\n if (Array.isArray(ext)) {\n for (let e of ext)\n inner(e, prec);\n }\n else if (ext instanceof CompartmentInstance) {\n if (newCompartments.has(ext.compartment))\n throw new RangeError(`Duplicate use of compartment in extensions`);\n let content = compartments.get(ext.compartment) || ext.inner;\n newCompartments.set(ext.compartment, content);\n inner(content, prec);\n }\n else if (ext instanceof PrecExtension) {\n inner(ext.inner, ext.prec);\n }\n else if (ext instanceof StateField) {\n result[prec].push(ext);\n if (ext.provides)\n inner(ext.provides, prec);\n }\n else if (ext instanceof FacetProvider) {\n result[prec].push(ext);\n if (ext.facet.extensions)\n inner(ext.facet.extensions, Prec_.default);\n }\n else {\n let content = ext.extension;\n if (!content)\n throw new Error(`Unrecognized extension value in extension set (${ext}). This sometimes happens because multiple instances of @codemirror/state are loaded, breaking instanceof checks.`);\n inner(content, prec);\n }\n }\n inner(extension, Prec_.default);\n return result.reduce((a, b) => a.concat(b));\n}\nfunction ensureAddr(state, addr) {\n if (addr & 1)\n return 2 /* SlotStatus.Computed */;\n let idx = addr >> 1;\n let status = state.status[idx];\n if (status == 4 /* SlotStatus.Computing */)\n throw new Error(\"Cyclic dependency between fields and/or facets\");\n if (status & 2 /* SlotStatus.Computed */)\n return status;\n state.status[idx] = 4 /* SlotStatus.Computing */;\n let changed = state.computeSlot(state, state.config.dynamicSlots[idx]);\n return state.status[idx] = 2 /* SlotStatus.Computed */ | changed;\n}\nfunction getAddr(state, addr) {\n return addr & 1 ? state.config.staticValues[addr >> 1] : state.values[addr >> 1];\n}\n\nconst languageData = /*@__PURE__*/Facet.define();\nconst allowMultipleSelections = /*@__PURE__*/Facet.define({\n combine: values => values.some(v => v),\n static: true\n});\nconst lineSeparator = /*@__PURE__*/Facet.define({\n combine: values => values.length ? values[0] : undefined,\n static: true\n});\nconst changeFilter = /*@__PURE__*/Facet.define();\nconst transactionFilter = /*@__PURE__*/Facet.define();\nconst transactionExtender = /*@__PURE__*/Facet.define();\nconst readOnly = /*@__PURE__*/Facet.define({\n combine: values => values.length ? values[0] : false\n});\n\n/**\nAnnotations are tagged values that are used to add metadata to\ntransactions in an extensible way. They should be used to model\nthings that effect the entire transaction (such as its [time\nstamp](https://codemirror.net/6/docs/ref/#state.Transaction^time) or information about its\n[origin](https://codemirror.net/6/docs/ref/#state.Transaction^userEvent)). For effects that happen\n_alongside_ the other changes made by the transaction, [state\neffects](https://codemirror.net/6/docs/ref/#state.StateEffect) are more appropriate.\n*/\nclass Annotation {\n /**\n @internal\n */\n constructor(\n /**\n The annotation type.\n */\n type, \n /**\n The value of this annotation.\n */\n value) {\n this.type = type;\n this.value = value;\n }\n /**\n Define a new type of annotation.\n */\n static define() { return new AnnotationType(); }\n}\n/**\nMarker that identifies a type of [annotation](https://codemirror.net/6/docs/ref/#state.Annotation).\n*/\nclass AnnotationType {\n /**\n Create an instance of this annotation.\n */\n of(value) { return new Annotation(this, value); }\n}\n/**\nRepresentation of a type of state effect. Defined with\n[`StateEffect.define`](https://codemirror.net/6/docs/ref/#state.StateEffect^define).\n*/\nclass StateEffectType {\n /**\n @internal\n */\n constructor(\n // The `any` types in these function types are there to work\n // around TypeScript issue #37631, where the type guard on\n // `StateEffect.is` mysteriously stops working when these properly\n // have type `Value`.\n /**\n @internal\n */\n map) {\n this.map = map;\n }\n /**\n Create a [state effect](https://codemirror.net/6/docs/ref/#state.StateEffect) instance of this\n type.\n */\n of(value) { return new StateEffect(this, value); }\n}\n/**\nState effects can be used to represent additional effects\nassociated with a [transaction](https://codemirror.net/6/docs/ref/#state.Transaction.effects). They\nare often useful to model changes to custom [state\nfields](https://codemirror.net/6/docs/ref/#state.StateField), when those changes aren't implicit in\ndocument or selection changes.\n*/\nclass StateEffect {\n /**\n @internal\n */\n constructor(\n /**\n @internal\n */\n type, \n /**\n The value of this effect.\n */\n value) {\n this.type = type;\n this.value = value;\n }\n /**\n Map this effect through a position mapping. Will return\n `undefined` when that ends up deleting the effect.\n */\n map(mapping) {\n let mapped = this.type.map(this.value, mapping);\n return mapped === undefined ? undefined : mapped == this.value ? this : new StateEffect(this.type, mapped);\n }\n /**\n Tells you whether this effect object is of a given\n [type](https://codemirror.net/6/docs/ref/#state.StateEffectType).\n */\n is(type) { return this.type == type; }\n /**\n Define a new effect type. The type parameter indicates the type\n of values that his effect holds. It should be a type that\n doesn't include `undefined`, since that is used in\n [mapping](https://codemirror.net/6/docs/ref/#state.StateEffect.map) to indicate that an effect is\n removed.\n */\n static define(spec = {}) {\n return new StateEffectType(spec.map || (v => v));\n }\n /**\n Map an array of effects through a change set.\n */\n static mapEffects(effects, mapping) {\n if (!effects.length)\n return effects;\n let result = [];\n for (let effect of effects) {\n let mapped = effect.map(mapping);\n if (mapped)\n result.push(mapped);\n }\n return result;\n }\n}\n/**\nThis effect can be used to reconfigure the root extensions of\nthe editor. Doing this will discard any extensions\n[appended](https://codemirror.net/6/docs/ref/#state.StateEffect^appendConfig), but does not reset\nthe content of [reconfigured](https://codemirror.net/6/docs/ref/#state.Compartment.reconfigure)\ncompartments.\n*/\nStateEffect.reconfigure = /*@__PURE__*/StateEffect.define();\n/**\nAppend extensions to the top-level configuration of the editor.\n*/\nStateEffect.appendConfig = /*@__PURE__*/StateEffect.define();\n/**\nChanges to the editor state are grouped into transactions.\nTypically, a user action creates a single transaction, which may\ncontain any number of document changes, may change the selection,\nor have other effects. Create a transaction by calling\n[`EditorState.update`](https://codemirror.net/6/docs/ref/#state.EditorState.update), or immediately\ndispatch one by calling\n[`EditorView.dispatch`](https://codemirror.net/6/docs/ref/#view.EditorView.dispatch).\n*/\nclass Transaction {\n constructor(\n /**\n The state from which the transaction starts.\n */\n startState, \n /**\n The document changes made by this transaction.\n */\n changes, \n /**\n The selection set by this transaction, or undefined if it\n doesn't explicitly set a selection.\n */\n selection, \n /**\n The effects added to the transaction.\n */\n effects, \n /**\n @internal\n */\n annotations, \n /**\n Whether the selection should be scrolled into view after this\n transaction is dispatched.\n */\n scrollIntoView) {\n this.startState = startState;\n this.changes = changes;\n this.selection = selection;\n this.effects = effects;\n this.annotations = annotations;\n this.scrollIntoView = scrollIntoView;\n /**\n @internal\n */\n this._doc = null;\n /**\n @internal\n */\n this._state = null;\n if (selection)\n checkSelection(selection, changes.newLength);\n if (!annotations.some((a) => a.type == Transaction.time))\n this.annotations = annotations.concat(Transaction.time.of(Date.now()));\n }\n /**\n @internal\n */\n static create(startState, changes, selection, effects, annotations, scrollIntoView) {\n return new Transaction(startState, changes, selection, effects, annotations, scrollIntoView);\n }\n /**\n The new document produced by the transaction. Contrary to\n [`.state`](https://codemirror.net/6/docs/ref/#state.Transaction.state)`.doc`, accessing this won't\n force the entire new state to be computed right away, so it is\n recommended that [transaction\n filters](https://codemirror.net/6/docs/ref/#state.EditorState^transactionFilter) use this getter\n when they need to look at the new document.\n */\n get newDoc() {\n return this._doc || (this._doc = this.changes.apply(this.startState.doc));\n }\n /**\n The new selection produced by the transaction. If\n [`this.selection`](https://codemirror.net/6/docs/ref/#state.Transaction.selection) is undefined,\n this will [map](https://codemirror.net/6/docs/ref/#state.EditorSelection.map) the start state's\n current selection through the changes made by the transaction.\n */\n get newSelection() {\n return this.selection || this.startState.selection.map(this.changes);\n }\n /**\n The new state created by the transaction. Computed on demand\n (but retained for subsequent access), so it is recommended not to\n access it in [transaction\n filters](https://codemirror.net/6/docs/ref/#state.EditorState^transactionFilter) when possible.\n */\n get state() {\n if (!this._state)\n this.startState.applyTransaction(this);\n return this._state;\n }\n /**\n Get the value of the given annotation type, if any.\n */\n annotation(type) {\n for (let ann of this.annotations)\n if (ann.type == type)\n return ann.value;\n return undefined;\n }\n /**\n Indicates whether the transaction changed the document.\n */\n get docChanged() { return !this.changes.empty; }\n /**\n Indicates whether this transaction reconfigures the state\n (through a [configuration compartment](https://codemirror.net/6/docs/ref/#state.Compartment) or\n with a top-level configuration\n [effect](https://codemirror.net/6/docs/ref/#state.StateEffect^reconfigure).\n */\n get reconfigured() { return this.startState.config != this.state.config; }\n /**\n Returns true if the transaction has a [user\n event](https://codemirror.net/6/docs/ref/#state.Transaction^userEvent) annotation that is equal to\n or more specific than `event`. For example, if the transaction\n has `\"select.pointer\"` as user event, `\"select\"` and\n `\"select.pointer\"` will match it.\n */\n isUserEvent(event) {\n let e = this.annotation(Transaction.userEvent);\n return !!(e && (e == event || e.length > event.length && e.slice(0, event.length) == event && e[event.length] == \".\"));\n }\n}\n/**\nAnnotation used to store transaction timestamps. Automatically\nadded to every transaction, holding `Date.now()`.\n*/\nTransaction.time = /*@__PURE__*/Annotation.define();\n/**\nAnnotation used to associate a transaction with a user interface\nevent. Holds a string identifying the event, using a\ndot-separated format to support attaching more specific\ninformation. The events used by the core libraries are:\n\n - `\"input\"` when content is entered\n - `\"input.type\"` for typed input\n - `\"input.type.compose\"` for composition\n - `\"input.paste\"` for pasted input\n - `\"input.drop\"` when adding content with drag-and-drop\n - `\"input.complete\"` when autocompleting\n - `\"delete\"` when the user deletes content\n - `\"delete.selection\"` when deleting the selection\n - `\"delete.forward\"` when deleting forward from the selection\n - `\"delete.backward\"` when deleting backward from the selection\n - `\"delete.cut\"` when cutting to the clipboard\n - `\"move\"` when content is moved\n - `\"move.drop\"` when content is moved within the editor through drag-and-drop\n - `\"select\"` when explicitly changing the selection\n - `\"select.pointer\"` when selecting with a mouse or other pointing device\n - `\"undo\"` and `\"redo\"` for history actions\n\nUse [`isUserEvent`](https://codemirror.net/6/docs/ref/#state.Transaction.isUserEvent) to check\nwhether the annotation matches a given event.\n*/\nTransaction.userEvent = /*@__PURE__*/Annotation.define();\n/**\nAnnotation indicating whether a transaction should be added to\nthe undo history or not.\n*/\nTransaction.addToHistory = /*@__PURE__*/Annotation.define();\n/**\nAnnotation indicating (when present and true) that a transaction\nrepresents a change made by some other actor, not the user. This\nis used, for example, to tag other people's changes in\ncollaborative editing.\n*/\nTransaction.remote = /*@__PURE__*/Annotation.define();\nfunction joinRanges(a, b) {\n let result = [];\n for (let iA = 0, iB = 0;;) {\n let from, to;\n if (iA < a.length && (iB == b.length || b[iB] >= a[iA])) {\n from = a[iA++];\n to = a[iA++];\n }\n else if (iB < b.length) {\n from = b[iB++];\n to = b[iB++];\n }\n else\n return result;\n if (!result.length || result[result.length - 1] < from)\n result.push(from, to);\n else if (result[result.length - 1] < to)\n result[result.length - 1] = to;\n }\n}\nfunction mergeTransaction(a, b, sequential) {\n var _a;\n let mapForA, mapForB, changes;\n if (sequential) {\n mapForA = b.changes;\n mapForB = ChangeSet.empty(b.changes.length);\n changes = a.changes.compose(b.changes);\n }\n else {\n mapForA = b.changes.map(a.changes);\n mapForB = a.changes.mapDesc(b.changes, true);\n changes = a.changes.compose(mapForA);\n }\n return {\n changes,\n selection: b.selection ? b.selection.map(mapForB) : (_a = a.selection) === null || _a === void 0 ? void 0 : _a.map(mapForA),\n effects: StateEffect.mapEffects(a.effects, mapForA).concat(StateEffect.mapEffects(b.effects, mapForB)),\n annotations: a.annotations.length ? a.annotations.concat(b.annotations) : b.annotations,\n scrollIntoView: a.scrollIntoView || b.scrollIntoView\n };\n}\nfunction resolveTransactionInner(state, spec, docSize) {\n let sel = spec.selection, annotations = asArray(spec.annotations);\n if (spec.userEvent)\n annotations = annotations.concat(Transaction.userEvent.of(spec.userEvent));\n return {\n changes: spec.changes instanceof ChangeSet ? spec.changes\n : ChangeSet.of(spec.changes || [], docSize, state.facet(lineSeparator)),\n selection: sel && (sel instanceof EditorSelection ? sel : EditorSelection.single(sel.anchor, sel.head)),\n effects: asArray(spec.effects),\n annotations,\n scrollIntoView: !!spec.scrollIntoView\n };\n}\nfunction resolveTransaction(state, specs, filter) {\n let s = resolveTransactionInner(state, specs.length ? specs[0] : {}, state.doc.length);\n if (specs.length && specs[0].filter === false)\n filter = false;\n for (let i = 1; i < specs.length; i++) {\n if (specs[i].filter === false)\n filter = false;\n let seq = !!specs[i].sequential;\n s = mergeTransaction(s, resolveTransactionInner(state, specs[i], seq ? s.changes.newLength : state.doc.length), seq);\n }\n let tr = Transaction.create(state, s.changes, s.selection, s.effects, s.annotations, s.scrollIntoView);\n return extendTransaction(filter ? filterTransaction(tr) : tr);\n}\n// Finish a transaction by applying filters if necessary.\nfunction filterTransaction(tr) {\n let state = tr.startState;\n // Change filters\n let result = true;\n for (let filter of state.facet(changeFilter)) {\n let value = filter(tr);\n if (value === false) {\n result = false;\n break;\n }\n if (Array.isArray(value))\n result = result === true ? value : joinRanges(result, value);\n }\n if (result !== true) {\n let changes, back;\n if (result === false) {\n back = tr.changes.invertedDesc;\n changes = ChangeSet.empty(state.doc.length);\n }\n else {\n let filtered = tr.changes.filter(result);\n changes = filtered.changes;\n back = filtered.filtered.mapDesc(filtered.changes).invertedDesc;\n }\n tr = Transaction.create(state, changes, tr.selection && tr.selection.map(back), StateEffect.mapEffects(tr.effects, back), tr.annotations, tr.scrollIntoView);\n }\n // Transaction filters\n let filters = state.facet(transactionFilter);\n for (let i = filters.length - 1; i >= 0; i--) {\n let filtered = filters[i](tr);\n if (filtered instanceof Transaction)\n tr = filtered;\n else if (Array.isArray(filtered) && filtered.length == 1 && filtered[0] instanceof Transaction)\n tr = filtered[0];\n else\n tr = resolveTransaction(state, asArray(filtered), false);\n }\n return tr;\n}\nfunction extendTransaction(tr) {\n let state = tr.startState, extenders = state.facet(transactionExtender), spec = tr;\n for (let i = extenders.length - 1; i >= 0; i--) {\n let extension = extenders[i](tr);\n if (extension && Object.keys(extension).length)\n spec = mergeTransaction(spec, resolveTransactionInner(state, extension, tr.changes.newLength), true);\n }\n return spec == tr ? tr : Transaction.create(state, tr.changes, tr.selection, spec.effects, spec.annotations, spec.scrollIntoView);\n}\nconst none = [];\nfunction asArray(value) {\n return value == null ? none : Array.isArray(value) ? value : [value];\n}\n\n/**\nThe categories produced by a [character\ncategorizer](https://codemirror.net/6/docs/ref/#state.EditorState.charCategorizer). These are used\ndo things like selecting by word.\n*/\nvar CharCategory = /*@__PURE__*/(function (CharCategory) {\n /**\n Word characters.\n */\n CharCategory[CharCategory[\"Word\"] = 0] = \"Word\";\n /**\n Whitespace.\n */\n CharCategory[CharCategory[\"Space\"] = 1] = \"Space\";\n /**\n Anything else.\n */\n CharCategory[CharCategory[\"Other\"] = 2] = \"Other\";\nreturn CharCategory})(CharCategory || (CharCategory = {}));\nconst nonASCIISingleCaseWordChar = /[\\u00df\\u0587\\u0590-\\u05f4\\u0600-\\u06ff\\u3040-\\u309f\\u30a0-\\u30ff\\u3400-\\u4db5\\u4e00-\\u9fcc\\uac00-\\ud7af]/;\nlet wordChar;\ntry {\n wordChar = /*@__PURE__*/new RegExp(\"[\\\\p{Alphabetic}\\\\p{Number}_]\", \"u\");\n}\ncatch (_) { }\nfunction hasWordChar(str) {\n if (wordChar)\n return wordChar.test(str);\n for (let i = 0; i < str.length; i++) {\n let ch = str[i];\n if (/\\w/.test(ch) || ch > \"\\x80\" && (ch.toUpperCase() != ch.toLowerCase() || nonASCIISingleCaseWordChar.test(ch)))\n return true;\n }\n return false;\n}\nfunction makeCategorizer(wordChars) {\n return (char) => {\n if (!/\\S/.test(char))\n return CharCategory.Space;\n if (hasWordChar(char))\n return CharCategory.Word;\n for (let i = 0; i < wordChars.length; i++)\n if (char.indexOf(wordChars[i]) > -1)\n return CharCategory.Word;\n return CharCategory.Other;\n };\n}\n\n/**\nThe editor state class is a persistent (immutable) data structure.\nTo update a state, you [create](https://codemirror.net/6/docs/ref/#state.EditorState.update) a\n[transaction](https://codemirror.net/6/docs/ref/#state.Transaction), which produces a _new_ state\ninstance, without modifying the original object.\n\nAs such, _never_ mutate properties of a state directly. That'll\njust break things.\n*/\nclass EditorState {\n constructor(\n /**\n @internal\n */\n config, \n /**\n The current document.\n */\n doc, \n /**\n The current selection.\n */\n selection, \n /**\n @internal\n */\n values, computeSlot, tr) {\n this.config = config;\n this.doc = doc;\n this.selection = selection;\n this.values = values;\n this.status = config.statusTemplate.slice();\n this.computeSlot = computeSlot;\n // Fill in the computed state immediately, so that further queries\n // for it made during the update return this state\n if (tr)\n tr._state = this;\n for (let i = 0; i < this.config.dynamicSlots.length; i++)\n ensureAddr(this, i << 1);\n this.computeSlot = null;\n }\n field(field, require = true) {\n let addr = this.config.address[field.id];\n if (addr == null) {\n if (require)\n throw new RangeError(\"Field is not present in this state\");\n return undefined;\n }\n ensureAddr(this, addr);\n return getAddr(this, addr);\n }\n /**\n Create a [transaction](https://codemirror.net/6/docs/ref/#state.Transaction) that updates this\n state. Any number of [transaction specs](https://codemirror.net/6/docs/ref/#state.TransactionSpec)\n can be passed. Unless\n [`sequential`](https://codemirror.net/6/docs/ref/#state.TransactionSpec.sequential) is set, the\n [changes](https://codemirror.net/6/docs/ref/#state.TransactionSpec.changes) (if any) of each spec\n are assumed to start in the _current_ document (not the document\n produced by previous specs), and its\n [selection](https://codemirror.net/6/docs/ref/#state.TransactionSpec.selection) and\n [effects](https://codemirror.net/6/docs/ref/#state.TransactionSpec.effects) are assumed to refer\n to the document created by its _own_ changes. The resulting\n transaction contains the combined effect of all the different\n specs. For [selection](https://codemirror.net/6/docs/ref/#state.TransactionSpec.selection), later\n specs take precedence over earlier ones.\n */\n update(...specs) {\n return resolveTransaction(this, specs, true);\n }\n /**\n @internal\n */\n applyTransaction(tr) {\n let conf = this.config, { base, compartments } = conf;\n for (let effect of tr.effects) {\n if (effect.is(Compartment.reconfigure)) {\n if (conf) {\n compartments = new Map;\n conf.compartments.forEach((val, key) => compartments.set(key, val));\n conf = null;\n }\n compartments.set(effect.value.compartment, effect.value.extension);\n }\n else if (effect.is(StateEffect.reconfigure)) {\n conf = null;\n base = effect.value;\n }\n else if (effect.is(StateEffect.appendConfig)) {\n conf = null;\n base = asArray(base).concat(effect.value);\n }\n }\n let startValues;\n if (!conf) {\n conf = Configuration.resolve(base, compartments, this);\n let intermediateState = new EditorState(conf, this.doc, this.selection, conf.dynamicSlots.map(() => null), (state, slot) => slot.reconfigure(state, this), null);\n startValues = intermediateState.values;\n }\n else {\n startValues = tr.startState.values.slice();\n }\n let selection = tr.startState.facet(allowMultipleSelections) ? tr.newSelection : tr.newSelection.asSingle();\n new EditorState(conf, tr.newDoc, selection, startValues, (state, slot) => slot.update(state, tr), tr);\n }\n /**\n Create a [transaction spec](https://codemirror.net/6/docs/ref/#state.TransactionSpec) that\n replaces every selection range with the given content.\n */\n replaceSelection(text) {\n if (typeof text == \"string\")\n text = this.toText(text);\n return this.changeByRange(range => ({ changes: { from: range.from, to: range.to, insert: text },\n range: EditorSelection.cursor(range.from + text.length) }));\n }\n /**\n Create a set of changes and a new selection by running the given\n function for each range in the active selection. The function\n can return an optional set of changes (in the coordinate space\n of the start document), plus an updated range (in the coordinate\n space of the document produced by the call's own changes). This\n method will merge all the changes and ranges into a single\n changeset and selection, and return it as a [transaction\n spec](https://codemirror.net/6/docs/ref/#state.TransactionSpec), which can be passed to\n [`update`](https://codemirror.net/6/docs/ref/#state.EditorState.update).\n */\n changeByRange(f) {\n let sel = this.selection;\n let result1 = f(sel.ranges[0]);\n let changes = this.changes(result1.changes), ranges = [result1.range];\n let effects = asArray(result1.effects);\n for (let i = 1; i < sel.ranges.length; i++) {\n let result = f(sel.ranges[i]);\n let newChanges = this.changes(result.changes), newMapped = newChanges.map(changes);\n for (let j = 0; j < i; j++)\n ranges[j] = ranges[j].map(newMapped);\n let mapBy = changes.mapDesc(newChanges, true);\n ranges.push(result.range.map(mapBy));\n changes = changes.compose(newMapped);\n effects = StateEffect.mapEffects(effects, newMapped).concat(StateEffect.mapEffects(asArray(result.effects), mapBy));\n }\n return {\n changes,\n selection: EditorSelection.create(ranges, sel.mainIndex),\n effects\n };\n }\n /**\n Create a [change set](https://codemirror.net/6/docs/ref/#state.ChangeSet) from the given change\n description, taking the state's document length and line\n separator into account.\n */\n changes(spec = []) {\n if (spec instanceof ChangeSet)\n return spec;\n return ChangeSet.of(spec, this.doc.length, this.facet(EditorState.lineSeparator));\n }\n /**\n Using the state's [line\n separator](https://codemirror.net/6/docs/ref/#state.EditorState^lineSeparator), create a\n [`Text`](https://codemirror.net/6/docs/ref/#state.Text) instance from the given string.\n */\n toText(string) {\n return Text.of(string.split(this.facet(EditorState.lineSeparator) || DefaultSplit));\n }\n /**\n Return the given range of the document as a string.\n */\n sliceDoc(from = 0, to = this.doc.length) {\n return this.doc.sliceString(from, to, this.lineBreak);\n }\n /**\n Get the value of a state [facet](https://codemirror.net/6/docs/ref/#state.Facet).\n */\n facet(facet) {\n let addr = this.config.address[facet.id];\n if (addr == null)\n return facet.default;\n ensureAddr(this, addr);\n return getAddr(this, addr);\n }\n /**\n Convert this state to a JSON-serializable object. When custom\n fields should be serialized, you can pass them in as an object\n mapping property names (in the resulting object, which should\n not use `doc` or `selection`) to fields.\n */\n toJSON(fields) {\n let result = {\n doc: this.sliceDoc(),\n selection: this.selection.toJSON()\n };\n if (fields)\n for (let prop in fields) {\n let value = fields[prop];\n if (value instanceof StateField && this.config.address[value.id] != null)\n result[prop] = value.spec.toJSON(this.field(fields[prop]), this);\n }\n return result;\n }\n /**\n Deserialize a state from its JSON representation. When custom\n fields should be deserialized, pass the same object you passed\n to [`toJSON`](https://codemirror.net/6/docs/ref/#state.EditorState.toJSON) when serializing as\n third argument.\n */\n static fromJSON(json, config = {}, fields) {\n if (!json || typeof json.doc != \"string\")\n throw new RangeError(\"Invalid JSON representation for EditorState\");\n let fieldInit = [];\n if (fields)\n for (let prop in fields) {\n if (Object.prototype.hasOwnProperty.call(json, prop)) {\n let field = fields[prop], value = json[prop];\n fieldInit.push(field.init(state => field.spec.fromJSON(value, state)));\n }\n }\n return EditorState.create({\n doc: json.doc,\n selection: EditorSelection.fromJSON(json.selection),\n extensions: config.extensions ? fieldInit.concat([config.extensions]) : fieldInit\n });\n }\n /**\n Create a new state. You'll usually only need this when\n initializing an editor\u2014updated states are created by applying\n transactions.\n */\n static create(config = {}) {\n let configuration = Configuration.resolve(config.extensions || [], new Map);\n let doc = config.doc instanceof Text ? config.doc\n : Text.of((config.doc || \"\").split(configuration.staticFacet(EditorState.lineSeparator) || DefaultSplit));\n let selection = !config.selection ? EditorSelection.single(0)\n : config.selection instanceof EditorSelection ? config.selection\n : EditorSelection.single(config.selection.anchor, config.selection.head);\n checkSelection(selection, doc.length);\n if (!configuration.staticFacet(allowMultipleSelections))\n selection = selection.asSingle();\n return new EditorState(configuration, doc, selection, configuration.dynamicSlots.map(() => null), (state, slot) => slot.create(state), null);\n }\n /**\n The size (in columns) of a tab in the document, determined by\n the [`tabSize`](https://codemirror.net/6/docs/ref/#state.EditorState^tabSize) facet.\n */\n get tabSize() { return this.facet(EditorState.tabSize); }\n /**\n Get the proper [line-break](https://codemirror.net/6/docs/ref/#state.EditorState^lineSeparator)\n string for this state.\n */\n get lineBreak() { return this.facet(EditorState.lineSeparator) || \"\\n\"; }\n /**\n Returns true when the editor is\n [configured](https://codemirror.net/6/docs/ref/#state.EditorState^readOnly) to be read-only.\n */\n get readOnly() { return this.facet(readOnly); }\n /**\n Look up a translation for the given phrase (via the\n [`phrases`](https://codemirror.net/6/docs/ref/#state.EditorState^phrases) facet), or return the\n original string if no translation is found.\n \n If additional arguments are passed, they will be inserted in\n place of markers like `$1` (for the first value) and `$2`, etc.\n A single `$` is equivalent to `$1`, and `$$` will produce a\n literal dollar sign.\n */\n phrase(phrase, ...insert) {\n for (let map of this.facet(EditorState.phrases))\n if (Object.prototype.hasOwnProperty.call(map, phrase)) {\n phrase = map[phrase];\n break;\n }\n if (insert.length)\n phrase = phrase.replace(/\\$(\\$|\\d*)/g, (m, i) => {\n if (i == \"$\")\n return \"$\";\n let n = +(i || 1);\n return !n || n > insert.length ? m : insert[n - 1];\n });\n return phrase;\n }\n /**\n Find the values for a given language data field, provided by the\n the [`languageData`](https://codemirror.net/6/docs/ref/#state.EditorState^languageData) facet.\n \n Examples of language data fields are...\n \n - [`\"commentTokens\"`](https://codemirror.net/6/docs/ref/#commands.CommentTokens) for specifying\n comment syntax.\n - [`\"autocomplete\"`](https://codemirror.net/6/docs/ref/#autocomplete.autocompletion^config.override)\n for providing language-specific completion sources.\n - [`\"wordChars\"`](https://codemirror.net/6/docs/ref/#state.EditorState.charCategorizer) for adding\n characters that should be considered part of words in this\n language.\n - [`\"closeBrackets\"`](https://codemirror.net/6/docs/ref/#autocomplete.CloseBracketConfig) controls\n bracket closing behavior.\n */\n languageDataAt(name, pos, side = -1) {\n let values = [];\n for (let provider of this.facet(languageData)) {\n for (let result of provider(this, pos, side)) {\n if (Object.prototype.hasOwnProperty.call(result, name))\n values.push(result[name]);\n }\n }\n return values;\n }\n /**\n Return a function that can categorize strings (expected to\n represent a single [grapheme cluster](https://codemirror.net/6/docs/ref/#state.findClusterBreak))\n into one of:\n \n - Word (contains an alphanumeric character or a character\n explicitly listed in the local language's `\"wordChars\"`\n language data, which should be a string)\n - Space (contains only whitespace)\n - Other (anything else)\n */\n charCategorizer(at) {\n let chars = this.languageDataAt(\"wordChars\", at);\n return makeCategorizer(chars.length ? chars[0] : \"\");\n }\n /**\n Find the word at the given position, meaning the range\n containing all [word](https://codemirror.net/6/docs/ref/#state.CharCategory.Word) characters\n around it. If no word characters are adjacent to the position,\n this returns null.\n */\n wordAt(pos) {\n let { text, from, length } = this.doc.lineAt(pos);\n let cat = this.charCategorizer(pos);\n let start = pos - from, end = pos - from;\n while (start > 0) {\n let prev = findClusterBreak(text, start, false);\n if (cat(text.slice(prev, start)) != CharCategory.Word)\n break;\n start = prev;\n }\n while (end < length) {\n let next = findClusterBreak(text, end);\n if (cat(text.slice(end, next)) != CharCategory.Word)\n break;\n end = next;\n }\n return start == end ? null : EditorSelection.range(start + from, end + from);\n }\n}\n/**\nA facet that, when enabled, causes the editor to allow multiple\nranges to be selected. Be careful though, because by default the\neditor relies on the native DOM selection, which cannot handle\nmultiple selections. An extension like\n[`drawSelection`](https://codemirror.net/6/docs/ref/#view.drawSelection) can be used to make\nsecondary selections visible to the user.\n*/\nEditorState.allowMultipleSelections = allowMultipleSelections;\n/**\nConfigures the tab size to use in this state. The first\n(highest-precedence) value of the facet is used. If no value is\ngiven, this defaults to 4.\n*/\nEditorState.tabSize = /*@__PURE__*/Facet.define({\n combine: values => values.length ? values[0] : 4\n});\n/**\nThe line separator to use. By default, any of `\"\\n\"`, `\"\\r\\n\"`\nand `\"\\r\"` is treated as a separator when splitting lines, and\nlines are joined with `\"\\n\"`.\n\nWhen you configure a value here, only that precise separator\nwill be used, allowing you to round-trip documents through the\neditor without normalizing line separators.\n*/\nEditorState.lineSeparator = lineSeparator;\n/**\nThis facet controls the value of the\n[`readOnly`](https://codemirror.net/6/docs/ref/#state.EditorState.readOnly) getter, which is\nconsulted by commands and extensions that implement editing\nfunctionality to determine whether they should apply. It\ndefaults to false, but when its highest-precedence value is\n`true`, such functionality disables itself.\n\nNot to be confused with\n[`EditorView.editable`](https://codemirror.net/6/docs/ref/#view.EditorView^editable), which\ncontrols whether the editor's DOM is set to be editable (and\nthus focusable).\n*/\nEditorState.readOnly = readOnly;\n/**\nRegisters translation phrases. The\n[`phrase`](https://codemirror.net/6/docs/ref/#state.EditorState.phrase) method will look through\nall objects registered with this facet to find translations for\nits argument.\n*/\nEditorState.phrases = /*@__PURE__*/Facet.define({\n compare(a, b) {\n let kA = Object.keys(a), kB = Object.keys(b);\n return kA.length == kB.length && kA.every(k => a[k] == b[k]);\n }\n});\n/**\nA facet used to register [language\ndata](https://codemirror.net/6/docs/ref/#state.EditorState.languageDataAt) providers.\n*/\nEditorState.languageData = languageData;\n/**\nFacet used to register change filters, which are called for each\ntransaction (unless explicitly\n[disabled](https://codemirror.net/6/docs/ref/#state.TransactionSpec.filter)), and can suppress\npart of the transaction's changes.\n\nSuch a function can return `true` to indicate that it doesn't\nwant to do anything, `false` to completely stop the changes in\nthe transaction, or a set of ranges in which changes should be\nsuppressed. Such ranges are represented as an array of numbers,\nwith each pair of two numbers indicating the start and end of a\nrange. So for example `[10, 20, 100, 110]` suppresses changes\nbetween 10 and 20, and between 100 and 110.\n*/\nEditorState.changeFilter = changeFilter;\n/**\nFacet used to register a hook that gets a chance to update or\nreplace transaction specs before they are applied. This will\nonly be applied for transactions that don't have\n[`filter`](https://codemirror.net/6/docs/ref/#state.TransactionSpec.filter) set to `false`. You\ncan either return a single transaction spec (possibly the input\ntransaction), or an array of specs (which will be combined in\nthe same way as the arguments to\n[`EditorState.update`](https://codemirror.net/6/docs/ref/#state.EditorState.update)).\n\nWhen possible, it is recommended to avoid accessing\n[`Transaction.state`](https://codemirror.net/6/docs/ref/#state.Transaction.state) in a filter,\nsince it will force creation of a state that will then be\ndiscarded again, if the transaction is actually filtered.\n\n(This functionality should be used with care. Indiscriminately\nmodifying transaction is likely to break something or degrade\nthe user experience.)\n*/\nEditorState.transactionFilter = transactionFilter;\n/**\nThis is a more limited form of\n[`transactionFilter`](https://codemirror.net/6/docs/ref/#state.EditorState^transactionFilter),\nwhich can only add\n[annotations](https://codemirror.net/6/docs/ref/#state.TransactionSpec.annotations) and\n[effects](https://codemirror.net/6/docs/ref/#state.TransactionSpec.effects). _But_, this type\nof filter runs even if the transaction has disabled regular\n[filtering](https://codemirror.net/6/docs/ref/#state.TransactionSpec.filter), making it suitable\nfor effects that don't need to touch the changes or selection,\nbut do want to process every transaction.\n\nExtenders run _after_ filters, when both are present.\n*/\nEditorState.transactionExtender = transactionExtender;\nCompartment.reconfigure = /*@__PURE__*/StateEffect.define();\n\n/**\nUtility function for combining behaviors to fill in a config\nobject from an array of provided configs. `defaults` should hold\ndefault values for all optional fields in `Config`.\n\nThe function will, by default, error\nwhen a field gets two values that aren't `===`-equal, but you can\nprovide combine functions per field to do something else.\n*/\nfunction combineConfig(configs, defaults, // Should hold only the optional properties of Config, but I haven't managed to express that\ncombine = {}) {\n let result = {};\n for (let config of configs)\n for (let key of Object.keys(config)) {\n let value = config[key], current = result[key];\n if (current === undefined)\n result[key] = value;\n else if (current === value || value === undefined) ; // No conflict\n else if (Object.hasOwnProperty.call(combine, key))\n result[key] = combine[key](current, value);\n else\n throw new Error(\"Config merge conflict for field \" + key);\n }\n for (let key in defaults)\n if (result[key] === undefined)\n result[key] = defaults[key];\n return result;\n}\n\n/**\nEach range is associated with a value, which must inherit from\nthis class.\n*/\nclass RangeValue {\n /**\n Compare this value with another value. Used when comparing\n rangesets. The default implementation compares by identity.\n Unless you are only creating a fixed number of unique instances\n of your value type, it is a good idea to implement this\n properly.\n */\n eq(other) { return this == other; }\n /**\n Create a [range](https://codemirror.net/6/docs/ref/#state.Range) with this value.\n */\n range(from, to = from) { return Range.create(from, to, this); }\n}\nRangeValue.prototype.startSide = RangeValue.prototype.endSide = 0;\nRangeValue.prototype.point = false;\nRangeValue.prototype.mapMode = MapMode.TrackDel;\nfunction cmpVal(a, b) {\n return a == b || a.constructor == b.constructor && a.eq(b);\n}\n/**\nA range associates a value with a range of positions.\n*/\nclass Range {\n constructor(\n /**\n The range's start position.\n */\n from, \n /**\n Its end position.\n */\n to, \n /**\n The value associated with this range.\n */\n value) {\n this.from = from;\n this.to = to;\n this.value = value;\n }\n /**\n @internal\n */\n static create(from, to, value) {\n return new Range(from, to, value);\n }\n}\nfunction cmpRange(a, b) {\n return a.from - b.from || a.value.startSide - b.value.startSide;\n}\nclass Chunk {\n constructor(from, to, value, \n // Chunks are marked with the largest point that occurs\n // in them (or -1 for no points), so that scans that are\n // only interested in points (such as the\n // heightmap-related logic) can skip range-only chunks.\n maxPoint) {\n this.from = from;\n this.to = to;\n this.value = value;\n this.maxPoint = maxPoint;\n }\n get length() { return this.to[this.to.length - 1]; }\n // Find the index of the given position and side. Use the ranges'\n // `from` pos when `end == false`, `to` when `end == true`.\n findIndex(pos, side, end, startAt = 0) {\n let arr = end ? this.to : this.from;\n for (let lo = startAt, hi = arr.length;;) {\n if (lo == hi)\n return lo;\n let mid = (lo + hi) >> 1;\n let diff = arr[mid] - pos || (end ? this.value[mid].endSide : this.value[mid].startSide) - side;\n if (mid == lo)\n return diff >= 0 ? lo : hi;\n if (diff >= 0)\n hi = mid;\n else\n lo = mid + 1;\n }\n }\n between(offset, from, to, f) {\n for (let i = this.findIndex(from, -1000000000 /* C.Far */, true), e = this.findIndex(to, 1000000000 /* C.Far */, false, i); i < e; i++)\n if (f(this.from[i] + offset, this.to[i] + offset, this.value[i]) === false)\n return false;\n }\n map(offset, changes) {\n let value = [], from = [], to = [], newPos = -1, maxPoint = -1;\n for (let i = 0; i < this.value.length; i++) {\n let val = this.value[i], curFrom = this.from[i] + offset, curTo = this.to[i] + offset, newFrom, newTo;\n if (curFrom == curTo) {\n let mapped = changes.mapPos(curFrom, val.startSide, val.mapMode);\n if (mapped == null)\n continue;\n newFrom = newTo = mapped;\n if (val.startSide != val.endSide) {\n newTo = changes.mapPos(curFrom, val.endSide);\n if (newTo < newFrom)\n continue;\n }\n }\n else {\n newFrom = changes.mapPos(curFrom, val.startSide);\n newTo = changes.mapPos(curTo, val.endSide);\n if (newFrom > newTo || newFrom == newTo && val.startSide > 0 && val.endSide <= 0)\n continue;\n }\n if ((newTo - newFrom || val.endSide - val.startSide) < 0)\n continue;\n if (newPos < 0)\n newPos = newFrom;\n if (val.point)\n maxPoint = Math.max(maxPoint, newTo - newFrom);\n value.push(val);\n from.push(newFrom - newPos);\n to.push(newTo - newPos);\n }\n return { mapped: value.length ? new Chunk(from, to, value, maxPoint) : null, pos: newPos };\n }\n}\n/**\nA range set stores a collection of [ranges](https://codemirror.net/6/docs/ref/#state.Range) in a\nway that makes them efficient to [map](https://codemirror.net/6/docs/ref/#state.RangeSet.map) and\n[update](https://codemirror.net/6/docs/ref/#state.RangeSet.update). This is an immutable data\nstructure.\n*/\nclass RangeSet {\n constructor(\n /**\n @internal\n */\n chunkPos, \n /**\n @internal\n */\n chunk, \n /**\n @internal\n */\n nextLayer, \n /**\n @internal\n */\n maxPoint) {\n this.chunkPos = chunkPos;\n this.chunk = chunk;\n this.nextLayer = nextLayer;\n this.maxPoint = maxPoint;\n }\n /**\n @internal\n */\n static create(chunkPos, chunk, nextLayer, maxPoint) {\n return new RangeSet(chunkPos, chunk, nextLayer, maxPoint);\n }\n /**\n @internal\n */\n get length() {\n let last = this.chunk.length - 1;\n return last < 0 ? 0 : Math.max(this.chunkEnd(last), this.nextLayer.length);\n }\n /**\n The number of ranges in the set.\n */\n get size() {\n if (this.isEmpty)\n return 0;\n let size = this.nextLayer.size;\n for (let chunk of this.chunk)\n size += chunk.value.length;\n return size;\n }\n /**\n @internal\n */\n chunkEnd(index) {\n return this.chunkPos[index] + this.chunk[index].length;\n }\n /**\n Update the range set, optionally adding new ranges or filtering\n out existing ones.\n \n (Note: The type parameter is just there as a kludge to work\n around TypeScript variance issues that prevented `RangeSet`\n from being a subtype of `RangeSet` when `X` is a subtype of\n `Y`.)\n */\n update(updateSpec) {\n let { add = [], sort = false, filterFrom = 0, filterTo = this.length } = updateSpec;\n let filter = updateSpec.filter;\n if (add.length == 0 && !filter)\n return this;\n if (sort)\n add = add.slice().sort(cmpRange);\n if (this.isEmpty)\n return add.length ? RangeSet.of(add) : this;\n let cur = new LayerCursor(this, null, -1).goto(0), i = 0, spill = [];\n let builder = new RangeSetBuilder();\n while (cur.value || i < add.length) {\n if (i < add.length && (cur.from - add[i].from || cur.startSide - add[i].value.startSide) >= 0) {\n let range = add[i++];\n if (!builder.addInner(range.from, range.to, range.value))\n spill.push(range);\n }\n else if (cur.rangeIndex == 1 && cur.chunkIndex < this.chunk.length &&\n (i == add.length || this.chunkEnd(cur.chunkIndex) < add[i].from) &&\n (!filter || filterFrom > this.chunkEnd(cur.chunkIndex) || filterTo < this.chunkPos[cur.chunkIndex]) &&\n builder.addChunk(this.chunkPos[cur.chunkIndex], this.chunk[cur.chunkIndex])) {\n cur.nextChunk();\n }\n else {\n if (!filter || filterFrom > cur.to || filterTo < cur.from || filter(cur.from, cur.to, cur.value)) {\n if (!builder.addInner(cur.from, cur.to, cur.value))\n spill.push(Range.create(cur.from, cur.to, cur.value));\n }\n cur.next();\n }\n }\n return builder.finishInner(this.nextLayer.isEmpty && !spill.length ? RangeSet.empty\n : this.nextLayer.update({ add: spill, filter, filterFrom, filterTo }));\n }\n /**\n Map this range set through a set of changes, return the new set.\n */\n map(changes) {\n if (changes.empty || this.isEmpty)\n return this;\n let chunks = [], chunkPos = [], maxPoint = -1;\n for (let i = 0; i < this.chunk.length; i++) {\n let start = this.chunkPos[i], chunk = this.chunk[i];\n let touch = changes.touchesRange(start, start + chunk.length);\n if (touch === false) {\n maxPoint = Math.max(maxPoint, chunk.maxPoint);\n chunks.push(chunk);\n chunkPos.push(changes.mapPos(start));\n }\n else if (touch === true) {\n let { mapped, pos } = chunk.map(start, changes);\n if (mapped) {\n maxPoint = Math.max(maxPoint, mapped.maxPoint);\n chunks.push(mapped);\n chunkPos.push(pos);\n }\n }\n }\n let next = this.nextLayer.map(changes);\n return chunks.length == 0 ? next : new RangeSet(chunkPos, chunks, next || RangeSet.empty, maxPoint);\n }\n /**\n Iterate over the ranges that touch the region `from` to `to`,\n calling `f` for each. There is no guarantee that the ranges will\n be reported in any specific order. When the callback returns\n `false`, iteration stops.\n */\n between(from, to, f) {\n if (this.isEmpty)\n return;\n for (let i = 0; i < this.chunk.length; i++) {\n let start = this.chunkPos[i], chunk = this.chunk[i];\n if (to >= start && from <= start + chunk.length &&\n chunk.between(start, from - start, to - start, f) === false)\n return;\n }\n this.nextLayer.between(from, to, f);\n }\n /**\n Iterate over the ranges in this set, in order, including all\n ranges that end at or after `from`.\n */\n iter(from = 0) {\n return HeapCursor.from([this]).goto(from);\n }\n /**\n @internal\n */\n get isEmpty() { return this.nextLayer == this; }\n /**\n Iterate over the ranges in a collection of sets, in order,\n starting from `from`.\n */\n static iter(sets, from = 0) {\n return HeapCursor.from(sets).goto(from);\n }\n /**\n Iterate over two groups of sets, calling methods on `comparator`\n to notify it of possible differences.\n */\n static compare(oldSets, newSets, \n /**\n This indicates how the underlying data changed between these\n ranges, and is needed to synchronize the iteration.\n */\n textDiff, comparator, \n /**\n Can be used to ignore all non-point ranges, and points below\n the given size. When -1, all ranges are compared.\n */\n minPointSize = -1) {\n let a = oldSets.filter(set => set.maxPoint > 0 || !set.isEmpty && set.maxPoint >= minPointSize);\n let b = newSets.filter(set => set.maxPoint > 0 || !set.isEmpty && set.maxPoint >= minPointSize);\n let sharedChunks = findSharedChunks(a, b, textDiff);\n let sideA = new SpanCursor(a, sharedChunks, minPointSize);\n let sideB = new SpanCursor(b, sharedChunks, minPointSize);\n textDiff.iterGaps((fromA, fromB, length) => compare(sideA, fromA, sideB, fromB, length, comparator));\n if (textDiff.empty && textDiff.length == 0)\n compare(sideA, 0, sideB, 0, 0, comparator);\n }\n /**\n Compare the contents of two groups of range sets, returning true\n if they are equivalent in the given range.\n */\n static eq(oldSets, newSets, from = 0, to) {\n if (to == null)\n to = 1000000000 /* C.Far */ - 1;\n let a = oldSets.filter(set => !set.isEmpty && newSets.indexOf(set) < 0);\n let b = newSets.filter(set => !set.isEmpty && oldSets.indexOf(set) < 0);\n if (a.length != b.length)\n return false;\n if (!a.length)\n return true;\n let sharedChunks = findSharedChunks(a, b);\n let sideA = new SpanCursor(a, sharedChunks, 0).goto(from), sideB = new SpanCursor(b, sharedChunks, 0).goto(from);\n for (;;) {\n if (sideA.to != sideB.to ||\n !sameValues(sideA.active, sideB.active) ||\n sideA.point && (!sideB.point || !cmpVal(sideA.point, sideB.point)))\n return false;\n if (sideA.to > to)\n return true;\n sideA.next();\n sideB.next();\n }\n }\n /**\n Iterate over a group of range sets at the same time, notifying\n the iterator about the ranges covering every given piece of\n content. Returns the open count (see\n [`SpanIterator.span`](https://codemirror.net/6/docs/ref/#state.SpanIterator.span)) at the end\n of the iteration.\n */\n static spans(sets, from, to, iterator, \n /**\n When given and greater than -1, only points of at least this\n size are taken into account.\n */\n minPointSize = -1) {\n let cursor = new SpanCursor(sets, null, minPointSize).goto(from), pos = from;\n let openRanges = cursor.openStart;\n for (;;) {\n let curTo = Math.min(cursor.to, to);\n if (cursor.point) {\n let active = cursor.activeForPoint(cursor.to);\n let openCount = cursor.pointFrom < from ? active.length + 1\n : cursor.point.startSide < 0 ? active.length\n : Math.min(active.length, openRanges);\n iterator.point(pos, curTo, cursor.point, active, openCount, cursor.pointRank);\n openRanges = Math.min(cursor.openEnd(curTo), active.length);\n }\n else if (curTo > pos) {\n iterator.span(pos, curTo, cursor.active, openRanges);\n openRanges = cursor.openEnd(curTo);\n }\n if (cursor.to > to)\n return openRanges + (cursor.point && cursor.to > to ? 1 : 0);\n pos = cursor.to;\n cursor.next();\n }\n }\n /**\n Create a range set for the given range or array of ranges. By\n default, this expects the ranges to be _sorted_ (by start\n position and, if two start at the same position,\n `value.startSide`). You can pass `true` as second argument to\n cause the method to sort them.\n */\n static of(ranges, sort = false) {\n let build = new RangeSetBuilder();\n for (let range of ranges instanceof Range ? [ranges] : sort ? lazySort(ranges) : ranges)\n build.add(range.from, range.to, range.value);\n return build.finish();\n }\n /**\n Join an array of range sets into a single set.\n */\n static join(sets) {\n if (!sets.length)\n return RangeSet.empty;\n let result = sets[sets.length - 1];\n for (let i = sets.length - 2; i >= 0; i--) {\n for (let layer = sets[i]; layer != RangeSet.empty; layer = layer.nextLayer)\n result = new RangeSet(layer.chunkPos, layer.chunk, result, Math.max(layer.maxPoint, result.maxPoint));\n }\n return result;\n }\n}\n/**\nThe empty set of ranges.\n*/\nRangeSet.empty = /*@__PURE__*/new RangeSet([], [], null, -1);\nfunction lazySort(ranges) {\n if (ranges.length > 1)\n for (let prev = ranges[0], i = 1; i < ranges.length; i++) {\n let cur = ranges[i];\n if (cmpRange(prev, cur) > 0)\n return ranges.slice().sort(cmpRange);\n prev = cur;\n }\n return ranges;\n}\nRangeSet.empty.nextLayer = RangeSet.empty;\n/**\nA range set builder is a data structure that helps build up a\n[range set](https://codemirror.net/6/docs/ref/#state.RangeSet) directly, without first allocating\nan array of [`Range`](https://codemirror.net/6/docs/ref/#state.Range) objects.\n*/\nclass RangeSetBuilder {\n finishChunk(newArrays) {\n this.chunks.push(new Chunk(this.from, this.to, this.value, this.maxPoint));\n this.chunkPos.push(this.chunkStart);\n this.chunkStart = -1;\n this.setMaxPoint = Math.max(this.setMaxPoint, this.maxPoint);\n this.maxPoint = -1;\n if (newArrays) {\n this.from = [];\n this.to = [];\n this.value = [];\n }\n }\n /**\n Create an empty builder.\n */\n constructor() {\n this.chunks = [];\n this.chunkPos = [];\n this.chunkStart = -1;\n this.last = null;\n this.lastFrom = -1000000000 /* C.Far */;\n this.lastTo = -1000000000 /* C.Far */;\n this.from = [];\n this.to = [];\n this.value = [];\n this.maxPoint = -1;\n this.setMaxPoint = -1;\n this.nextLayer = null;\n }\n /**\n Add a range. Ranges should be added in sorted (by `from` and\n `value.startSide`) order.\n */\n add(from, to, value) {\n if (!this.addInner(from, to, value))\n (this.nextLayer || (this.nextLayer = new RangeSetBuilder)).add(from, to, value);\n }\n /**\n @internal\n */\n addInner(from, to, value) {\n let diff = from - this.lastTo || value.startSide - this.last.endSide;\n if (diff <= 0 && (from - this.lastFrom || value.startSide - this.last.startSide) < 0)\n throw new Error(\"Ranges must be added sorted by `from` position and `startSide`\");\n if (diff < 0)\n return false;\n if (this.from.length == 250 /* C.ChunkSize */)\n this.finishChunk(true);\n if (this.chunkStart < 0)\n this.chunkStart = from;\n this.from.push(from - this.chunkStart);\n this.to.push(to - this.chunkStart);\n this.last = value;\n this.lastFrom = from;\n this.lastTo = to;\n this.value.push(value);\n if (value.point)\n this.maxPoint = Math.max(this.maxPoint, to - from);\n return true;\n }\n /**\n @internal\n */\n addChunk(from, chunk) {\n if ((from - this.lastTo || chunk.value[0].startSide - this.last.endSide) < 0)\n return false;\n if (this.from.length)\n this.finishChunk(true);\n this.setMaxPoint = Math.max(this.setMaxPoint, chunk.maxPoint);\n this.chunks.push(chunk);\n this.chunkPos.push(from);\n let last = chunk.value.length - 1;\n this.last = chunk.value[last];\n this.lastFrom = chunk.from[last] + from;\n this.lastTo = chunk.to[last] + from;\n return true;\n }\n /**\n Finish the range set. Returns the new set. The builder can't be\n used anymore after this has been called.\n */\n finish() { return this.finishInner(RangeSet.empty); }\n /**\n @internal\n */\n finishInner(next) {\n if (this.from.length)\n this.finishChunk(false);\n if (this.chunks.length == 0)\n return next;\n let result = RangeSet.create(this.chunkPos, this.chunks, this.nextLayer ? this.nextLayer.finishInner(next) : next, this.setMaxPoint);\n this.from = null; // Make sure further `add` calls produce errors\n return result;\n }\n}\nfunction findSharedChunks(a, b, textDiff) {\n let inA = new Map();\n for (let set of a)\n for (let i = 0; i < set.chunk.length; i++)\n if (set.chunk[i].maxPoint <= 0)\n inA.set(set.chunk[i], set.chunkPos[i]);\n let shared = new Set();\n for (let set of b)\n for (let i = 0; i < set.chunk.length; i++) {\n let known = inA.get(set.chunk[i]);\n if (known != null && (textDiff ? textDiff.mapPos(known) : known) == set.chunkPos[i] &&\n !(textDiff === null || textDiff === void 0 ? void 0 : textDiff.touchesRange(known, known + set.chunk[i].length)))\n shared.add(set.chunk[i]);\n }\n return shared;\n}\nclass LayerCursor {\n constructor(layer, skip, minPoint, rank = 0) {\n this.layer = layer;\n this.skip = skip;\n this.minPoint = minPoint;\n this.rank = rank;\n }\n get startSide() { return this.value ? this.value.startSide : 0; }\n get endSide() { return this.value ? this.value.endSide : 0; }\n goto(pos, side = -1000000000 /* C.Far */) {\n this.chunkIndex = this.rangeIndex = 0;\n this.gotoInner(pos, side, false);\n return this;\n }\n gotoInner(pos, side, forward) {\n while (this.chunkIndex < this.layer.chunk.length) {\n let next = this.layer.chunk[this.chunkIndex];\n if (!(this.skip && this.skip.has(next) ||\n this.layer.chunkEnd(this.chunkIndex) < pos ||\n next.maxPoint < this.minPoint))\n break;\n this.chunkIndex++;\n forward = false;\n }\n if (this.chunkIndex < this.layer.chunk.length) {\n let rangeIndex = this.layer.chunk[this.chunkIndex].findIndex(pos - this.layer.chunkPos[this.chunkIndex], side, true);\n if (!forward || this.rangeIndex < rangeIndex)\n this.setRangeIndex(rangeIndex);\n }\n this.next();\n }\n forward(pos, side) {\n if ((this.to - pos || this.endSide - side) < 0)\n this.gotoInner(pos, side, true);\n }\n next() {\n for (;;) {\n if (this.chunkIndex == this.layer.chunk.length) {\n this.from = this.to = 1000000000 /* C.Far */;\n this.value = null;\n break;\n }\n else {\n let chunkPos = this.layer.chunkPos[this.chunkIndex], chunk = this.layer.chunk[this.chunkIndex];\n let from = chunkPos + chunk.from[this.rangeIndex];\n this.from = from;\n this.to = chunkPos + chunk.to[this.rangeIndex];\n this.value = chunk.value[this.rangeIndex];\n this.setRangeIndex(this.rangeIndex + 1);\n if (this.minPoint < 0 || this.value.point && this.to - this.from >= this.minPoint)\n break;\n }\n }\n }\n setRangeIndex(index) {\n if (index == this.layer.chunk[this.chunkIndex].value.length) {\n this.chunkIndex++;\n if (this.skip) {\n while (this.chunkIndex < this.layer.chunk.length && this.skip.has(this.layer.chunk[this.chunkIndex]))\n this.chunkIndex++;\n }\n this.rangeIndex = 0;\n }\n else {\n this.rangeIndex = index;\n }\n }\n nextChunk() {\n this.chunkIndex++;\n this.rangeIndex = 0;\n this.next();\n }\n compare(other) {\n return this.from - other.from || this.startSide - other.startSide || this.rank - other.rank ||\n this.to - other.to || this.endSide - other.endSide;\n }\n}\nclass HeapCursor {\n constructor(heap) {\n this.heap = heap;\n }\n static from(sets, skip = null, minPoint = -1) {\n let heap = [];\n for (let i = 0; i < sets.length; i++) {\n for (let cur = sets[i]; !cur.isEmpty; cur = cur.nextLayer) {\n if (cur.maxPoint >= minPoint)\n heap.push(new LayerCursor(cur, skip, minPoint, i));\n }\n }\n return heap.length == 1 ? heap[0] : new HeapCursor(heap);\n }\n get startSide() { return this.value ? this.value.startSide : 0; }\n goto(pos, side = -1000000000 /* C.Far */) {\n for (let cur of this.heap)\n cur.goto(pos, side);\n for (let i = this.heap.length >> 1; i >= 0; i--)\n heapBubble(this.heap, i);\n this.next();\n return this;\n }\n forward(pos, side) {\n for (let cur of this.heap)\n cur.forward(pos, side);\n for (let i = this.heap.length >> 1; i >= 0; i--)\n heapBubble(this.heap, i);\n if ((this.to - pos || this.value.endSide - side) < 0)\n this.next();\n }\n next() {\n if (this.heap.length == 0) {\n this.from = this.to = 1000000000 /* C.Far */;\n this.value = null;\n this.rank = -1;\n }\n else {\n let top = this.heap[0];\n this.from = top.from;\n this.to = top.to;\n this.value = top.value;\n this.rank = top.rank;\n if (top.value)\n top.next();\n heapBubble(this.heap, 0);\n }\n }\n}\nfunction heapBubble(heap, index) {\n for (let cur = heap[index];;) {\n let childIndex = (index << 1) + 1;\n if (childIndex >= heap.length)\n break;\n let child = heap[childIndex];\n if (childIndex + 1 < heap.length && child.compare(heap[childIndex + 1]) >= 0) {\n child = heap[childIndex + 1];\n childIndex++;\n }\n if (cur.compare(child) < 0)\n break;\n heap[childIndex] = cur;\n heap[index] = child;\n index = childIndex;\n }\n}\nclass SpanCursor {\n constructor(sets, skip, minPoint) {\n this.minPoint = minPoint;\n this.active = [];\n this.activeTo = [];\n this.activeRank = [];\n this.minActive = -1;\n // A currently active point range, if any\n this.point = null;\n this.pointFrom = 0;\n this.pointRank = 0;\n this.to = -1000000000 /* C.Far */;\n this.endSide = 0;\n // The amount of open active ranges at the start of the iterator.\n // Not including points.\n this.openStart = -1;\n this.cursor = HeapCursor.from(sets, skip, minPoint);\n }\n goto(pos, side = -1000000000 /* C.Far */) {\n this.cursor.goto(pos, side);\n this.active.length = this.activeTo.length = this.activeRank.length = 0;\n this.minActive = -1;\n this.to = pos;\n this.endSide = side;\n this.openStart = -1;\n this.next();\n return this;\n }\n forward(pos, side) {\n while (this.minActive > -1 && (this.activeTo[this.minActive] - pos || this.active[this.minActive].endSide - side) < 0)\n this.removeActive(this.minActive);\n this.cursor.forward(pos, side);\n }\n removeActive(index) {\n remove(this.active, index);\n remove(this.activeTo, index);\n remove(this.activeRank, index);\n this.minActive = findMinIndex(this.active, this.activeTo);\n }\n addActive(trackOpen) {\n let i = 0, { value, to, rank } = this.cursor;\n // Organize active marks by rank first, then by size\n while (i < this.activeRank.length && (rank - this.activeRank[i] || to - this.activeTo[i]) > 0)\n i++;\n insert(this.active, i, value);\n insert(this.activeTo, i, to);\n insert(this.activeRank, i, rank);\n if (trackOpen)\n insert(trackOpen, i, this.cursor.from);\n this.minActive = findMinIndex(this.active, this.activeTo);\n }\n // After calling this, if `this.point` != null, the next range is a\n // point. Otherwise, it's a regular range, covered by `this.active`.\n next() {\n let from = this.to, wasPoint = this.point;\n this.point = null;\n let trackOpen = this.openStart < 0 ? [] : null;\n for (;;) {\n let a = this.minActive;\n if (a > -1 && (this.activeTo[a] - this.cursor.from || this.active[a].endSide - this.cursor.startSide) < 0) {\n if (this.activeTo[a] > from) {\n this.to = this.activeTo[a];\n this.endSide = this.active[a].endSide;\n break;\n }\n this.removeActive(a);\n if (trackOpen)\n remove(trackOpen, a);\n }\n else if (!this.cursor.value) {\n this.to = this.endSide = 1000000000 /* C.Far */;\n break;\n }\n else if (this.cursor.from > from) {\n this.to = this.cursor.from;\n this.endSide = this.cursor.startSide;\n break;\n }\n else {\n let nextVal = this.cursor.value;\n if (!nextVal.point) { // Opening a range\n this.addActive(trackOpen);\n this.cursor.next();\n }\n else if (wasPoint && this.cursor.to == this.to && this.cursor.from < this.cursor.to) {\n // Ignore any non-empty points that end precisely at the end of the prev point\n this.cursor.next();\n }\n else { // New point\n this.point = nextVal;\n this.pointFrom = this.cursor.from;\n this.pointRank = this.cursor.rank;\n this.to = this.cursor.to;\n this.endSide = nextVal.endSide;\n this.cursor.next();\n this.forward(this.to, this.endSide);\n break;\n }\n }\n }\n if (trackOpen) {\n this.openStart = 0;\n for (let i = trackOpen.length - 1; i >= 0 && trackOpen[i] < from; i--)\n this.openStart++;\n }\n }\n activeForPoint(to) {\n if (!this.active.length)\n return this.active;\n let active = [];\n for (let i = this.active.length - 1; i >= 0; i--) {\n if (this.activeRank[i] < this.pointRank)\n break;\n if (this.activeTo[i] > to || this.activeTo[i] == to && this.active[i].endSide >= this.point.endSide)\n active.push(this.active[i]);\n }\n return active.reverse();\n }\n openEnd(to) {\n let open = 0;\n for (let i = this.activeTo.length - 1; i >= 0 && this.activeTo[i] > to; i--)\n open++;\n return open;\n }\n}\nfunction compare(a, startA, b, startB, length, comparator) {\n a.goto(startA);\n b.goto(startB);\n let endB = startB + length;\n let pos = startB, dPos = startB - startA;\n let bounds = !!comparator.boundChange;\n for (let boundChange = false;;) {\n let dEnd = (a.to + dPos) - b.to, diff = dEnd || a.endSide - b.endSide;\n let end = diff < 0 ? a.to + dPos : b.to, clipEnd = Math.min(end, endB);\n let point = a.point || b.point;\n if (point) {\n if (!(a.point && b.point && cmpVal(a.point, b.point) &&\n sameValues(a.activeForPoint(a.to), b.activeForPoint(b.to))))\n comparator.comparePoint(pos, clipEnd, a.point, b.point);\n boundChange = false;\n }\n else {\n if (boundChange)\n comparator.boundChange(pos);\n if (clipEnd > pos && !sameValues(a.active, b.active))\n comparator.compareRange(pos, clipEnd, a.active, b.active);\n if (bounds && clipEnd < endB && (dEnd || a.openEnd(end) != b.openEnd(end)))\n boundChange = true;\n }\n if (end > endB)\n break;\n pos = end;\n if (diff <= 0)\n a.next();\n if (diff >= 0)\n b.next();\n }\n}\nfunction sameValues(a, b) {\n if (a.length != b.length)\n return false;\n for (let i = 0; i < a.length; i++)\n if (a[i] != b[i] && !cmpVal(a[i], b[i]))\n return false;\n return true;\n}\nfunction remove(array, index) {\n for (let i = index, e = array.length - 1; i < e; i++)\n array[i] = array[i + 1];\n array.pop();\n}\nfunction insert(array, index, value) {\n for (let i = array.length - 1; i >= index; i--)\n array[i + 1] = array[i];\n array[index] = value;\n}\nfunction findMinIndex(value, array) {\n let found = -1, foundPos = 1000000000 /* C.Far */;\n for (let i = 0; i < array.length; i++)\n if ((array[i] - foundPos || value[i].endSide - value[found].endSide) < 0) {\n found = i;\n foundPos = array[i];\n }\n return found;\n}\n\n/**\nCount the column position at the given offset into the string,\ntaking extending characters and tab size into account.\n*/\nfunction countColumn(string, tabSize, to = string.length) {\n let n = 0;\n for (let i = 0; i < to && i < string.length;) {\n if (string.charCodeAt(i) == 9) {\n n += tabSize - (n % tabSize);\n i++;\n }\n else {\n n++;\n i = findClusterBreak(string, i);\n }\n }\n return n;\n}\n/**\nFind the offset that corresponds to the given column position in a\nstring, taking extending characters and tab size into account. By\ndefault, the string length is returned when it is too short to\nreach the column. Pass `strict` true to make it return -1 in that\nsituation.\n*/\nfunction findColumn(string, col, tabSize, strict) {\n for (let i = 0, n = 0;;) {\n if (n >= col)\n return i;\n if (i == string.length)\n break;\n n += string.charCodeAt(i) == 9 ? tabSize - (n % tabSize) : 1;\n i = findClusterBreak(string, i);\n }\n return strict === true ? -1 : string.length;\n}\n\nexport { Annotation, AnnotationType, ChangeDesc, ChangeSet, CharCategory, Compartment, EditorSelection, EditorState, Facet, Line, MapMode, Prec, Range, RangeSet, RangeSetBuilder, RangeValue, SelectionRange, StateEffect, StateEffectType, StateField, Text, Transaction, codePointAt, codePointSize, combineConfig, countColumn, findClusterBreak, findColumn, fromCodePoint };\n", "const C = \"\\u037c\"\nconst COUNT = typeof Symbol == \"undefined\" ? \"__\" + C : Symbol.for(C)\nconst SET = typeof Symbol == \"undefined\" ? \"__styleSet\" + Math.floor(Math.random() * 1e8) : Symbol(\"styleSet\")\nconst top = typeof globalThis != \"undefined\" ? globalThis : typeof window != \"undefined\" ? window : {}\n\n// :: - Style modules encapsulate a set of CSS rules defined from\n// JavaScript. Their definitions are only available in a given DOM\n// root after it has been _mounted_ there with `StyleModule.mount`.\n//\n// Style modules should be created once and stored somewhere, as\n// opposed to re-creating them every time you need them. The amount of\n// CSS rules generated for a given DOM root is bounded by the amount\n// of style modules that were used. So to avoid leaking rules, don't\n// create these dynamically, but treat them as one-time allocations.\nexport class StyleModule {\n // :: (Object
    {{ template "public-navigation" . }} -
    {{ if .HTML }}
    {{ .HTML }}
    {{ else if eq .Media "image" }}{{ .Title }}{{ else if eq .Media "pdf" }}{{ else if eq .Media "audio" }}{{ else if eq .Media "video" }}{{ else if .Text }}
    {{ .Text }}
    {{ end }}
    +
    {{ if .HTML }}
    {{ .HTML }}
    {{ else if eq .Media "image" }}{{ .Title }}{{ else if eq .Media "pdf" }}{{ else if eq .Media "audio" }}{{ else if eq .Media "video" }}{{ else if .Text }}
    {{ .Text }}
    {{ end }}
    {{ end }} diff --git a/templates/pages/browser.gohtml b/templates/pages/browser.gohtml index 6d37cdb..acf9746 100644 --- a/templates/pages/browser.gohtml +++ b/templates/pages/browser.gohtml @@ -1,9 +1,10 @@ {{ define "browser.gohtml" }} - + + Browser @@ -58,9 +59,25 @@ .browser-main { padding: 1rem; + min-width: 0; overflow: auto; } +.browser-address { display:flex; gap:.35rem; align-items:center; flex:1; min-width:8rem; max-width:42rem; margin:0 .8rem; } +.browser-address input { flex:1; min-width:0; width:100%; font-family:monospace; background:var(--input-bg); color:var(--fg); border:1px solid var(--border-soft); border-radius:.3rem; padding:.4rem .6rem; } +.browser-address-error { color:var(--danger,#c34747); margin:0; position:absolute; top:100%; background:var(--bg); } +.topbar:has(.browser-address){position:relative}.browser-main{padding-bottom:4.5rem} +@media(max-width:1100px){.topbar .browser-address{display:none}} +.browser-create-dialog { box-sizing:border-box; width:min(32rem,calc(100vw - 2rem)); color:var(--fg); background:var(--bg); border:1px solid var(--border-soft); border-radius:.6rem; padding:1rem; } +.browser-create-dialog::backdrop { background:rgb(0 0 0 / .45); } +.browser-create-dialog h2 { font-size:1.1rem; margin:0 0 .8rem; } +.browser-create-dialog label { display:grid; gap:.35rem; } +.browser-create-dialog [hidden] { display:none !important; } +.browser-create-dialog input { box-sizing:border-box; width:100%; padding:.5rem; color:var(--input-fg); background:var(--input-bg); border:1px solid var(--border-soft); } +.browser-create-dialog p { font-size:.85rem; } +.browser-create-dialog footer { display:flex; justify-content:flex-end; gap:.5rem; } +.browser-empty { color:var(--muted); } + .browser-toolbar { display: flex; gap: 0.5rem; @@ -259,7 +276,7 @@ position: fixed; z-index: 1200; left: 1rem; - bottom: 1rem; + bottom: max(4.5rem, calc(env(safe-area-inset-bottom) + 4rem)); width: min(22rem, calc(100vw - 2rem)); display: grid; gap: .45rem; @@ -371,10 +388,24 @@ display: none; } +@media(max-width:800px){ + .browser-page .topbar{flex-wrap:wrap;gap:.3rem;padding:.5rem} + .browser-page .topbar .brand{flex:1 0 100%} + .browser-page .topbar>nav{order:5;flex:1 0 100%;flex-wrap:nowrap;overflow:auto;gap:0} + .browser-page .topbar>nav>a{font-size:.85rem;padding:.25rem .4rem} + .browser-page .topbar .city-map-launch{margin:0;flex-shrink:0} + .browser-page .topbar>button{flex-shrink:0} + .browser-layout,.browser-layout.no-sidebar,.browser-layout.no-preview,.browser-layout.no-sidebar.no-preview{display:block;min-height:calc(100dvh - 8rem)} + .browser-layout .browser-sidebar,.browser-layout .browser-preview,.browser-layout .resize-handle{display:none} + .browser-layout.mobile-sidebar .browser-sidebar,.browser-layout.mobile-preview .browser-preview{display:block;position:fixed;inset:8rem .5rem 4.5rem;z-index:60;box-sizing:border-box;border:1px solid var(--border);background:var(--bg);box-shadow:0 1rem 2rem #0006} + .browser-main{padding:.65rem .65rem 4.5rem} + .browser-grid{grid-template-columns:repeat(auto-fill,minmax(8rem,1fr))} +} + - +
    @@ -386,6 +417,12 @@ {{ .Label }} {{ end }} +
    + + + + +
    {{ template "auth-menu" . }}
    @@ -422,6 +459,9 @@
    + + + -
    -{{ .Path }} -
    +{{ if not .Entries }}

    {{ T "browser.empty" }}

    {{ end }}
    @@ -614,6 +652,12 @@ function toggleSidebar() { const layout = document.getElementById("browser-layout") + if (matchMedia('(max-width:800px)').matches) { + layout.classList.toggle('mobile-sidebar') + layout.classList.remove('mobile-preview') + return + } + layout.classList.toggle("no-sidebar") localStorage.setItem( @@ -626,6 +670,13 @@ function togglePreview() { const layout = document.getElementById("browser-layout") + if (matchMedia('(max-width:800px)').matches) { + layout.classList.toggle('mobile-preview') + layout.classList.remove('mobile-sidebar') + updatePaneButtons() + return + } + layout.classList.toggle("no-preview") localStorage.setItem( @@ -645,7 +696,7 @@ function updatePaneButtons() { if (previewBtn) { const hidden = - layout.classList.contains("no-preview") + matchMedia('(max-width:800px)').matches ? !layout.classList.contains('mobile-preview') : layout.classList.contains("no-preview") previewBtn.textContent = hidden ? "◨" : "◧" @@ -902,7 +953,7 @@ async function ctxPaste() { clipboard = null } if (!clipboard?.paths?.length) return - const directory = document.querySelector(".browser-path").textContent.trim() || "/" + const directory = document.querySelector("[data-browser-address]").dataset.path || "/" for (const sourcePath of clipboard.paths) { const basename = sourcePath.replace(/\/+$/, "").split("/").pop() const sourceDirectory = sourcePath.slice(0, sourcePath.lastIndexOf("/")) || "/" @@ -1061,6 +1112,17 @@ restoreViewMode() restorePanes() + +
    +

    {{ T "browser.new" }}

    + + +

    {{ T "browser.create_help" }}

    +

    {{ T "browser.folder_help" }}

    + +
    +
    +
    diff --git a/templates/pages/city_map.gohtml b/templates/pages/city_map.gohtml index b05d3d8..8141773 100644 --- a/templates/pages/city_map.gohtml +++ b/templates/pages/city_map.gohtml @@ -21,4 +21,4 @@

    {{ end }} {{ template "city-governance" . }}
    -{{ end }} +{{ end }} diff --git a/templates/pages/city_node.gohtml b/templates/pages/city_node.gohtml index 2b74bdb..712528f 100644 --- a/templates/pages/city_node.gohtml +++ b/templates/pages/city_node.gohtml @@ -1,11 +1,11 @@ {{ define "city_node.gohtml" }} -{{ .Node.Title }} · {{ .WikiName }} +{{ .Node.DisplayTitle }} · {{ .WikiName }} {{ template "social-metadata" . }}
    {{ T "city.map" }}{{ template "auth-menu" . }}
    -
    -

    {{ .Node.Kind }} · {{ .Node.State }}

    {{ .Node.Title }}

    {{ template "city-governance" . }}

    {{ .Node.Description }}

    +
    +

    {{ .Node.Kind }} · {{ .Node.State }}

    {{ .Node.DisplayTitle }}

    {{ template "city-governance" . }}

    {{ .Node.Description }}

    UUID
    {{ .Node.UUID }}
    {{ T "config.name" }}
    {{ .Node.Name }}
    Path
    {{ if .CanPlan }}
    {{ else }}{{ .Node.PathPrefix }}{{ end }}
    Version
    {{ .Node.Version }}
    {{ if eq .Node.Kind "spot" }}

    Pages

    {{ end }}

    {{ T "city.show_on_map" }}

    diff --git a/templates/pages/config.gohtml b/templates/pages/config.gohtml index 77832f0..6d076c2 100644 --- a/templates/pages/config.gohtml +++ b/templates/pages/config.gohtml @@ -1165,7 +1165,10 @@ Vim
    {{ range .AuthPrincipals }}{{ else }}{{ end }}
    {{ T "auth.display_name" }}{{ T "auth.email" }}{{ T "config.status" }}{{ T "config.operations" }}
    {{ .DisplayName }}{{ .Email }}{{ T (printf "auth.state_%s" .State) }}
    —
    -

    {{ T "auth.sessions" }}

    {{ range .AuthSessions }}{{ else }}{{ end }}
    {{ T "auth.account" }}{{ T "auth.auth_time" }}{{ T "auth.last_seen" }}{{ T "auth.expires" }}MFA
    {{ .DisplayName }}{{ formatUnixMS .AuthTimeMS }}{{ formatUnixMS .LastSeenAtMS }}{{ formatUnixMS .ExpiresAtMS }}{{ .MFALevel }}
    —
    +

    {{ T "auth.sessions" }}

    + +

    {{ T "auth.session_policy_help" }}

    +
    {{ range .AuthSessions }}{{ else }}{{ end }}
    {{ T "auth.account" }}{{ T "auth.auth_time" }}{{ T "auth.last_seen" }}{{ T "auth.expires" }}MFA
    {{ .DisplayName }}{{ formatUnixMS .AuthTimeMS }}{{ formatUnixMS .LastSeenAtMS }}{{ formatUnixMS .ExpiresAtMS }}{{ .MFALevel }}
    —

    {{ T "config.event_definitions" }}

    diff --git a/templates/pages/editor.gohtml b/templates/pages/editor.gohtml index 468dc8f..a4fab6e 100644 --- a/templates/pages/editor.gohtml +++ b/templates/pages/editor.gohtml @@ -28,13 +28,13 @@ @@ -51,6 +51,10 @@
    MODE - + {{ .EditType }}
    + Ln 1, Col 1 @@ -228,6 +236,7 @@ Ln 1, Col 1
    + - + + @@ -301,27 +311,7 @@ window.addEventListener( setTimeout(() => cm.focus(), 0) window.PagesWikiGetEditorText = - () => cm.state.doc.toString() - - const syncTextarea = - () => { - textarea.value = - cm.state.doc.toString() - } - - const oldSaveNow = - window.PagesEditor && window.PagesEditor.saveNow - - if (oldSaveNow) { - window.PagesEditor.saveNow = - function() { - syncTextarea() - return oldSaveNow.apply( - window.PagesEditor, - arguments - ) - } - } + () => window.PagesWikiCodeMirror.state.doc.toString() } ) @@ -430,6 +420,7 @@ window.addEventListener( ) function switchEditorFormat(format) { + if (window.PagesEditor?.switchFormat) return window.PagesEditor.switchFormat(format) const url = new URL(location.href) diff --git a/templates/pages/home.gohtml b/templates/pages/home.gohtml index 87039dc..19065dd 100644 --- a/templates/pages/home.gohtml +++ b/templates/pages/home.gohtml @@ -13,6 +13,7 @@ + @@ -25,7 +26,7 @@