#!/usr/bin/env python3
"""Pages Wiki data CLI.
Provides one-shot, rsync-like transfers and an interactive semantic-path shell.
Uses only the Python standard library.
"""
from __future__ import annotations
import difflib
import fcntl
import fnmatch
import glob
import hashlib
import html
import json
import mimetypes
import os
from pathlib import Path
import posixpath
import re
import shlex
import subprocess
import sqlite3
import sys
import tempfile
import time
import uuid
try:
import readline
except ImportError:
readline = None
class CLIError(Exception):
pass
def looks_like_text(value: str) -> bool:
if not value:
return True
controls = sum(
1
for char in value
if ord(char) < 32 and char not in "\n\r\t\f"
)
return controls * 100 <= len(value) * 2
def decode_text(payload: bytes) -> tuple[str, str] | None:
candidates: list[tuple[str, str]] = []
if payload.startswith(b"\xef\xbb\xbf"):
candidates.append(("utf-8-sig", "utf-8"))
elif payload.startswith(b"\xff\xfe"):
candidates.append(("utf-16", "utf-16le"))
elif payload.startswith(b"\xfe\xff"):
candidates.append(("utf-16", "utf-16be"))
else:
candidates.append(("utf-8", "utf-8"))
if b"\x1b$" in payload:
candidates.append(("iso2022_jp", "iso-2022-jp"))
candidates.extend(
[
("shift_jis", "shift_jis"),
("euc_jp", "euc-jp"),
]
)
for codec, label in candidates:
try:
value = payload.decode(codec)
except UnicodeDecodeError:
continue
if looks_like_text(value):
return value, label
return None
def detect_content_type(payload: bytes, filename: str = "") -> str:
guessed = mimetypes.guess_type(filename)[0] if filename else None
signatures = (
(b"%PDF-", "application/pdf"),
(b"\x89PNG\r\n\x1a\n", "image/png"),
(b"\xff\xd8\xff", "image/jpeg"),
(b"GIF87a", "image/gif"),
(b"GIF89a", "image/gif"),
(b"\x1f\x8b", "application/gzip"),
(b"BZh", "application/x-bzip2"),
(b"\xfd7zXZ\x00", "application/x-xz"),
(b"7z\xbc\xaf\x27\x1c", "application/x-7z-compressed"),
(b"\x7fELF", "application/x-executable"),
(b"\x00asm", "application/wasm"),
(b"SQLite format 3\x00", "application/vnd.sqlite3"),
(b"OggS", "application/ogg"),
(b"fLaC", "audio/flac"),
)
for signature, content_type in signatures:
if payload.startswith(signature):
return content_type
if payload.startswith(b"RIFF") and payload[8:12] == b"WEBP":
return "image/webp"
if len(payload) >= 12 and payload[4:8] == b"ftyp":
return "video/mp4"
if payload.startswith(b"PK\x03\x04"):
if guessed and (
guessed.startswith("application/vnd.")
or guessed == "application/epub+zip"
):
return guessed
return "application/zip"
decoded = decode_text(payload)
if decoded is None:
return guessed or "application/octet-stream"
text, charset = decoded
first_line = text.splitlines()[0].lower() if text.splitlines() else ""
if first_line.startswith("#!") and "python" in first_line:
return f"text/x-python; charset={charset}"
if first_line.startswith("#!") and any(
shell in first_line for shell in ("/sh", "bash", "zsh")
):
return f"application/x-sh; charset={charset}"
if first_line.startswith("#!") and any(
runtime in first_line for runtime in ("node", "deno")
):
return f"application/javascript; charset={charset}"
try:
json.loads(text)
except (json.JSONDecodeError, ValueError):
pass
else:
return f"application/json; charset={charset}"
if text.lstrip().startswith(" str:
"""Convert common Markdown constructs to canonical HTML."""
def inline(value: str) -> str:
value = html.escape(value, quote=False)
value = re.sub(
r"`([^`]+)`",
lambda match: f"{match.group(1)}",
value,
)
value = re.sub(
r"\[([^]]+)]\(([^)]+)\)",
lambda match: (
f''
f"{match.group(1)}"
),
value,
)
value = re.sub(r"\*\*([^*]+)\*\*", r"\1", value)
value = re.sub(r"(?\1", value)
return value
output: list[str] = []
paragraph: list[str] = []
list_kind: str | None = None
in_code = False
code_language = ""
code_lines: list[str] = []
def flush_paragraph() -> None:
if paragraph:
output.append("
" + inline(" ".join(paragraph)) + "
") paragraph.clear() def close_list() -> None: nonlocal list_kind if list_kind: output.append(f"{list_kind}>") list_kind = None for raw_line in source.splitlines(): line = raw_line.rstrip() if line.startswith("```"): flush_paragraph() close_list() if in_code: language_class = ( f' class="language-{html.escape(code_language, quote=True)}"' if code_language else "" ) output.append( f""
+ html.escape("\n".join(code_lines))
+ ""
)
code_lines.clear()
in_code = False
else:
in_code = True
code_language = line[3:].strip()
continue
if in_code:
code_lines.append(raw_line)
continue
if not line.strip():
flush_paragraph()
close_list()
continue
heading = re.match(r"^(#{1,6})\s+(.+)$", line)
if heading:
flush_paragraph()
close_list()
level = len(heading.group(1))
output.append(f"") continue paragraph.append(line.strip()) flush_paragraph() close_list() if in_code: output.append("" + inline(line[2:]) + "
" + html.escape("\n".join(code_lines)) + "")
return "\n".join(output)
HELP = """\
Usage:
pageswiki-cli COMMAND [ARG ...]
pageswiki-cli ADDRESS
pageswiki-cli shell
Commands:
ls [PATH] list direct children
cd PATH change semantic working path (shell)
mv SOURCE DEST move a path and its descendants
view [ADDRESS] show metadata and payload
cat [ADDRESS] write only the payload to stdout
edit [ADDRESS] edit through $EDITOR and write changes
get ADDRESS [LOCAL] download unless content is unchanged
put LOCAL ADDRESS upload unless content is unchanged
mget REMOTE_PATTERN [LOCAL_DIR]
download matching semantic paths
mput LOCAL_GLOB REMOTE_DIR
upload matching local files
revision [ADDRESS] list Page revisions
diff REVISION REVISION unified Document diff
restore REVISION create a new Revision from an old one
link ADDRESS PATH bind another path to a Page
unlink PATH [ADDRESS] remove a path binding
domain [NAME|UUID|HOST] show or select the active Domain
extension [ID] [on|off] list or toggle a system extension
pwd print semantic working path
help show this help
exit | quit leave the interactive shell
Addresses:
path:/semantic/path
uuid:PAGE-UUID
rev:REVISION-UUID
asset:TYPE:NAME
Environment:
PAGESWIKI_DATA_DIR directory containing pages.db and documents/
PAGESWIKI_DB explicit SQLite database path
PAGESWIKI_DOCUMENTS_DIR explicit content-addressed Document directory
PAGESWIKI_LANG language for put/revision selection (default: und)
PAGESWIKI_DOMAIN Domain name, UUID, or host (default: default)
PAGER pager used by view on an interactive terminal
EDITOR editor used by edit
PAGESWIKI_HISTORY interactive history file
"""
class PagesWikiCLI:
command_names = (
"ls",
"cd",
"mv",
"view",
"cat",
"edit",
"get",
"put",
"mget",
"mput",
"revision",
"diff",
"restore",
"link",
"unlink",
"domain",
"extension",
"pwd",
"help",
"exit",
"quit",
)
def __init__(self) -> None:
script_dir = Path(__file__).resolve().parent
self.data_dir = Path(
os.environ.get("PAGESWIKI_DATA_DIR", script_dir)
).resolve()
self.database_path = Path(
os.environ.get("PAGESWIKI_DB", self.data_dir / "pages.db")
).resolve()
self.documents = Path(
os.environ.get(
"PAGESWIKI_DOCUMENTS_DIR",
self.data_dir / "documents",
)
).resolve()
self.language = os.environ.get("PAGESWIKI_LANG", "und").strip() or "und"
if not re.fullmatch(
r"(?:und|[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*)",
self.language,
):
raise CLIError(f"invalid language tag: {self.language}")
self.cwd = "/"
self._completion_matches: list[str] = []
if not self.database_path.is_file():
raise CLIError(f"SQLite database not found: {self.database_path}")
# Coordinate with offline DR snapshots/promotions. Do this before any
# schema repair or content write. A standby never permits this CLI.
self._instance_lock = None
try:
lock_path = self.database_path.parent / ".pageswiki-instance.lock"
self._instance_lock = lock_path.open("rb") if lock_path.exists() else lock_path.open("a+b")
fcntl.flock(self._instance_lock.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
for root in {self.data_dir, self.database_path.parent, self.documents.parent}:
config_path = root / "dr-node.json"
if config_path.exists():
config = json.loads(config_path.read_text())
if not isinstance(config, dict) or config.get("version") != 1 or config.get("role") != "primary":
raise CLIError("DR standby or invalid configuration: data CLI is disabled")
if root != self.database_path.parent or self.documents != root / "documents":
raise CLIError("DR requires DB and documents in the same instance data directory")
except (OSError, ValueError, CLIError) as exc:
if self._instance_lock is not None:
self._instance_lock.close()
raise CLIError(f"DR maintenance guard: {exc}") from exc
self.db = sqlite3.connect(
self.database_path,
timeout=30,
isolation_level="DEFERRED",
)
self.db.row_factory = sqlite3.Row
self.db.execute("PRAGMA busy_timeout = 30000")
self.db.execute("PRAGMA foreign_keys = ON")
self._check_schema()
self.domain_uuid = ""
self.domain_name = ""
self.select_domain(os.environ.get("PAGESWIKI_DOMAIN", "default"))
def close(self) -> None:
self.db.close()
if self._instance_lock is not None:
self._instance_lock.close()
def _check_schema(self) -> None:
required = {"objects", "paths", "revisions"}
rows = self.db.execute(
"SELECT name FROM sqlite_master WHERE type = 'table'"
)
existing = {row["name"] for row in rows}
missing = sorted(required - existing)
if missing:
raise CLIError(
"database schema is not initialized: "
+ ", ".join(missing)
+ "; start Pages Wiki once"
)
path_columns = {
row["name"] for row in self.db.execute("PRAGMA table_info(paths)")
}
if "domain_uuid" not in path_columns:
raise CLIError(
"paths are not Domain-scoped; start Pages Wiki once to migrate"
)
with self.db:
self.db.execute(
"""
CREATE TABLE IF NOT EXISTS assets (
domain_uuid TEXT NOT NULL,
address TEXT NOT NULL,
object_uuid TEXT NOT NULL,
head_document_hash TEXT,
content_type TEXT NOT NULL DEFAULT 'application/octet-stream',
filename TEXT NOT NULL DEFAULT '',
size_bytes INTEGER NOT NULL DEFAULT 0,
created_at_ms INTEGER NOT NULL,
updated_at_ms INTEGER NOT NULL,
PRIMARY KEY (domain_uuid, address),
UNIQUE (object_uuid)
)
"""
)
revision_columns = {
row["name"]
for row in self.db.execute("PRAGMA table_info(revisions)")
}
if "content_type" not in revision_columns:
self.db.execute(
"ALTER TABLE revisions ADD COLUMN content_type "
"TEXT NOT NULL DEFAULT 'text/html; charset=utf-8'"
)
if "filename" not in revision_columns:
self.db.execute(
"ALTER TABLE revisions ADD COLUMN filename "
"TEXT NOT NULL DEFAULT ''"
)
def select_domain(self, selector: str) -> None:
selector = selector.strip() or "default"
row = self.db.execute(
"""
SELECT
d.uuid,
d.name,
COALESCE(d.placement, 'local') AS placement
FROM domains d
JOIN campuses c ON c.uuid = d.campus_uuid
WHERE d.enabled = 1 AND c.enabled = 1
AND (
d.uuid = ?
OR lower(d.name) = lower(?)
OR lower(
CASE
WHEN d.domain = '' THEN c.fqdn
WHEN instr(d.domain, '.') > 0 THEN d.domain
ELSE d.domain || '.' || c.fqdn
END
) = lower(?)
)
ORDER BY CASE WHEN d.name = 'default' THEN 0 ELSE 1 END, d.id
LIMIT 1
""",
(selector, selector, selector),
).fetchone()
if row is None:
raise CLIError(f"Domain not found or disabled: {selector}")
if row["placement"] != "local":
raise CLIError(
"Domain is remote and has no local Path namespace"
)
self.domain_uuid = row["uuid"]
self.domain_name = row["name"]
self.cwd = "/"
@staticmethod
def hash_bytes(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
@staticmethod
def now_ms() -> int:
return time.time_ns() // 1_000_000
@staticmethod
def strip_scheme(value: str, scheme: str) -> str:
prefix = scheme + ":"
return value[len(prefix) :] if value.startswith(prefix) else value
def semantic_path(self, raw: str | None) -> str:
if raw is None or raw == "":
return self.cwd
raw = self.strip_scheme(raw, "path")
if not raw.startswith("/"):
raw = posixpath.join(self.cwd, raw)
normalized = posixpath.normpath(raw)
return "/" if normalized == "." else "/" + normalized.lstrip("/")
def document_path(self, document_hash: str) -> Path:
normalized = document_hash.lower()
if (
len(normalized) != 64
or any(ch not in "0123456789abcdef" for ch in normalized)
):
raise CLIError(f"invalid Document SHA-256: {document_hash}")
return self.documents / "sha256" / normalized[:2] / normalized
def read_document(
self,
document_hash: str,
compatibility_body: str | bytes | None = None,
) -> bytes:
path = self.document_path(document_hash)
if path.is_file():
payload = path.read_bytes()
actual = self.hash_bytes(payload)
if actual != document_hash.lower():
raise CLIError(f"Document hash mismatch: {path}")
return payload
if compatibility_body is None:
raise CLIError(f"Document is missing: {path}")
if isinstance(compatibility_body, str):
payload = compatibility_body.encode()
else:
payload = compatibility_body
if self.hash_bytes(payload) != document_hash.lower():
raise CLIError(
"SQLite compatibility body does not match Document hash"
)
return payload
def write_document(self, payload: bytes) -> str:
document_hash = self.hash_bytes(payload)
path = self.document_path(document_hash)
path.parent.mkdir(parents=True, exist_ok=True)
if path.exists():
existing = path.read_bytes()
if self.hash_bytes(existing) != document_hash:
raise CLIError(f"existing Document is corrupt: {path}")
return document_hash
descriptor, temporary = tempfile.mkstemp(
prefix=".document-",
dir=path.parent,
)
try:
with os.fdopen(descriptor, "wb") as stream:
stream.write(payload)
stream.flush()
os.fsync(stream.fileno())
os.chmod(temporary, 0o644)
os.replace(temporary, path)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
return document_hash
def resolve_page(self, address: str | None) -> str:
target = address or self.cwd
if target.startswith("uuid:"):
page_uuid = self.strip_scheme(target, "uuid")
row = self.db.execute(
"SELECT uuid FROM objects WHERE uuid = ?",
(page_uuid,),
).fetchone()
if row is None:
raise CLIError(f"Page/Object not found: {page_uuid}")
return page_uuid
if target.startswith("asset:"):
row = self.db.execute(
"SELECT object_uuid FROM assets WHERE domain_uuid = ? AND address = ?",
(self.domain_uuid, target),
).fetchone()
if row is None:
raise CLIError(f"Asset not found: {target}")
return row["object_uuid"]
path = self.semantic_path(target)
row = self.db.execute(
"SELECT object_uuid FROM paths WHERE domain_uuid = ? AND path = ?",
(self.domain_uuid, path),
).fetchone()
if row is None:
raise CLIError(f"Semantic Path not found: {path}")
return row["object_uuid"]
def revision_row(self, address: str) -> sqlite3.Row:
revision_uuid = self.strip_scheme(address, "rev")
row = self.db.execute(
"""
SELECT
uuid,
object_uuid,
COALESCE(document_hash, '') AS document_hash,
COALESCE(NULLIF(lang, ''), 'und') AS lang,
COALESCE(NULLIF(content_type, ''), 'text/html; charset=utf-8')
AS content_type,
COALESCE(filename, '') AS filename,
COALESCE(body, '') AS body,
COALESCE(message, '') AS message,
COALESCE(created_at_ms, COALESCE(created_at, 0) * 1000)
AS created_at_ms
FROM revisions
WHERE uuid = ?
""",
(revision_uuid,),
).fetchone()
if row is None:
raise CLIError(f"Revision not found: {revision_uuid}")
return row
def latest_revision(
self,
page_uuid: str,
language: str | None = None,
) -> sqlite3.Row | None:
preferred = language or self.language
for candidate in dict.fromkeys((preferred, "und")):
row = self.db.execute(
"""
SELECT
uuid,
object_uuid,
COALESCE(document_hash, '') AS document_hash,
COALESCE(NULLIF(lang, ''), 'und') AS lang,
COALESCE(NULLIF(content_type, ''), 'text/html; charset=utf-8')
AS content_type,
COALESCE(filename, '') AS filename,
COALESCE(body, '') AS body,
COALESCE(message, '') AS message,
COALESCE(
created_at_ms,
COALESCE(created_at, 0) * 1000
) AS created_at_ms
FROM revisions
WHERE object_uuid = ?
AND COALESCE(NULLIF(lang, ''), 'und') = ?
ORDER BY
COALESCE(
created_at_ms,
COALESCE(created_at, 0) * 1000
) DESC,
rowid DESC
LIMIT 1
""",
(page_uuid, candidate),
).fetchone()
if row is not None:
return row
return None
def payload_for_address(self, address: str | None) -> tuple[bytes, str]:
target = address or self.cwd
if target.startswith("rev:"):
row = self.revision_row(target)
return (
self.read_document(row["document_hash"], row["body"]),
row["document_hash"],
)
if target.startswith("asset:"):
row = self.db.execute(
"""
SELECT head_document_hash
FROM assets
WHERE domain_uuid = ? AND address = ?
""",
(self.domain_uuid, target),
).fetchone()
if row is None:
raise CLIError(f"Asset not found: {target}")
document_hash = row["head_document_hash"] or ""
if not document_hash:
return b"", self.hash_bytes(b"")
return self.read_document(document_hash), document_hash
page_uuid = self.resolve_page(target)
row = self.latest_revision(page_uuid)
if row is None:
return b"", self.hash_bytes(b"")
return (
self.read_document(row["document_hash"], row["body"]),
row["document_hash"],
)
def create_page(self, path: str) -> str:
page_uuid = str(uuid.uuid4())
now = self.now_ms()
with self.db:
self.db.execute(
"""
INSERT INTO objects (uuid, kind, created_at)
VALUES (?, 'page', ?)
""",
(page_uuid, now // 1000),
)
self.db.execute(
"""
INSERT INTO paths (
domain_uuid,
path,
object_uuid,
created_at,
is_primary
)
VALUES (?, ?, ?, ?, 1)
""",
(self.domain_uuid, path, page_uuid, now // 1000),
)
return page_uuid
def save_revision(
self,
page_uuid: str,
payload: bytes,
message: str,
language: str | None = None,
content_type: str = "text/html; charset=utf-8",
filename: str = "",
force: bool = False,
) -> tuple[str, str]:
lang = language or self.language
document_hash = self.write_document(payload)
latest = self.latest_revision(page_uuid, lang)
if (
not force
and latest is not None
and latest["document_hash"] == document_hash
and latest["content_type"] == content_type
and latest["filename"] == filename
):
return "skipped", latest["uuid"]
try:
body: str | bytes = payload.decode("utf-8")
except UnicodeDecodeError:
body = payload
revision_uuid = str(uuid.uuid4())
now = self.now_ms()
with self.db:
self.db.execute(
"""
INSERT INTO revisions (
uuid,
object_uuid,
body,
document_hash,
lang,
content_type,
filename,
message,
created_at_ms,
created_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
revision_uuid,
page_uuid,
body,
document_hash,
lang,
content_type,
filename,
message,
now,
now // 1000,
),
)
return "updated", revision_uuid
def put_asset(self, address: str, payload: bytes, filename: str = "", content_type: str | None = None) -> str:
content_type = content_type or detect_content_type(payload, filename)
document_hash = self.write_document(payload)
row = self.db.execute(
"""
SELECT object_uuid, COALESCE(head_document_hash, '') AS head, content_type, filename
FROM assets
WHERE domain_uuid = ? AND address = ?
""",
(self.domain_uuid, address),
).fetchone()
if row is not None and row["head"] == document_hash and row["content_type"] == content_type and row["filename"] == filename:
return "skipped"
now = self.now_ms()
with self.db:
self.db.execute("""CREATE TABLE IF NOT EXISTS asset_head_history (
id INTEGER PRIMARY KEY AUTOINCREMENT, domain_uuid TEXT NOT NULL,
address TEXT NOT NULL, action TEXT NOT NULL DEFAULT 'put',
document_hash TEXT NOT NULL, content_type TEXT NOT NULL DEFAULT 'application/octet-stream',
filename TEXT NOT NULL DEFAULT '', size_bytes INTEGER NOT NULL DEFAULT 0,
actor_uuid TEXT NOT NULL DEFAULT '', created_at_ms INTEGER NOT NULL)""")
if row is None:
object_uuid = str(uuid.uuid4())
self.db.execute(
"""
INSERT INTO objects (uuid, kind, created_at)
VALUES (?, 'asset', ?)
""",
(object_uuid, now // 1000),
)
self.db.execute(
"""
INSERT INTO assets (
domain_uuid,
address,
object_uuid,
head_document_hash,
content_type,
filename,
size_bytes,
created_at_ms,
updated_at_ms
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
self.domain_uuid,
address,
object_uuid,
document_hash,
content_type,
filename,
len(payload),
now,
now,
),
)
else:
result = self.db.execute(
"""
UPDATE assets
SET head_document_hash = ?, size_bytes = ?, updated_at_ms = ?, content_type = ?, filename = ?
WHERE domain_uuid = ? AND address = ? AND head_document_hash = ?
""",
(document_hash, len(payload), now, content_type, filename, self.domain_uuid, address, row["head"]),
)
if result.rowcount != 1:
raise CLIError("asset HEAD conflict; reload before saving")
self.db.execute("""INSERT INTO asset_head_history
(domain_uuid,address,action,document_hash,content_type,filename,size_bytes,actor_uuid,created_at_ms)
VALUES (?,?,'put',?,?,?,?, 'cli',?)""",
(self.domain_uuid,address,document_hash,content_type,filename,len(payload),now))
return "updated"
def command_ls(self, args: list[str]) -> None:
if len(args) > 1:
raise CLIError("usage: ls [PATH]")
parent = self.semantic_path(args[0] if args else None)
prefix = "/" if parent == "/" else parent + "/"
rows = self.db.execute(
"""SELECT path, object_uuid FROM paths
WHERE domain_uuid = ? AND path LIKE ? ORDER BY path""",
(self.domain_uuid, prefix + "%"),
)
children: dict[str, dict[str, object]] = {}
for row in rows:
remainder = row["path"][len(prefix) :]
if not remainder:
continue
name, separator, _ = remainder.partition("/")
is_directory = bool(separator)
entry = children.setdefault(
name,
{
"directory": False,
"object_uuid": "",
},
)
if is_directory:
entry["directory"] = True
else:
entry["object_uuid"] = row["object_uuid"]
exact = self.db.execute(
"SELECT object_uuid FROM paths WHERE domain_uuid = ? AND path = ?",
(self.domain_uuid, parent),
).fetchone()
if not children and exact is None:
raise CLIError(f"Semantic Path not found: {parent}")
for name, entry in children.items():
is_directory = bool(entry["directory"])
object_uuid = str(entry["object_uuid"])
suffix = "/" if is_directory else ""
identity = f"\t{object_uuid}" if object_uuid else ""
print(f"{name}{suffix}{identity}")
def command_cd(self, args: list[str]) -> None:
if len(args) != 1:
raise CLIError("usage: cd PATH")
path = self.semantic_path(args[0])
prefix = "/" if path == "/" else path + "/"
row = self.db.execute(
"""
SELECT 1
FROM paths
WHERE domain_uuid = ? AND (path = ? OR path LIKE ?)
LIMIT 1
""",
(self.domain_uuid, path, prefix + "%"),
).fetchone()
if row is None and path != "/":
raise CLIError(f"Semantic Path not found: {path}")
self.cwd = path
def command_mv(self, args: list[str]) -> None:
if len(args) != 2:
raise CLIError("usage: mv SOURCE DEST")
source = self.semantic_path(args[0])
destination = self.semantic_path(args[1])
if source == "/" or destination == "/":
raise CLIError("moving the semantic root is not allowed")
if destination == source or destination.startswith(source + "/"):
raise CLIError("destination cannot be inside source")
rows = list(
self.db.execute(
"""
SELECT path
FROM paths
WHERE domain_uuid = ? AND (path = ? OR path LIKE ?)
ORDER BY LENGTH(path)
""",
(self.domain_uuid, source, source + "/%"),
)
)
if not rows:
raise CLIError(f"Semantic Path not found: {source}")
source_paths = {row["path"] for row in rows}
mappings = [
(path, destination + path[len(source) :])
for path in source_paths
]
destinations = {new for _, new in mappings}
placeholders = ",".join("?" for _ in destinations)
collisions = list(
self.db.execute(
f"""SELECT path FROM paths
WHERE domain_uuid = ? AND path IN ({placeholders})""",
(self.domain_uuid, *destinations),
)
)
collisions = [
row["path"] for row in collisions if row["path"] not in source_paths
]
if collisions:
raise CLIError("destination exists: " + ", ".join(collisions))
temporary_prefix = f"/.__pageswiki_mv_{uuid.uuid4().hex}"
with self.db:
temporary = []
for old, new in mappings:
temp = temporary_prefix + old
self.db.execute(
"""UPDATE paths SET path = ?
WHERE domain_uuid = ? AND path = ?""",
(temp, self.domain_uuid, old),
)
temporary.append((temp, new))
for temp, new in temporary:
self.db.execute(
"""UPDATE paths SET path = ?
WHERE domain_uuid = ? AND path = ?""",
(new, self.domain_uuid, temp),
)
if self.cwd == source or self.cwd.startswith(source + "/"):
self.cwd = destination + self.cwd[len(source) :]
print(f"moved {len(mappings)} path(s)")
def print_page_info(self, page_uuid: str) -> None:
row = self.db.execute(
"""
SELECT uuid, COALESCE(kind, '') AS kind, COALESCE(created_at, 0) AS created
FROM objects WHERE uuid = ?
""",
(page_uuid,),
).fetchone()
if row is None:
raise CLIError(f"Page/Object not found: {page_uuid}")
print(f"Page/Object UUID: {row['uuid']}")
print(f"Kind: {row['kind']}")
print("Semantic Paths:")
for path in self.db.execute(
"""
SELECT path, COALESCE(is_primary, 0) AS primary_path
FROM paths WHERE domain_uuid = ? AND object_uuid = ?
ORDER BY primary_path DESC, path
""",
(self.domain_uuid, page_uuid),
):
marker = "*" if path["primary_path"] else "-"
print(f" {marker} {path['path']}")
def command_view(self, args: list[str]) -> None:
if len(args) > 1:
raise CLIError("usage: view [ADDRESS]")
target = args[0] if args else self.cwd
if target.startswith("rev:"):
row = self.revision_row(target)
print(f"Revision UUID: {row['uuid']}")
print(f"Page UUID: {row['object_uuid']}")
print(f"Document SHA-256: {row['document_hash']}")
print(f"Language: {row['lang']}")
print(f"Content-Type: {row['content_type']}")
print(f"Filename: {row['filename'] or '-'}")
print(f"Created at (ms): {row['created_at_ms']}")
print(f"Message: {row['message']}")
elif target.startswith("asset:"):
row = self.db.execute(
"""
SELECT object_uuid, COALESCE(head_document_hash, '') AS hash
FROM assets WHERE domain_uuid = ? AND address = ?
""",
(self.domain_uuid, target),
).fetchone()
if row is None:
raise CLIError(f"Asset not found: {target}")
print(f"Asset: {target}")
print(f"Page/Object UUID: {row['object_uuid']}")
print(f"Document SHA-256: {row['hash']}")
else:
self.print_page_info(self.resolve_page(target))
payload, _ = self.payload_for_address(target)
print("--- Document ---")
self.output_payload(payload, use_pager=True)
def output_payload(self, payload: bytes, use_pager: bool = False) -> None:
pager = os.environ.get("PAGER", "").strip()
if use_pager and pager and sys.stdout.isatty():
try:
process = subprocess.run(
shlex.split(pager),
input=payload,
check=False,
)
except OSError as error:
raise CLIError(f"cannot start PAGER: {error}") from error
if process.returncode != 0:
raise CLIError(
f"PAGER exited with status {process.returncode}"
)
return
sys.stdout.flush()
sys.stdout.buffer.write(payload)
if use_pager and payload and not payload.endswith(b"\n"):
sys.stdout.buffer.write(b"\n")
sys.stdout.buffer.flush()
def command_cat(self, args: list[str]) -> None:
if len(args) > 1:
raise CLIError("usage: cat [ADDRESS]")
target = args[0] if args else self.cwd
payload, _ = self.payload_for_address(target)
self.output_payload(payload)
def command_edit(self, args: list[str]) -> None:
if len(args) > 1:
raise CLIError("usage: edit [ADDRESS]")
target = args[0] if args else self.cwd
if target.startswith("rev:"):
raise CLIError("Revision addresses are immutable")
editor = os.environ.get("EDITOR", "").strip()
if not editor:
raise CLIError("EDITOR is not set")
original, _ = self.payload_for_address(target)
suffix = self.local_default(target).suffix
descriptor, temporary = tempfile.mkstemp(
prefix="pageswiki-edit-",
suffix=suffix,
)
try:
with os.fdopen(descriptor, "wb") as stream:
stream.write(original)
stream.flush()
os.fsync(stream.fileno())
try:
process = subprocess.run(
[*shlex.split(editor), temporary],
check=False,
)
except OSError as error:
raise CLIError(f"cannot start EDITOR: {error}") from error
if process.returncode != 0:
raise CLIError(
f"EDITOR exited with status {process.returncode}"
)
edited = Path(temporary).read_bytes()
self.put_payload(edited, target, f"EDITOR:{editor}")
finally:
try:
os.unlink(temporary)
except FileNotFoundError:
pass
def local_default(self, address: str) -> Path:
if address.startswith("asset:"):
return Path(address.removeprefix("asset:").replace(":", "-"))
if address.startswith("rev:"):
return Path(address.removeprefix("rev:"))
path = self.semantic_path(address)
return Path(posixpath.basename(path.rstrip("/")) or "root")
def write_local(self, destination: Path, payload: bytes) -> str:
if destination.exists() and destination.is_file():
if self.hash_bytes(destination.read_bytes()) == self.hash_bytes(payload):
return "skipped"
destination.parent.mkdir(parents=True, exist_ok=True)
descriptor, temporary = tempfile.mkstemp(
prefix=".pageswiki-",
dir=destination.parent,
)
try:
with os.fdopen(descriptor, "wb") as stream:
stream.write(payload)
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, destination)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
return "updated"
def command_get(self, args: list[str]) -> None:
if not 1 <= len(args) <= 2:
raise CLIError("usage: get ADDRESS [LOCAL]")
address = args[0]
payload, document_hash = self.payload_for_address(address)
if len(args) == 2 and args[1] == "-":
sys.stdout.buffer.write(payload)
return
destination = Path(args[1]) if len(args) == 2 else self.local_default(address)
status = self.write_local(destination, payload)
print(
f"{status}: {address} -> {destination} "
f"({len(payload)} bytes, {document_hash})"
)
def put_payload(
self,
payload: bytes,
address: str,
source: str,
content_type: str | None = None,
) -> None:
if address.startswith("asset:"):
status = self.put_asset(address, payload, Path(source).name, content_type)
print(f"{status}: {source} -> {address} ({len(payload)} bytes)")
return
if address.startswith("rev:"):
raise CLIError("Revision addresses are immutable")
if address.startswith("uuid:"):
page_uuid = self.resolve_page(address)
created = False
destination = address
else:
path = self.semantic_path(address)
try:
page_uuid = self.resolve_page(path)
created = False
except CLIError:
page_uuid = self.create_page(path)
created = True
destination = f"path:{path}"
filename = "" if source == "-" else Path(source).name
if source.startswith("EDITOR:"):
latest = self.latest_revision(page_uuid)
if latest is not None:
content_type = latest["content_type"]
filename = latest["filename"]
if content_type is None:
detection_name = filename or self.semantic_path(address).rsplit("/", 1)[-1]
content_type = detect_content_type(payload, detection_name)
status, revision_uuid = self.save_revision(
page_uuid,
payload,
f"cli put {source}",
content_type=content_type,
filename=filename,
)
if created:
status = "created"
print(
f"{status}: {source} -> {destination} "
f"(Revision {revision_uuid})"
)
def command_put(self, args: list[str]) -> None:
if len(args) != 2:
raise CLIError("usage: put LOCAL ADDRESS")
source, address = args
payload = sys.stdin.buffer.read() if source == "-" else Path(source).read_bytes()
payload, address, content_type = self.prepare_upload(
payload,
address,
source,
)
self.put_payload(payload, address, source, content_type)
def command_mget(self, args: list[str]) -> None:
if not 1 <= len(args) <= 2:
raise CLIError("usage: mget REMOTE_PATTERN [LOCAL_DIR]")
pattern = self.semantic_path(args[0])
destination = Path(args[1]) if len(args) == 2 else Path(".")
paths = [
row["path"]
for row in self.db.execute(
"SELECT path FROM paths WHERE domain_uuid = ? ORDER BY path",
(self.domain_uuid,),
)
if fnmatch.fnmatch(row["path"], pattern)
]
if not paths:
raise CLIError(f"no Semantic Paths match: {pattern}")
wildcard_at = min(
(pattern.find(ch) for ch in "*?[" if ch in pattern),
default=len(pattern),
)
base = pattern[:wildcard_at].rstrip("/")
base = posixpath.dirname(base) if base else "/"
for path in paths:
relative = posixpath.relpath(path, base)
self.command_get([f"path:{path}", str(destination / relative)])
def command_mput(self, args: list[str]) -> None:
if len(args) != 2:
raise CLIError("usage: mput LOCAL_GLOB REMOTE_DIR")
pattern, remote_dir_raw = args
sources = [Path(item) for item in glob.glob(pattern, recursive=True)]
sources = [item for item in sources if item.is_file()]
if not sources:
raise CLIError(f"no local files match: {pattern}")
remote_dir = self.semantic_path(remote_dir_raw)
wildcard_at = min(
(pattern.find(ch) for ch in "*?[" if ch in pattern),
default=len(pattern),
)
base_raw = pattern[:wildcard_at]
if not base_raw:
base = Path(".")
elif base_raw.endswith(("/", os.sep)):
base = Path(base_raw)
else:
base = Path(base_raw).parent
for source in sources:
try:
relative = source.relative_to(base)
except ValueError:
relative = Path(source.name)
remote = posixpath.join(remote_dir, relative.as_posix())
payload, address, content_type = self.prepare_upload(
source.read_bytes(),
f"path:{remote}",
str(source),
)
self.put_payload(payload, address, str(source), content_type)
def prepare_upload(
self,
payload: bytes,
address: str,
source: str,
) -> tuple[bytes, str, str | None]:
suffix = Path(source).suffix.lower()
if suffix not in {".md", ".markdown"}:
return payload, address, None
try:
markdown = payload.decode("utf-8")
except UnicodeDecodeError as exc:
raise CLIError(f"Markdown must be UTF-8: {source}") from exc
if address.startswith("path:"):
address = re.sub(r"(?i)\.(?:md|markdown)$", "", address)
elif not any(address.startswith(prefix) for prefix in ("uuid:", "asset:", "rev:")):
address = re.sub(r"(?i)\.(?:md|markdown)$", "", address)
return (
markdown_to_html(markdown).encode("utf-8"),
address,
"text/html; charset=utf-8",
)
def command_revision(self, args: list[str]) -> None:
if len(args) > 1:
raise CLIError("usage: revision [ADDRESS]")
page_uuid = self.resolve_page(args[0] if args else self.cwd)
self.print_page_info(page_uuid)
print("Revisions:")
for row in self.db.execute(
"""
SELECT
uuid,
COALESCE(NULLIF(lang, ''), 'und') AS lang,
COALESCE(document_hash, '') AS hash,
COALESCE(message, '') AS message,
COALESCE(
created_at_ms,
COALESCE(created_at, 0) * 1000
) AS created_at_ms
FROM revisions
WHERE object_uuid = ?
ORDER BY created_at_ms DESC, rowid DESC
""",
(page_uuid,),
):
print(
f" {row['uuid']} {row['lang']} "
f"{row['created_at_ms']} {row['hash']} {row['message']}"
)
def command_diff(self, args: list[str]) -> None:
if len(args) != 2:
raise CLIError("usage: diff REVISION REVISION")
before = self.revision_row(args[0])
after = self.revision_row(args[1])
before_payload = self.read_document(
before["document_hash"],
before["body"],
).decode("utf-8", errors="replace")
after_payload = self.read_document(
after["document_hash"],
after["body"],
).decode("utf-8", errors="replace")
sys.stdout.writelines(
difflib.unified_diff(
before_payload.splitlines(keepends=True),
after_payload.splitlines(keepends=True),
fromfile=before["uuid"],
tofile=after["uuid"],
)
)
def command_restore(self, args: list[str]) -> None:
if len(args) != 1:
raise CLIError("usage: restore REVISION")
row = self.revision_row(args[0])
payload = self.read_document(row["document_hash"], row["body"])
_, revision_uuid = self.save_revision(
row["object_uuid"],
payload,
f"cli restore {row['uuid']}",
row["lang"],
force=True,
)
print(f"restored: {row['uuid']} -> {revision_uuid}")
def command_link(self, args: list[str]) -> None:
if len(args) != 2:
raise CLIError("usage: link ADDRESS PATH")
page_uuid = self.resolve_page(args[0])
path = self.semantic_path(args[1])
now = self.now_ms()
try:
with self.db:
self.db.execute(
"""
INSERT INTO paths (
domain_uuid,
path,
object_uuid,
created_at,
is_primary
)
VALUES (?, ?, ?, ?, 0)
""",
(self.domain_uuid, path, page_uuid, now // 1000),
)
except sqlite3.IntegrityError as error:
raise CLIError(f"Semantic Path already exists: {path}") from error
print(f"linked: path:{path} -> uuid:{page_uuid}")
def command_unlink(self, args: list[str]) -> None:
if not 1 <= len(args) <= 2:
raise CLIError("usage: unlink PATH [ADDRESS]")
path = self.semantic_path(args[0])
parameters: list[str] = [self.domain_uuid, path]
query = "DELETE FROM paths WHERE domain_uuid = ? AND path = ?"
if len(args) == 2:
query += " AND object_uuid = ?"
parameters.append(self.resolve_page(args[1]))
with self.db:
cursor = self.db.execute(query, parameters)
if cursor.rowcount == 0:
raise CLIError(f"Semantic Path binding not found: {path}")
print(f"unlinked: path:{path}")
def command_domain(self, args: list[str]) -> None:
if len(args) > 1:
raise CLIError("usage: domain [NAME|UUID|HOST]")
if args:
self.select_domain(args[0])
print(f"{self.domain_name}\t{self.domain_uuid}")
def command_extension(self, args: list[str]) -> None:
if len(args) > 2:
raise CLIError("usage: extension [ID] [on|off]")
row = self.db.execute("SELECT value FROM config WHERE domain_uuid=? AND key=?", (self.domain_uuid, "system_extension_states")).fetchone()
states = json.loads(row[0]) if row and row[0] else {}
if not args:
for extension_id, enabled in sorted(states.items()):
print(f"{extension_id}\t{'enabled' if enabled else 'disabled'}")
print("(unlisted extensions use their manifest default; viewer.gecko: disabled, others: enabled)")
return
extension_id = args[0].strip()
if not re.fullmatch(r"[a-z][a-z0-9._-]{1,63}", extension_id):
raise CLIError("invalid extension ID")
if len(args) == 1:
print(f"{extension_id}\t{'enabled' if states.get(extension_id, extension_id != 'viewer.gecko') else 'disabled'}")
return
value = args[1].lower()
if value not in {"on", "off"}:
raise CLIError("usage: extension ID on|off")
states[extension_id] = value == "on"
payload = json.dumps(states, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
with self.db:
self.db.execute("INSERT INTO config(domain_uuid,key,value,updated_at) VALUES(?,?,?,strftime('%s','now')) ON CONFLICT(domain_uuid,key) DO UPDATE SET value=excluded.value,updated_at=excluded.updated_at", (self.domain_uuid, "system_extension_states", payload))
print(f"{extension_id}\t{value}")
def inspect_address(self, address: str) -> None:
if address.startswith(("rev:", "asset:")):
self.command_view([address])
return
page_uuid = self.resolve_page(address)
self.command_revision([f"uuid:{page_uuid}"])
def remote_completion_candidates(self) -> list[str]:
candidates: set[str] = set()
for row in self.db.execute(
"SELECT path FROM paths WHERE domain_uuid = ? ORDER BY path",
(self.domain_uuid,),
):
path = row["path"]
candidates.add(path)
candidates.add(f"path:{path}")
relative = posixpath.relpath(path, self.cwd)
if relative != "." and not relative.startswith("../"):
candidates.add(relative)
for row in self.db.execute("SELECT uuid FROM objects ORDER BY uuid"):
candidates.add(f"uuid:{row['uuid']}")
for row in self.db.execute(
"""
SELECT uuid
FROM revisions
ORDER BY
COALESCE(created_at_ms, COALESCE(created_at, 0) * 1000) DESC,
rowid DESC
"""
):
candidates.add(f"rev:{row['uuid']}")
for row in self.db.execute(
"SELECT address FROM assets WHERE domain_uuid = ? ORDER BY address",
(self.domain_uuid,),
):
candidates.add(row["address"])
return sorted(candidates)
@staticmethod
def local_completion_candidates(text: str) -> list[str]:
expanded = os.path.expanduser(text or ".")
pattern = expanded + "*"
candidates = []
for match in glob.glob(pattern):
candidate = match
if text.startswith("~"):
home = str(Path.home())
if candidate.startswith(home):
candidate = "~" + candidate[len(home) :]
if os.path.isdir(match):
candidate += os.sep
candidates.append(candidate)
return sorted(candidates)
def complete(
self,
text: str,
state: int,
) -> str | None:
if readline is None:
return None
if state == 0:
line = readline.get_line_buffer()
begin = readline.get_begidx()
prefix = line[:begin]
try:
words = shlex.split(prefix)
except ValueError:
words = prefix.split()
if not words:
candidates = list(self.command_names)
else:
command = words[0]
argument_position = len(words)
local_argument = (
command in {"put", "mput"} and argument_position == 1
) or (
command in {"get", "mget"} and argument_position == 2
)
if local_argument:
candidates = self.local_completion_candidates(text)
else:
candidates = self.remote_completion_candidates()
self._completion_matches = [
candidate for candidate in candidates if candidate.startswith(text)
]
if state < len(self._completion_matches):
return self._completion_matches[state]
return None
def setup_readline(self) -> Path | None:
if readline is None:
return None
history_path = Path(
os.environ.get(
"PAGESWIKI_HISTORY",
Path.home() / ".pageswiki_history",
)
).expanduser()
try:
readline.read_history_file(history_path)
except FileNotFoundError:
pass
readline.set_history_length(1000)
readline.set_completer(self.complete)
readline.set_completer_delims(" \t\n")
if "libedit" in (readline.__doc__ or ""):
readline.parse_and_bind("bind ^I rl_complete")
else:
readline.parse_and_bind("tab: complete")
return history_path
def dispatch(self, words: list[str], interactive: bool = False) -> bool:
if not words:
return True
if len(words) == 1 and ":" in words[0] and words[0].split(":", 1)[0] in {
"path",
"uuid",
"rev",
"asset",
}:
self.inspect_address(words[0])
return True
command, args = words[0], words[1:]
commands = {
"ls": self.command_ls,
"cd": self.command_cd,
"mv": self.command_mv,
"view": self.command_view,
"cat": self.command_cat,
"edit": self.command_edit,
"get": self.command_get,
"put": self.command_put,
"mget": self.command_mget,
"mput": self.command_mput,
"revision": self.command_revision,
"diff": self.command_diff,
"restore": self.command_restore,
"link": self.command_link,
"unlink": self.command_unlink,
"domain": self.command_domain,
"extension": self.command_extension,
}
if command in {"exit", "quit"}:
return False
if command == "help":
print(HELP)
return True
if command == "pwd":
if args:
raise CLIError("usage: pwd")
print(self.cwd)
return True
if command == "shell":
if interactive:
raise CLIError("already in interactive shell")
self.shell()
return True
handler = commands.get(command)
if handler is None:
raise CLIError(f"unknown command: {command}")
handler(args)
return True
def shell(self) -> None:
history_path = self.setup_readline()
try:
while True:
try:
line = input(f"pages:{self.domain_name}:{self.cwd}> ")
except EOFError:
print()
return
if readline is not None and line.strip():
history_length = readline.get_current_history_length()
previous = (
readline.get_history_item(history_length)
if history_length > 0
else None
)
if previous != line:
readline.add_history(line)
try:
words = shlex.split(line)
if not self.dispatch(words, interactive=True):
return
except (CLIError, OSError, sqlite3.Error) as error:
print(f"error: {error}", file=sys.stderr)
finally:
if readline is not None and history_path is not None:
history_path.parent.mkdir(parents=True, exist_ok=True)
try:
readline.write_history_file(history_path)
except OSError as error:
print(
f"warning: cannot write history: {error}",
file=sys.stderr,
)
def main() -> int:
cli: PagesWikiCLI | None = None
try:
arguments = sys.argv[1:]
if arguments == ["help"] or arguments == ["--help"] or arguments == ["-h"]:
print(HELP)
return 0
if not arguments:
if not sys.stdin.isatty():
print(HELP)
return 2
cli = PagesWikiCLI()
if not arguments:
cli.shell()
return 0
cli.dispatch(arguments)
return 0
except (CLIError, OSError, sqlite3.Error) as error:
print(f"pageswiki-cli: {error}", file=sys.stderr)
return 1
finally:
if cli is not None:
cli.close()
if __name__ == "__main__":
raise SystemExit(main())