feat: add browser upload queue and trusted sessions with bounded projections
This commit is contained in:
1 parent
84d52390f9
commit
b90aef79d0
79 files changed
+1962
-196
No files matched your search
@@ -1,7 +1,7 @@
|
||||
import {spawnSync} from 'node:child_process';
|
||||
import {pathToFileURL} from 'node:url';
|
||||
|
||||
export const images = ['pageswiki-app', 'pageswiki-adaptor-gateway', 'pageswiki-excalidraw-viewer'];
|
||||
export const images = ['pageswiki-app', 'pageswiki-adaptor-gateway', 'pageswiki-excalidraw-viewer', 'pageswiki-gecko-relay'];
|
||||
// SemVer 2.0.0: numeric prerelease identifiers cannot have leading zeroes.
|
||||
const number = '(0|[1-9][0-9]*)';
|
||||
const identifier = '(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)';
|
||||
|
||||
@@ -37,19 +37,19 @@ function registry() {
|
||||
return {client, entries, copies};
|
||||
}
|
||||
|
||||
test('promotes three sources by digest and retry does not rewrite', () => {
|
||||
test('promotes all sources by digest and retry does not rewrite', () => {
|
||||
const {client, copies} = registry();
|
||||
const plan = promote(options, client, () => {});
|
||||
assert.equal(copies.length, 3);
|
||||
assert.equal(copies.length, images.length);
|
||||
for (const item of plan) assert.ok(item.source.endsWith(`@${item.digest}`));
|
||||
promote(options, client, () => {});
|
||||
assert.equal(copies.length, 3);
|
||||
assert.equal(copies.length, images.length);
|
||||
});
|
||||
|
||||
test('missing source, wrong revision or existing release conflict prevents all writes', () => {
|
||||
for (const failure of ['missing', 'revision', 'conflict']) {
|
||||
const {client, entries, copies} = registry();
|
||||
const last = `${options.prefix}/${images[2]}`;
|
||||
const last = `${options.prefix}/${images.at(-1)}`;
|
||||
if (failure === 'missing') entries.delete(`${last}:sha-${revision}`);
|
||||
if (failure === 'revision') client.revision = () => 'b'.repeat(40);
|
||||
if (failure === 'conflict') entries.set(`${last}:v0.1.0`, `sha256:${'f'.repeat(64)}`);
|
||||
@@ -68,7 +68,7 @@ test('partial promotion is safely resumable', () => {
|
||||
assert.throws(() => promote(options, client, () => {}), /unavailable/);
|
||||
client.copy = copy;
|
||||
promote(options, client, () => {});
|
||||
assert.equal(copies.length, 3);
|
||||
assert.equal(copies.length, images.length);
|
||||
});
|
||||
|
||||
test('destination digest mismatch and concurrent overwrite are failures', () => {
|
||||
@@ -79,7 +79,7 @@ test('destination digest mismatch and concurrent overwrite are failures', () =>
|
||||
const get = other.client.digest;
|
||||
let reads = 0;
|
||||
other.client.digest = (ref, optional) => {
|
||||
if (ref.endsWith(':v0.1.0') && ++reads > 3) return `sha256:${'f'.repeat(64)}`;
|
||||
if (ref.endsWith(':v0.1.0') && ++reads > images.length) return `sha256:${'f'.repeat(64)}`;
|
||||
return get(ref, optional);
|
||||
};
|
||||
assert.throws(() => promote(options, other.client), /concurrently/);
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
Gecko WASM v0.0.1 distribution for PagesWiki
|
||||
|
||||
The gecko.js and gecko.wasm.zst binaries are unmodified upstream release files.
|
||||
PagesWiki's relay and UI are separate software. This is not an official Mozilla
|
||||
Firefox distribution and no Mozilla trademarks or logos are licensed here.
|
||||
|
||||
Corresponding source and build recipe:
|
||||
wrapper.tar.gz: HeyPuter/firefox-wasm eadcb5fa828d7e8b54ae2df13b6b3208f6553944
|
||||
firefox.tar.gz: MercuryWorkshop/firefox 2e1e835a5da5907e5ceef19aade58a7493f1e34d
|
||||
emscripten.tar.gz: Emscripten 6.0.1 (runtime and bundled libc sources/notices)
|
||||
rust-src.tar.xz: Rust 1.95.0 standard library; .crate files are all registry
|
||||
dependencies of library/Cargo.lock, checked against that lock's checksums.
|
||||
zstddec.tgz: zstddec 0.2.0, MIT and BSD-3-Clause notices.
|
||||
wrapper-MPL.txt: MPL-2.0 declaration added by upstream at
|
||||
ac40f56c85060bd49dfc36ca60490f97dd270c50. The wrapper's core source and
|
||||
Emscripten patches were unchanged between these commits.
|
||||
vex-sdk-LICENSE.txt: MIT notice from the crate's source commit, omitted from
|
||||
the published crate. Fortanix's in-source copyright/MPL header is retained.
|
||||
NotoSansCJKjp-Regular.otf: unmodified Noto Sans CJK JP Sans2.004, commit
|
||||
523d033d6cb47f4a80c58a35753646f5c3608a78. Embedded copyright notice:
|
||||
© 2014-2021 Adobe (http://www.adobe.com/).
|
||||
SIL Open Font License 1.1, reproduced in NotoSansCJK-LICENSE.txt. No font
|
||||
names or embedded notices are modified. Source repository: notofonts/noto-cjk.
|
||||
|
||||
Extract wrapper.tar.gz into a working directory, put the extracted Firefox
|
||||
source at firefox/, and follow the included README and Makefile. Use the fixed
|
||||
Firefox revision above, Emscripten 6.0.1, Rust 1.95.0 and its rust-src, and the
|
||||
included pnpm-lock.yaml. make emsdk applies the included WasmFS patches;
|
||||
vendor-std-deps.py verifies the standard library crates. The upstream Makefile,
|
||||
mozconfig and build scripts describe the release build. Building is
|
||||
not performed by the PagesWiki runtime. The upstream build timestamp is not
|
||||
fixed; byte-for-byte reproducibility is not claimed.
|
||||
|
||||
The source archives preserve their original licenses, copyrights and patches.
|
||||
The *.notices.txt files collect license, copying, copyright, notice and author
|
||||
files from those archives. Mozilla's license.html is converted to inert readable
|
||||
text with ALL conditional sections included, a conservative superset. Original
|
||||
HTML and build conditionals remain in firefox.tar.gz. Liberation fonts (SIL OFL
|
||||
1.1, reserved names retained) are covered by that archive's PDF.js font notice.
|
||||
Gecko's source archive includes its vendored third-party/Rust source and notices.
|
||||
|
||||
Upstream source URLs and exact hashes/sizes are recorded in manifest.json.
|
||||
Keep this complete bundle available to recipients whenever the engine is served,
|
||||
and preserve the source offer after disabling the feature. Do not deploy a pair
|
||||
of engine files alone. Do not remove notices because a component is believed
|
||||
unused without reviewing the actual build. No additional warranty is provided.
|
||||
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build an offline, hash-locked Gecko distribution without extracting source trees.
|
||||
|
||||
Python 3.11+. Sources and notices remain available alongside the engine. The
|
||||
output is a new directory: never replace a bundle mounted by a running relay.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import html
|
||||
from html.parser import HTMLParser
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import shutil
|
||||
import tarfile
|
||||
import tomllib
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def digest(path):
|
||||
with path.open('rb') as stream:
|
||||
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def obtain(cache, name, url, expected, offline=False):
|
||||
path = cache / name
|
||||
if not path.exists():
|
||||
if offline:
|
||||
raise ValueError('missing offline input: ' + name)
|
||||
temporary = path.with_suffix(path.suffix + '.part')
|
||||
with urllib.request.urlopen(url, timeout=60) as response, temporary.open('wb') as target:
|
||||
shutil.copyfileobj(response, target, 1 << 20)
|
||||
if digest(temporary) != expected:
|
||||
temporary.unlink()
|
||||
raise ValueError('download checksum mismatch: ' + name)
|
||||
temporary.rename(path)
|
||||
if digest(path) != expected:
|
||||
raise ValueError('cached checksum mismatch: ' + name)
|
||||
return path
|
||||
|
||||
|
||||
class NoticeText(HTMLParser):
|
||||
"""Readable superset of Mozilla's conditional notice, without active HTML."""
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.parts = []
|
||||
self.skip = 0
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag in ('script', 'style'):
|
||||
self.skip += 1
|
||||
if not self.skip and tag in ('p', 'li', 'pre', 'h1', 'h2', 'h3', 'br', 'div'):
|
||||
self.parts.append('\n')
|
||||
if not self.skip and tag == 'a':
|
||||
href = dict(attrs).get('href', '')
|
||||
if href.startswith(('https://', 'http://')):
|
||||
self.parts.append(' [' + href + '] ')
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag in ('script', 'style') and self.skip:
|
||||
self.skip -= 1
|
||||
|
||||
def handle_data(self, value):
|
||||
if not self.skip:
|
||||
self.parts.append(value)
|
||||
|
||||
|
||||
def notices(archive):
|
||||
chunks = []
|
||||
with tarfile.open(archive) as source:
|
||||
for member in source:
|
||||
name = PurePosixPath(member.name).name
|
||||
if not member.isfile() or not re.match(r'(?i)^(licen[cs]e|copying|copyright|notice|authors)([._-].*|$)', name):
|
||||
continue
|
||||
if member.size > 8 << 20:
|
||||
raise ValueError('oversized notice: ' + member.name)
|
||||
raw = source.extractfile(member).read().decode('utf-8', errors='replace')
|
||||
if name == 'license.html':
|
||||
parser = NoticeText()
|
||||
# Include ALL conditional branches (a conservative superset).
|
||||
parser.feed(re.sub(r'^#.*$', '', raw, flags=re.M))
|
||||
raw = ''.join(parser.parts)
|
||||
chunks.append('\n\n===== ' + member.name + ' =====\n\n' + raw)
|
||||
if not chunks:
|
||||
if archive.name == 'fortanix-sgx-abi-0.6.1.crate':
|
||||
raw = member_bytes(archive, 'src/lib.rs').decode()
|
||||
return raw.split('*/', 1)[0] + '*/\nFull MPL-2.0 text: wrapper-MPL.txt\n'
|
||||
if archive.name == 'vex-sdk-0.27.1.crate':
|
||||
return 'MIT license and copyright notice: vex-sdk-LICENSE.txt\nOriginal metadata:\n' + member_bytes(archive, 'Cargo.toml').decode()
|
||||
raise ValueError('no notices found: ' + archive.name)
|
||||
return ''.join(chunks)
|
||||
|
||||
|
||||
def member_bytes(archive, suffix):
|
||||
with tarfile.open(archive) as source:
|
||||
matches = [m for m in source if m.isfile() and m.name.endswith('/' + suffix)]
|
||||
if len(matches) != 1 or matches[0].size > 64 << 20:
|
||||
raise ValueError('ambiguous/missing member: ' + suffix)
|
||||
return source.extractfile(matches[0]).read()
|
||||
|
||||
|
||||
def build(cache, output, offline=False):
|
||||
if output.exists():
|
||||
raise ValueError('output already exists; choose a new immutable directory')
|
||||
lock = json.loads((ROOT / 'sources.lock.json').read_text())
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
inputs = {entry['file']: obtain(cache, entry['file'], entry['url'], entry['sha256'], offline) for entry in lock['inputs']}
|
||||
# Rust build-std vendors its own Cargo.lock packages, not just Gecko crates.
|
||||
cargo = tomllib.loads(member_bytes(inputs['rust-src.tar.xz'], 'library/Cargo.lock').decode())
|
||||
crates = []
|
||||
for package in cargo['package']:
|
||||
if 'source' not in package:
|
||||
continue
|
||||
if package['source'] != 'registry+https://github.com/rust-lang/crates.io-index':
|
||||
raise ValueError('unsupported Rust dependency source')
|
||||
name = package['name'] + '-' + package['version'] + '.crate'
|
||||
url = 'https://static.crates.io/crates/' + package['name'] + '/' + name
|
||||
crates.append((name, obtain(cache, name, url, package['checksum'], offline), url))
|
||||
output.mkdir(parents=True)
|
||||
records = []
|
||||
|
||||
def record(name, origin, kind):
|
||||
records.append(dict(file=name, sha256=digest(output / name), size=(output / name).stat().st_size, origin=origin, kind=kind))
|
||||
|
||||
for entry in lock['inputs']:
|
||||
name = entry['file']
|
||||
shutil.copyfile(inputs[name], output / name)
|
||||
record(name, entry['url'], 'notice' if name.endswith('.txt') else 'source' if name != 'engine.tar.gz' else 'upstream-binary')
|
||||
if name.endswith(('.tar.gz', '.tar.xz', '.tgz')) and name not in ('engine.tar.gz', 'wrapper.tar.gz'):
|
||||
target = name + '.notices.txt'
|
||||
(output / target).write_text(notices(inputs[name]), encoding='utf-8')
|
||||
record(target, name, 'notice')
|
||||
for name, source, url in crates:
|
||||
shutil.copyfile(source, output / name)
|
||||
record(name, url, 'source')
|
||||
target = name + '.notices.txt'
|
||||
(output / target).write_text(notices(source), encoding='utf-8')
|
||||
record(target, name, 'notice')
|
||||
for name in ('gecko.js', 'gecko.wasm.zst'):
|
||||
(output / name).write_bytes(member_bytes(inputs['engine.tar.gz'], 'dist/' + name))
|
||||
record(name, 'engine.tar.gz', 'engine')
|
||||
if records[-1]['sha256'] != lock['engine'][name]:
|
||||
raise ValueError('engine payload mismatch: ' + name)
|
||||
# This file describes preservation of originals and source/build relationships.
|
||||
shutil.copyfile(ROOT / 'README.txt', output / 'README.txt')
|
||||
record('README.txt', 'PagesWiki tools/gecko/README.txt', 'notice')
|
||||
rows = ''.join('<tr><td><a href="' + html.escape(r['file'], quote=True) + '">' + html.escape(r['file']) + '</a></td><td>' + r['kind'] + '</td><td>' + str(r['size']) + '</td><td><code>' + r['sha256'] + '</code></td></tr>' for r in records)
|
||||
(output / 'index.html').write_text('<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width"><title>Gecko sources and licenses</title><h1>Gecko sources and licenses</h1><p>Unmodified upstream Gecko WASM v0.0.1. Not an official Mozilla Firefox distribution. <a href="README.txt">Source and build instructions</a>. <a href="manifest.json">SHA-256 manifest</a>.</p><p>Notices include all build-conditional sections and vendored source notices; some components may not be linked. Originals remain in the source archives. License texts and copyright notices are not replaced by this index.</p><table><thead><tr><th>File</th><th>Kind</th><th>Bytes</th><th>SHA-256</th></tr></thead><tbody>' + rows + '</tbody></table></html>', encoding='utf-8')
|
||||
record('index.html', 'generated index', 'index')
|
||||
(output / 'manifest.json').write_text(json.dumps(dict(format='pageswiki.gecko-distribution.v1', version='v0.0.1', files=records), indent=2, sort_keys=True) + '\n')
|
||||
print('manifest SHA-256:', digest(output / 'manifest.json'), flush=True)
|
||||
print('files:', len(records), 'bytes:', sum(r['size'] for r in records), flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--cache', type=Path, required=True)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--offline', action='store_true')
|
||||
args = parser.parse_args()
|
||||
build(args.cache, args.output, args.offline)
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"engine": {
|
||||
"gecko.js": "e5aeebffd21106eccf23b1219316221add5e95085bc5a1bfb941117cfcd75ef7",
|
||||
"gecko.wasm.zst": "ebd6c75179152624d9644865e11792ed25cadc714d9e0985f412baf9c637a086"
|
||||
},
|
||||
"inputs": [
|
||||
{"file":"engine.tar.gz","url":"https://github.com/HeyPuter/firefox-wasm/releases/download/v0.0.1/gecko.js-v0.0.1.tar.gz","sha256":"bac28056a00ef6e5fb4345117e73d3629e75aca00dac71389fabbce39c018a73"},
|
||||
{"file":"wrapper.tar.gz","url":"https://codeload.github.com/HeyPuter/firefox-wasm/tar.gz/eadcb5fa828d7e8b54ae2df13b6b3208f6553944","sha256":"60078bb4c68ed97e95b5720b9833a9454aac0668fb069aa53393e96628e845e1"},
|
||||
{"file":"firefox.tar.gz","url":"https://codeload.github.com/MercuryWorkshop/firefox/tar.gz/2e1e835a5da5907e5ceef19aade58a7493f1e34d","sha256":"d3d3907c34e498e2a1ef2ada1d11797a4e8075b49ab45b1968aa49d5cddc3096"},
|
||||
{"file":"emscripten.tar.gz","url":"https://codeload.github.com/emscripten-core/emscripten/tar.gz/refs/tags/6.0.1","sha256":"0187d75d8ff8c5345fcf37deb92c24b7ebd2eb1fe04339c21ea2bcd9d81cc681"},
|
||||
{"file":"rust-src.tar.xz","url":"https://static.rust-lang.org/dist/rust-src-1.95.0.tar.xz","sha256":"67b09138c8db96afc4bbfc69ea771ac9a091fd777698acb43f6dfd9fb7dea363"},
|
||||
{"file":"zstddec.tgz","url":"https://registry.npmjs.org/zstddec/-/zstddec-0.2.0.tgz","sha256":"4708fc2e97b5ae6e8c2545ceabdd965bfe8858a5064fce30380eeb4c6ba3f41d"},
|
||||
{"file":"wrapper-MPL.txt","url":"https://raw.githubusercontent.com/HeyPuter/firefox-wasm/ac40f56c85060bd49dfc36ca60490f97dd270c50/LICENSE","sha256":"3f3d9e0024b1921b067d6f7f88deb4a60cbe7a78e76c64e3f1d7fc3b779b9d04"},
|
||||
{"file":"vex-sdk-LICENSE.txt","url":"https://raw.githubusercontent.com/vexide/vex-sdk/6c0b83d35b6fc31a7607efe456e387df45476314/LICENSE.md","sha256":"7bc7d5de2a0b793d4e1adb7ffcb99387776adc1f15f055bc3b667ea449249e6a"},
|
||||
{"file":"NotoSansCJKjp-Regular.otf","url":"https://raw.githubusercontent.com/notofonts/noto-cjk/523d033d6cb47f4a80c58a35753646f5c3608a78/Sans/OTF/Japanese/NotoSansCJKjp-Regular.otf","sha256":"68a3fc98800b2a27b371f2fb79991daf3633bd89309d4ffaa6946fd587f375b5"},
|
||||
{"file":"NotoSansCJK-LICENSE.txt","url":"https://raw.githubusercontent.com/notofonts/noto-cjk/523d033d6cb47f4a80c58a35753646f5c3608a78/LICENSE","sha256":"6a73f9541c2de74158c0e7cf6b0a58ef774f5a780bf191f2d7ec9cc53efe2bf2"}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import hashlib
|
||||
import io
|
||||
from pathlib import Path
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import bundle
|
||||
|
||||
|
||||
class BundleTest(unittest.TestCase):
|
||||
def test_notice_superset_is_inert(self):
|
||||
p = bundle.NoticeText()
|
||||
p.feed('<style>hidden</style><h1>Licenses</h1><p>Copyright & authors</p><script>bad()</script><a href="https://example.org/license">Terms</a>')
|
||||
result = ''.join(p.parts)
|
||||
self.assertIn('Copyright & authors', result)
|
||||
self.assertIn('https://example.org/license', result)
|
||||
self.assertNotIn('bad()', result)
|
||||
self.assertNotIn('hidden', result)
|
||||
|
||||
def test_hash_rejects_changed_cached_source(self):
|
||||
with tempfile.TemporaryDirectory() as name:
|
||||
cache = Path(name)
|
||||
(cache / 'source').write_bytes(b'changed')
|
||||
with self.assertRaisesRegex(ValueError, 'checksum'):
|
||||
bundle.obtain(cache, 'source', 'https://invalid.test', '0' * 64, True)
|
||||
self.assertEqual(bundle.obtain(cache, 'source', '', hashlib.sha256(b'changed').hexdigest(), True), cache / 'source')
|
||||
|
||||
def test_archive_is_read_not_extracted(self):
|
||||
with tempfile.TemporaryDirectory() as name:
|
||||
path = Path(name) / 'source.tar.gz'
|
||||
with tarfile.open(path, 'w:gz') as archive:
|
||||
member = tarfile.TarInfo('../../LICENSE')
|
||||
data = b'Copyright owner. Permission notice.'
|
||||
member.size = len(data)
|
||||
archive.addfile(member, io.BytesIO(data))
|
||||
member = tarfile.TarInfo('src/license.html')
|
||||
data = b'#ifdef OPTIONAL\n<p>Optional terms</p>\n#else\n<p>Other terms</p>\n#endif'
|
||||
member.size = len(data)
|
||||
archive.addfile(member, io.BytesIO(data))
|
||||
notice = bundle.notices(path)
|
||||
self.assertIn('Copyright owner', notice)
|
||||
self.assertIn('Optional terms', notice)
|
||||
self.assertIn('Other terms', notice)
|
||||
self.assertNotIn('#ifdef', notice)
|
||||
self.assertEqual(list(Path(name).iterdir()), [path])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user