Files
pages/tools/gecko/bundle.py
T
Anya Tokugawa b90aef79d0
Build and publish container images / images (push) Successful in 11m37s
Release container images / release (push) Successful in 22s
feat: add browser upload queue and trusted sessions with bounded projections
2026-11-08 20:27:04 +09:00

164 lines
8.1 KiB
Python

#!/usr/bin/env python3
"""Build an offline, hash-locked Gecko distribution without extracting source trees.
Python 3.11+. Sources and notices remain available alongside the engine. The
output is a new directory: never replace a bundle mounted by a running relay.
"""
import argparse
import hashlib
import html
from html.parser import HTMLParser
import json
from pathlib import Path, PurePosixPath
import re
import shutil
import tarfile
import tomllib
import urllib.request
ROOT = Path(__file__).resolve().parent
def digest(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def obtain(cache, name, url, expected, offline=False):
path = cache / name
if not path.exists():
if offline:
raise ValueError('missing offline input: ' + name)
temporary = path.with_suffix(path.suffix + '.part')
with urllib.request.urlopen(url, timeout=60) as response, temporary.open('wb') as target:
shutil.copyfileobj(response, target, 1 << 20)
if digest(temporary) != expected:
temporary.unlink()
raise ValueError('download checksum mismatch: ' + name)
temporary.rename(path)
if digest(path) != expected:
raise ValueError('cached checksum mismatch: ' + name)
return path
class NoticeText(HTMLParser):
"""Readable superset of Mozilla's conditional notice, without active HTML."""
def __init__(self):
super().__init__(convert_charrefs=True)
self.parts = []
self.skip = 0
def handle_starttag(self, tag, attrs):
if tag in ('script', 'style'):
self.skip += 1
if not self.skip and tag in ('p', 'li', 'pre', 'h1', 'h2', 'h3', 'br', 'div'):
self.parts.append('\n')
if not self.skip and tag == 'a':
href = dict(attrs).get('href', '')
if href.startswith(('https://', 'http://')):
self.parts.append(' [' + href + '] ')
def handle_endtag(self, tag):
if tag in ('script', 'style') and self.skip:
self.skip -= 1
def handle_data(self, value):
if not self.skip:
self.parts.append(value)
def notices(archive):
chunks = []
with tarfile.open(archive) as source:
for member in source:
name = PurePosixPath(member.name).name
if not member.isfile() or not re.match(r'(?i)^(licen[cs]e|copying|copyright|notice|authors)([._-].*|$)', name):
continue
if member.size > 8 << 20:
raise ValueError('oversized notice: ' + member.name)
raw = source.extractfile(member).read().decode('utf-8', errors='replace')
if name == 'license.html':
parser = NoticeText()
# Include ALL conditional branches (a conservative superset).
parser.feed(re.sub(r'^#.*$', '', raw, flags=re.M))
raw = ''.join(parser.parts)
chunks.append('\n\n===== ' + member.name + ' =====\n\n' + raw)
if not chunks:
if archive.name == 'fortanix-sgx-abi-0.6.1.crate':
raw = member_bytes(archive, 'src/lib.rs').decode()
return raw.split('*/', 1)[0] + '*/\nFull MPL-2.0 text: wrapper-MPL.txt\n'
if archive.name == 'vex-sdk-0.27.1.crate':
return 'MIT license and copyright notice: vex-sdk-LICENSE.txt\nOriginal metadata:\n' + member_bytes(archive, 'Cargo.toml').decode()
raise ValueError('no notices found: ' + archive.name)
return ''.join(chunks)
def member_bytes(archive, suffix):
with tarfile.open(archive) as source:
matches = [m for m in source if m.isfile() and m.name.endswith('/' + suffix)]
if len(matches) != 1 or matches[0].size > 64 << 20:
raise ValueError('ambiguous/missing member: ' + suffix)
return source.extractfile(matches[0]).read()
def build(cache, output, offline=False):
if output.exists():
raise ValueError('output already exists; choose a new immutable directory')
lock = json.loads((ROOT / 'sources.lock.json').read_text())
cache.mkdir(parents=True, exist_ok=True)
inputs = {entry['file']: obtain(cache, entry['file'], entry['url'], entry['sha256'], offline) for entry in lock['inputs']}
# Rust build-std vendors its own Cargo.lock packages, not just Gecko crates.
cargo = tomllib.loads(member_bytes(inputs['rust-src.tar.xz'], 'library/Cargo.lock').decode())
crates = []
for package in cargo['package']:
if 'source' not in package:
continue
if package['source'] != 'registry+https://github.com/rust-lang/crates.io-index':
raise ValueError('unsupported Rust dependency source')
name = package['name'] + '-' + package['version'] + '.crate'
url = 'https://static.crates.io/crates/' + package['name'] + '/' + name
crates.append((name, obtain(cache, name, url, package['checksum'], offline), url))
output.mkdir(parents=True)
records = []
def record(name, origin, kind):
records.append(dict(file=name, sha256=digest(output / name), size=(output / name).stat().st_size, origin=origin, kind=kind))
for entry in lock['inputs']:
name = entry['file']
shutil.copyfile(inputs[name], output / name)
record(name, entry['url'], 'notice' if name.endswith('.txt') else 'source' if name != 'engine.tar.gz' else 'upstream-binary')
if name.endswith(('.tar.gz', '.tar.xz', '.tgz')) and name not in ('engine.tar.gz', 'wrapper.tar.gz'):
target = name + '.notices.txt'
(output / target).write_text(notices(inputs[name]), encoding='utf-8')
record(target, name, 'notice')
for name, source, url in crates:
shutil.copyfile(source, output / name)
record(name, url, 'source')
target = name + '.notices.txt'
(output / target).write_text(notices(source), encoding='utf-8')
record(target, name, 'notice')
for name in ('gecko.js', 'gecko.wasm.zst'):
(output / name).write_bytes(member_bytes(inputs['engine.tar.gz'], 'dist/' + name))
record(name, 'engine.tar.gz', 'engine')
if records[-1]['sha256'] != lock['engine'][name]:
raise ValueError('engine payload mismatch: ' + name)
# This file describes preservation of originals and source/build relationships.
shutil.copyfile(ROOT / 'README.txt', output / 'README.txt')
record('README.txt', 'PagesWiki tools/gecko/README.txt', 'notice')
rows = ''.join('<tr><td><a href="' + html.escape(r['file'], quote=True) + '">' + html.escape(r['file']) + '</a></td><td>' + r['kind'] + '</td><td>' + str(r['size']) + '</td><td><code>' + r['sha256'] + '</code></td></tr>' for r in records)
(output / 'index.html').write_text('<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width"><title>Gecko sources and licenses</title><h1>Gecko sources and licenses</h1><p>Unmodified upstream Gecko WASM v0.0.1. Not an official Mozilla Firefox distribution. <a href="README.txt">Source and build instructions</a>. <a href="manifest.json">SHA-256 manifest</a>.</p><p>Notices include all build-conditional sections and vendored source notices; some components may not be linked. Originals remain in the source archives. License texts and copyright notices are not replaced by this index.</p><table><thead><tr><th>File</th><th>Kind</th><th>Bytes</th><th>SHA-256</th></tr></thead><tbody>' + rows + '</tbody></table></html>', encoding='utf-8')
record('index.html', 'generated index', 'index')
(output / 'manifest.json').write_text(json.dumps(dict(format='pageswiki.gecko-distribution.v1', version='v0.0.1', files=records), indent=2, sort_keys=True) + '\n')
print('manifest SHA-256:', digest(output / 'manifest.json'), flush=True)
print('files:', len(records), 'bytes:', sum(r['size'] for r in records), flush=True)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--cache', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--offline', action='store_true')
args = parser.parse_args()
build(args.cache, args.output, args.offline)