164 lines
8.1 KiB
Python
164 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Build an offline, hash-locked Gecko distribution without extracting source trees.
|
|
|
|
Python 3.11+. Sources and notices remain available alongside the engine. The
|
|
output is a new directory: never replace a bundle mounted by a running relay.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import html
|
|
from html.parser import HTMLParser
|
|
import json
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import shutil
|
|
import tarfile
|
|
import tomllib
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
|
|
|
|
def digest(path):
|
|
with path.open('rb') as stream:
|
|
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
|
|
|
|
|
def obtain(cache, name, url, expected, offline=False):
|
|
path = cache / name
|
|
if not path.exists():
|
|
if offline:
|
|
raise ValueError('missing offline input: ' + name)
|
|
temporary = path.with_suffix(path.suffix + '.part')
|
|
with urllib.request.urlopen(url, timeout=60) as response, temporary.open('wb') as target:
|
|
shutil.copyfileobj(response, target, 1 << 20)
|
|
if digest(temporary) != expected:
|
|
temporary.unlink()
|
|
raise ValueError('download checksum mismatch: ' + name)
|
|
temporary.rename(path)
|
|
if digest(path) != expected:
|
|
raise ValueError('cached checksum mismatch: ' + name)
|
|
return path
|
|
|
|
|
|
class NoticeText(HTMLParser):
|
|
"""Readable superset of Mozilla's conditional notice, without active HTML."""
|
|
def __init__(self):
|
|
super().__init__(convert_charrefs=True)
|
|
self.parts = []
|
|
self.skip = 0
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
if tag in ('script', 'style'):
|
|
self.skip += 1
|
|
if not self.skip and tag in ('p', 'li', 'pre', 'h1', 'h2', 'h3', 'br', 'div'):
|
|
self.parts.append('\n')
|
|
if not self.skip and tag == 'a':
|
|
href = dict(attrs).get('href', '')
|
|
if href.startswith(('https://', 'http://')):
|
|
self.parts.append(' [' + href + '] ')
|
|
|
|
def handle_endtag(self, tag):
|
|
if tag in ('script', 'style') and self.skip:
|
|
self.skip -= 1
|
|
|
|
def handle_data(self, value):
|
|
if not self.skip:
|
|
self.parts.append(value)
|
|
|
|
|
|
def notices(archive):
|
|
chunks = []
|
|
with tarfile.open(archive) as source:
|
|
for member in source:
|
|
name = PurePosixPath(member.name).name
|
|
if not member.isfile() or not re.match(r'(?i)^(licen[cs]e|copying|copyright|notice|authors)([._-].*|$)', name):
|
|
continue
|
|
if member.size > 8 << 20:
|
|
raise ValueError('oversized notice: ' + member.name)
|
|
raw = source.extractfile(member).read().decode('utf-8', errors='replace')
|
|
if name == 'license.html':
|
|
parser = NoticeText()
|
|
# Include ALL conditional branches (a conservative superset).
|
|
parser.feed(re.sub(r'^#.*$', '', raw, flags=re.M))
|
|
raw = ''.join(parser.parts)
|
|
chunks.append('\n\n===== ' + member.name + ' =====\n\n' + raw)
|
|
if not chunks:
|
|
if archive.name == 'fortanix-sgx-abi-0.6.1.crate':
|
|
raw = member_bytes(archive, 'src/lib.rs').decode()
|
|
return raw.split('*/', 1)[0] + '*/\nFull MPL-2.0 text: wrapper-MPL.txt\n'
|
|
if archive.name == 'vex-sdk-0.27.1.crate':
|
|
return 'MIT license and copyright notice: vex-sdk-LICENSE.txt\nOriginal metadata:\n' + member_bytes(archive, 'Cargo.toml').decode()
|
|
raise ValueError('no notices found: ' + archive.name)
|
|
return ''.join(chunks)
|
|
|
|
|
|
def member_bytes(archive, suffix):
|
|
with tarfile.open(archive) as source:
|
|
matches = [m for m in source if m.isfile() and m.name.endswith('/' + suffix)]
|
|
if len(matches) != 1 or matches[0].size > 64 << 20:
|
|
raise ValueError('ambiguous/missing member: ' + suffix)
|
|
return source.extractfile(matches[0]).read()
|
|
|
|
|
|
def build(cache, output, offline=False):
|
|
if output.exists():
|
|
raise ValueError('output already exists; choose a new immutable directory')
|
|
lock = json.loads((ROOT / 'sources.lock.json').read_text())
|
|
cache.mkdir(parents=True, exist_ok=True)
|
|
inputs = {entry['file']: obtain(cache, entry['file'], entry['url'], entry['sha256'], offline) for entry in lock['inputs']}
|
|
# Rust build-std vendors its own Cargo.lock packages, not just Gecko crates.
|
|
cargo = tomllib.loads(member_bytes(inputs['rust-src.tar.xz'], 'library/Cargo.lock').decode())
|
|
crates = []
|
|
for package in cargo['package']:
|
|
if 'source' not in package:
|
|
continue
|
|
if package['source'] != 'registry+https://github.com/rust-lang/crates.io-index':
|
|
raise ValueError('unsupported Rust dependency source')
|
|
name = package['name'] + '-' + package['version'] + '.crate'
|
|
url = 'https://static.crates.io/crates/' + package['name'] + '/' + name
|
|
crates.append((name, obtain(cache, name, url, package['checksum'], offline), url))
|
|
output.mkdir(parents=True)
|
|
records = []
|
|
|
|
def record(name, origin, kind):
|
|
records.append(dict(file=name, sha256=digest(output / name), size=(output / name).stat().st_size, origin=origin, kind=kind))
|
|
|
|
for entry in lock['inputs']:
|
|
name = entry['file']
|
|
shutil.copyfile(inputs[name], output / name)
|
|
record(name, entry['url'], 'notice' if name.endswith('.txt') else 'source' if name != 'engine.tar.gz' else 'upstream-binary')
|
|
if name.endswith(('.tar.gz', '.tar.xz', '.tgz')) and name not in ('engine.tar.gz', 'wrapper.tar.gz'):
|
|
target = name + '.notices.txt'
|
|
(output / target).write_text(notices(inputs[name]), encoding='utf-8')
|
|
record(target, name, 'notice')
|
|
for name, source, url in crates:
|
|
shutil.copyfile(source, output / name)
|
|
record(name, url, 'source')
|
|
target = name + '.notices.txt'
|
|
(output / target).write_text(notices(source), encoding='utf-8')
|
|
record(target, name, 'notice')
|
|
for name in ('gecko.js', 'gecko.wasm.zst'):
|
|
(output / name).write_bytes(member_bytes(inputs['engine.tar.gz'], 'dist/' + name))
|
|
record(name, 'engine.tar.gz', 'engine')
|
|
if records[-1]['sha256'] != lock['engine'][name]:
|
|
raise ValueError('engine payload mismatch: ' + name)
|
|
# This file describes preservation of originals and source/build relationships.
|
|
shutil.copyfile(ROOT / 'README.txt', output / 'README.txt')
|
|
record('README.txt', 'PagesWiki tools/gecko/README.txt', 'notice')
|
|
rows = ''.join('<tr><td><a href="' + html.escape(r['file'], quote=True) + '">' + html.escape(r['file']) + '</a></td><td>' + r['kind'] + '</td><td>' + str(r['size']) + '</td><td><code>' + r['sha256'] + '</code></td></tr>' for r in records)
|
|
(output / 'index.html').write_text('<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width"><title>Gecko sources and licenses</title><h1>Gecko sources and licenses</h1><p>Unmodified upstream Gecko WASM v0.0.1. Not an official Mozilla Firefox distribution. <a href="README.txt">Source and build instructions</a>. <a href="manifest.json">SHA-256 manifest</a>.</p><p>Notices include all build-conditional sections and vendored source notices; some components may not be linked. Originals remain in the source archives. License texts and copyright notices are not replaced by this index.</p><table><thead><tr><th>File</th><th>Kind</th><th>Bytes</th><th>SHA-256</th></tr></thead><tbody>' + rows + '</tbody></table></html>', encoding='utf-8')
|
|
record('index.html', 'generated index', 'index')
|
|
(output / 'manifest.json').write_text(json.dumps(dict(format='pageswiki.gecko-distribution.v1', version='v0.0.1', files=records), indent=2, sort_keys=True) + '\n')
|
|
print('manifest SHA-256:', digest(output / 'manifest.json'), flush=True)
|
|
print('files:', len(records), 'bytes:', sum(r['size'] for r in records), flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--cache', type=Path, required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--offline', action='store_true')
|
|
args = parser.parse_args()
|
|
build(args.cache, args.output, args.offline)
|