Serve smaller Zstd Wasm for Breeze web and Telegram

This commit is contained in:
waozi committed 2026-11-19 18:30:47 -03:00
1 parent d3628b76f6
commit 3423cba76d
3 files changed
+228 -25

No files matched your search

+108 -15
View File
@@ -1,32 +1,111 @@
#!/usr/bin/env python3
"""Package large browser files within the Pages per-file limit."""
from __future__ import annotations
import argparse
import ctypes
import filecmp
import gzip
import hashlib
import json
import shutil
from ctypes.util import find_library
from pathlib import Path
MAX_FILE_BYTES = 25 * 1024 * 1024
def prepare(path: Path, limit: int) -> None:
class ZstdEncoder:
"""Use the host libzstd when it is installed. Browsers decode this natively."""
def __init__(self, library: ctypes.CDLL):
self.library = library
size = ctypes.c_size_t
pointer = ctypes.c_void_p
signatures = {
"createCCtx": (pointer, []),
"freeCCtx": (size, [pointer]),
"CCtx_setParameter": (size, [pointer, ctypes.c_int, ctypes.c_int]),
"compressBound": (size, [size]),
"compress2": (size, [pointer, pointer, size, ctypes.c_char_p, size]),
"isError": (ctypes.c_uint, [size]),
"getErrorName": (ctypes.c_char_p, [size]),
}
for name, (result, parameters) in signatures.items():
function = getattr(library, "ZSTD_" + name)
function.restype = result
function.argtypes = parameters
def checked(self, result: int) -> int:
if self.library.ZSTD_isError(result):
raise ValueError(self.library.ZSTD_getErrorName(result).decode())
return result
def encode(self, path: Path, limit: int) -> list[bytes]:
zstd = self.library
context = zstd.ZSTD_createCCtx()
if not context:
raise MemoryError("Could not allocate a Zstd compression context")
parts = []
try:
self.checked(zstd.ZSTD_CCtx_setParameter(context, 100, 22)) # compressionLevel
self.checked(zstd.ZSTD_CCtx_setParameter(context, 101, 23)) # windowLog, RFC 9659
with path.open("rb") as source:
while block := source.read(min(limit, MAX_FILE_BYTES)):
output = ctypes.create_string_buffer(zstd.ZSTD_compressBound(len(block)))
size = self.checked(zstd.ZSTD_compress2(context, output, len(output), block, len(block)))
frame = output[:size]
for offset in range(0, size, limit):
parts.append(frame[offset:offset + limit])
finally:
zstd.ZSTD_freeCCtx(context)
return parts
def load_zstd() -> ZstdEncoder | None:
name = find_library("zstd")
if name:
try:
return ZstdEncoder(ctypes.CDLL(name))
except (OSError, AttributeError):
pass
return None
def write_parts(path: Path, coding: str, parts: list[bytes]) -> list[str]:
sha256 = hashlib.sha256()
for part in parts:
sha256.update(part)
digest = sha256.hexdigest()
names = []
for index, part in enumerate(parts):
name = f"{path.name}.{coding}.{digest[:16]}.part-{index:03d}"
path.with_name(name).write_bytes(part)
names.append(name)
manifest = {"parts": names, "sha256": digest, "size": sum(map(len, parts))}
suffix = ".parts.json" if coding == "gz" else ".zstd.parts.json"
manifest_path = path.with_name(path.name + suffix)
manifest_path.write_text(json.dumps(manifest, sort_keys=True) + "\n")
return names + [manifest_path.name]
def prepare(path: Path, limit: int, zstd: ZstdEncoder | None = None) -> list[str]:
if not path.exists() or path.stat().st_size <= limit:
return
return []
compressed = gzip.compress(path.read_bytes(), compresslevel=9, mtime=0)
if len(compressed) <= limit:
path.with_name(path.name + ".gz").write_bytes(compressed)
output = path.with_name(path.name + ".gz")
output.write_bytes(compressed)
names = [output.name]
else:
digest = hashlib.sha256(compressed).hexdigest()
parts = []
for index, offset in enumerate(range(0, len(compressed), limit)):
name = f"{path.name}.gz.{digest[:16]}.part-{index:03d}"
path.with_name(name).write_bytes(compressed[offset:offset + limit])
parts.append(name)
manifest = {"parts": parts, "sha256": digest, "size": len(compressed)}
path.with_name(path.name + ".parts.json").write_text(
json.dumps(manifest, sort_keys=True) + "\n")
names = write_parts(path, "gz", [compressed[i:i + limit] for i in range(0, len(compressed), limit)])
if zstd:
parts = zstd.encode(path, limit)
if sum(map(len, parts)) < len(compressed):
names.extend(write_parts(path, "zstd", parts))
path.unlink()
return names
def main() -> None:
@@ -36,9 +115,23 @@ def main() -> None:
args = parser.parse_args()
if args.max_file_bytes <= 0:
parser.error("--max-file-bytes must be positive")
for app in ("web", "telegram"):
for name in ("index.data", "index.wasm"):
prepare(args.root / "build" / app / name, args.max_file_bytes)
web = args.root / "build" / "web"
telegram = args.root / "build" / "telegram"
for directory in (web, telegram):
prepare(directory / "index.data", args.max_file_bytes)
web_wasm = web / "index.wasm"
telegram_wasm = telegram / "index.wasm"
same_wasm = web_wasm.is_file() and telegram_wasm.is_file() and filecmp.cmp(
web_wasm, telegram_wasm, shallow=False)
encoder = load_zstd()
prepared = prepare(web_wasm, args.max_file_bytes, encoder)
if same_wasm:
if prepared:
for name in prepared:
shutil.copyfile(web / name, telegram / name)
telegram_wasm.unlink()
else:
prepare(telegram_wasm, args.max_file_bytes, encoder)
for path in args.root.rglob("*"):
if path.is_file() and path.stat().st_size > args.max_file_bytes:
raise SystemExit(f"Site asset exceeds the Pages limit: {path}")
+46 -9
View File
@@ -6,20 +6,44 @@ function assetRequest(url, request, method = 'GET') {
return new Request(url, { method, headers });
}
async function multipartAsset(url, request, env, manifestResponse) {
const manifest = await manifestResponse.json();
function preferZstd(request) {
const accepted = request.cf?.clientAcceptEncoding ?? request.headers.get('Accept-Encoding') ?? '';
const quality = new Map();
for (const item of accepted.toLowerCase().split(',')) {
const [coding, parameter] = item.trim().split(';');
if (parameter === undefined) {
quality.set(coding, 1);
} else {
const match = /^\s*q\s*=\s*(0(?:\.\d{0,3})?|1(?:\.0{0,3})?)\s*$/.exec(parameter);
if (match) quality.set(coding.trim(), Number(match[1]));
}
}
const zstd = quality.get('zstd') ?? 0;
return zstd > 0 && zstd >= (quality.get('gzip') ?? quality.get('*') ?? 0);
}
function varyWasm(headers) {
headers.append('Vary', 'Accept-Encoding');
headers.append('Cache-Control', 'no-transform');
}
async function multipartAsset(url, request, env, manifestResponse, coding) {
const manifest = await manifestResponse.json().catch(() => null);
const basename = url.pathname.slice(url.pathname.lastIndexOf('/') + 1);
if (!Array.isArray(manifest.parts) || manifest.parts.length === 0 ||
const extension = coding === 'zstd' ? 'zstd' : 'gz';
if (!manifest || !Array.isArray(manifest.parts) || manifest.parts.length === 0 ||
!manifest.parts.every(part => typeof part === 'string' &&
part.startsWith(basename + '.gz.') && !/[\\/]/.test(part)) ||
!/^[a-f0-9]{64}$/.test(manifest.sha256)) {
part.startsWith(basename + '.' + extension + '.') && !/[\\/]/.test(part)) ||
!/^[a-f0-9]{64}$/.test(manifest.sha256) ||
(coding === 'zstd' && (!Number.isSafeInteger(manifest.size) || manifest.size <= 0))) {
return new Response('Invalid asset manifest', { status: 502 });
}
const headers = new Headers(manifestResponse.headers);
headers.delete('Content-Length');
headers.delete('Content-Range');
headers.delete('Accept-Ranges');
headers.set('Content-Encoding', 'gzip');
headers.set('Content-Encoding', coding);
if (basename.endsWith('.wasm')) varyWasm(headers);
headers.set('Content-Type', basename.endsWith('.wasm')
? 'application/wasm' : 'application/octet-stream');
const etag = '"' + manifest.sha256 + '"';
@@ -31,7 +55,10 @@ async function multipartAsset(url, request, env, manifestResponse) {
if (request.method === 'HEAD') {
return new Response(null, { headers });
}
const { readable, writable } = new TransformStream();
if (coding === 'zstd') headers.set('Content-Length', String(manifest.size));
// Fixed length avoids HTTP/1.1 chunked Zstd issues in WebKit.
const { readable, writable } = coding === 'zstd' && typeof FixedLengthStream === 'function'
? new FixedLengthStream(manifest.size) : new TransformStream();
async function pump() {
for (const part of manifest.parts) {
const partUrl = new URL(url);
@@ -54,11 +81,20 @@ export default {
const url = new URL(request.url);
if (['GET', 'HEAD'].includes(request.method) &&
/^\/build\/(web|telegram)\/index\.(data|wasm)$/.test(url.pathname)) {
const wasm = url.pathname.endsWith('.wasm');
const manifestUrl = new URL(url);
manifestUrl.pathname += '.parts.json';
if (wasm && preferZstd(request)) {
manifestUrl.pathname = url.pathname + '.zstd.parts.json';
const manifest = await env.ASSETS.fetch(assetRequest(manifestUrl, request));
if (manifest.ok && manifest.headers.get('Content-Type')?.includes('application/json')) {
const response = await multipartAsset(url, request, env, manifest, 'zstd');
if (response.status !== 502) return response;
}
}
manifestUrl.pathname = url.pathname + '.parts.json';
const manifest = await env.ASSETS.fetch(assetRequest(manifestUrl, request));
if (manifest.ok && manifest.headers.get('Content-Type')?.includes('application/json')) {
return multipartAsset(url, request, env, manifest);
return multipartAsset(url, request, env, manifest, 'gzip');
}
const compressed = new URL(url);
compressed.pathname += '.gz';
@@ -66,6 +102,7 @@ export default {
if (response.ok && !response.headers.get('Content-Type')?.includes('text/html')) {
const headers = new Headers(response.headers);
headers.set('Content-Encoding', 'gzip');
if (wasm) varyWasm(headers);
headers.set('Content-Type', url.pathname.endsWith('.wasm')
? 'application/wasm' : 'application/octet-stream');
return new Response(response.body, {
+74 -1
View File
@@ -5,9 +5,11 @@ import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, readdirSync, rmSyn
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { spawnSync } from 'node:child_process';
import { gunzipSync } from 'node:zlib';
import * as zlib from 'node:zlib';
import { Script } from 'node:vm';
const { gunzipSync, zstdDecompressSync } = zlib;
const root = resolve(import.meta.dirname, '..');
const temporary = mkdtempSync(join(tmpdir(), 'inbe-site-assets-'));
const source = readFileSync(join(root, 'site/static/_worker.js'));
@@ -121,6 +123,76 @@ window.originalBootstrapRan = true;
console.log('Mini App generated bootstrap: syntax, SDK absence/older features, theme and gestures passed');
}
async function verifyWasmNegotiation() {
const partLimit = 96 * 1024;
const block = Array.from({ length: 3 }, (_, frame) => {
const noise = Buffer.concat(Array.from({ length: 1920 }, (_, i) =>
createHash('sha256').update(`${frame}-${i}`).digest()));
const repeat = noise.subarray(0, 36 * 1024);
return Buffer.concat([repeat, repeat, noise.subarray(repeat.length)]);
});
const original = Buffer.concat(block);
for (const app of ['web', 'telegram']) {
const directory = join(temporary, 'build', app);
rmSync(directory, { recursive: true, force: true });
mkdirSync(directory, { recursive: true });
writeFileSync(join(directory, 'index.wasm'), original);
}
const result = spawnSync('python3', [join(root, 'scripts/prepare-site-assets.py'), temporary,
'--max-file-bytes', String(partLimit)], { encoding: 'utf8' });
assert.equal(result.status, 0, result.stdout + result.stderr);
const library = spawnSync('python3', ['-c', `import runpy, sys
stage = runpy.run_path(sys.argv[1])
print('available' if stage['load_zstd']() else 'missing')`,
join(root, 'scripts/prepare-site-assets.py')], { encoding: 'utf8' });
assert.equal(library.status, 0, library.stdout + library.stderr);
for (const app of ['web', 'telegram']) {
const path = `/build/${app}/index.wasm`;
const directory = join(temporary, 'build', app);
const gzip = await serve(path, { headers: { 'Accept-Encoding': 'gzip, zstd;q=0' } });
assert.equal(gzip.headers.get('Content-Encoding'), 'gzip');
assert.ok(gzip.headers.get('Vary').toLowerCase().includes('accept-encoding'));
assert.deepEqual(gunzipSync(Buffer.from(await gzip.arrayBuffer())), original);
const zstdManifest = join(directory, 'index.wasm.zstd.parts.json');
if (library.stdout.trim() === 'missing') {
assert.ok(!existsSync(zstdManifest));
assert.equal((await serve(path, { method: 'HEAD', headers: { 'Accept-Encoding': 'zstd, gzip' } }))
.headers.get('Content-Encoding'), 'gzip');
continue;
}
const requestForZstd = new Request('https://inbe.example' + path + '?v=test', {
headers: { 'Accept-Encoding': 'gzip, br', Range: 'bytes=0-9', 'If-None-Match': gzip.headers.get('ETag') },
});
requestForZstd.cf = { clientAcceptEncoding: 'gzip;q=0.8, ZSTD;q=0.9' };
const zstd = await worker.fetch(requestForZstd, { ASSETS: assets });
assert.equal(zstd.status, 200);
assert.equal(zstd.headers.get('Content-Encoding'), 'zstd');
assert.equal(zstd.headers.get('Content-Type'), 'application/wasm');
assert.match(zstd.headers.get('Cache-Control'), /no-transform/);
const encoded = Buffer.from(await zstd.arrayBuffer());
assert.equal(zstd.headers.get('Content-Length'), String(encoded.length));
assert.equal(zstd.headers.get('ETag'), '"' + createHash('sha256').update(encoded).digest('hex') + '"');
assert.ok(encoded.length < +JSON.parse(readFileSync(join(directory, 'index.wasm.parts.json'))).size);
if (typeof zstdDecompressSync === 'function') {
assert.deepEqual(zstdDecompressSync(encoded), original);
}
const manifest = JSON.parse(readFileSync(zstdManifest));
assert.ok(manifest.parts.length > 1);
assert.ok(manifest.parts.every(part => readFileSync(join(directory, part)).byteLength <= partLimit));
const head = await serve(path, { method: 'HEAD', headers: { 'Accept-Encoding': 'zstd, gzip' } });
assert.equal(head.headers.get('ETag'), zstd.headers.get('ETag'));
assert.equal((await head.arrayBuffer()).byteLength, 0);
const cached = await serve(path, { headers: { 'Accept-Encoding': 'gzip, zstd', 'If-None-Match': zstd.headers.get('ETag') } });
assert.equal(cached.status, 304);
const gzipPreferred = await serve(path, { method: 'HEAD', headers: { 'Accept-Encoding': 'zstd;q=0.5, gzip' } });
assert.equal(gzipPreferred.headers.get('Content-Encoding'), 'gzip');
writeFileSync(zstdManifest, '{');
const fallback = await serve(path, { method: 'HEAD', headers: { 'Accept-Encoding': 'zstd, gzip' } });
assert.equal(fallback.headers.get('ETag'), gzip.headers.get('ETag'));
}
console.log('Wasm variants: Zstd framing, negotiation, gzip fallback, asset limit and validators passed');
}
try {
verifyTelegramBootstrap();
for (const app of ['web', 'telegram']) {
@@ -173,6 +245,7 @@ try {
rmSync(join(directory, 'index.wasm.gz'));
assert.equal(await (await serve('/build/web/index.wasm')).text(), 'small wasm');
console.log('Site assets: both apps, data/Wasm, streaming gzip, cache, HEAD, ranges and missing pieces passed');
await verifyWasmNegotiation();
} finally {
rmSync(temporary, { recursive: true, force: true });
}