From 3423cba76d40e90ee4f91c37961785a2a42cd7ae Mon Sep 17 00:00:00 2001 From: Waozi Date: Thu, 19 Nov 2026 18:30:47 -0300 Subject: [PATCH] Serve smaller Zstd Wasm for Breeze web and Telegram --- scripts/prepare-site-assets.py | 123 +++++++++++++++++++++++++++++---- site/static/_worker.js | 55 ++++++++++++--- tests/site_assets_test.mjs | 75 +++++++++++++++++++- 3 files changed, 228 insertions(+), 25 deletions(-) diff --git a/scripts/prepare-site-assets.py b/scripts/prepare-site-assets.py index 4690a5814..8a5c107d6 100644 --- a/scripts/prepare-site-assets.py +++ b/scripts/prepare-site-assets.py @@ -1,32 +1,111 @@ #!/usr/bin/env python3 """Package large browser files within the Pages per-file limit.""" +from __future__ import annotations + import argparse +import ctypes +import filecmp import gzip import hashlib import json +import shutil +from ctypes.util import find_library from pathlib import Path MAX_FILE_BYTES = 25 * 1024 * 1024 -def prepare(path: Path, limit: int) -> None: +class ZstdEncoder: + """Use the host libzstd when it is installed. Browsers decode this natively.""" + + def __init__(self, library: ctypes.CDLL): + self.library = library + size = ctypes.c_size_t + pointer = ctypes.c_void_p + signatures = { + "createCCtx": (pointer, []), + "freeCCtx": (size, [pointer]), + "CCtx_setParameter": (size, [pointer, ctypes.c_int, ctypes.c_int]), + "compressBound": (size, [size]), + "compress2": (size, [pointer, pointer, size, ctypes.c_char_p, size]), + "isError": (ctypes.c_uint, [size]), + "getErrorName": (ctypes.c_char_p, [size]), + } + for name, (result, parameters) in signatures.items(): + function = getattr(library, "ZSTD_" + name) + function.restype = result + function.argtypes = parameters + + def checked(self, result: int) -> int: + if self.library.ZSTD_isError(result): + raise ValueError(self.library.ZSTD_getErrorName(result).decode()) + return result + + def encode(self, path: Path, limit: int) -> list[bytes]: + zstd = self.library + context = zstd.ZSTD_createCCtx() + if not context: + raise MemoryError("Could not allocate a Zstd compression context") + parts = [] + try: + self.checked(zstd.ZSTD_CCtx_setParameter(context, 100, 22)) # compressionLevel + self.checked(zstd.ZSTD_CCtx_setParameter(context, 101, 23)) # windowLog, RFC 9659 + with path.open("rb") as source: + while block := source.read(min(limit, MAX_FILE_BYTES)): + output = ctypes.create_string_buffer(zstd.ZSTD_compressBound(len(block))) + size = self.checked(zstd.ZSTD_compress2(context, output, len(output), block, len(block))) + frame = output[:size] + for offset in range(0, size, limit): + parts.append(frame[offset:offset + limit]) + finally: + zstd.ZSTD_freeCCtx(context) + return parts + + +def load_zstd() -> ZstdEncoder | None: + name = find_library("zstd") + if name: + try: + return ZstdEncoder(ctypes.CDLL(name)) + except (OSError, AttributeError): + pass + return None + + +def write_parts(path: Path, coding: str, parts: list[bytes]) -> list[str]: + sha256 = hashlib.sha256() + for part in parts: + sha256.update(part) + digest = sha256.hexdigest() + names = [] + for index, part in enumerate(parts): + name = f"{path.name}.{coding}.{digest[:16]}.part-{index:03d}" + path.with_name(name).write_bytes(part) + names.append(name) + manifest = {"parts": names, "sha256": digest, "size": sum(map(len, parts))} + suffix = ".parts.json" if coding == "gz" else ".zstd.parts.json" + manifest_path = path.with_name(path.name + suffix) + manifest_path.write_text(json.dumps(manifest, sort_keys=True) + "\n") + return names + [manifest_path.name] + + +def prepare(path: Path, limit: int, zstd: ZstdEncoder | None = None) -> list[str]: if not path.exists() or path.stat().st_size <= limit: - return + return [] compressed = gzip.compress(path.read_bytes(), compresslevel=9, mtime=0) if len(compressed) <= limit: - path.with_name(path.name + ".gz").write_bytes(compressed) + output = path.with_name(path.name + ".gz") + output.write_bytes(compressed) + names = [output.name] else: - digest = hashlib.sha256(compressed).hexdigest() - parts = [] - for index, offset in enumerate(range(0, len(compressed), limit)): - name = f"{path.name}.gz.{digest[:16]}.part-{index:03d}" - path.with_name(name).write_bytes(compressed[offset:offset + limit]) - parts.append(name) - manifest = {"parts": parts, "sha256": digest, "size": len(compressed)} - path.with_name(path.name + ".parts.json").write_text( - json.dumps(manifest, sort_keys=True) + "\n") + names = write_parts(path, "gz", [compressed[i:i + limit] for i in range(0, len(compressed), limit)]) + if zstd: + parts = zstd.encode(path, limit) + if sum(map(len, parts)) < len(compressed): + names.extend(write_parts(path, "zstd", parts)) path.unlink() + return names def main() -> None: @@ -36,9 +115,23 @@ def main() -> None: args = parser.parse_args() if args.max_file_bytes <= 0: parser.error("--max-file-bytes must be positive") - for app in ("web", "telegram"): - for name in ("index.data", "index.wasm"): - prepare(args.root / "build" / app / name, args.max_file_bytes) + web = args.root / "build" / "web" + telegram = args.root / "build" / "telegram" + for directory in (web, telegram): + prepare(directory / "index.data", args.max_file_bytes) + web_wasm = web / "index.wasm" + telegram_wasm = telegram / "index.wasm" + same_wasm = web_wasm.is_file() and telegram_wasm.is_file() and filecmp.cmp( + web_wasm, telegram_wasm, shallow=False) + encoder = load_zstd() + prepared = prepare(web_wasm, args.max_file_bytes, encoder) + if same_wasm: + if prepared: + for name in prepared: + shutil.copyfile(web / name, telegram / name) + telegram_wasm.unlink() + else: + prepare(telegram_wasm, args.max_file_bytes, encoder) for path in args.root.rglob("*"): if path.is_file() and path.stat().st_size > args.max_file_bytes: raise SystemExit(f"Site asset exceeds the Pages limit: {path}") diff --git a/site/static/_worker.js b/site/static/_worker.js index 0ff007bb5..4d6800d7e 100644 --- a/site/static/_worker.js +++ b/site/static/_worker.js @@ -6,20 +6,44 @@ function assetRequest(url, request, method = 'GET') { return new Request(url, { method, headers }); } -async function multipartAsset(url, request, env, manifestResponse) { - const manifest = await manifestResponse.json(); +function preferZstd(request) { + const accepted = request.cf?.clientAcceptEncoding ?? request.headers.get('Accept-Encoding') ?? ''; + const quality = new Map(); + for (const item of accepted.toLowerCase().split(',')) { + const [coding, parameter] = item.trim().split(';'); + if (parameter === undefined) { + quality.set(coding, 1); + } else { + const match = /^\s*q\s*=\s*(0(?:\.\d{0,3})?|1(?:\.0{0,3})?)\s*$/.exec(parameter); + if (match) quality.set(coding.trim(), Number(match[1])); + } + } + const zstd = quality.get('zstd') ?? 0; + return zstd > 0 && zstd >= (quality.get('gzip') ?? quality.get('*') ?? 0); +} + +function varyWasm(headers) { + headers.append('Vary', 'Accept-Encoding'); + headers.append('Cache-Control', 'no-transform'); +} + +async function multipartAsset(url, request, env, manifestResponse, coding) { + const manifest = await manifestResponse.json().catch(() => null); const basename = url.pathname.slice(url.pathname.lastIndexOf('/') + 1); - if (!Array.isArray(manifest.parts) || manifest.parts.length === 0 || + const extension = coding === 'zstd' ? 'zstd' : 'gz'; + if (!manifest || !Array.isArray(manifest.parts) || manifest.parts.length === 0 || !manifest.parts.every(part => typeof part === 'string' && - part.startsWith(basename + '.gz.') && !/[\\/]/.test(part)) || - !/^[a-f0-9]{64}$/.test(manifest.sha256)) { + part.startsWith(basename + '.' + extension + '.') && !/[\\/]/.test(part)) || + !/^[a-f0-9]{64}$/.test(manifest.sha256) || + (coding === 'zstd' && (!Number.isSafeInteger(manifest.size) || manifest.size <= 0))) { return new Response('Invalid asset manifest', { status: 502 }); } const headers = new Headers(manifestResponse.headers); headers.delete('Content-Length'); headers.delete('Content-Range'); headers.delete('Accept-Ranges'); - headers.set('Content-Encoding', 'gzip'); + headers.set('Content-Encoding', coding); + if (basename.endsWith('.wasm')) varyWasm(headers); headers.set('Content-Type', basename.endsWith('.wasm') ? 'application/wasm' : 'application/octet-stream'); const etag = '"' + manifest.sha256 + '"'; @@ -31,7 +55,10 @@ async function multipartAsset(url, request, env, manifestResponse) { if (request.method === 'HEAD') { return new Response(null, { headers }); } - const { readable, writable } = new TransformStream(); + if (coding === 'zstd') headers.set('Content-Length', String(manifest.size)); + // Fixed length avoids HTTP/1.1 chunked Zstd issues in WebKit. + const { readable, writable } = coding === 'zstd' && typeof FixedLengthStream === 'function' + ? new FixedLengthStream(manifest.size) : new TransformStream(); async function pump() { for (const part of manifest.parts) { const partUrl = new URL(url); @@ -54,11 +81,20 @@ export default { const url = new URL(request.url); if (['GET', 'HEAD'].includes(request.method) && /^\/build\/(web|telegram)\/index\.(data|wasm)$/.test(url.pathname)) { + const wasm = url.pathname.endsWith('.wasm'); const manifestUrl = new URL(url); - manifestUrl.pathname += '.parts.json'; + if (wasm && preferZstd(request)) { + manifestUrl.pathname = url.pathname + '.zstd.parts.json'; + const manifest = await env.ASSETS.fetch(assetRequest(manifestUrl, request)); + if (manifest.ok && manifest.headers.get('Content-Type')?.includes('application/json')) { + const response = await multipartAsset(url, request, env, manifest, 'zstd'); + if (response.status !== 502) return response; + } + } + manifestUrl.pathname = url.pathname + '.parts.json'; const manifest = await env.ASSETS.fetch(assetRequest(manifestUrl, request)); if (manifest.ok && manifest.headers.get('Content-Type')?.includes('application/json')) { - return multipartAsset(url, request, env, manifest); + return multipartAsset(url, request, env, manifest, 'gzip'); } const compressed = new URL(url); compressed.pathname += '.gz'; @@ -66,6 +102,7 @@ export default { if (response.ok && !response.headers.get('Content-Type')?.includes('text/html')) { const headers = new Headers(response.headers); headers.set('Content-Encoding', 'gzip'); + if (wasm) varyWasm(headers); headers.set('Content-Type', url.pathname.endsWith('.wasm') ? 'application/wasm' : 'application/octet-stream'); return new Response(response.body, { diff --git a/tests/site_assets_test.mjs b/tests/site_assets_test.mjs index b6af20d1e..2c9aff401 100644 --- a/tests/site_assets_test.mjs +++ b/tests/site_assets_test.mjs @@ -5,9 +5,11 @@ import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, readdirSync, rmSyn import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { spawnSync } from 'node:child_process'; -import { gunzipSync } from 'node:zlib'; +import * as zlib from 'node:zlib'; import { Script } from 'node:vm'; +const { gunzipSync, zstdDecompressSync } = zlib; + const root = resolve(import.meta.dirname, '..'); const temporary = mkdtempSync(join(tmpdir(), 'inbe-site-assets-')); const source = readFileSync(join(root, 'site/static/_worker.js')); @@ -121,6 +123,76 @@ window.originalBootstrapRan = true; console.log('Mini App generated bootstrap: syntax, SDK absence/older features, theme and gestures passed'); } +async function verifyWasmNegotiation() { + const partLimit = 96 * 1024; + const block = Array.from({ length: 3 }, (_, frame) => { + const noise = Buffer.concat(Array.from({ length: 1920 }, (_, i) => + createHash('sha256').update(`${frame}-${i}`).digest())); + const repeat = noise.subarray(0, 36 * 1024); + return Buffer.concat([repeat, repeat, noise.subarray(repeat.length)]); + }); + const original = Buffer.concat(block); + for (const app of ['web', 'telegram']) { + const directory = join(temporary, 'build', app); + rmSync(directory, { recursive: true, force: true }); + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, 'index.wasm'), original); + } + const result = spawnSync('python3', [join(root, 'scripts/prepare-site-assets.py'), temporary, + '--max-file-bytes', String(partLimit)], { encoding: 'utf8' }); + assert.equal(result.status, 0, result.stdout + result.stderr); + const library = spawnSync('python3', ['-c', `import runpy, sys +stage = runpy.run_path(sys.argv[1]) +print('available' if stage['load_zstd']() else 'missing')`, + join(root, 'scripts/prepare-site-assets.py')], { encoding: 'utf8' }); + assert.equal(library.status, 0, library.stdout + library.stderr); + for (const app of ['web', 'telegram']) { + const path = `/build/${app}/index.wasm`; + const directory = join(temporary, 'build', app); + const gzip = await serve(path, { headers: { 'Accept-Encoding': 'gzip, zstd;q=0' } }); + assert.equal(gzip.headers.get('Content-Encoding'), 'gzip'); + assert.ok(gzip.headers.get('Vary').toLowerCase().includes('accept-encoding')); + assert.deepEqual(gunzipSync(Buffer.from(await gzip.arrayBuffer())), original); + const zstdManifest = join(directory, 'index.wasm.zstd.parts.json'); + if (library.stdout.trim() === 'missing') { + assert.ok(!existsSync(zstdManifest)); + assert.equal((await serve(path, { method: 'HEAD', headers: { 'Accept-Encoding': 'zstd, gzip' } })) + .headers.get('Content-Encoding'), 'gzip'); + continue; + } + const requestForZstd = new Request('https://inbe.example' + path + '?v=test', { + headers: { 'Accept-Encoding': 'gzip, br', Range: 'bytes=0-9', 'If-None-Match': gzip.headers.get('ETag') }, + }); + requestForZstd.cf = { clientAcceptEncoding: 'gzip;q=0.8, ZSTD;q=0.9' }; + const zstd = await worker.fetch(requestForZstd, { ASSETS: assets }); + assert.equal(zstd.status, 200); + assert.equal(zstd.headers.get('Content-Encoding'), 'zstd'); + assert.equal(zstd.headers.get('Content-Type'), 'application/wasm'); + assert.match(zstd.headers.get('Cache-Control'), /no-transform/); + const encoded = Buffer.from(await zstd.arrayBuffer()); + assert.equal(zstd.headers.get('Content-Length'), String(encoded.length)); + assert.equal(zstd.headers.get('ETag'), '"' + createHash('sha256').update(encoded).digest('hex') + '"'); + assert.ok(encoded.length < +JSON.parse(readFileSync(join(directory, 'index.wasm.parts.json'))).size); + if (typeof zstdDecompressSync === 'function') { + assert.deepEqual(zstdDecompressSync(encoded), original); + } + const manifest = JSON.parse(readFileSync(zstdManifest)); + assert.ok(manifest.parts.length > 1); + assert.ok(manifest.parts.every(part => readFileSync(join(directory, part)).byteLength <= partLimit)); + const head = await serve(path, { method: 'HEAD', headers: { 'Accept-Encoding': 'zstd, gzip' } }); + assert.equal(head.headers.get('ETag'), zstd.headers.get('ETag')); + assert.equal((await head.arrayBuffer()).byteLength, 0); + const cached = await serve(path, { headers: { 'Accept-Encoding': 'gzip, zstd', 'If-None-Match': zstd.headers.get('ETag') } }); + assert.equal(cached.status, 304); + const gzipPreferred = await serve(path, { method: 'HEAD', headers: { 'Accept-Encoding': 'zstd;q=0.5, gzip' } }); + assert.equal(gzipPreferred.headers.get('Content-Encoding'), 'gzip'); + writeFileSync(zstdManifest, '{'); + const fallback = await serve(path, { method: 'HEAD', headers: { 'Accept-Encoding': 'zstd, gzip' } }); + assert.equal(fallback.headers.get('ETag'), gzip.headers.get('ETag')); + } + console.log('Wasm variants: Zstd framing, negotiation, gzip fallback, asset limit and validators passed'); +} + try { verifyTelegramBootstrap(); for (const app of ['web', 'telegram']) { @@ -173,6 +245,7 @@ try { rmSync(join(directory, 'index.wasm.gz')); assert.equal(await (await serve('/build/web/index.wasm')).text(), 'small wasm'); console.log('Site assets: both apps, data/Wasm, streaming gzip, cache, HEAD, ranges and missing pieces passed'); + await verifyWasmNegotiation(); } finally { rmSync(temporary, { recursive: true, force: true }); }