Serve smaller Zstd Wasm for Breeze web and Telegram
This commit is contained in:
1 parent
d3628b76f6
commit
3423cba76d
3 files changed
+228
-25
No files matched your search
+108
-15
@@ -1,32 +1,111 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Package large browser files within the Pages per-file limit."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import filecmp
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
from ctypes.util import find_library
|
||||
from pathlib import Path
|
||||
|
||||
MAX_FILE_BYTES = 25 * 1024 * 1024
|
||||
|
||||
|
||||
def prepare(path: Path, limit: int) -> None:
|
||||
class ZstdEncoder:
|
||||
"""Use the host libzstd when it is installed. Browsers decode this natively."""
|
||||
|
||||
def __init__(self, library: ctypes.CDLL):
|
||||
self.library = library
|
||||
size = ctypes.c_size_t
|
||||
pointer = ctypes.c_void_p
|
||||
signatures = {
|
||||
"createCCtx": (pointer, []),
|
||||
"freeCCtx": (size, [pointer]),
|
||||
"CCtx_setParameter": (size, [pointer, ctypes.c_int, ctypes.c_int]),
|
||||
"compressBound": (size, [size]),
|
||||
"compress2": (size, [pointer, pointer, size, ctypes.c_char_p, size]),
|
||||
"isError": (ctypes.c_uint, [size]),
|
||||
"getErrorName": (ctypes.c_char_p, [size]),
|
||||
}
|
||||
for name, (result, parameters) in signatures.items():
|
||||
function = getattr(library, "ZSTD_" + name)
|
||||
function.restype = result
|
||||
function.argtypes = parameters
|
||||
|
||||
def checked(self, result: int) -> int:
|
||||
if self.library.ZSTD_isError(result):
|
||||
raise ValueError(self.library.ZSTD_getErrorName(result).decode())
|
||||
return result
|
||||
|
||||
def encode(self, path: Path, limit: int) -> list[bytes]:
|
||||
zstd = self.library
|
||||
context = zstd.ZSTD_createCCtx()
|
||||
if not context:
|
||||
raise MemoryError("Could not allocate a Zstd compression context")
|
||||
parts = []
|
||||
try:
|
||||
self.checked(zstd.ZSTD_CCtx_setParameter(context, 100, 22)) # compressionLevel
|
||||
self.checked(zstd.ZSTD_CCtx_setParameter(context, 101, 23)) # windowLog, RFC 9659
|
||||
with path.open("rb") as source:
|
||||
while block := source.read(min(limit, MAX_FILE_BYTES)):
|
||||
output = ctypes.create_string_buffer(zstd.ZSTD_compressBound(len(block)))
|
||||
size = self.checked(zstd.ZSTD_compress2(context, output, len(output), block, len(block)))
|
||||
frame = output[:size]
|
||||
for offset in range(0, size, limit):
|
||||
parts.append(frame[offset:offset + limit])
|
||||
finally:
|
||||
zstd.ZSTD_freeCCtx(context)
|
||||
return parts
|
||||
|
||||
|
||||
def load_zstd() -> ZstdEncoder | None:
|
||||
name = find_library("zstd")
|
||||
if name:
|
||||
try:
|
||||
return ZstdEncoder(ctypes.CDLL(name))
|
||||
except (OSError, AttributeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def write_parts(path: Path, coding: str, parts: list[bytes]) -> list[str]:
|
||||
sha256 = hashlib.sha256()
|
||||
for part in parts:
|
||||
sha256.update(part)
|
||||
digest = sha256.hexdigest()
|
||||
names = []
|
||||
for index, part in enumerate(parts):
|
||||
name = f"{path.name}.{coding}.{digest[:16]}.part-{index:03d}"
|
||||
path.with_name(name).write_bytes(part)
|
||||
names.append(name)
|
||||
manifest = {"parts": names, "sha256": digest, "size": sum(map(len, parts))}
|
||||
suffix = ".parts.json" if coding == "gz" else ".zstd.parts.json"
|
||||
manifest_path = path.with_name(path.name + suffix)
|
||||
manifest_path.write_text(json.dumps(manifest, sort_keys=True) + "\n")
|
||||
return names + [manifest_path.name]
|
||||
|
||||
|
||||
def prepare(path: Path, limit: int, zstd: ZstdEncoder | None = None) -> list[str]:
|
||||
if not path.exists() or path.stat().st_size <= limit:
|
||||
return
|
||||
return []
|
||||
compressed = gzip.compress(path.read_bytes(), compresslevel=9, mtime=0)
|
||||
if len(compressed) <= limit:
|
||||
path.with_name(path.name + ".gz").write_bytes(compressed)
|
||||
output = path.with_name(path.name + ".gz")
|
||||
output.write_bytes(compressed)
|
||||
names = [output.name]
|
||||
else:
|
||||
digest = hashlib.sha256(compressed).hexdigest()
|
||||
parts = []
|
||||
for index, offset in enumerate(range(0, len(compressed), limit)):
|
||||
name = f"{path.name}.gz.{digest[:16]}.part-{index:03d}"
|
||||
path.with_name(name).write_bytes(compressed[offset:offset + limit])
|
||||
parts.append(name)
|
||||
manifest = {"parts": parts, "sha256": digest, "size": len(compressed)}
|
||||
path.with_name(path.name + ".parts.json").write_text(
|
||||
json.dumps(manifest, sort_keys=True) + "\n")
|
||||
names = write_parts(path, "gz", [compressed[i:i + limit] for i in range(0, len(compressed), limit)])
|
||||
if zstd:
|
||||
parts = zstd.encode(path, limit)
|
||||
if sum(map(len, parts)) < len(compressed):
|
||||
names.extend(write_parts(path, "zstd", parts))
|
||||
path.unlink()
|
||||
return names
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@@ -36,9 +115,23 @@ def main() -> None:
|
||||
args = parser.parse_args()
|
||||
if args.max_file_bytes <= 0:
|
||||
parser.error("--max-file-bytes must be positive")
|
||||
for app in ("web", "telegram"):
|
||||
for name in ("index.data", "index.wasm"):
|
||||
prepare(args.root / "build" / app / name, args.max_file_bytes)
|
||||
web = args.root / "build" / "web"
|
||||
telegram = args.root / "build" / "telegram"
|
||||
for directory in (web, telegram):
|
||||
prepare(directory / "index.data", args.max_file_bytes)
|
||||
web_wasm = web / "index.wasm"
|
||||
telegram_wasm = telegram / "index.wasm"
|
||||
same_wasm = web_wasm.is_file() and telegram_wasm.is_file() and filecmp.cmp(
|
||||
web_wasm, telegram_wasm, shallow=False)
|
||||
encoder = load_zstd()
|
||||
prepared = prepare(web_wasm, args.max_file_bytes, encoder)
|
||||
if same_wasm:
|
||||
if prepared:
|
||||
for name in prepared:
|
||||
shutil.copyfile(web / name, telegram / name)
|
||||
telegram_wasm.unlink()
|
||||
else:
|
||||
prepare(telegram_wasm, args.max_file_bytes, encoder)
|
||||
for path in args.root.rglob("*"):
|
||||
if path.is_file() and path.stat().st_size > args.max_file_bytes:
|
||||
raise SystemExit(f"Site asset exceeds the Pages limit: {path}")
|
||||
|
||||
+46
-9
@@ -6,20 +6,44 @@ function assetRequest(url, request, method = 'GET') {
|
||||
return new Request(url, { method, headers });
|
||||
}
|
||||
|
||||
async function multipartAsset(url, request, env, manifestResponse) {
|
||||
const manifest = await manifestResponse.json();
|
||||
function preferZstd(request) {
|
||||
const accepted = request.cf?.clientAcceptEncoding ?? request.headers.get('Accept-Encoding') ?? '';
|
||||
const quality = new Map();
|
||||
for (const item of accepted.toLowerCase().split(',')) {
|
||||
const [coding, parameter] = item.trim().split(';');
|
||||
if (parameter === undefined) {
|
||||
quality.set(coding, 1);
|
||||
} else {
|
||||
const match = /^\s*q\s*=\s*(0(?:\.\d{0,3})?|1(?:\.0{0,3})?)\s*$/.exec(parameter);
|
||||
if (match) quality.set(coding.trim(), Number(match[1]));
|
||||
}
|
||||
}
|
||||
const zstd = quality.get('zstd') ?? 0;
|
||||
return zstd > 0 && zstd >= (quality.get('gzip') ?? quality.get('*') ?? 0);
|
||||
}
|
||||
|
||||
function varyWasm(headers) {
|
||||
headers.append('Vary', 'Accept-Encoding');
|
||||
headers.append('Cache-Control', 'no-transform');
|
||||
}
|
||||
|
||||
async function multipartAsset(url, request, env, manifestResponse, coding) {
|
||||
const manifest = await manifestResponse.json().catch(() => null);
|
||||
const basename = url.pathname.slice(url.pathname.lastIndexOf('/') + 1);
|
||||
if (!Array.isArray(manifest.parts) || manifest.parts.length === 0 ||
|
||||
const extension = coding === 'zstd' ? 'zstd' : 'gz';
|
||||
if (!manifest || !Array.isArray(manifest.parts) || manifest.parts.length === 0 ||
|
||||
!manifest.parts.every(part => typeof part === 'string' &&
|
||||
part.startsWith(basename + '.gz.') && !/[\\/]/.test(part)) ||
|
||||
!/^[a-f0-9]{64}$/.test(manifest.sha256)) {
|
||||
part.startsWith(basename + '.' + extension + '.') && !/[\\/]/.test(part)) ||
|
||||
!/^[a-f0-9]{64}$/.test(manifest.sha256) ||
|
||||
(coding === 'zstd' && (!Number.isSafeInteger(manifest.size) || manifest.size <= 0))) {
|
||||
return new Response('Invalid asset manifest', { status: 502 });
|
||||
}
|
||||
const headers = new Headers(manifestResponse.headers);
|
||||
headers.delete('Content-Length');
|
||||
headers.delete('Content-Range');
|
||||
headers.delete('Accept-Ranges');
|
||||
headers.set('Content-Encoding', 'gzip');
|
||||
headers.set('Content-Encoding', coding);
|
||||
if (basename.endsWith('.wasm')) varyWasm(headers);
|
||||
headers.set('Content-Type', basename.endsWith('.wasm')
|
||||
? 'application/wasm' : 'application/octet-stream');
|
||||
const etag = '"' + manifest.sha256 + '"';
|
||||
@@ -31,7 +55,10 @@ async function multipartAsset(url, request, env, manifestResponse) {
|
||||
if (request.method === 'HEAD') {
|
||||
return new Response(null, { headers });
|
||||
}
|
||||
const { readable, writable } = new TransformStream();
|
||||
if (coding === 'zstd') headers.set('Content-Length', String(manifest.size));
|
||||
// Fixed length avoids HTTP/1.1 chunked Zstd issues in WebKit.
|
||||
const { readable, writable } = coding === 'zstd' && typeof FixedLengthStream === 'function'
|
||||
? new FixedLengthStream(manifest.size) : new TransformStream();
|
||||
async function pump() {
|
||||
for (const part of manifest.parts) {
|
||||
const partUrl = new URL(url);
|
||||
@@ -54,11 +81,20 @@ export default {
|
||||
const url = new URL(request.url);
|
||||
if (['GET', 'HEAD'].includes(request.method) &&
|
||||
/^\/build\/(web|telegram)\/index\.(data|wasm)$/.test(url.pathname)) {
|
||||
const wasm = url.pathname.endsWith('.wasm');
|
||||
const manifestUrl = new URL(url);
|
||||
manifestUrl.pathname += '.parts.json';
|
||||
if (wasm && preferZstd(request)) {
|
||||
manifestUrl.pathname = url.pathname + '.zstd.parts.json';
|
||||
const manifest = await env.ASSETS.fetch(assetRequest(manifestUrl, request));
|
||||
if (manifest.ok && manifest.headers.get('Content-Type')?.includes('application/json')) {
|
||||
const response = await multipartAsset(url, request, env, manifest, 'zstd');
|
||||
if (response.status !== 502) return response;
|
||||
}
|
||||
}
|
||||
manifestUrl.pathname = url.pathname + '.parts.json';
|
||||
const manifest = await env.ASSETS.fetch(assetRequest(manifestUrl, request));
|
||||
if (manifest.ok && manifest.headers.get('Content-Type')?.includes('application/json')) {
|
||||
return multipartAsset(url, request, env, manifest);
|
||||
return multipartAsset(url, request, env, manifest, 'gzip');
|
||||
}
|
||||
const compressed = new URL(url);
|
||||
compressed.pathname += '.gz';
|
||||
@@ -66,6 +102,7 @@ export default {
|
||||
if (response.ok && !response.headers.get('Content-Type')?.includes('text/html')) {
|
||||
const headers = new Headers(response.headers);
|
||||
headers.set('Content-Encoding', 'gzip');
|
||||
if (wasm) varyWasm(headers);
|
||||
headers.set('Content-Type', url.pathname.endsWith('.wasm')
|
||||
? 'application/wasm' : 'application/octet-stream');
|
||||
return new Response(response.body, {
|
||||
|
||||
@@ -5,9 +5,11 @@ import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, readdirSync, rmSyn
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { gunzipSync } from 'node:zlib';
|
||||
import * as zlib from 'node:zlib';
|
||||
import { Script } from 'node:vm';
|
||||
|
||||
const { gunzipSync, zstdDecompressSync } = zlib;
|
||||
|
||||
const root = resolve(import.meta.dirname, '..');
|
||||
const temporary = mkdtempSync(join(tmpdir(), 'inbe-site-assets-'));
|
||||
const source = readFileSync(join(root, 'site/static/_worker.js'));
|
||||
@@ -121,6 +123,76 @@ window.originalBootstrapRan = true;
|
||||
console.log('Mini App generated bootstrap: syntax, SDK absence/older features, theme and gestures passed');
|
||||
}
|
||||
|
||||
async function verifyWasmNegotiation() {
|
||||
const partLimit = 96 * 1024;
|
||||
const block = Array.from({ length: 3 }, (_, frame) => {
|
||||
const noise = Buffer.concat(Array.from({ length: 1920 }, (_, i) =>
|
||||
createHash('sha256').update(`${frame}-${i}`).digest()));
|
||||
const repeat = noise.subarray(0, 36 * 1024);
|
||||
return Buffer.concat([repeat, repeat, noise.subarray(repeat.length)]);
|
||||
});
|
||||
const original = Buffer.concat(block);
|
||||
for (const app of ['web', 'telegram']) {
|
||||
const directory = join(temporary, 'build', app);
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
mkdirSync(directory, { recursive: true });
|
||||
writeFileSync(join(directory, 'index.wasm'), original);
|
||||
}
|
||||
const result = spawnSync('python3', [join(root, 'scripts/prepare-site-assets.py'), temporary,
|
||||
'--max-file-bytes', String(partLimit)], { encoding: 'utf8' });
|
||||
assert.equal(result.status, 0, result.stdout + result.stderr);
|
||||
const library = spawnSync('python3', ['-c', `import runpy, sys
|
||||
stage = runpy.run_path(sys.argv[1])
|
||||
print('available' if stage['load_zstd']() else 'missing')`,
|
||||
join(root, 'scripts/prepare-site-assets.py')], { encoding: 'utf8' });
|
||||
assert.equal(library.status, 0, library.stdout + library.stderr);
|
||||
for (const app of ['web', 'telegram']) {
|
||||
const path = `/build/${app}/index.wasm`;
|
||||
const directory = join(temporary, 'build', app);
|
||||
const gzip = await serve(path, { headers: { 'Accept-Encoding': 'gzip, zstd;q=0' } });
|
||||
assert.equal(gzip.headers.get('Content-Encoding'), 'gzip');
|
||||
assert.ok(gzip.headers.get('Vary').toLowerCase().includes('accept-encoding'));
|
||||
assert.deepEqual(gunzipSync(Buffer.from(await gzip.arrayBuffer())), original);
|
||||
const zstdManifest = join(directory, 'index.wasm.zstd.parts.json');
|
||||
if (library.stdout.trim() === 'missing') {
|
||||
assert.ok(!existsSync(zstdManifest));
|
||||
assert.equal((await serve(path, { method: 'HEAD', headers: { 'Accept-Encoding': 'zstd, gzip' } }))
|
||||
.headers.get('Content-Encoding'), 'gzip');
|
||||
continue;
|
||||
}
|
||||
const requestForZstd = new Request('https://inbe.example' + path + '?v=test', {
|
||||
headers: { 'Accept-Encoding': 'gzip, br', Range: 'bytes=0-9', 'If-None-Match': gzip.headers.get('ETag') },
|
||||
});
|
||||
requestForZstd.cf = { clientAcceptEncoding: 'gzip;q=0.8, ZSTD;q=0.9' };
|
||||
const zstd = await worker.fetch(requestForZstd, { ASSETS: assets });
|
||||
assert.equal(zstd.status, 200);
|
||||
assert.equal(zstd.headers.get('Content-Encoding'), 'zstd');
|
||||
assert.equal(zstd.headers.get('Content-Type'), 'application/wasm');
|
||||
assert.match(zstd.headers.get('Cache-Control'), /no-transform/);
|
||||
const encoded = Buffer.from(await zstd.arrayBuffer());
|
||||
assert.equal(zstd.headers.get('Content-Length'), String(encoded.length));
|
||||
assert.equal(zstd.headers.get('ETag'), '"' + createHash('sha256').update(encoded).digest('hex') + '"');
|
||||
assert.ok(encoded.length < +JSON.parse(readFileSync(join(directory, 'index.wasm.parts.json'))).size);
|
||||
if (typeof zstdDecompressSync === 'function') {
|
||||
assert.deepEqual(zstdDecompressSync(encoded), original);
|
||||
}
|
||||
const manifest = JSON.parse(readFileSync(zstdManifest));
|
||||
assert.ok(manifest.parts.length > 1);
|
||||
assert.ok(manifest.parts.every(part => readFileSync(join(directory, part)).byteLength <= partLimit));
|
||||
const head = await serve(path, { method: 'HEAD', headers: { 'Accept-Encoding': 'zstd, gzip' } });
|
||||
assert.equal(head.headers.get('ETag'), zstd.headers.get('ETag'));
|
||||
assert.equal((await head.arrayBuffer()).byteLength, 0);
|
||||
const cached = await serve(path, { headers: { 'Accept-Encoding': 'gzip, zstd', 'If-None-Match': zstd.headers.get('ETag') } });
|
||||
assert.equal(cached.status, 304);
|
||||
const gzipPreferred = await serve(path, { method: 'HEAD', headers: { 'Accept-Encoding': 'zstd;q=0.5, gzip' } });
|
||||
assert.equal(gzipPreferred.headers.get('Content-Encoding'), 'gzip');
|
||||
writeFileSync(zstdManifest, '{');
|
||||
const fallback = await serve(path, { method: 'HEAD', headers: { 'Accept-Encoding': 'zstd, gzip' } });
|
||||
assert.equal(fallback.headers.get('ETag'), gzip.headers.get('ETag'));
|
||||
}
|
||||
console.log('Wasm variants: Zstd framing, negotiation, gzip fallback, asset limit and validators passed');
|
||||
}
|
||||
|
||||
try {
|
||||
verifyTelegramBootstrap();
|
||||
for (const app of ['web', 'telegram']) {
|
||||
@@ -173,6 +245,7 @@ try {
|
||||
rmSync(join(directory, 'index.wasm.gz'));
|
||||
assert.equal(await (await serve('/build/web/index.wasm')).text(), 'small wasm');
|
||||
console.log('Site assets: both apps, data/Wasm, streaming gzip, cache, HEAD, ranges and missing pieces passed');
|
||||
await verifyWasmNegotiation();
|
||||
} finally {
|
||||
rmSync(temporary, { recursive: true, force: true });
|
||||
}
|
||||
Reference in new issue
Block a user