Merge pending Breeze work: Photos as its own app, Opus voice packs, 2.1.63 notes
Photos downloads from Apps instead of shipping in Breeze, practice and guide voice clips move to Ogg Opus, dependencies update (Ziran, Kryon, Apps, Arcade, practices), and the 2.1.63 version and store notes are prepared. The Practice tab note is translated in every locale.
This commit is contained in:
1 parent
7790878186
commit
dc1d003cd8
197 files changed
+27843
-26670
No files matched your search
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Re-encode the practices' Ogg Vorbis voice clips as Ogg Opus, in place.
|
||||
|
||||
Each clip keeps its packaged path and .ogg name, so downloaded packs, pack
|
||||
lists and the app's lookups stay the same; Kryon reads the codec from the
|
||||
content. Without --apply this only checks every clip and reports the sizes
|
||||
it would produce.
|
||||
|
||||
With --apply, each original goes to the practices checkout's ignored
|
||||
backup/opus-voice/ first, with backup/opus-voice/checksums.json recording the
|
||||
original and new SHA-256 of every clip. A converted clip replaces its
|
||||
original only after ffprobe reads it as mono Opus that plays exactly as many
|
||||
samples as the Vorbis stream (its final granule, or fewer when the stream ends
|
||||
before it): ffmpeg's padded last frame is trimmed at the end
|
||||
(scripts/ogg_opus.py), and decoding the new clip must give that count. The
|
||||
narration receipts then take each new clip's SHA-256, size and length; their
|
||||
text hashes stay, so no line is marked to be spoken again.
|
||||
|
||||
Usage: convert-practice-voice-opus.py PRACTICES_CHECKOUT [--apply] [--bitrate 16k]
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CONTENT = ROOT / 'apps/practice/content.json'
|
||||
PREFIX = 'assets/practices/'
|
||||
_spec = importlib.util.spec_from_file_location('ogg_opus', ROOT / 'scripts/ogg_opus.py')
|
||||
ogg_opus = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(ogg_opus)
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def probe(path):
|
||||
"""(codec, channels, samples at 48 kHz) as ffprobe reads the stream.
|
||||
|
||||
A Vorbis stream's duration_ts is its own sample count (its final granule),
|
||||
which already leaves out the encoder's padding."""
|
||||
out = subprocess.run(['ffprobe', '-v', 'error', '-select_streams', 'a:0',
|
||||
'-show_entries', 'stream=codec_name,channels,sample_rate,duration_ts', '-of', 'json',
|
||||
str(path)], check=True, capture_output=True, text=True).stdout
|
||||
stream = json.loads(out)['streams'][0]
|
||||
rate = int(stream['sample_rate'])
|
||||
samples = int(stream.get('duration_ts') or 0)
|
||||
if ogg_opus.OPUS_RATE % rate:
|
||||
raise ValueError(f'{path}: {rate} Hz does not divide 48 kHz')
|
||||
return stream['codec_name'], int(stream['channels']), samples * (ogg_opus.OPUS_RATE // rate)
|
||||
|
||||
|
||||
def decoded_frames(path):
|
||||
"""Samples ffmpeg's decoder actually plays from the clip, at 48 kHz."""
|
||||
raw = subprocess.run(['ffmpeg', '-nostdin', '-v', 'error', '-i', str(path), '-ac', '1',
|
||||
'-ar', str(ogg_opus.OPUS_RATE), '-f', 'f32le', '-'], check=True, capture_output=True).stdout
|
||||
return len(raw) // 4
|
||||
|
||||
|
||||
def encode(source, bitrate, frames, work):
|
||||
"""The clip as Ogg Opus that plays exactly frames samples at 48 kHz."""
|
||||
target = work / 'clip.opus'
|
||||
subprocess.run(['ffmpeg', '-nostdin', '-v', 'error', '-y', '-i', str(source),
|
||||
'-ac', '1', '-c:a', 'libopus', '-b:a', bitrate, '-application', 'voip',
|
||||
'-vbr', 'on', '-map_metadata', '-1', '-f', 'ogg', str(target)],
|
||||
check=True, capture_output=True)
|
||||
target.write_bytes(ogg_opus.end_trim(target.read_bytes(), frames))
|
||||
return target
|
||||
|
||||
|
||||
def clips(practices):
|
||||
voice = [path for path in sorted(practices.rglob('*.ogg'))
|
||||
if not {'.git', 'backup', 'work', 'build'} & set(path.relative_to(practices).parts)]
|
||||
if not voice:
|
||||
raise SystemExit(f'No .ogg clips under {practices}')
|
||||
return voice
|
||||
|
||||
|
||||
def ignored(practices, path):
|
||||
return subprocess.run(['git', '-C', str(practices), 'check-ignore', '-q', str(path)]).returncode == 0
|
||||
|
||||
|
||||
def update_receipts(changed):
|
||||
"""Point every narration receipt row at its converted clip."""
|
||||
narration = json.loads(CONTENT.read_text())['narration']
|
||||
updated = 0
|
||||
for settings in narration.values():
|
||||
path = ROOT / settings['receipt']
|
||||
receipt = json.loads(path.read_text())
|
||||
dirty = False
|
||||
for row in receipt.get('clips', {}).values():
|
||||
new = changed.get(row.get('file', ''))
|
||||
if new is None:
|
||||
continue
|
||||
row['sha256'], row['bytes'], row['seconds'] = new['sha256'], new['bytes'], round(new['seconds'], 2)
|
||||
dirty = True
|
||||
updated += 1
|
||||
if dirty:
|
||||
temporary = path.with_suffix('.json.part')
|
||||
temporary.write_text(json.dumps(receipt, indent=2, ensure_ascii=False) + '\n')
|
||||
os.replace(temporary, path)
|
||||
return updated
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('practices', type=Path)
|
||||
parser.add_argument('--apply', action='store_true')
|
||||
parser.add_argument('--bitrate', default='16k')
|
||||
args = parser.parse_args()
|
||||
practices = args.practices.resolve()
|
||||
backup = practices / 'backup/opus-voice'
|
||||
if args.apply and not ignored(practices, backup / 'probe'):
|
||||
raise SystemExit(f'{backup} is not ignored by git; refusing to write backups there')
|
||||
manifest_path = backup / 'checksums.json'
|
||||
manifest = json.loads(manifest_path.read_text()) if manifest_path.is_file() else {}
|
||||
before = after = converted = skipped = 0
|
||||
failures = []
|
||||
changed = {}
|
||||
with tempfile.TemporaryDirectory(dir=ROOT / 'build', prefix='opus-voice-') as scratch:
|
||||
work = Path(scratch)
|
||||
for path in clips(practices):
|
||||
relative = path.relative_to(practices).as_posix()
|
||||
data = path.read_bytes()
|
||||
codec, channels, frames = probe(path)
|
||||
if codec == 'opus':
|
||||
skipped += 1
|
||||
continue
|
||||
if codec != 'vorbis' or channels != 1 or frames <= 0:
|
||||
failures.append(f'{relative}: {codec} with {channels} channels and {frames} samples')
|
||||
continue
|
||||
# A stream that ends before its final granule plays only the
|
||||
# samples it holds; those are the recording.
|
||||
frames = min(frames, decoded_frames(path))
|
||||
try:
|
||||
target = encode(path, args.bitrate, frames, work)
|
||||
except ValueError as error:
|
||||
failures.append(f'{relative}: {error}')
|
||||
continue
|
||||
new = target.read_bytes()
|
||||
new_codec, new_channels, _ = probe(target)
|
||||
played = decoded_frames(target)
|
||||
if new_codec != 'opus' or new_channels != 1 or ogg_opus.opus_frames(new) != frames or played != frames:
|
||||
failures.append(f'{relative}: {new_codec}/{new_channels} plays {played} samples, not {frames}')
|
||||
continue
|
||||
timing = frames / ogg_opus.OPUS_RATE
|
||||
before += len(data)
|
||||
after += len(new)
|
||||
converted += 1
|
||||
if not args.apply:
|
||||
continue
|
||||
saved = backup / relative
|
||||
saved.parent.mkdir(parents=True, exist_ok=True)
|
||||
if not saved.exists():
|
||||
saved.write_bytes(data)
|
||||
if digest(saved.read_bytes()) != digest(data):
|
||||
failures.append(f'{relative}: a different backup already exists')
|
||||
continue
|
||||
manifest[relative] = {'vorbis_sha256': digest(data), 'opus_sha256': digest(new),
|
||||
'vorbis_bytes': len(data), 'opus_bytes': len(new)}
|
||||
temporary = path.with_name(path.name + '.opus-part')
|
||||
temporary.write_bytes(new)
|
||||
os.replace(temporary, path)
|
||||
changed[PREFIX + relative] = {'sha256': digest(new), 'bytes': len(new), 'seconds': timing}
|
||||
if args.apply:
|
||||
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
manifest_path.write_text(json.dumps(dict(sorted(manifest.items())), indent=1) + '\n')
|
||||
rows = update_receipts(changed) if args.apply else 0
|
||||
mode = 'Converted' if args.apply else 'Would convert'
|
||||
print(f'{mode} {converted} clips: {before / 1e6:.2f} MB of Vorbis to {after / 1e6:.2f} MB of Opus'
|
||||
+ (f' ({100 - after * 100 / before:.0f}% smaller)' if before else '')
|
||||
+ f'; {skipped} already Opus; {rows} receipt rows updated.')
|
||||
if failures:
|
||||
print('\n'.join(failures), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -31,6 +31,7 @@ import unicodedata
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
import ogg_opus
|
||||
from practice_media import media_path, packaged_path, writable_media
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -105,6 +106,9 @@ def narration_keys(config):
|
||||
|
||||
|
||||
def text_hash(config, text):
|
||||
# A recording's identity: its spoken text and speech settings. The clip is
|
||||
# Ogg Opus now; "vorbis_bitrate" stays in this identity only so the 2026
|
||||
# re-encode of existing recordings did not mark every line to be spoken again.
|
||||
settings = {name: config[name] for name in (
|
||||
"model", "voice", "pcm_sample_rate", "sample_rate", "vorbis_bitrate")}
|
||||
settings["text"] = text
|
||||
@@ -191,26 +195,35 @@ def speak(config, key, text):
|
||||
|
||||
|
||||
def encode(config, pcm, work):
|
||||
"""Raw speech to a small mono Ogg Vorbis clip without its silences."""
|
||||
"""Raw speech to a small mono Ogg Opus clip without its silences."""
|
||||
raw = work / "speech.pcm"
|
||||
raw.write_bytes(pcm)
|
||||
speech = work / "speech-clip-rate.pcm"
|
||||
clip = work / "clip.ogg"
|
||||
trim = ("silenceremove=start_periods=1:start_threshold=-50dB:start_silence=0.05,"
|
||||
"areverse,"
|
||||
"silenceremove=start_periods=1:start_threshold=-50dB:start_silence=0.15,"
|
||||
"areverse")
|
||||
# The trimmed speech at the clip's rate first, so its exact length is known.
|
||||
rate = int(config["sample_rate"])
|
||||
subprocess.run(["ffmpeg", "-v", "error", "-y", "-f", "s16le",
|
||||
"-ar", str(config["pcm_sample_rate"]), "-ac", "1", "-i", str(raw),
|
||||
"-af", trim, "-ar", str(config["sample_rate"]), "-ac", "1",
|
||||
"-c:a", "libvorbis", "-b:a", config["vorbis_bitrate"], str(clip)],
|
||||
"-af", trim, "-ar", str(rate), "-ac", "1", "-c:a", "pcm_s16le",
|
||||
"-f", "s16le", str(speech)], check=True, capture_output=True)
|
||||
frames = speech.stat().st_size // 2 * (ogg_opus.OPUS_RATE // rate)
|
||||
subprocess.run(["ffmpeg", "-v", "error", "-y", "-f", "s16le",
|
||||
"-ar", str(rate), "-ac", "1", "-i", str(speech),
|
||||
"-c:a", "libopus", "-b:a", config.get("opus_bitrate", "16k"),
|
||||
"-application", "voip", "-vbr", "on", "-map_metadata", "-1", "-f", "ogg", str(clip)],
|
||||
check=True, capture_output=True)
|
||||
# ffmpeg writes its padded last frame's length; end at the speech's own.
|
||||
clip.write_bytes(ogg_opus.end_trim(clip.read_bytes(), frames))
|
||||
return clip
|
||||
|
||||
|
||||
def seconds(path):
|
||||
return float(subprocess.check_output(["ffprobe", "-v", "error",
|
||||
"-show_entries", "format=duration", "-of", "csv=p=0", str(path)],
|
||||
text=True))
|
||||
"""How long the Ogg Opus clip plays, from its own trimmed timing."""
|
||||
return ogg_opus.opus_seconds(path.read_bytes())
|
||||
|
||||
|
||||
def transcribe(config, key, clip, code, work):
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Ogg Opus timing and exact end trimming for the spoken clips (RFC 7845).
|
||||
|
||||
ffmpeg's encoder pads a clip's last frame and writes the padded length as the
|
||||
stream's final granule position, so a short line gains up to ~60 ms of
|
||||
near-silent codec tail. end_trim sets the final granule to the source's own
|
||||
sample count, as opusenc does, so decoders stop exactly where the speech
|
||||
recording ends and the clip lasts as long as its source.
|
||||
"""
|
||||
|
||||
OPUS_RATE = 48000
|
||||
|
||||
|
||||
def _crc_table():
|
||||
table = []
|
||||
for byte in range(256):
|
||||
value = byte << 24
|
||||
for _ in range(8):
|
||||
value = ((value << 1) ^ 0x04C11DB7) if value & 0x80000000 else value << 1
|
||||
table.append(value & 0xFFFFFFFF)
|
||||
return table
|
||||
|
||||
|
||||
_CRC = _crc_table()
|
||||
|
||||
|
||||
def ogg_crc(page):
|
||||
value = 0
|
||||
for byte in page:
|
||||
value = ((value << 8) & 0xFFFFFFFF) ^ _CRC[((value >> 24) ^ byte) & 0xFF]
|
||||
return value
|
||||
|
||||
|
||||
def pages(data):
|
||||
"""(offset, length, header_type, granule) of every Ogg page in data."""
|
||||
found = []
|
||||
at = 0
|
||||
while at < len(data):
|
||||
if data[at:at + 4] != b'OggS' or at + 27 > len(data):
|
||||
raise ValueError(f'no Ogg page at byte {at}')
|
||||
count = data[at + 26]
|
||||
body = 27 + count
|
||||
length = body + sum(data[at + 27:at + body])
|
||||
if at + length > len(data):
|
||||
raise ValueError('truncated Ogg page')
|
||||
granule = int.from_bytes(data[at + 6:at + 14], 'little', signed=True)
|
||||
found.append((at, length, data[at + 5], granule))
|
||||
at += length
|
||||
return found
|
||||
|
||||
|
||||
def pre_skip(data):
|
||||
body = 27 + data[26]
|
||||
if data[:4] != b'OggS' or data[body:body + 8] != b'OpusHead':
|
||||
raise ValueError('first page is not OpusHead')
|
||||
return int.from_bytes(data[body + 10:body + 12], 'little')
|
||||
|
||||
|
||||
def opus_frames(data):
|
||||
"""Samples at 48 kHz the stream plays: final granule minus pre-skip."""
|
||||
final = pages(data)[-1][3]
|
||||
skip = pre_skip(data)
|
||||
if final <= skip:
|
||||
raise ValueError('no audio after pre-skip')
|
||||
return final - skip
|
||||
|
||||
|
||||
def opus_seconds(data):
|
||||
return opus_frames(data) / OPUS_RATE
|
||||
|
||||
|
||||
def end_trim(data, frames):
|
||||
"""data with its final granule at pre-skip + frames (48 kHz samples).
|
||||
|
||||
Only the encoder's padding in the last page may go: the stream must hold at
|
||||
least frames samples, and no earlier page may already lie past them.
|
||||
"""
|
||||
found = pages(data)
|
||||
offset, length, header_type, final = found[-1]
|
||||
if not header_type & 4:
|
||||
raise ValueError('last page does not end the stream')
|
||||
target = pre_skip(data) + frames
|
||||
if final < target:
|
||||
raise ValueError(f'stream holds {final - pre_skip(data)} samples, fewer than {frames}')
|
||||
earlier = max((granule for _, _, _, granule in found[:-1] if granule >= 0), default=0)
|
||||
if earlier > target:
|
||||
raise ValueError('trimming would reach into an earlier page')
|
||||
page = bytearray(data[offset:offset + length])
|
||||
page[6:14] = target.to_bytes(8, 'little', signed=True)
|
||||
page[22:26] = bytes(4)
|
||||
page[22:26] = ogg_crc(page).to_bytes(4, 'little')
|
||||
return data[:offset] + bytes(page) + data[offset + length:]
|
||||
@@ -31,7 +31,9 @@ PY
|
||||
packages=${1:-build/packages}
|
||||
for path in kryon=kryon/src/ui kryon=kryon/src/backend apps=apps/src \
|
||||
crypto=crypto/src entity=entity/src plot=plot/src qrcode=qrcode/src \
|
||||
oqs=oqs/src mingcute=mingcute/src daochi_client=daochi-client; do
|
||||
oqs=oqs/src mingcute=mingcute/src daochi_client=daochi-client \
|
||||
webp=webp/src image_codec=image-codec/src avif=avif/src \
|
||||
avif=avif/src/bridge audio=audio/src; do
|
||||
directory="$packages/${path#*=}"
|
||||
if [ "${path%%=*}" = daochi_client ] && [ -n "${DAOCHI_CLIENT_DIR:-}" ]; then
|
||||
directory=$DAOCHI_CLIENT_DIR
|
||||
|
||||
@@ -94,6 +94,18 @@ link kss kss
|
||||
link oqs oqs
|
||||
link raylib raylib
|
||||
link libwebp libwebp
|
||||
# Kryon's image decoders: codec packages and the pinned native sources they
|
||||
# bridge. Kryon's CMake recipes receive these paths instead of resolving them
|
||||
# against Kryon's own lock.
|
||||
link webp webp
|
||||
link image-codec image-codec
|
||||
link avif avif
|
||||
link libavif libavif
|
||||
link libaom libaom
|
||||
# Kryon's Ogg Opus decoder: the audio package and its pinned native sources.
|
||||
link audio audio
|
||||
link libopus libopus
|
||||
link libsamplerate libsamplerate
|
||||
link game2d game2d
|
||||
link qrcode qrcode
|
||||
link plot plot
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
# exact content it commits.
|
||||
#
|
||||
# The container is the Test job's machine: Ubuntu 24.04 with
|
||||
# .github/apt/test-packages.txt and Node 22 (.github/local-ci/Containerfile).
|
||||
# .github/apt/test-packages.txt and Bun (.github/local-ci/provision.sh), which
|
||||
# Harmony's container service provisions from the pinned base image.
|
||||
# It materializes one fixed commit in this checkout’s private overlay, including
|
||||
# every tracked file. Source directories are recreated in the overlay so later
|
||||
# edits and new files in the canonical checkout cannot enter the tests. Peers
|
||||
@@ -70,13 +71,34 @@ exec 8>"$cache/cache.lock"
|
||||
echo "release-ready: waiting for the shared build cache"
|
||||
flock 8
|
||||
mkdir -p "$cache/home" "$output/build"
|
||||
|
||||
# The image runs JavaScript with Bun only; nothing in it executes Node. The
|
||||
# tests use this machine's Bun (JS_RUNTIME, as for make), kept separate from
|
||||
# the immutable source overlay.
|
||||
runtime_directory="$cache/js-runtime"
|
||||
mkdir -p "$runtime_directory"
|
||||
runtime=$(command -v "${JS_RUNTIME:-bun}" 2>/dev/null || true)
|
||||
if [ -z "$runtime" ] && [ -z "${JS_RUNTIME:-}" ] && [ -x "$HOME/.bun/bin/bun" ]; then
|
||||
runtime=$HOME/.bun/bin/bun
|
||||
fi
|
||||
if [ -z "$runtime" ] || [ "$(basename "$runtime")" != bun ]; then
|
||||
echo "release-ready: set JS_RUNTIME to an installed Bun executable; the CI image has no Node" >&2
|
||||
exit 2
|
||||
fi
|
||||
cp "$runtime" "$runtime_directory/bun"
|
||||
chmod 755 "$runtime_directory/bun"
|
||||
container_runtime=/opt/js-runtime/bun
|
||||
container_test_flags=${JS_TEST_FLAGS:-test}
|
||||
container_linter=${ADDONS_LINTER:-"/opt/js-runtime/bun x --bun addons-linter"}
|
||||
commit=$(git rev-parse "${RELEASE_READY_REVISION:-HEAD}^{commit}")
|
||||
image_tag=$({
|
||||
git show "$commit:.github/local-ci/Containerfile"
|
||||
git show "$commit:.github/local-ci/base-image"
|
||||
git show "$commit:.github/local-ci/provision.sh"
|
||||
git show "$commit:.github/apt/test-packages.txt"
|
||||
git show "$commit:.github/apt/web-packages.txt"
|
||||
git show "$commit:.github/apt/windows-packages.txt"
|
||||
git show "$commit:.github/emscripten-version"
|
||||
} | sha256sum | cut -c1-12)
|
||||
} | sha256sum | cut -c1-12)-bun
|
||||
image=localhost/inbe-ci:$image_tag
|
||||
|
||||
# Read the candidate from Git objects, never the changing files on disk.
|
||||
@@ -173,12 +195,31 @@ printf '[containers]\nkeyring = false\n' > "$output/containers.conf"
|
||||
CONTAINERS_CONF_OVERRIDE="$root/$output/containers.conf"
|
||||
export CONTAINERS_CONF_OVERRIDE
|
||||
|
||||
if ! podman image exists "$image"; then
|
||||
echo "release-ready: building $image"
|
||||
git archive "$commit:.github" | podman build -t "$image" -f local-ci/Containerfile -
|
||||
# Replace the images of older package lists; one still in use stays.
|
||||
podman images --format '{{.Repository}}:{{.Tag}}' localhost/inbe-ci \
|
||||
| grep -v ":$image_tag\$" | xargs -r podman rmi >/dev/null 2>&1 || true
|
||||
if podman image exists "$image"; then
|
||||
:
|
||||
else
|
||||
image_status=$?
|
||||
if [ "$image_status" -gt 1 ]; then
|
||||
echo "release-ready: the container engine could not inspect the CI image" >&2
|
||||
exit "$image_status"
|
||||
fi
|
||||
# The service runs the candidate's recipe on the pinned base image and
|
||||
# commits the result with its provenance. Both inputs are hashed files
|
||||
# under build/, never the changing checkout.
|
||||
echo "release-ready: provisioning $image"
|
||||
base=$(git show "$commit:.github/local-ci/base-image")
|
||||
provision=$root/$output/provision
|
||||
rm -rf "$provision"
|
||||
mkdir -p "$provision"
|
||||
git archive --format=tar -o "$provision/context.tar" "$commit:.github"
|
||||
git show "$commit:.github/local-ci/provision.sh" > "$provision/provision.sh"
|
||||
podman pull "$base"
|
||||
podman provision "$base" "$image" \
|
||||
"$provision/context.tar" "$(sha256sum < "$provision/context.tar" | cut -c1-64)" \
|
||||
"$provision/provision.sh" "$(sha256sum < "$provision/provision.sh" | cut -c1-64)" \
|
||||
--env EM_NODE_JS=/usr/local/bin/bun --env EMSDK_NODE=/usr/local/bin/bun \
|
||||
--env EM_CONFIG=/opt/emsdk/.emscripten \
|
||||
--env PATH=/opt/emsdk/upstream/emscripten:/usr/local/go/bin:/usr/local/bin:/usr/bin:/bin
|
||||
fi
|
||||
|
||||
rm -f "$output/stamp.json"
|
||||
@@ -206,10 +247,14 @@ podman run --rm --pull=never --userns=keep-id \
|
||||
-v "$root:/src:O" \
|
||||
-v "$root/$output/build:/src/build" \
|
||||
-v "$cache/home:/home/ubuntu" \
|
||||
-v "$runtime_directory:/opt/js-runtime:ro" \
|
||||
-v "$sync_mount_source:/sync-server:O" \
|
||||
$android_mount \
|
||||
-e HOME=/home/ubuntu -e XDG_CACHE_HOME=/home/ubuntu/.cache \
|
||||
-e GRADLE_USER_HOME=/home/ubuntu/.gradle \
|
||||
-e JS_RUNTIME="$container_runtime" -e JS_TEST_FLAGS="$container_test_flags" \
|
||||
-e ADDONS_LINTER="$container_linter" \
|
||||
-e EM_NODE_JS="$container_runtime" -e EMSDK_NODE="$container_runtime" \
|
||||
-e CI=true -e RELEASE_READY_COMMIT="$commit" \
|
||||
-e RELEASE_READY_INCREMENTAL="$incremental" \
|
||||
-e RELEASE_READY_SNAPSHOT="$snapshot" \
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
// Finite browser-fixture operations. A deadline also aborts the owned transport.
|
||||
export function withDeadline(operation, milliseconds, message, parentSignal) {
|
||||
if (!Number.isFinite(milliseconds) || milliseconds <= 0)
|
||||
return Promise.reject(new Error('deadline must be a positive finite duration'));
|
||||
return new Promise((resolve, reject) => {
|
||||
const controller = new AbortController();
|
||||
let finished = false;
|
||||
const finish = (error, value) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
clearTimeout(timer);
|
||||
parentSignal?.removeEventListener('abort', parentAborted);
|
||||
if (error) reject(error); else resolve(value);
|
||||
};
|
||||
const abort = reason => {
|
||||
controller.abort(reason);
|
||||
finish(reason);
|
||||
};
|
||||
const parentAborted = () => abort(parentSignal.reason || new Error('fixture aborted'));
|
||||
const timer = setTimeout(() => abort(new Error(message)), milliseconds);
|
||||
if (parentSignal?.aborted) {
|
||||
parentAborted();
|
||||
return;
|
||||
}
|
||||
parentSignal?.addEventListener('abort', parentAborted, { once: true });
|
||||
Promise.resolve().then(() => operation(controller.signal)).then(
|
||||
value => finish(null, value), error => finish(error));
|
||||
});
|
||||
}
|
||||
|
||||
export function fetchWithDeadline(url, options, milliseconds, signal, consume = response => response) {
|
||||
return withDeadline(async requestSignal => {
|
||||
const response = await fetch(url, { ...options, signal: requestSignal });
|
||||
return await consume(response);
|
||||
}, milliseconds, `HTTP request timed out after ${milliseconds}ms: ${url}`, signal);
|
||||
}
|
||||
|
||||
export function connectProtocol(url, label, milliseconds, signal) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const ws = new WebSocket(url);
|
||||
const pending = new Map();
|
||||
const events = [];
|
||||
let nextId = 1;
|
||||
let closed = '';
|
||||
const fail = error => {
|
||||
if (closed) return;
|
||||
closed = error.message;
|
||||
clearTimeout(connectTimer);
|
||||
signal?.removeEventListener('abort', aborted);
|
||||
reject(error);
|
||||
for (const item of pending.values()) {
|
||||
clearTimeout(item.timer);
|
||||
item.reject(new Error(`${item.method}: ${closed}`));
|
||||
}
|
||||
pending.clear();
|
||||
try { ws.close(); } catch {}
|
||||
};
|
||||
const aborted = () => fail(signal.reason || new Error('fixture aborted'));
|
||||
const connectTimer = setTimeout(() => fail(new Error(
|
||||
`${label} connection timed out after ${milliseconds}ms`)), milliseconds);
|
||||
if (signal?.aborted) {
|
||||
aborted();
|
||||
return;
|
||||
}
|
||||
signal?.addEventListener('abort', aborted, { once: true });
|
||||
ws.addEventListener('open', () => {
|
||||
clearTimeout(connectTimer);
|
||||
if (closed) return;
|
||||
resolve({
|
||||
events,
|
||||
send(method, params = {}) {
|
||||
if (closed) return Promise.reject(new Error(`${method}: ${closed}`));
|
||||
const id = nextId++;
|
||||
return new Promise((resolveSend, rejectSend) => {
|
||||
const timer = setTimeout(() => fail(new Error(
|
||||
`${label} ${method} timed out after ${milliseconds}ms`)), milliseconds);
|
||||
pending.set(id, { resolve: resolveSend, reject: rejectSend, method, timer });
|
||||
try { ws.send(JSON.stringify({ id, method, params })); }
|
||||
catch (error) { fail(error); }
|
||||
});
|
||||
},
|
||||
close() { fail(new Error(`${label} connection closed by fixture`)); }
|
||||
});
|
||||
});
|
||||
ws.addEventListener('message', event => {
|
||||
let message;
|
||||
try { message = JSON.parse(event.data); }
|
||||
catch { fail(new Error(`${label} returned invalid JSON`)); return; }
|
||||
const item = pending.get(message.id);
|
||||
if (item) {
|
||||
pending.delete(message.id);
|
||||
clearTimeout(item.timer);
|
||||
if (message.error)
|
||||
item.reject(new Error(`${item.method}: ${message.error.message || message.message || message.error}`));
|
||||
else item.resolve(message.result || {});
|
||||
} else if (message.method) events.push(message);
|
||||
});
|
||||
ws.addEventListener('error', event => fail(new Error(event.message || `${label} WebSocket error`)));
|
||||
ws.addEventListener('close', event => fail(new Error(`${label} connection closed (${event.code})`)));
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
export function validateChromiumSocketPath(directory) {
|
||||
const socket = `${directory}/org.chromium.Chromium.XXXXXX/SingletonSocket`;
|
||||
if (Buffer.byteLength(socket) >= 108)
|
||||
throw new Error('WEB_SMOKE_PROFILE_ROOT is too long for Chromium Unix sockets; choose a shorter directory inside this project');
|
||||
}
|
||||
+26
-91
@@ -2,7 +2,6 @@
|
||||
|
||||
import { createServer } from 'node:http';
|
||||
import { createReadStream, mkdirSync, mkdtempSync, rmSync, existsSync, accessSync, constants, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, normalize, resolve, sep } from 'node:path';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { Readable } from 'node:stream';
|
||||
@@ -11,6 +10,7 @@ import { verifyAppearanceCanvas, verifySunSalutationCanvas } from './web-sun-sal
|
||||
import { verifyCanvasTextResolution } from './web-canvas-text-test.mjs';
|
||||
import { waitForFirefoxBidi } from './firefox-startup.mjs';
|
||||
import { musicAudioInstrumentation, verifyMusicAcrossTabs } from './web-music-test.mjs';
|
||||
import { connectProtocol, fetchWithDeadline, withDeadline, validateChromiumSocketPath } from './web-smoke-protocol.mjs';
|
||||
|
||||
if (typeof WebSocket === 'undefined') {
|
||||
console.error('web smoke: FAIL: this script needs the global WebSocket API. Use Node >= 21, or pass --experimental-websocket on Node 20.');
|
||||
@@ -38,13 +38,17 @@ const allowWebglDisabled = process.env.WEB_SMOKE_ALLOW_WEBGL_DISABLED
|
||||
const useNoSandbox = process.env.WEB_SMOKE_NO_SANDBOX
|
||||
? !/^(0|false|no)$/i.test(process.env.WEB_SMOKE_NO_SANDBOX)
|
||||
: !!process.env.CI || (typeof process.getuid === 'function' && process.getuid() === 0);
|
||||
const userDataDir = mkdtempSync(join(tmpdir(), 'inbe-web-smoke-'));
|
||||
const profileRoot = resolve(process.env.WEB_SMOKE_PROFILE_ROOT || 'build/scratch');
|
||||
mkdirSync(profileRoot, { recursive: true });
|
||||
const userDataDir = mkdtempSync(join(profileRoot, 'profile-'));
|
||||
let smokeSignal;
|
||||
const browserEnvironment = { ...process.env };
|
||||
delete browserEnvironment.DISPLAY;
|
||||
delete browserEnvironment.WAYLAND_DISPLAY;
|
||||
delete browserEnvironment.XAUTHORITY;
|
||||
delete browserEnvironment.DBUS_SESSION_BUS_ADDRESS;
|
||||
browserEnvironment.YUE_DESKTOP_RECOVERY = '0';
|
||||
browserEnvironment.TMPDIR = profileRoot;
|
||||
const mime = new Map([
|
||||
['.html', 'text/html; charset=utf-8'],
|
||||
['.js', 'text/javascript; charset=utf-8'],
|
||||
@@ -259,13 +263,14 @@ async function waitForDevtools(chrome, stderrLines) {
|
||||
|
||||
try {
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
smokeSignal?.throwIfAborted();
|
||||
if (exited)
|
||||
throw formatChromeLaunchError(stderrLines);
|
||||
|
||||
const port = readDevtoolsPort();
|
||||
if (port > 0) {
|
||||
try {
|
||||
const response = await fetch(`http://127.0.0.1:${port}/json/version`);
|
||||
const response = await fetchWithDeadline(`http://127.0.0.1:${port}/json/version`, {}, timeoutMs, smokeSignal);
|
||||
if (response.ok)
|
||||
return port;
|
||||
} catch {}
|
||||
@@ -283,99 +288,19 @@ async function waitForDevtools(chrome, stderrLines) {
|
||||
}
|
||||
|
||||
async function jsonRequest(url, options = {}) {
|
||||
const response = await fetch(url, options);
|
||||
if (!response.ok)
|
||||
throw new Error(`${options.method || 'GET'} ${url} failed: ${response.status}`);
|
||||
return response.json();
|
||||
return fetchWithDeadline(url, options, timeoutMs, smokeSignal, async response => {
|
||||
if (!response.ok)
|
||||
throw new Error(`${options.method || 'GET'} ${url} failed: ${response.status}`);
|
||||
return response.json();
|
||||
});
|
||||
}
|
||||
|
||||
function connect(wsUrl) {
|
||||
return new Promise((resolveConnect, reject) => {
|
||||
const ws = new WebSocket(wsUrl);
|
||||
const pending = new Map();
|
||||
const events = [];
|
||||
let nextId = 1;
|
||||
|
||||
ws.addEventListener('open', () => {
|
||||
resolveConnect({
|
||||
events,
|
||||
send(method, params = {}) {
|
||||
const id = nextId++;
|
||||
ws.send(JSON.stringify({ id, method, params }));
|
||||
return new Promise((resolveSend, rejectSend) => {
|
||||
pending.set(id, { resolve: resolveSend, reject: rejectSend, method });
|
||||
});
|
||||
},
|
||||
close() {
|
||||
ws.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
ws.addEventListener('message', event => {
|
||||
const message = JSON.parse(event.data);
|
||||
if (message.id && pending.has(message.id)) {
|
||||
const item = pending.get(message.id);
|
||||
pending.delete(message.id);
|
||||
if (message.error)
|
||||
item.reject(new Error(`${item.method}: ${message.error.message}`));
|
||||
else
|
||||
item.resolve(message.result || {});
|
||||
} else if (message.method) {
|
||||
events.push(message);
|
||||
}
|
||||
});
|
||||
ws.addEventListener('error', reject);
|
||||
});
|
||||
return connectProtocol(wsUrl, 'Chrome CDP', timeoutMs, smokeSignal);
|
||||
}
|
||||
|
||||
function connectBidi(wsUrl) {
|
||||
return new Promise((resolveConnect, reject) => {
|
||||
const ws = new WebSocket(wsUrl);
|
||||
const pending = new Map();
|
||||
const events = [];
|
||||
let nextId = 1;
|
||||
let closed = '';
|
||||
|
||||
ws.addEventListener('open', () => {
|
||||
resolveConnect({
|
||||
events,
|
||||
send(method, params = {}) {
|
||||
if (closed)
|
||||
return Promise.reject(new Error(`${method}: ${closed}`));
|
||||
const id = nextId++;
|
||||
ws.send(JSON.stringify({ id, method, params }));
|
||||
return new Promise((resolveSend, rejectSend) => {
|
||||
pending.set(id, { resolve: resolveSend, reject: rejectSend, method });
|
||||
});
|
||||
},
|
||||
close() {
|
||||
ws.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
ws.addEventListener('message', event => {
|
||||
const message = JSON.parse(event.data);
|
||||
if (message.id && pending.has(message.id)) {
|
||||
const item = pending.get(message.id);
|
||||
pending.delete(message.id);
|
||||
if (message.error)
|
||||
item.reject(new Error(`${item.method}: ${message.message || message.error || JSON.stringify(message)}`));
|
||||
else
|
||||
item.resolve(message.result || {});
|
||||
} else if (message.method) {
|
||||
events.push(message);
|
||||
}
|
||||
});
|
||||
ws.addEventListener('error', event => reject(new Error(event.message || `WebSocket error for ${wsUrl}`)));
|
||||
// A dropped connection fails waiting and later commands instead of
|
||||
// leaving them unanswered.
|
||||
ws.addEventListener('close', event => {
|
||||
closed = `Firefox closed the BiDi connection (${event.code})`;
|
||||
for (const item of pending.values())
|
||||
item.reject(new Error(`${item.method}: ${closed}`));
|
||||
pending.clear();
|
||||
});
|
||||
});
|
||||
return connectProtocol(wsUrl, 'Firefox BiDi', timeoutMs, smokeSignal);
|
||||
}
|
||||
|
||||
function eventText(event) {
|
||||
@@ -1646,6 +1571,7 @@ async function verifyHabitsClickDoesNotReload(client) {
|
||||
}
|
||||
|
||||
let chrome;
|
||||
const browserStderr = [];
|
||||
let client;
|
||||
let failure;
|
||||
|
||||
@@ -1897,8 +1823,14 @@ function firefoxPageClient(bidi, context) {
|
||||
}
|
||||
|
||||
try {
|
||||
const totalTimeoutMs = Number(process.env.WEB_SMOKE_TOTAL_TIMEOUT_MS || 15 * 60 * 1000);
|
||||
await withDeadline(async signal => {
|
||||
smokeSignal = signal;
|
||||
const port = await listen();
|
||||
smokeSignal.throwIfAborted();
|
||||
const browser = resolveBrowser();
|
||||
if (process.platform === 'linux' && /(?:chromium(?:-browser)?|google-chrome|chrome)(?:-stable)?$/.test(browser))
|
||||
validateChromiumSocketPath(profileRoot);
|
||||
if (isFirefoxBrowser(browser)) {
|
||||
writeFirefoxPrefs();
|
||||
const args = [
|
||||
@@ -1979,7 +1911,7 @@ try {
|
||||
if (useNoSandbox)
|
||||
args.splice(1, 0, '--no-sandbox');
|
||||
|
||||
const chromeStderr = [];
|
||||
const chromeStderr = browserStderr;
|
||||
chrome = spawn(browser, args, { stdio: ['ignore', 'ignore', 'pipe'], detached: true, env: browserEnvironment });
|
||||
chrome.stderr.setEncoding('utf8');
|
||||
chrome.stderr.on('data', chunk => {
|
||||
@@ -2051,8 +1983,11 @@ try {
|
||||
if (rejected.length)
|
||||
throw new Error(`Application rejected ${rejected.length} UI frames: ${rejected.join(' | ')}`);
|
||||
console.log(`web smoke: PASS (${renderer})`);
|
||||
}, totalTimeoutMs, `web smoke exceeded its ${totalTimeoutMs}ms fixture deadline`);
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
if (browserStderr.length)
|
||||
console.error(`web smoke browser stderr (exit ${chrome?.exitCode}, signal ${chrome?.signalCode}):\n${browserStderr.join('').slice(-4000)}`);
|
||||
const recent = (client?.events || []).map(eventText).filter(Boolean).slice(-30);
|
||||
if (recent.length)
|
||||
console.error(`web smoke browser logs:\n${recent.join('\n').slice(-12000)}`);
|
||||
|
||||
Reference in new issue
Block a user