Speed up CI: shard the tests, build the web app once, cache toolchains

- make test runs as three parallel shards (scripts/test-shard.sh), so
  the test job takes about a third as long.
- One job builds the web app, packages it for Chrome and Firefox, and
  runs the smoke test in both, instead of two jobs building it twice.
- Emscripten is pinned to the version CI already used (6.0.10) and
  cached, including the libraries it builds on first use.
- The built Ziran compiler is cached by the toolchain commit instead of
  the whole lock, so a Kryon bump no longer rebuilds it.
This commit is contained in:
waozi committed 2026-09-29 19:20:12 -03:00
1 parent b8c6ed63fe
commit 94a3133c97
8 files changed
+142 -125

No files matched your search

+42
View File
@@ -0,0 +1,42 @@
name: Set up Emscripten
description: >
Install the Emscripten SDK version named in .github/emscripten-version and
put it on PATH. The SDK, including the system libraries and ports it builds
on first use, is cached per version, so a job only downloads it when the
pinned version changes.
runs:
using: composite
steps:
- name: Read the pinned Emscripten version
id: version
shell: bash
run: echo "version=$(tr -d '[:space:]' < .github/emscripten-version)" >> "$GITHUB_OUTPUT"
- name: Restore the Emscripten SDK
uses: actions/cache@v4
with:
path: ~/emsdk
key: emsdk-${{ env.ImageOS }}-${{ runner.arch }}-${{ steps.version.outputs.version }}-${{ hashFiles('scripts/ensure-emscripten-zlib.sh') }}
restore-keys: |
emsdk-${{ env.ImageOS }}-${{ runner.arch }}-${{ steps.version.outputs.version }}-
- name: Install and activate Emscripten
shell: bash
env:
EMSCRIPTEN_VERSION: ${{ steps.version.outputs.version }}
run: |
set -euo pipefail
if [ ! -x "$HOME/emsdk/emsdk" ]; then
rm -rf "$HOME/emsdk"
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
fi
if [ ! -x "$HOME/emsdk/upstream/emscripten/emcc" ]; then
"$HOME/emsdk/emsdk" install "$EMSCRIPTEN_VERSION"
fi
"$HOME/emsdk/emsdk" activate "$EMSCRIPTEN_VERSION"
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder"
+22 -7
View File
@@ -1,22 +1,37 @@
name: Fetch Ziran packages
description: >
Fetch the packages pinned in ziran.lock. The built Ziran compiler and the
package checkouts are cached by the lock's hash, so a job only rebuilds and
clones when a dependency actually changes.
Fetch the packages pinned in ziran.lock. The built Ziran compiler is cached
by the toolchain commit alone, so bumping another package such as Kryon
keeps it; the package checkouts are cached by the whole lock and fall back
to the previous checkouts, so a bump fetches only what changed.
runs:
using: composite
steps:
- name: Restore the Ziran toolchain and package sources
- name: Read the Ziran toolchain commit
id: toolchain
shell: sh
run: |
commit=$(python3 -c 'import json; print(json.load(open("ziran.lock"))["toolchain"]["commit"])')
echo "commit=$commit" >> "$GITHUB_OUTPUT"
- name: Restore the Ziran toolchain
uses: actions/cache@v4
with:
path: |
~/.cache/ziran/sources
build/ziran-bootstrap
build/ziran-toolchain
key: ziran-${{ env.ImageOS }}-${{ runner.arch }}-${{ hashFiles('ziran.lock') }}
key: ziran-toolchain-${{ env.ImageOS }}-${{ runner.arch }}-${{ steps.toolchain.outputs.commit }}
restore-keys: |
ziran-${{ env.ImageOS }}-${{ runner.arch }}-
ziran-toolchain-${{ env.ImageOS }}-${{ runner.arch }}-
- name: Restore the package sources
uses: actions/cache@v4
with:
path: ~/.cache/ziran/sources
key: ziran-sources-${{ env.ImageOS }}-${{ runner.arch }}-${{ hashFiles('ziran.lock') }}
restore-keys: |
ziran-sources-${{ env.ImageOS }}-${{ runner.arch }}-
- name: Fetch Ziran packages
shell: sh
+1
View File
@@ -0,0 +1 @@
6.0.10
+22 -79
View File
@@ -65,12 +65,17 @@ jobs:
echo "all package URLs HTTPS - ok"
test:
name: Test
# make test split across runners; each shard is a third of its targets.
name: Test (${{ matrix.shard }}/3)
runs-on: ubuntu-latest
# Secret Scan runs beside the tests; nothing here publishes, so it need not gate them.
needs:
- guard-package-urls
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
shard: [0, 1, 2]
env:
APT_CACHE_DIR: /tmp/apt-cache
@@ -102,9 +107,10 @@ jobs:
run: .github/scripts/install-apt-packages.sh .github/apt/test-packages.txt
- name: Run Tests
run: make test
run: sh scripts/test-shard.sh ${{ matrix.shard }} 3
- name: Check out sync test server
if: matrix.shard == 0
uses: actions/checkout@v4
with:
repository: kryonlabs/daochi
@@ -113,22 +119,25 @@ jobs:
submodules: recursive
- name: Set up Go for sync test server
if: matrix.shard == 0
uses: actions/setup-go@v5
with:
go-version-file: .test-server/go.mod
cache-dependency-path: .test-server/go.sum
- name: Verify sync and recovery
if: matrix.shard == 0
run: |
make -C .test-server build
make sync-server-test DAOCHI_BIN="$GITHUB_WORKSPACE/.test-server/daochi"
build-chrome-extension:
name: Build Chrome Extension
browser-addons:
# Builds the web app once, packages it for Chrome and Firefox, and runs
# the smoke test in both browsers. It runs beside the tests; a failure in
# either still fails the run.
name: Build Browser Add-ons
runs-on: ubuntu-latest
timeout-minutes: 60
# Builds beside the tests instead of after them; a failure in either
# still fails the run.
needs:
- guard-package-urls
env:
@@ -162,18 +171,9 @@ jobs:
run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt
- name: Set up Emscripten
run: |
set -euo pipefail
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
"$HOME/emsdk/emsdk" install latest
"$HOME/emsdk/emsdk" activate latest
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder"
uses: ./.github/actions/emscripten
- name: Build Chrome Extension ZIP
- name: Build Chrome and Firefox ZIPs
run: |
set -euo pipefail
export PATH="$INBE_NODE_BIN:$PATH"
@@ -181,75 +181,18 @@ jobs:
make package-unpackaged-assets
make chrome-web-store
test -f build/dist/inbe-chrome-web-store.zip
make firefox-addons
test -f build/dist/inbe-firefox-addons.zip
make firefox-addons-lint
# The web build renders on canvas, so this is also the canvas smoke
# test; `make web-canvas` would rebuild and run it twice more.
- name: Run Web Smoke Test
# test.
- name: Run Web Smoke Test (Chrome)
run: |
set -euo pipefail
export PATH="$INBE_NODE_BIN:$PATH"
make web-smoke-test
build-firefox-addons:
name: Build Firefox Add-on
runs-on: ubuntu-latest
timeout-minutes: 60
# Builds beside the tests instead of after them; a failure in either
# still fails the run.
needs:
- guard-package-urls
env:
APT_CACHE_DIR: /tmp/apt-cache
steps:
- name: Checkout Repository
uses: actions/checkout@v4
with:
fetch-depth: 1
lfs: true
- name: Fetch Ziran packages
uses: ./.github/actions/ziran-packages
- name: Set up Node.js
uses: actions/setup-node@v4
with:
node-version: 22
- name: Cache Web apt packages
uses: actions/cache@v4
with:
path: ${{ env.APT_CACHE_DIR }}
key: web-apt-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/apt/web-packages.txt') }}
restore-keys: |
web-apt-${{ runner.os }}-${{ runner.arch }}-
- name: Install Web Dependencies
timeout-minutes: 15
run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt
- name: Set up Emscripten
run: |
set -euo pipefail
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
"$HOME/emsdk/emsdk" install latest
"$HOME/emsdk/emsdk" activate latest
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder"
- name: Build Firefox Add-on ZIP
run: |
set -euo pipefail
export PATH="$INBE_NODE_BIN:$PATH"
node --version
make package-unpackaged-assets
make firefox-addons
test -f build/dist/inbe-firefox-addons.zip
make firefox-addons-lint
- name: Install Firefox (Mozilla build)
timeout-minutes: 15
run: |
+1 -9
View File
@@ -51,15 +51,7 @@ jobs:
- name: Set Up Emscripten
if: ${{ !inputs.released_web }}
run: |
set -euo pipefail
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
"$HOME/emsdk/emsdk" install latest
"$HOME/emsdk/emsdk" activate latest
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
uses: ./.github/actions/emscripten
- name: Download Released Web App
if: ${{ inputs.released_web }}
+3 -30
View File
@@ -111,16 +111,7 @@ jobs:
run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt
- name: Set up Emscripten
run: |
set -euo pipefail
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
"$HOME/emsdk/emsdk" install latest
"$HOME/emsdk/emsdk" activate latest
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder"
uses: ./.github/actions/emscripten
- name: Build Web ZIP
run: |
@@ -175,16 +166,7 @@ jobs:
run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt
- name: Set up Emscripten
run: |
set -euo pipefail
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
"$HOME/emsdk/emsdk" install latest
"$HOME/emsdk/emsdk" activate latest
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder"
uses: ./.github/actions/emscripten
- name: Build Chrome Extension ZIP
run: |
@@ -239,16 +221,7 @@ jobs:
run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt
- name: Set up Emscripten
run: |
set -euo pipefail
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
"$HOME/emsdk/emsdk" install latest
"$HOME/emsdk/emsdk" activate latest
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder"
uses: ./.github/actions/emscripten
- name: Build Firefox Add-on ZIP
run: |
+4
View File
@@ -1146,6 +1146,10 @@ storage-literals-check:
bash ./scripts/check-storage-literals.sh
test: clean-text-api-check package-check secret-check storage-literals-check font-bundle-check audio-test-fixture-check embedded-image-assets-check
test: screenshot-scene-test
.PHONY: screenshot-scene-test
screenshot-scene-test:
bash ./tests/screenshot_scene_test.sh
audio-test-fixture-check:
+47
View File
@@ -0,0 +1,47 @@
#!/bin/sh
# Run one shard of `make test`: every COUNT-th of its prerequisites, starting
# at INDEX (0-based). CI runs the shards on separate runners at once.
#
# sh scripts/test-shard.sh INDEX COUNT
# sh scripts/test-shard.sh --list # every test target, one per line
set -eu
root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$root"
targets() {
# make's database lists all of test's prerequisites on one line, with
# .WAIT markers between them because test runs them one at a time.
make -pq test 2>/dev/null |
awk '/^test:/ { for (i = 2; i <= NF; i++) if ($i != ".WAIT") print $i; exit }' |
awk '!seen[$0]++'
}
if [ "${1:-}" = --list ]; then
targets
exit 0
fi
if [ $# -ne 2 ]; then
echo "usage: sh scripts/test-shard.sh INDEX COUNT | --list" >&2
exit 2
fi
index=$1
count=$2
case "$index$count" in
*[!0-9]*) echo "test-shard: INDEX and COUNT must be numbers" >&2; exit 2 ;;
esac
if [ "$count" -lt 1 ] || [ "$index" -ge "$count" ]; then
echo "test-shard: need 0 <= INDEX < COUNT" >&2
exit 2
fi
all=$(targets)
total=$(printf '%s\n' "$all" | grep -c .)
if [ "$total" -lt 1 ]; then
echo "test-shard: found no test targets" >&2
exit 1
fi
shard=$(printf '%s\n' "$all" | awk -v shard="$index" -v count="$count" '(NR - 1) % count == shard')
echo "test-shard: $index of $count runs $(printf '%s\n' "$shard" | grep -c .) of $total targets" >&2
# shellcheck disable=SC2086
exec make $shard