Speed up CI: shard the tests, build the web app once, cache toolchains
- make test runs as three parallel shards (scripts/test-shard.sh), so the test job takes about a third as long. - One job builds the web app, packages it for Chrome and Firefox, and runs the smoke test in both, instead of two jobs building it twice. - Emscripten is pinned to the version CI already used (6.0.10) and cached, including the libraries it builds on first use. - The built Ziran compiler is cached by the toolchain commit instead of the whole lock, so a Kryon bump no longer rebuilds it.
This commit is contained in:
1 parent
b8c6ed63fe
commit
94a3133c97
8 files changed
+142
-125
No files matched your search
@@ -0,0 +1,42 @@
|
||||
name: Set up Emscripten
|
||||
description: >
|
||||
Install the Emscripten SDK version named in .github/emscripten-version and
|
||||
put it on PATH. The SDK, including the system libraries and ports it builds
|
||||
on first use, is cached per version, so a job only downloads it when the
|
||||
pinned version changes.
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Read the pinned Emscripten version
|
||||
id: version
|
||||
shell: bash
|
||||
run: echo "version=$(tr -d '[:space:]' < .github/emscripten-version)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Restore the Emscripten SDK
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/emsdk
|
||||
key: emsdk-${{ env.ImageOS }}-${{ runner.arch }}-${{ steps.version.outputs.version }}-${{ hashFiles('scripts/ensure-emscripten-zlib.sh') }}
|
||||
restore-keys: |
|
||||
emsdk-${{ env.ImageOS }}-${{ runner.arch }}-${{ steps.version.outputs.version }}-
|
||||
|
||||
- name: Install and activate Emscripten
|
||||
shell: bash
|
||||
env:
|
||||
EMSCRIPTEN_VERSION: ${{ steps.version.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ ! -x "$HOME/emsdk/emsdk" ]; then
|
||||
rm -rf "$HOME/emsdk"
|
||||
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
|
||||
fi
|
||||
if [ ! -x "$HOME/emsdk/upstream/emscripten/emcc" ]; then
|
||||
"$HOME/emsdk/emsdk" install "$EMSCRIPTEN_VERSION"
|
||||
fi
|
||||
"$HOME/emsdk/emsdk" activate "$EMSCRIPTEN_VERSION"
|
||||
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
|
||||
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
|
||||
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
|
||||
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
|
||||
bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder"
|
||||
@@ -1,22 +1,37 @@
|
||||
name: Fetch Ziran packages
|
||||
description: >
|
||||
Fetch the packages pinned in ziran.lock. The built Ziran compiler and the
|
||||
package checkouts are cached by the lock's hash, so a job only rebuilds and
|
||||
clones when a dependency actually changes.
|
||||
Fetch the packages pinned in ziran.lock. The built Ziran compiler is cached
|
||||
by the toolchain commit alone, so bumping another package such as Kryon
|
||||
keeps it; the package checkouts are cached by the whole lock and fall back
|
||||
to the previous checkouts, so a bump fetches only what changed.
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Restore the Ziran toolchain and package sources
|
||||
- name: Read the Ziran toolchain commit
|
||||
id: toolchain
|
||||
shell: sh
|
||||
run: |
|
||||
commit=$(python3 -c 'import json; print(json.load(open("ziran.lock"))["toolchain"]["commit"])')
|
||||
echo "commit=$commit" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Restore the Ziran toolchain
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/ziran/sources
|
||||
build/ziran-bootstrap
|
||||
build/ziran-toolchain
|
||||
key: ziran-${{ env.ImageOS }}-${{ runner.arch }}-${{ hashFiles('ziran.lock') }}
|
||||
key: ziran-toolchain-${{ env.ImageOS }}-${{ runner.arch }}-${{ steps.toolchain.outputs.commit }}
|
||||
restore-keys: |
|
||||
ziran-${{ env.ImageOS }}-${{ runner.arch }}-
|
||||
ziran-toolchain-${{ env.ImageOS }}-${{ runner.arch }}-
|
||||
|
||||
- name: Restore the package sources
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/ziran/sources
|
||||
key: ziran-sources-${{ env.ImageOS }}-${{ runner.arch }}-${{ hashFiles('ziran.lock') }}
|
||||
restore-keys: |
|
||||
ziran-sources-${{ env.ImageOS }}-${{ runner.arch }}-
|
||||
|
||||
- name: Fetch Ziran packages
|
||||
shell: sh
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
6.0.10
|
||||
+22
-79
@@ -65,12 +65,17 @@ jobs:
|
||||
echo "all package URLs HTTPS - ok"
|
||||
|
||||
test:
|
||||
name: Test
|
||||
# make test split across runners; each shard is a third of its targets.
|
||||
name: Test (${{ matrix.shard }}/3)
|
||||
runs-on: ubuntu-latest
|
||||
# Secret Scan runs beside the tests; nothing here publishes, so it need not gate them.
|
||||
needs:
|
||||
- guard-package-urls
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [0, 1, 2]
|
||||
env:
|
||||
APT_CACHE_DIR: /tmp/apt-cache
|
||||
|
||||
@@ -102,9 +107,10 @@ jobs:
|
||||
run: .github/scripts/install-apt-packages.sh .github/apt/test-packages.txt
|
||||
|
||||
- name: Run Tests
|
||||
run: make test
|
||||
run: sh scripts/test-shard.sh ${{ matrix.shard }} 3
|
||||
|
||||
- name: Check out sync test server
|
||||
if: matrix.shard == 0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: kryonlabs/daochi
|
||||
@@ -113,22 +119,25 @@ jobs:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Go for sync test server
|
||||
if: matrix.shard == 0
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: .test-server/go.mod
|
||||
cache-dependency-path: .test-server/go.sum
|
||||
|
||||
- name: Verify sync and recovery
|
||||
if: matrix.shard == 0
|
||||
run: |
|
||||
make -C .test-server build
|
||||
make sync-server-test DAOCHI_BIN="$GITHUB_WORKSPACE/.test-server/daochi"
|
||||
|
||||
build-chrome-extension:
|
||||
name: Build Chrome Extension
|
||||
browser-addons:
|
||||
# Builds the web app once, packages it for Chrome and Firefox, and runs
|
||||
# the smoke test in both browsers. It runs beside the tests; a failure in
|
||||
# either still fails the run.
|
||||
name: Build Browser Add-ons
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
# Builds beside the tests instead of after them; a failure in either
|
||||
# still fails the run.
|
||||
needs:
|
||||
- guard-package-urls
|
||||
env:
|
||||
@@ -162,18 +171,9 @@ jobs:
|
||||
run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt
|
||||
|
||||
- name: Set up Emscripten
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
|
||||
"$HOME/emsdk/emsdk" install latest
|
||||
"$HOME/emsdk/emsdk" activate latest
|
||||
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
|
||||
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
|
||||
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
|
||||
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
|
||||
bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder"
|
||||
uses: ./.github/actions/emscripten
|
||||
|
||||
- name: Build Chrome Extension ZIP
|
||||
- name: Build Chrome and Firefox ZIPs
|
||||
run: |
|
||||
set -euo pipefail
|
||||
export PATH="$INBE_NODE_BIN:$PATH"
|
||||
@@ -181,75 +181,18 @@ jobs:
|
||||
make package-unpackaged-assets
|
||||
make chrome-web-store
|
||||
test -f build/dist/inbe-chrome-web-store.zip
|
||||
make firefox-addons
|
||||
test -f build/dist/inbe-firefox-addons.zip
|
||||
make firefox-addons-lint
|
||||
|
||||
# The web build renders on canvas, so this is also the canvas smoke
|
||||
# test; `make web-canvas` would rebuild and run it twice more.
|
||||
- name: Run Web Smoke Test
|
||||
# test.
|
||||
- name: Run Web Smoke Test (Chrome)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
export PATH="$INBE_NODE_BIN:$PATH"
|
||||
make web-smoke-test
|
||||
|
||||
build-firefox-addons:
|
||||
name: Build Firefox Add-on
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
# Builds beside the tests instead of after them; a failure in either
|
||||
# still fails the run.
|
||||
needs:
|
||||
- guard-package-urls
|
||||
env:
|
||||
APT_CACHE_DIR: /tmp/apt-cache
|
||||
|
||||
steps:
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
lfs: true
|
||||
|
||||
- name: Fetch Ziran packages
|
||||
uses: ./.github/actions/ziran-packages
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
- name: Cache Web apt packages
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ${{ env.APT_CACHE_DIR }}
|
||||
key: web-apt-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/apt/web-packages.txt') }}
|
||||
restore-keys: |
|
||||
web-apt-${{ runner.os }}-${{ runner.arch }}-
|
||||
|
||||
- name: Install Web Dependencies
|
||||
timeout-minutes: 15
|
||||
run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt
|
||||
|
||||
- name: Set up Emscripten
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
|
||||
"$HOME/emsdk/emsdk" install latest
|
||||
"$HOME/emsdk/emsdk" activate latest
|
||||
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
|
||||
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
|
||||
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
|
||||
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
|
||||
bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder"
|
||||
|
||||
- name: Build Firefox Add-on ZIP
|
||||
run: |
|
||||
set -euo pipefail
|
||||
export PATH="$INBE_NODE_BIN:$PATH"
|
||||
node --version
|
||||
make package-unpackaged-assets
|
||||
make firefox-addons
|
||||
test -f build/dist/inbe-firefox-addons.zip
|
||||
make firefox-addons-lint
|
||||
|
||||
- name: Install Firefox (Mozilla build)
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
|
||||
@@ -51,15 +51,7 @@ jobs:
|
||||
|
||||
- name: Set Up Emscripten
|
||||
if: ${{ !inputs.released_web }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
|
||||
"$HOME/emsdk/emsdk" install latest
|
||||
"$HOME/emsdk/emsdk" activate latest
|
||||
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
|
||||
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
|
||||
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
|
||||
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
|
||||
uses: ./.github/actions/emscripten
|
||||
|
||||
- name: Download Released Web App
|
||||
if: ${{ inputs.released_web }}
|
||||
|
||||
@@ -111,16 +111,7 @@ jobs:
|
||||
run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt
|
||||
|
||||
- name: Set up Emscripten
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
|
||||
"$HOME/emsdk/emsdk" install latest
|
||||
"$HOME/emsdk/emsdk" activate latest
|
||||
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
|
||||
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
|
||||
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
|
||||
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
|
||||
bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder"
|
||||
uses: ./.github/actions/emscripten
|
||||
|
||||
- name: Build Web ZIP
|
||||
run: |
|
||||
@@ -175,16 +166,7 @@ jobs:
|
||||
run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt
|
||||
|
||||
- name: Set up Emscripten
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
|
||||
"$HOME/emsdk/emsdk" install latest
|
||||
"$HOME/emsdk/emsdk" activate latest
|
||||
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
|
||||
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
|
||||
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
|
||||
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
|
||||
bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder"
|
||||
uses: ./.github/actions/emscripten
|
||||
|
||||
- name: Build Chrome Extension ZIP
|
||||
run: |
|
||||
@@ -239,16 +221,7 @@ jobs:
|
||||
run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt
|
||||
|
||||
- name: Set up Emscripten
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk"
|
||||
"$HOME/emsdk/emsdk" install latest
|
||||
"$HOME/emsdk/emsdk" activate latest
|
||||
echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV"
|
||||
echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV"
|
||||
echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV"
|
||||
echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH"
|
||||
bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder"
|
||||
uses: ./.github/actions/emscripten
|
||||
|
||||
- name: Build Firefox Add-on ZIP
|
||||
run: |
|
||||
|
||||
@@ -1146,6 +1146,10 @@ storage-literals-check:
|
||||
bash ./scripts/check-storage-literals.sh
|
||||
|
||||
test: clean-text-api-check package-check secret-check storage-literals-check font-bundle-check audio-test-fixture-check embedded-image-assets-check
|
||||
test: screenshot-scene-test
|
||||
|
||||
.PHONY: screenshot-scene-test
|
||||
screenshot-scene-test:
|
||||
bash ./tests/screenshot_scene_test.sh
|
||||
|
||||
audio-test-fixture-check:
|
||||
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/bin/sh
|
||||
# Run one shard of `make test`: every COUNT-th of its prerequisites, starting
|
||||
# at INDEX (0-based). CI runs the shards on separate runners at once.
|
||||
#
|
||||
# sh scripts/test-shard.sh INDEX COUNT
|
||||
# sh scripts/test-shard.sh --list # every test target, one per line
|
||||
set -eu
|
||||
|
||||
root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$root"
|
||||
|
||||
targets() {
|
||||
# make's database lists all of test's prerequisites on one line, with
|
||||
# .WAIT markers between them because test runs them one at a time.
|
||||
make -pq test 2>/dev/null |
|
||||
awk '/^test:/ { for (i = 2; i <= NF; i++) if ($i != ".WAIT") print $i; exit }' |
|
||||
awk '!seen[$0]++'
|
||||
}
|
||||
|
||||
if [ "${1:-}" = --list ]; then
|
||||
targets
|
||||
exit 0
|
||||
fi
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: sh scripts/test-shard.sh INDEX COUNT | --list" >&2
|
||||
exit 2
|
||||
fi
|
||||
index=$1
|
||||
count=$2
|
||||
case "$index$count" in
|
||||
*[!0-9]*) echo "test-shard: INDEX and COUNT must be numbers" >&2; exit 2 ;;
|
||||
esac
|
||||
if [ "$count" -lt 1 ] || [ "$index" -ge "$count" ]; then
|
||||
echo "test-shard: need 0 <= INDEX < COUNT" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
all=$(targets)
|
||||
total=$(printf '%s\n' "$all" | grep -c .)
|
||||
if [ "$total" -lt 1 ]; then
|
||||
echo "test-shard: found no test targets" >&2
|
||||
exit 1
|
||||
fi
|
||||
shard=$(printf '%s\n' "$all" | awk -v shard="$index" -v count="$count" '(NR - 1) % count == shard')
|
||||
echo "test-shard: $index of $count runs $(printf '%s\n' "$shard" | grep -c .) of $total targets" >&2
|
||||
# shellcheck disable=SC2086
|
||||
exec make $shard
|
||||
Reference in new issue
Block a user