diff --git a/.github/actions/emscripten/action.yml b/.github/actions/emscripten/action.yml new file mode 100644 index 000000000..f114c29fb --- /dev/null +++ b/.github/actions/emscripten/action.yml @@ -0,0 +1,42 @@ +name: Set up Emscripten +description: > + Install the Emscripten SDK version named in .github/emscripten-version and + put it on PATH. The SDK, including the system libraries and ports it builds + on first use, is cached per version, so a job only downloads it when the + pinned version changes. + +runs: + using: composite + steps: + - name: Read the pinned Emscripten version + id: version + shell: bash + run: echo "version=$(tr -d '[:space:]' < .github/emscripten-version)" >> "$GITHUB_OUTPUT" + + - name: Restore the Emscripten SDK + uses: actions/cache@v4 + with: + path: ~/emsdk + key: emsdk-${{ env.ImageOS }}-${{ runner.arch }}-${{ steps.version.outputs.version }}-${{ hashFiles('scripts/ensure-emscripten-zlib.sh') }} + restore-keys: | + emsdk-${{ env.ImageOS }}-${{ runner.arch }}-${{ steps.version.outputs.version }}- + + - name: Install and activate Emscripten + shell: bash + env: + EMSCRIPTEN_VERSION: ${{ steps.version.outputs.version }} + run: | + set -euo pipefail + if [ ! -x "$HOME/emsdk/emsdk" ]; then + rm -rf "$HOME/emsdk" + git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk" + fi + if [ ! -x "$HOME/emsdk/upstream/emscripten/emcc" ]; then + "$HOME/emsdk/emsdk" install "$EMSCRIPTEN_VERSION" + fi + "$HOME/emsdk/emsdk" activate "$EMSCRIPTEN_VERSION" + echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV" + echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV" + echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV" + echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH" + bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder" diff --git a/.github/actions/ziran-packages/action.yml b/.github/actions/ziran-packages/action.yml index 661c84c42..6d3e15900 100644 --- a/.github/actions/ziran-packages/action.yml +++ b/.github/actions/ziran-packages/action.yml @@ -1,22 +1,37 @@ name: Fetch Ziran packages description: > - Fetch the packages pinned in ziran.lock. The built Ziran compiler and the - package checkouts are cached by the lock's hash, so a job only rebuilds and - clones when a dependency actually changes. + Fetch the packages pinned in ziran.lock. The built Ziran compiler is cached + by the toolchain commit alone, so bumping another package such as Kryon + keeps it; the package checkouts are cached by the whole lock and fall back + to the previous checkouts, so a bump fetches only what changed. runs: using: composite steps: - - name: Restore the Ziran toolchain and package sources + - name: Read the Ziran toolchain commit + id: toolchain + shell: sh + run: | + commit=$(python3 -c 'import json; print(json.load(open("ziran.lock"))["toolchain"]["commit"])') + echo "commit=$commit" >> "$GITHUB_OUTPUT" + + - name: Restore the Ziran toolchain uses: actions/cache@v4 with: path: | - ~/.cache/ziran/sources build/ziran-bootstrap build/ziran-toolchain - key: ziran-${{ env.ImageOS }}-${{ runner.arch }}-${{ hashFiles('ziran.lock') }} + key: ziran-toolchain-${{ env.ImageOS }}-${{ runner.arch }}-${{ steps.toolchain.outputs.commit }} restore-keys: | - ziran-${{ env.ImageOS }}-${{ runner.arch }}- + ziran-toolchain-${{ env.ImageOS }}-${{ runner.arch }}- + + - name: Restore the package sources + uses: actions/cache@v4 + with: + path: ~/.cache/ziran/sources + key: ziran-sources-${{ env.ImageOS }}-${{ runner.arch }}-${{ hashFiles('ziran.lock') }} + restore-keys: | + ziran-sources-${{ env.ImageOS }}-${{ runner.arch }}- - name: Fetch Ziran packages shell: sh diff --git a/.github/emscripten-version b/.github/emscripten-version new file mode 100644 index 000000000..c7d48f048 --- /dev/null +++ b/.github/emscripten-version @@ -0,0 +1 @@ +6.0.10 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c4dd1236..f266871d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,12 +65,17 @@ jobs: echo "all package URLs HTTPS - ok" test: - name: Test + # make test split across runners; each shard is a third of its targets. + name: Test (${{ matrix.shard }}/3) runs-on: ubuntu-latest # Secret Scan runs beside the tests; nothing here publishes, so it need not gate them. needs: - guard-package-urls timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2] env: APT_CACHE_DIR: /tmp/apt-cache @@ -102,9 +107,10 @@ jobs: run: .github/scripts/install-apt-packages.sh .github/apt/test-packages.txt - name: Run Tests - run: make test + run: sh scripts/test-shard.sh ${{ matrix.shard }} 3 - name: Check out sync test server + if: matrix.shard == 0 uses: actions/checkout@v4 with: repository: kryonlabs/daochi @@ -113,22 +119,25 @@ jobs: submodules: recursive - name: Set up Go for sync test server + if: matrix.shard == 0 uses: actions/setup-go@v5 with: go-version-file: .test-server/go.mod cache-dependency-path: .test-server/go.sum - name: Verify sync and recovery + if: matrix.shard == 0 run: | make -C .test-server build make sync-server-test DAOCHI_BIN="$GITHUB_WORKSPACE/.test-server/daochi" - build-chrome-extension: - name: Build Chrome Extension + browser-addons: + # Builds the web app once, packages it for Chrome and Firefox, and runs + # the smoke test in both browsers. It runs beside the tests; a failure in + # either still fails the run. + name: Build Browser Add-ons runs-on: ubuntu-latest timeout-minutes: 60 - # Builds beside the tests instead of after them; a failure in either - # still fails the run. needs: - guard-package-urls env: @@ -162,18 +171,9 @@ jobs: run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt - name: Set up Emscripten - run: | - set -euo pipefail - git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk" - "$HOME/emsdk/emsdk" install latest - "$HOME/emsdk/emsdk" activate latest - echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV" - echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV" - echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV" - echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH" - bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder" + uses: ./.github/actions/emscripten - - name: Build Chrome Extension ZIP + - name: Build Chrome and Firefox ZIPs run: | set -euo pipefail export PATH="$INBE_NODE_BIN:$PATH" @@ -181,75 +181,18 @@ jobs: make package-unpackaged-assets make chrome-web-store test -f build/dist/inbe-chrome-web-store.zip + make firefox-addons + test -f build/dist/inbe-firefox-addons.zip + make firefox-addons-lint # The web build renders on canvas, so this is also the canvas smoke - # test; `make web-canvas` would rebuild and run it twice more. - - name: Run Web Smoke Test + # test. + - name: Run Web Smoke Test (Chrome) run: | set -euo pipefail export PATH="$INBE_NODE_BIN:$PATH" make web-smoke-test - build-firefox-addons: - name: Build Firefox Add-on - runs-on: ubuntu-latest - timeout-minutes: 60 - # Builds beside the tests instead of after them; a failure in either - # still fails the run. - needs: - - guard-package-urls - env: - APT_CACHE_DIR: /tmp/apt-cache - - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - with: - fetch-depth: 1 - lfs: true - - - name: Fetch Ziran packages - uses: ./.github/actions/ziran-packages - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22 - - - name: Cache Web apt packages - uses: actions/cache@v4 - with: - path: ${{ env.APT_CACHE_DIR }} - key: web-apt-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/apt/web-packages.txt') }} - restore-keys: | - web-apt-${{ runner.os }}-${{ runner.arch }}- - - - name: Install Web Dependencies - timeout-minutes: 15 - run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt - - - name: Set up Emscripten - run: | - set -euo pipefail - git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk" - "$HOME/emsdk/emsdk" install latest - "$HOME/emsdk/emsdk" activate latest - echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV" - echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV" - echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV" - echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH" - bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder" - - - name: Build Firefox Add-on ZIP - run: | - set -euo pipefail - export PATH="$INBE_NODE_BIN:$PATH" - node --version - make package-unpackaged-assets - make firefox-addons - test -f build/dist/inbe-firefox-addons.zip - make firefox-addons-lint - - name: Install Firefox (Mozilla build) timeout-minutes: 15 run: | diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index c6d87514d..2de720f95 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -51,15 +51,7 @@ jobs: - name: Set Up Emscripten if: ${{ !inputs.released_web }} - run: | - set -euo pipefail - git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk" - "$HOME/emsdk/emsdk" install latest - "$HOME/emsdk/emsdk" activate latest - echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV" - echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV" - echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV" - echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH" + uses: ./.github/actions/emscripten - name: Download Released Web App if: ${{ inputs.released_web }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6b06fc671..a358c6968 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,16 +111,7 @@ jobs: run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt - name: Set up Emscripten - run: | - set -euo pipefail - git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk" - "$HOME/emsdk/emsdk" install latest - "$HOME/emsdk/emsdk" activate latest - echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV" - echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV" - echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV" - echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH" - bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder" + uses: ./.github/actions/emscripten - name: Build Web ZIP run: | @@ -175,16 +166,7 @@ jobs: run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt - name: Set up Emscripten - run: | - set -euo pipefail - git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk" - "$HOME/emsdk/emsdk" install latest - "$HOME/emsdk/emsdk" activate latest - echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV" - echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV" - echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV" - echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH" - bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder" + uses: ./.github/actions/emscripten - name: Build Chrome Extension ZIP run: | @@ -239,16 +221,7 @@ jobs: run: .github/scripts/install-apt-packages.sh .github/apt/web-packages.txt - name: Set up Emscripten - run: | - set -euo pipefail - git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/emsdk" - "$HOME/emsdk/emsdk" install latest - "$HOME/emsdk/emsdk" activate latest - echo "EMSDK=$HOME/emsdk" >> "$GITHUB_ENV" - echo "EM_CONFIG=$HOME/emsdk/.emscripten" >> "$GITHUB_ENV" - echo "INBE_NODE_BIN=$(dirname "$(command -v node)")" >> "$GITHUB_ENV" - echo "$HOME/emsdk/upstream/emscripten" >> "$GITHUB_PATH" - bash scripts/ensure-emscripten-zlib.sh "$HOME/emsdk/upstream/emscripten/embuilder" + uses: ./.github/actions/emscripten - name: Build Firefox Add-on ZIP run: | diff --git a/Makefile b/Makefile index e38a4f055..f149a4d5a 100644 --- a/Makefile +++ b/Makefile @@ -1146,6 +1146,10 @@ storage-literals-check: bash ./scripts/check-storage-literals.sh test: clean-text-api-check package-check secret-check storage-literals-check font-bundle-check audio-test-fixture-check embedded-image-assets-check +test: screenshot-scene-test + +.PHONY: screenshot-scene-test +screenshot-scene-test: bash ./tests/screenshot_scene_test.sh audio-test-fixture-check: diff --git a/scripts/test-shard.sh b/scripts/test-shard.sh new file mode 100755 index 000000000..462884623 --- /dev/null +++ b/scripts/test-shard.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# Run one shard of `make test`: every COUNT-th of its prerequisites, starting +# at INDEX (0-based). CI runs the shards on separate runners at once. +# +# sh scripts/test-shard.sh INDEX COUNT +# sh scripts/test-shard.sh --list # every test target, one per line +set -eu + +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$root" + +targets() { + # make's database lists all of test's prerequisites on one line, with + # .WAIT markers between them because test runs them one at a time. + make -pq test 2>/dev/null | + awk '/^test:/ { for (i = 2; i <= NF; i++) if ($i != ".WAIT") print $i; exit }' | + awk '!seen[$0]++' +} + +if [ "${1:-}" = --list ]; then + targets + exit 0 +fi +if [ $# -ne 2 ]; then + echo "usage: sh scripts/test-shard.sh INDEX COUNT | --list" >&2 + exit 2 +fi +index=$1 +count=$2 +case "$index$count" in + *[!0-9]*) echo "test-shard: INDEX and COUNT must be numbers" >&2; exit 2 ;; +esac +if [ "$count" -lt 1 ] || [ "$index" -ge "$count" ]; then + echo "test-shard: need 0 <= INDEX < COUNT" >&2 + exit 2 +fi + +all=$(targets) +total=$(printf '%s\n' "$all" | grep -c .) +if [ "$total" -lt 1 ]; then + echo "test-shard: found no test targets" >&2 + exit 1 +fi +shard=$(printf '%s\n' "$all" | awk -v shard="$index" -v count="$count" '(NR - 1) % count == shard') +echo "test-shard: $index of $count runs $(printf '%s\n' "$shard" | grep -c .) of $total targets" >&2 +# shellcheck disable=SC2086 +exec make $shard