#!/bin/sh # Run CI's checks here before a release, the way CI runs them, so a release # cannot go red after its phone check. `make release` requires a pass for the # exact content it commits. # # The container is the Test job's machine: Ubuntu 24.04 with # .github/apt/test-packages.txt and Bun (.github/local-ci/provision.sh), which # Harmony's container service provisions from the pinned base image. # It materializes one fixed commit in this checkout’s private overlay, including # every tracked file. Source directories are recreated in the overlay so later # edits and new files in the canonical checkout cannot enter the tests. Peers # keep working on master; nothing this container writes reaches their checkout. # Packages use the fixed lock without local overrides or untracked secrets. # Build outputs reuse build/release-ready/ or build/release-ready-phone/; a run # takes about 90 minutes. # # Covered: ci.yml Secret Scan, Package URLs, large-file guard, all three test # shards, Chrome/Firefox add-ons and smoke, the real pinned Mesh # sync server, and release.yml's package compatibility test. Platform release # builds remain separate. The server comes from its canonical checkout through # another private overlay; no temporary source checkout is created. # # sh scripts/release-ready.sh every covered job; writes the stamp # sh scripts/release-ready.sh TARGET... only these make targets; no stamp # RELEASE_READY_INCREMENTAL=1 (targets only) keeps the times of files the # commit did not change and the Gradle and CMake build directories between # runs, so a rebuild compiles only what changed (scripts/phone-install.sh). # RELEASE_READY_DNS optionally supplies a resolver when rootless DNS is broken. set -eu root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) cd "$root" # Legacy task/version names are aliases, never additional permanent outputs. requested_output=${RELEASE_READY_OUTPUT:-build/release-ready} output=$(python3 scripts/check-build-layout.py "$requested_output") || exit 2 if [ "$output" != "$requested_output" ]; then echo "release-ready: $requested_output reuses $output" fi # Lock before writing any output. Never follow an output symlink into another # purpose's files. There is deliberately no scan or cleanup of old folders. for directory in build "$output" "$output/build" build/release-ready build/release-ready/home; do if [ -L "$directory" ]; then echo "release-ready: refusing symlink $directory" >&2 exit 2 fi done mkdir -p "$output" if [ -L "$output/lock" ]; then echo "release-ready: refusing symlink $output/lock" >&2 exit 2 fi exec 9>"$output/lock" if ! flock -n 9; then echo "release-ready: another run is using $output" >&2 exit 1 fi # Every fixed-commit run uses the same home, including Ziran's URL+commit # source cache and Gradle's downloads. Hold this lease until Podman exits: # package tools may repair sources and build inside their cached checkouts. # Different output purposes can queue, but cannot mutate this cache together. cache="$root/build/release-ready" mkdir -p "$cache" if [ -L "$cache/cache.lock" ]; then echo "release-ready: refusing symlink $cache/cache.lock" >&2 exit 2 fi exec 8>"$cache/cache.lock" echo "release-ready: waiting for the shared build cache" flock 8 mkdir -p "$cache/home" "$output/build" # The image runs JavaScript with Bun only; nothing in it executes Node. The # tests use this machine's Bun (JS_RUNTIME, as for make), kept separate from # the immutable source overlay. runtime_directory="$cache/js-runtime" mkdir -p "$runtime_directory" runtime=$(command -v "${JS_RUNTIME:-bun}" 2>/dev/null || true) if [ -z "$runtime" ] && [ -z "${JS_RUNTIME:-}" ] && [ -x "$HOME/.bun/bin/bun" ]; then runtime=$HOME/.bun/bin/bun fi if [ -z "$runtime" ] || [ "$(basename "$runtime")" != bun ]; then echo "release-ready: set JS_RUNTIME to an installed Bun executable; the CI image has no Node" >&2 exit 2 fi cp "$runtime" "$runtime_directory/bun" chmod 755 "$runtime_directory/bun" container_runtime=/opt/js-runtime/bun container_test_flags=${JS_TEST_FLAGS:-test} container_linter=${ADDONS_LINTER:-"/opt/js-runtime/bun x --bun addons-linter"} commit=$(git rev-parse "${RELEASE_READY_REVISION:-HEAD}^{commit}") image_tag=$({ git show "$commit:.github/local-ci/base-image" git show "$commit:.github/local-ci/provision.sh" git show "$commit:.github/apt/test-packages.txt" git show "$commit:.github/apt/web-packages.txt" git show "$commit:.github/apt/windows-packages.txt" git show "$commit:.github/emscripten-version" } | sha256sum | cut -c1-12)-bun image=localhost/inbe-ci:$image_tag # Read the candidate from Git objects, never the changing files on disk. content=$(python3 - "$commit" <<'PY_CONTENT' import importlib.util import sys spec = importlib.util.spec_from_file_location("phone_check", "scripts/phone-check.py") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) print(module.release_content(module.ROOT, sys.argv[1])) PY_CONTENT ) snapshot=$(cat scripts/check-release-overlay.py) incremental="" if [ -n "${RELEASE_READY_INCREMENTAL:-}" ]; then [ "$#" -gt 0 ] || { echo "release-ready: RELEASE_READY_INCREMENTAL needs make targets; full runs start clean" >&2; exit 2; } incremental=1 fi # Runs inside the container, from this variable rather than the script file, # because resetting the overlay to HEAD may rewrite this script. inside='set -eu if [ -n "$RELEASE_READY_INCREMENTAL" ]; then python3 -c "$RELEASE_READY_SNAPSHOT" /src "$RELEASE_READY_COMMIT" build/incremental/source-times.json for directory in droid/.gradle droid/.kotlin droid/build droid/app/build droid/app/.cxx; do mkdir -p "build/incremental/$directory" ln -s "/src/build/incremental/$directory" "$directory" done else # Outputs this run builds from fresh times must not be trusted later # against the times recorded for another commit. rm -f build/incremental/source-times.json python3 -c "$RELEASE_READY_SNAPSHOT" /src "$RELEASE_READY_COMMIT" fi failed="" job() { name=$1 shift printf "\n==> %s\n" "$name" if ! "$@"; then failed="$failed $name" fi } package_urls() { test ! -f .gitmodules || { echo "Dependencies are Ziran packages, not submodules"; return 1; } bad=$(grep -nE "^[[:space:]]*git[[:space:]]*=" ziran.toml \ | grep -vE "^[0-9]+:[[:space:]]*git[[:space:]]*=[[:space:]]*\"https://" || true) test -z "$bad" || { echo "Non-HTTPS package URL(s):$bad"; return 1; } } printf "\n==> Fetch Ziran packages (locked)\n" sh scripts/packages.sh --locked if [ "$#" -gt 0 ]; then job "make $*" make -k "$@" else job "Secret Scan: app publisher" python3 scripts/check-package-publishers.py job "Secret Scan: version consistency" sh -c "python3 scripts/check-version.py && python3 tests/version_test.py" job "Secret Scan: tracked files and history" sh -c "python3 scripts/check-secrets.py --working-tree && python3 scripts/check-secrets.py --history" job "Package URLs are public HTTPS" package_urls job "Files over 1 MB" python3 scripts/check-file-size.py --since origin/master job "Test: make test" make -k test job "Test: make package-compat-test" make package-compat-test job "Browser add-ons: Chrome and Firefox" sh .github/local-ci/check-browser.sh job "Sync: pinned Mesh server" sh .github/local-ci/check-sync.sh fi if [ -n "$failed" ]; then printf "\nrelease-ready: FAILED:%s\n" "$failed" exit 1 fi git diff --exit-code "$RELEASE_READY_COMMIT" -- . if [ -n "$RELEASE_READY_INCREMENTAL" ]; then python3 -c "$RELEASE_READY_SNAPSHOT" --record-times /src build/incremental/source-times.json fi if [ "$#" -gt 0 ]; then printf "\nrelease-ready: make %s passed\n" "$*" else printf "\nrelease-ready: every covered local CI job passed\n" fi' sync_server="" sync_mount_source="$root/$output/sync-unused" mkdir -p "$sync_mount_source" if [ "$#" -eq 0 ]; then sync_server=${RELEASE_READY_MESH_SOURCE:-"$root/../../../ziranlang/mesh"} sync_server=$(CDPATH= cd -- "$sync_server" && pwd) sync_commit=$(git show "$commit:.github/sync-test-server-commit") git -C "$sync_server" cat-file -e "$sync_commit^{commit}" sync_mount_source=$sync_server fi # Containers here need no session keyring. Creating one counts against the # account's kernel key quota (200 keys), which long-lived sessions fill, and # a full quota stops every container from starting. printf '[containers]\nkeyring = false\n' > "$output/containers.conf" CONTAINERS_CONF_OVERRIDE="$root/$output/containers.conf" export CONTAINERS_CONF_OVERRIDE if podman image exists "$image"; then : else image_status=$? if [ "$image_status" -gt 1 ]; then echo "release-ready: the container engine could not inspect the CI image" >&2 exit "$image_status" fi # The service runs the candidate's recipe on the pinned base image and # commits the result with its provenance. Both inputs are hashed files # under build/, never the changing checkout. echo "release-ready: provisioning $image" base=$(git show "$commit:.github/local-ci/base-image") provision=$root/$output/provision rm -rf "$provision" mkdir -p "$provision" git archive --format=tar -o "$provision/context.tar" "$commit:.github" git show "$commit:.github/local-ci/provision.sh" > "$provision/provision.sh" podman pull "$base" podman provision "$base" "$image" \ "$provision/context.tar" "$(sha256sum < "$provision/context.tar" | cut -c1-64)" \ "$provision/provision.sh" "$(sha256sum < "$provision/provision.sh" | cut -c1-64)" \ --env EM_NODE_JS=/usr/local/bin/bun --env EMSDK_NODE=/usr/local/bin/bun \ --env EM_CONFIG=/opt/emsdk/.emscripten \ --env PATH=/opt/emsdk/upstream/emscripten:/usr/local/go/bin:/usr/local/bin:/usr/bin:/bin fi rm -f "$output/stamp.json" started=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo "release-ready: testing $(git log -1 --format='%h %s' "$commit")" # Hosted runners have the Android SDK and NDK; the JNI layout tests use it. android_sdk=${ANDROID_SDK_ROOT:-${ANDROID_HOME:-$HOME/Android/Sdk}} android_mount="" if [ -d "$android_sdk/ndk" ]; then android_mount="-v $android_sdk:/opt/android-sdk:ro -e ANDROID_HOME=/opt/android-sdk -e ANDROID_SDK_ROOT=/opt/android-sdk" fi status=0 dns_option="" if [ -n "${RELEASE_READY_DNS:-}" ]; then case "$RELEASE_READY_DNS" in *[!0-9a-fA-F.:]*) echo "release-ready: RELEASE_READY_DNS must be an IP address" >&2; exit 2 ;; esac dns_option="--dns=$RELEASE_READY_DNS" fi # shellcheck disable=SC2086 podman run --rm --pull=never --userns=keep-id \ $dns_option \ -v "$root:/src:O" \ -v "$root/$output/build:/src/build" \ -v "$cache/home:/home/ubuntu" \ -v "$runtime_directory:/opt/js-runtime:ro" \ -v "$sync_mount_source:/sync-server:O" \ $android_mount \ -e HOME=/home/ubuntu -e XDG_CACHE_HOME=/home/ubuntu/.cache \ -e GRADLE_USER_HOME=/home/ubuntu/.gradle \ -e JS_RUNTIME="$container_runtime" -e JS_TEST_FLAGS="$container_test_flags" \ -e ADDONS_LINTER="$container_linter" \ -e EM_NODE_JS="$container_runtime" -e EMSDK_NODE="$container_runtime" \ -e CI=true -e RELEASE_READY_COMMIT="$commit" \ -e RELEASE_READY_INCREMENTAL="$incremental" \ -e RELEASE_READY_SNAPSHOT="$snapshot" \ -w /src "$image" sh -c "$inside" release-ready "$@" || status=$? if [ "$status" -ne 0 ] || [ "$#" -gt 0 ]; then exit "$status" fi printf '{\n "passed": true,\n "commit": "%s",\n "content": "%s",\n "started": "%s",\n "finished": "%s"\n}\n' \ "$commit" "$content" "$started" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$output/stamp.json" echo "release-ready: recorded $output/stamp.json for $(git log -1 --format=%h "$commit")"