From adbb3c3116c1d1760064f0518b4df3ffa6c9fa2b Mon Sep 17 00:00:00 2001 From: Waozi Date: Fri, 23 Oct 2026 14:08:39 -0300 Subject: [PATCH] Stop the always-red runs and check pushes locally first A pushed version that is already released no longer starts the forty-minute release builds; CI still tests every push. A package version waiting for the next release is a notice in App packages instead of a failure. The web build keeps saving its storage while practice packs install (up to about ten seconds of retries), and the package compatibility check asks the node again while a deploy restarts it. The pre-push hook now runs the quick checks that match the pushed files (versions, translations and fonts, UI APIs, app bundles, workflows, script syntax); make pre-push-check runs them by hand. --- .githooks/pre-push | 1 + .github/workflows/release.yml | 43 ++++++++---- mk/tests-checks.mk | 5 ++ scripts/package-release.py | 15 +++-- scripts/pre-push-checks.py | 123 ++++++++++++++++++++++++++++++++++ src/web_boot.js | 12 ++-- tests/package_compat_test.py | 17 ++++- 7 files changed, 191 insertions(+), 25 deletions(-) create mode 100755 scripts/pre-push-checks.py diff --git a/.githooks/pre-push b/.githooks/pre-push index 4303630e3..68d5e01a8 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -15,3 +15,4 @@ else fi scripts/check-secrets.py --pre-push "$@" < "$tmp" +scripts/pre-push-checks.py < "$tmp" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 23dc92cec..64060de41 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,6 +52,7 @@ jobs: runs-on: ubuntu-latest outputs: reuse_artifacts: ${{ steps.reuse.outputs.reuse_artifacts }} + release_due: ${{ steps.due.outputs.release_due }} steps: - name: Require an immutable candidate env: @@ -68,6 +69,22 @@ jobs: ref: ${{ inputs.revision || github.sha }} fetch-depth: 0 + # A push of a version that is already released has nothing to release: + # CI tests it, and the platform builds wait for the next version. + - name: Release only a new version or a requested run + id: due + env: + EVENT: ${{ github.event_name }} + run: | + set -euo pipefail + version=$(python3 scripts/check-version.py --print-version) + if [ "$EVENT" = "workflow_dispatch" ] || ! git rev-parse -q --verify "refs/tags/v$version" >/dev/null; then + echo "release_due=true" >> "$GITHUB_OUTPUT" + else + echo "release_due=false" >> "$GITHUB_OUTPUT" + echo "::notice title=Release::v$version is already released; CI tests this push and no release builds run" + fi + - name: Verify identical source and every gate before reusing artifacts id: reuse env: @@ -168,7 +185,7 @@ jobs: python3 scripts/check-secrets.py --history test: - if: needs.secret-scan.outputs.reuse_artifacts != 'true' + if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true' name: Test (${{ matrix.shard }}/3) runs-on: ubuntu-latest needs: secret-scan @@ -227,7 +244,7 @@ jobs: fdroid: - if: needs.secret-scan.outputs.reuse_artifacts != 'true' + if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true' name: F-Droid needs: secret-scan uses: ./.github/workflows/fdroid.yml @@ -238,7 +255,7 @@ jobs: name: Verify F-Droid reproducibility runs-on: ubuntu-latest needs: [secret-scan, fdroid, build-android] - if: needs.secret-scan.outputs.reuse_artifacts != 'true' + if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true' timeout-minutes: 15 steps: - name: Download independent F-Droid build @@ -271,7 +288,7 @@ jobs: apksigcopier compare --unsigned "${signed[0]}" "${rebuilt[0]}" build-web: - if: needs.secret-scan.outputs.reuse_artifacts != 'true' + if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true' name: Build Web runs-on: ubuntu-latest timeout-minutes: 60 @@ -328,7 +345,7 @@ jobs: if-no-files-found: error build-chrome-extension: - if: needs.secret-scan.outputs.reuse_artifacts != 'true' + if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true' name: Build Chrome Extension runs-on: ubuntu-latest timeout-minutes: 60 @@ -385,7 +402,7 @@ jobs: if-no-files-found: error build-firefox-addons: - if: needs.secret-scan.outputs.reuse_artifacts != 'true' + if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true' name: Build Firefox Add-on runs-on: ubuntu-latest timeout-minutes: 60 @@ -443,7 +460,7 @@ jobs: if-no-files-found: error build-linux-appimage: - if: needs.secret-scan.outputs.reuse_artifacts != 'true' + if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true' name: Build Linux AppImage runs-on: ${{ matrix.runner }} timeout-minutes: 60 @@ -562,7 +579,7 @@ jobs: if-no-files-found: error build-linux-deb: - if: needs.secret-scan.outputs.reuse_artifacts != 'true' + if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true' name: Build Linux Deb runs-on: ${{ matrix.runner }} timeout-minutes: 60 @@ -624,7 +641,7 @@ jobs: if-no-files-found: error build-linux-rpm: - if: needs.secret-scan.outputs.reuse_artifacts != 'true' + if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true' name: Build Linux RPM runs-on: ${{ matrix.runner }} timeout-minutes: 60 @@ -686,7 +703,7 @@ jobs: if-no-files-found: error build-linux-snap: - if: needs.secret-scan.outputs.reuse_artifacts != 'true' + if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true' name: Build Linux Snap runs-on: ${{ matrix.runner }} timeout-minutes: 60 @@ -745,7 +762,7 @@ jobs: if-no-files-found: error build-linux-flatpak: - if: needs.secret-scan.outputs.reuse_artifacts != 'true' + if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true' name: Build Linux Flatpak runs-on: ${{ matrix.runner }} timeout-minutes: 60 @@ -804,7 +821,7 @@ jobs: if-no-files-found: error build-windows: - if: needs.secret-scan.outputs.reuse_artifacts != 'true' + if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true' name: Build Windows runs-on: ubuntu-latest timeout-minutes: 60 @@ -870,7 +887,7 @@ jobs: if-no-files-found: error build-android: - if: needs.secret-scan.outputs.reuse_artifacts != 'true' + if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true' name: Build Android runs-on: ubuntu-latest timeout-minutes: 60 diff --git a/mk/tests-checks.mk b/mk/tests-checks.mk index fa3200083..066891c96 100644 --- a/mk/tests-checks.mk +++ b/mk/tests-checks.mk @@ -16,6 +16,11 @@ clean-text-api-check: button-api-check: bash scripts/check-button-api.sh +# The checks the pre-push hook runs, for everything not yet on origin/master. +.PHONY: pre-push-check +pre-push-check: + python3 scripts/pre-push-checks.py --unpushed + .PHONY: version-test version-test: python3 tests/version_test.py diff --git a/scripts/package-release.py b/scripts/package-release.py index a517c00e2..21edf7cb3 100644 --- a/scripts/package-release.py +++ b/scripts/package-release.py @@ -609,13 +609,16 @@ def main(): if args.verify_origin: advanced, unversioned = published_changes(args.store, args.verify_origin) # A package that changed without a new version waits for - # its next version; that is not a failure. Only a new - # version that ships with an unversioned change fails, - # because the node accepts every app together or none. + # its next version; that is not a failure. The node + # accepts every app together or none, so a new version + # that ships beside an unversioned change waits too: the + # release, whose ./update_version.sh versions every + # package, publishes them together. if unversioned and advanced: - raise ValueError(", ".join(unversioned) + " changed without a new version, " - "so " + ", ".join(advanced) + " cannot be published with it; " - "./update_version.sh versions every package for the release") + print("::notice title=App packages::Waiting for the next release: " + + ", ".join(advanced) + " ships together with " + ", ".join(unversioned) + + ", which changed without a new version") + return if unversioned: print("::notice title=App packages::Not published: " + ", ".join(unversioned) + " changed without a new version and will publish with the next one") diff --git a/scripts/pre-push-checks.py b/scripts/pre-push-checks.py new file mode 100755 index 000000000..b7130632c --- /dev/null +++ b/scripts/pre-push-checks.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Run the quick local checks that match what a push changes. + +Git runs this from .githooks/pre-push with the pushed refs on stdin, so the +checks that fail most often on GitHub (versions, translations, fonts, the +canonical UI APIs, the app bundles and the workflows) fail here first, in a +minute or two. `make pre-push-check` runs it for everything not yet on +origin/master. The checks read the working tree; `git push --no-verify` +skips them. +""" + +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parent.parent +ZERO = "0" * 40 + + +def git(*arguments): + return subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True, + text=True, check=True).stdout + + +def pushed_ranges(lines): + """(base, tip) for each pushed ref that adds commits.""" + ranges = [] + for line in lines: + parts = line.split() + if len(parts) != 4 or parts[1] == ZERO: + continue + tip, remote = parts[1], parts[3] + if remote == ZERO: + remote = git("merge-base", tip, "origin/master").strip() + ranges.append((remote, tip)) + return ranges + + +def changed_files(ranges): + files = set() + for base, tip in ranges: + files.update(path for path in git("diff", "--name-only", base, tip).splitlines() if path) + return sorted(files) + + +def plan(files): + """The checks for these changed files, in the order they run.""" + def any_of(*prefixes): + return any(path.startswith(prefixes) for path in files) + + def any_suffix(*suffixes): + return any(path.endswith(suffixes) for path in files) + + checks = [("version files agree", ["python3", "scripts/check-version.py"]), + ("package versions", ["make", "-s", "package-version-test"])] + if any_of("locales/", "assets/fonts/") or any_suffix(".zi"): + checks.append(("translations and fonts", ["make", "-s", "locale-check"])) + if any_suffix(".zi"): + checks.append(("canonical UI APIs", ["make", "-s", "clean-text-api-check", "button-api-check"])) + if any_of("apps/", "src/cells/", "locales/"): + checks.append(("app bundles", ["make", "-s", "cells-test"])) + if any_of(".github/workflows/", "scripts/package-release.py", "scripts/release", + "scripts/check-version.py", "tests/release_test.py", "tests/package_workflow_test.py"): + checks.append(("release workflow", ["python3", "tests/release_test.py"])) + checks.append(("package workflow", ["python3", "tests/package_workflow_test.py"])) + python = [path for path in files if path.endswith(".py") and (ROOT / path).is_file()] + if python: + checks.append(("Python syntax", ["python3", "-m", "py_compile", *python])) + scripts = [path for path in files if path.endswith((".js", ".mjs")) and (ROOT / path).is_file()] + for path in scripts: + checks.append((path + " syntax", ["node", "--check", path])) + return checks + + +def run(checks, ranges): + failed = [] + for base, tip in ranges: + result = subprocess.run(["git", "diff", "--check", base, tip], cwd=ROOT, + capture_output=True, text=True) + if result.returncode != 0: + print(result.stdout + result.stderr, end="") + failed.append("whitespace in the pushed commits") + environment = dict(os.environ) + # Python caches under the checkout would be swept into commits. + environment["PYTHONDONTWRITEBYTECODE"] = "1" + for name, command in checks: + started = time.monotonic() + result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, env=environment) + seconds = time.monotonic() - started + if result.returncode == 0: + print(f"pre-push: {name} passed ({seconds:.0f} s)") + continue + output = (result.stdout + result.stderr).strip().splitlines() + print("\n".join(output[-25:])) + print(f"pre-push: {name} FAILED: {' '.join(command)}") + failed.append(name) + return failed + + +def main(): + if sys.argv[1:] == ["--unpushed"]: + lines = [f"HEAD {git('rev-parse', 'HEAD').strip()} refs/heads/master " + f"{git('rev-parse', 'origin/master').strip()}"] + else: + lines = sys.stdin.read().splitlines() + ranges = pushed_ranges(lines) + if not ranges: + return 0 + files = changed_files(ranges) + if not files: + return 0 + failed = run(plan(files), ranges) + if failed: + print("pre-push: not pushed. Fix: " + ", ".join(failed) + + " (git push --no-verify skips these checks)") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/web_boot.js b/src/web_boot.js index 33505ec73..90f6c14fe 100644 --- a/src/web_boot.js +++ b/src/web_boot.js @@ -237,18 +237,20 @@ function runStorageSync(retryDelay) { Module.__inbeStorageSyncLastError = message; Module.__inbeStorageSyncPending = true; console.error('IDBFS save failed: ' + message); - // A package can replace a temporary file while IDBFS enumerates it. - // Start a fresh snapshot after that race; persistent failures still - // resolve false, leaving the dirty state available to an explicit save. + // A package can replace a temporary file while IDBFS enumerates it, and + // practice packs keep installing for several seconds after start. Start + // a fresh snapshot after that race, waiting longer each time (about ten + // seconds in all); persistent failures still resolve false, leaving the + // dirty state available to an explicit save. var transient = error && (error.errno === 44 || error.name === 'NotFoundError' || error.name === 'AbortError'); var attempt = Module.__inbeStorageSyncRetry || 0; - if (transient && attempt < 3) { + if (transient && attempt < 8) { Module.__inbeStorageSyncRetry = attempt + 1; if (Module.__inbeStorageSyncTimer) clearTimeout(Module.__inbeStorageSyncTimer); Module.__inbeStorageSyncTimer = setTimeout(function() { Module.__inbeStorageSyncTimer = 0; runStorageSync(retryDelay); - }, 100 * Math.pow(2, attempt)); + }, Math.min(2000, 100 * Math.pow(2, attempt))); return; } finishStorageSync(false); diff --git a/tests/package_compat_test.py b/tests/package_compat_test.py index 85a1ce27f..d73f312c3 100644 --- a/tests/package_compat_test.py +++ b/tests/package_compat_test.py @@ -22,6 +22,7 @@ import stat import subprocess import sys import tempfile +import time import urllib.error import urllib.parse import urllib.request @@ -167,6 +168,20 @@ def published_release(origin, ledger, app_id, output): return bundle, metadata +def published_release_retried(origin, ledger, app_id, output, attempts=6, delay=15): + """The node restarts for a minute or so whenever it is deployed; ask + again for a while before deciding that it serves no release.""" + for attempt in range(attempts): + try: + return published_release(origin, ledger, app_id, output) + except (OSError, ValueError) as error: + if attempt == attempts - 1: + raise + print(f"forward {app_id}: the node did not answer ({error}); asking again") + time.sleep(delay) + return None + + def published_games(binary, origin, ledger, output): """Install and play the node's games, independently of Arcade's checkout. @@ -177,7 +192,7 @@ def published_games(binary, origin, ledger, output): environment["YUE_DESKTOP_RECOVERY"] = "0" checked = 0 for app_id, test in (("inbe.arcade", "tests/arcade_ui_test.sh"), ("inbe.chess", "tests/chess_ui_test.sh")): - published = published_release(origin, ledger, app_id, output) + published = published_release_retried(origin, ledger, app_id, output) if published is None: print(f"forward {app_id}: the node serves no release") continue