Stop the always-red runs and check pushes locally first

A pushed version that is already released no longer starts the forty-minute release builds; CI still tests every push. A package version waiting for the next release is a notice in App packages instead of a failure. The web build keeps saving its storage while practice packs install (up to about ten seconds of retries), and the package compatibility check asks the node again while a deploy restarts it. The pre-push hook now runs the quick checks that match the pushed files (versions, translations and fonts, UI APIs, app bundles, workflows, script syntax); make pre-push-check runs them by hand.
This commit is contained in:
waozi committed 2026-10-23 14:08:39 -03:00
1 parent 2bc7e5fe65
commit adbb3c3116
7 files changed
+191 -25

No files matched your search

+1
View File
@@ -15,3 +15,4 @@ else
fi fi
scripts/check-secrets.py --pre-push "$@" < "$tmp" scripts/check-secrets.py --pre-push "$@" < "$tmp"
scripts/pre-push-checks.py < "$tmp"
+30 -13
View File
@@ -52,6 +52,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs: outputs:
reuse_artifacts: ${{ steps.reuse.outputs.reuse_artifacts }} reuse_artifacts: ${{ steps.reuse.outputs.reuse_artifacts }}
release_due: ${{ steps.due.outputs.release_due }}
steps: steps:
- name: Require an immutable candidate - name: Require an immutable candidate
env: env:
@@ -68,6 +69,22 @@ jobs:
ref: ${{ inputs.revision || github.sha }} ref: ${{ inputs.revision || github.sha }}
fetch-depth: 0 fetch-depth: 0
# A push of a version that is already released has nothing to release:
# CI tests it, and the platform builds wait for the next version.
- name: Release only a new version or a requested run
id: due
env:
EVENT: ${{ github.event_name }}
run: |
set -euo pipefail
version=$(python3 scripts/check-version.py --print-version)
if [ "$EVENT" = "workflow_dispatch" ] || ! git rev-parse -q --verify "refs/tags/v$version" >/dev/null; then
echo "release_due=true" >> "$GITHUB_OUTPUT"
else
echo "release_due=false" >> "$GITHUB_OUTPUT"
echo "::notice title=Release::v$version is already released; CI tests this push and no release builds run"
fi
- name: Verify identical source and every gate before reusing artifacts - name: Verify identical source and every gate before reusing artifacts
id: reuse id: reuse
env: env:
@@ -168,7 +185,7 @@ jobs:
python3 scripts/check-secrets.py --history python3 scripts/check-secrets.py --history
test: test:
if: needs.secret-scan.outputs.reuse_artifacts != 'true' if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
name: Test (${{ matrix.shard }}/3) name: Test (${{ matrix.shard }}/3)
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: secret-scan needs: secret-scan
@@ -227,7 +244,7 @@ jobs:
fdroid: fdroid:
if: needs.secret-scan.outputs.reuse_artifacts != 'true' if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
name: F-Droid name: F-Droid
needs: secret-scan needs: secret-scan
uses: ./.github/workflows/fdroid.yml uses: ./.github/workflows/fdroid.yml
@@ -238,7 +255,7 @@ jobs:
name: Verify F-Droid reproducibility name: Verify F-Droid reproducibility
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [secret-scan, fdroid, build-android] needs: [secret-scan, fdroid, build-android]
if: needs.secret-scan.outputs.reuse_artifacts != 'true' if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
timeout-minutes: 15 timeout-minutes: 15
steps: steps:
- name: Download independent F-Droid build - name: Download independent F-Droid build
@@ -271,7 +288,7 @@ jobs:
apksigcopier compare --unsigned "${signed[0]}" "${rebuilt[0]}" apksigcopier compare --unsigned "${signed[0]}" "${rebuilt[0]}"
build-web: build-web:
if: needs.secret-scan.outputs.reuse_artifacts != 'true' if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
name: Build Web name: Build Web
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 60 timeout-minutes: 60
@@ -328,7 +345,7 @@ jobs:
if-no-files-found: error if-no-files-found: error
build-chrome-extension: build-chrome-extension:
if: needs.secret-scan.outputs.reuse_artifacts != 'true' if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
name: Build Chrome Extension name: Build Chrome Extension
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 60 timeout-minutes: 60
@@ -385,7 +402,7 @@ jobs:
if-no-files-found: error if-no-files-found: error
build-firefox-addons: build-firefox-addons:
if: needs.secret-scan.outputs.reuse_artifacts != 'true' if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
name: Build Firefox Add-on name: Build Firefox Add-on
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 60 timeout-minutes: 60
@@ -443,7 +460,7 @@ jobs:
if-no-files-found: error if-no-files-found: error
build-linux-appimage: build-linux-appimage:
if: needs.secret-scan.outputs.reuse_artifacts != 'true' if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
name: Build Linux AppImage name: Build Linux AppImage
runs-on: ${{ matrix.runner }} runs-on: ${{ matrix.runner }}
timeout-minutes: 60 timeout-minutes: 60
@@ -562,7 +579,7 @@ jobs:
if-no-files-found: error if-no-files-found: error
build-linux-deb: build-linux-deb:
if: needs.secret-scan.outputs.reuse_artifacts != 'true' if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
name: Build Linux Deb name: Build Linux Deb
runs-on: ${{ matrix.runner }} runs-on: ${{ matrix.runner }}
timeout-minutes: 60 timeout-minutes: 60
@@ -624,7 +641,7 @@ jobs:
if-no-files-found: error if-no-files-found: error
build-linux-rpm: build-linux-rpm:
if: needs.secret-scan.outputs.reuse_artifacts != 'true' if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
name: Build Linux RPM name: Build Linux RPM
runs-on: ${{ matrix.runner }} runs-on: ${{ matrix.runner }}
timeout-minutes: 60 timeout-minutes: 60
@@ -686,7 +703,7 @@ jobs:
if-no-files-found: error if-no-files-found: error
build-linux-snap: build-linux-snap:
if: needs.secret-scan.outputs.reuse_artifacts != 'true' if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
name: Build Linux Snap name: Build Linux Snap
runs-on: ${{ matrix.runner }} runs-on: ${{ matrix.runner }}
timeout-minutes: 60 timeout-minutes: 60
@@ -745,7 +762,7 @@ jobs:
if-no-files-found: error if-no-files-found: error
build-linux-flatpak: build-linux-flatpak:
if: needs.secret-scan.outputs.reuse_artifacts != 'true' if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
name: Build Linux Flatpak name: Build Linux Flatpak
runs-on: ${{ matrix.runner }} runs-on: ${{ matrix.runner }}
timeout-minutes: 60 timeout-minutes: 60
@@ -804,7 +821,7 @@ jobs:
if-no-files-found: error if-no-files-found: error
build-windows: build-windows:
if: needs.secret-scan.outputs.reuse_artifacts != 'true' if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
name: Build Windows name: Build Windows
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 60 timeout-minutes: 60
@@ -870,7 +887,7 @@ jobs:
if-no-files-found: error if-no-files-found: error
build-android: build-android:
if: needs.secret-scan.outputs.reuse_artifacts != 'true' if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
name: Build Android name: Build Android
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 60 timeout-minutes: 60
+5
View File
@@ -16,6 +16,11 @@ clean-text-api-check:
button-api-check: button-api-check:
bash scripts/check-button-api.sh bash scripts/check-button-api.sh
# The checks the pre-push hook runs, for everything not yet on origin/master.
.PHONY: pre-push-check
pre-push-check:
python3 scripts/pre-push-checks.py --unpushed
.PHONY: version-test .PHONY: version-test
version-test: version-test:
python3 tests/version_test.py python3 tests/version_test.py
+9 -6
View File
@@ -609,13 +609,16 @@ def main():
if args.verify_origin: if args.verify_origin:
advanced, unversioned = published_changes(args.store, args.verify_origin) advanced, unversioned = published_changes(args.store, args.verify_origin)
# A package that changed without a new version waits for # A package that changed without a new version waits for
# its next version; that is not a failure. Only a new # its next version; that is not a failure. The node
# version that ships with an unversioned change fails, # accepts every app together or none, so a new version
# because the node accepts every app together or none. # that ships beside an unversioned change waits too: the
# release, whose ./update_version.sh versions every
# package, publishes them together.
if unversioned and advanced: if unversioned and advanced:
raise ValueError(", ".join(unversioned) + " changed without a new version, " print("::notice title=App packages::Waiting for the next release: " +
"so " + ", ".join(advanced) + " cannot be published with it; " ", ".join(advanced) + " ships together with " + ", ".join(unversioned) +
"./update_version.sh versions every package for the release") ", which changed without a new version")
return
if unversioned: if unversioned:
print("::notice title=App packages::Not published: " + ", ".join(unversioned) + print("::notice title=App packages::Not published: " + ", ".join(unversioned) +
" changed without a new version and will publish with the next one") " changed without a new version and will publish with the next one")
+123
View File
@@ -0,0 +1,123 @@
#!/usr/bin/env python3
"""Run the quick local checks that match what a push changes.
Git runs this from .githooks/pre-push with the pushed refs on stdin, so the
checks that fail most often on GitHub (versions, translations, fonts, the
canonical UI APIs, the app bundles and the workflows) fail here first, in a
minute or two. `make pre-push-check` runs it for everything not yet on
origin/master. The checks read the working tree; `git push --no-verify`
skips them.
"""
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parent.parent
ZERO = "0" * 40
def git(*arguments):
return subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True,
text=True, check=True).stdout
def pushed_ranges(lines):
"""(base, tip) for each pushed ref that adds commits."""
ranges = []
for line in lines:
parts = line.split()
if len(parts) != 4 or parts[1] == ZERO:
continue
tip, remote = parts[1], parts[3]
if remote == ZERO:
remote = git("merge-base", tip, "origin/master").strip()
ranges.append((remote, tip))
return ranges
def changed_files(ranges):
files = set()
for base, tip in ranges:
files.update(path for path in git("diff", "--name-only", base, tip).splitlines() if path)
return sorted(files)
def plan(files):
"""The checks for these changed files, in the order they run."""
def any_of(*prefixes):
return any(path.startswith(prefixes) for path in files)
def any_suffix(*suffixes):
return any(path.endswith(suffixes) for path in files)
checks = [("version files agree", ["python3", "scripts/check-version.py"]),
("package versions", ["make", "-s", "package-version-test"])]
if any_of("locales/", "assets/fonts/") or any_suffix(".zi"):
checks.append(("translations and fonts", ["make", "-s", "locale-check"]))
if any_suffix(".zi"):
checks.append(("canonical UI APIs", ["make", "-s", "clean-text-api-check", "button-api-check"]))
if any_of("apps/", "src/cells/", "locales/"):
checks.append(("app bundles", ["make", "-s", "cells-test"]))
if any_of(".github/workflows/", "scripts/package-release.py", "scripts/release",
"scripts/check-version.py", "tests/release_test.py", "tests/package_workflow_test.py"):
checks.append(("release workflow", ["python3", "tests/release_test.py"]))
checks.append(("package workflow", ["python3", "tests/package_workflow_test.py"]))
python = [path for path in files if path.endswith(".py") and (ROOT / path).is_file()]
if python:
checks.append(("Python syntax", ["python3", "-m", "py_compile", *python]))
scripts = [path for path in files if path.endswith((".js", ".mjs")) and (ROOT / path).is_file()]
for path in scripts:
checks.append((path + " syntax", ["node", "--check", path]))
return checks
def run(checks, ranges):
failed = []
for base, tip in ranges:
result = subprocess.run(["git", "diff", "--check", base, tip], cwd=ROOT,
capture_output=True, text=True)
if result.returncode != 0:
print(result.stdout + result.stderr, end="")
failed.append("whitespace in the pushed commits")
environment = dict(os.environ)
# Python caches under the checkout would be swept into commits.
environment["PYTHONDONTWRITEBYTECODE"] = "1"
for name, command in checks:
started = time.monotonic()
result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, env=environment)
seconds = time.monotonic() - started
if result.returncode == 0:
print(f"pre-push: {name} passed ({seconds:.0f} s)")
continue
output = (result.stdout + result.stderr).strip().splitlines()
print("\n".join(output[-25:]))
print(f"pre-push: {name} FAILED: {' '.join(command)}")
failed.append(name)
return failed
def main():
if sys.argv[1:] == ["--unpushed"]:
lines = [f"HEAD {git('rev-parse', 'HEAD').strip()} refs/heads/master "
f"{git('rev-parse', 'origin/master').strip()}"]
else:
lines = sys.stdin.read().splitlines()
ranges = pushed_ranges(lines)
if not ranges:
return 0
files = changed_files(ranges)
if not files:
return 0
failed = run(plan(files), ranges)
if failed:
print("pre-push: not pushed. Fix: " + ", ".join(failed) +
" (git push --no-verify skips these checks)")
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
+7 -5
View File
@@ -237,18 +237,20 @@ function runStorageSync(retryDelay) {
Module.__inbeStorageSyncLastError = message; Module.__inbeStorageSyncLastError = message;
Module.__inbeStorageSyncPending = true; Module.__inbeStorageSyncPending = true;
console.error('IDBFS save failed: ' + message); console.error('IDBFS save failed: ' + message);
// A package can replace a temporary file while IDBFS enumerates it. // A package can replace a temporary file while IDBFS enumerates it, and
// Start a fresh snapshot after that race; persistent failures still // practice packs keep installing for several seconds after start. Start
// resolve false, leaving the dirty state available to an explicit save. // a fresh snapshot after that race, waiting longer each time (about ten
// seconds in all); persistent failures still resolve false, leaving the
// dirty state available to an explicit save.
var transient = error && (error.errno === 44 || error.name === 'NotFoundError' || error.name === 'AbortError'); var transient = error && (error.errno === 44 || error.name === 'NotFoundError' || error.name === 'AbortError');
var attempt = Module.__inbeStorageSyncRetry || 0; var attempt = Module.__inbeStorageSyncRetry || 0;
if (transient && attempt < 3) { if (transient && attempt < 8) {
Module.__inbeStorageSyncRetry = attempt + 1; Module.__inbeStorageSyncRetry = attempt + 1;
if (Module.__inbeStorageSyncTimer) clearTimeout(Module.__inbeStorageSyncTimer); if (Module.__inbeStorageSyncTimer) clearTimeout(Module.__inbeStorageSyncTimer);
Module.__inbeStorageSyncTimer = setTimeout(function() { Module.__inbeStorageSyncTimer = setTimeout(function() {
Module.__inbeStorageSyncTimer = 0; Module.__inbeStorageSyncTimer = 0;
runStorageSync(retryDelay); runStorageSync(retryDelay);
}, 100 * Math.pow(2, attempt)); }, Math.min(2000, 100 * Math.pow(2, attempt)));
return; return;
} }
finishStorageSync(false); finishStorageSync(false);
+16 -1
View File
@@ -22,6 +22,7 @@ import stat
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import time
import urllib.error import urllib.error
import urllib.parse import urllib.parse
import urllib.request import urllib.request
@@ -167,6 +168,20 @@ def published_release(origin, ledger, app_id, output):
return bundle, metadata return bundle, metadata
def published_release_retried(origin, ledger, app_id, output, attempts=6, delay=15):
"""The node restarts for a minute or so whenever it is deployed; ask
again for a while before deciding that it serves no release."""
for attempt in range(attempts):
try:
return published_release(origin, ledger, app_id, output)
except (OSError, ValueError) as error:
if attempt == attempts - 1:
raise
print(f"forward {app_id}: the node did not answer ({error}); asking again")
time.sleep(delay)
return None
def published_games(binary, origin, ledger, output): def published_games(binary, origin, ledger, output):
"""Install and play the node's games, independently of Arcade's checkout. """Install and play the node's games, independently of Arcade's checkout.
@@ -177,7 +192,7 @@ def published_games(binary, origin, ledger, output):
environment["YUE_DESKTOP_RECOVERY"] = "0" environment["YUE_DESKTOP_RECOVERY"] = "0"
checked = 0 checked = 0
for app_id, test in (("inbe.arcade", "tests/arcade_ui_test.sh"), ("inbe.chess", "tests/chess_ui_test.sh")): for app_id, test in (("inbe.arcade", "tests/arcade_ui_test.sh"), ("inbe.chess", "tests/chess_ui_test.sh")):
published = published_release(origin, ledger, app_id, output) published = published_release_retried(origin, ledger, app_id, output)
if published is None: if published is None:
print(f"forward {app_id}: the node serves no release") print(f"forward {app_id}: the node serves no release")
continue continue