Stop the always-red runs and check pushes locally first
A pushed version that is already released no longer starts the forty-minute release builds; CI still tests every push. A package version waiting for the next release is a notice in App packages instead of a failure. The web build keeps saving its storage while practice packs install (up to about ten seconds of retries), and the package compatibility check asks the node again while a deploy restarts it. The pre-push hook now runs the quick checks that match the pushed files (versions, translations and fonts, UI APIs, app bundles, workflows, script syntax); make pre-push-check runs them by hand.
This commit is contained in:
1 parent
2bc7e5fe65
commit
adbb3c3116
7 files changed
+191
-25
No files matched your search
@@ -15,3 +15,4 @@ else
|
||||
fi
|
||||
|
||||
scripts/check-secrets.py --pre-push "$@" < "$tmp"
|
||||
scripts/pre-push-checks.py < "$tmp"
|
||||
@@ -52,6 +52,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
reuse_artifacts: ${{ steps.reuse.outputs.reuse_artifacts }}
|
||||
release_due: ${{ steps.due.outputs.release_due }}
|
||||
steps:
|
||||
- name: Require an immutable candidate
|
||||
env:
|
||||
@@ -68,6 +69,22 @@ jobs:
|
||||
ref: ${{ inputs.revision || github.sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
# A push of a version that is already released has nothing to release:
|
||||
# CI tests it, and the platform builds wait for the next version.
|
||||
- name: Release only a new version or a requested run
|
||||
id: due
|
||||
env:
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
version=$(python3 scripts/check-version.py --print-version)
|
||||
if [ "$EVENT" = "workflow_dispatch" ] || ! git rev-parse -q --verify "refs/tags/v$version" >/dev/null; then
|
||||
echo "release_due=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "release_due=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice title=Release::v$version is already released; CI tests this push and no release builds run"
|
||||
fi
|
||||
|
||||
- name: Verify identical source and every gate before reusing artifacts
|
||||
id: reuse
|
||||
env:
|
||||
@@ -168,7 +185,7 @@ jobs:
|
||||
python3 scripts/check-secrets.py --history
|
||||
|
||||
test:
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
|
||||
name: Test (${{ matrix.shard }}/3)
|
||||
runs-on: ubuntu-latest
|
||||
needs: secret-scan
|
||||
@@ -227,7 +244,7 @@ jobs:
|
||||
|
||||
|
||||
fdroid:
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
|
||||
name: F-Droid
|
||||
needs: secret-scan
|
||||
uses: ./.github/workflows/fdroid.yml
|
||||
@@ -238,7 +255,7 @@ jobs:
|
||||
name: Verify F-Droid reproducibility
|
||||
runs-on: ubuntu-latest
|
||||
needs: [secret-scan, fdroid, build-android]
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Download independent F-Droid build
|
||||
@@ -271,7 +288,7 @@ jobs:
|
||||
apksigcopier compare --unsigned "${signed[0]}" "${rebuilt[0]}"
|
||||
|
||||
build-web:
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
|
||||
name: Build Web
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
@@ -328,7 +345,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
build-chrome-extension:
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
|
||||
name: Build Chrome Extension
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
@@ -385,7 +402,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
build-firefox-addons:
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
|
||||
name: Build Firefox Add-on
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
@@ -443,7 +460,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
build-linux-appimage:
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
|
||||
name: Build Linux AppImage
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 60
|
||||
@@ -562,7 +579,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
build-linux-deb:
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
|
||||
name: Build Linux Deb
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 60
|
||||
@@ -624,7 +641,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
build-linux-rpm:
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
|
||||
name: Build Linux RPM
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 60
|
||||
@@ -686,7 +703,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
build-linux-snap:
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
|
||||
name: Build Linux Snap
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 60
|
||||
@@ -745,7 +762,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
build-linux-flatpak:
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
|
||||
name: Build Linux Flatpak
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 60
|
||||
@@ -804,7 +821,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
build-windows:
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
|
||||
name: Build Windows
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
@@ -870,7 +887,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
build-android:
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
|
||||
if: needs.secret-scan.outputs.reuse_artifacts != 'true' && needs.secret-scan.outputs.release_due == 'true'
|
||||
name: Build Android
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
@@ -16,6 +16,11 @@ clean-text-api-check:
|
||||
button-api-check:
|
||||
bash scripts/check-button-api.sh
|
||||
|
||||
# The checks the pre-push hook runs, for everything not yet on origin/master.
|
||||
.PHONY: pre-push-check
|
||||
pre-push-check:
|
||||
python3 scripts/pre-push-checks.py --unpushed
|
||||
|
||||
.PHONY: version-test
|
||||
version-test:
|
||||
python3 tests/version_test.py
|
||||
|
||||
@@ -609,13 +609,16 @@ def main():
|
||||
if args.verify_origin:
|
||||
advanced, unversioned = published_changes(args.store, args.verify_origin)
|
||||
# A package that changed without a new version waits for
|
||||
# its next version; that is not a failure. Only a new
|
||||
# version that ships with an unversioned change fails,
|
||||
# because the node accepts every app together or none.
|
||||
# its next version; that is not a failure. The node
|
||||
# accepts every app together or none, so a new version
|
||||
# that ships beside an unversioned change waits too: the
|
||||
# release, whose ./update_version.sh versions every
|
||||
# package, publishes them together.
|
||||
if unversioned and advanced:
|
||||
raise ValueError(", ".join(unversioned) + " changed without a new version, "
|
||||
"so " + ", ".join(advanced) + " cannot be published with it; "
|
||||
"./update_version.sh versions every package for the release")
|
||||
print("::notice title=App packages::Waiting for the next release: " +
|
||||
", ".join(advanced) + " ships together with " + ", ".join(unversioned) +
|
||||
", which changed without a new version")
|
||||
return
|
||||
if unversioned:
|
||||
print("::notice title=App packages::Not published: " + ", ".join(unversioned) +
|
||||
" changed without a new version and will publish with the next one")
|
||||
|
||||
Executable
+123
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the quick local checks that match what a push changes.
|
||||
|
||||
Git runs this from .githooks/pre-push with the pushed refs on stdin, so the
|
||||
checks that fail most often on GitHub (versions, translations, fonts, the
|
||||
canonical UI APIs, the app bundles and the workflows) fail here first, in a
|
||||
minute or two. `make pre-push-check` runs it for everything not yet on
|
||||
origin/master. The checks read the working tree; `git push --no-verify`
|
||||
skips them.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
ZERO = "0" * 40
|
||||
|
||||
|
||||
def git(*arguments):
|
||||
return subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True,
|
||||
text=True, check=True).stdout
|
||||
|
||||
|
||||
def pushed_ranges(lines):
|
||||
"""(base, tip) for each pushed ref that adds commits."""
|
||||
ranges = []
|
||||
for line in lines:
|
||||
parts = line.split()
|
||||
if len(parts) != 4 or parts[1] == ZERO:
|
||||
continue
|
||||
tip, remote = parts[1], parts[3]
|
||||
if remote == ZERO:
|
||||
remote = git("merge-base", tip, "origin/master").strip()
|
||||
ranges.append((remote, tip))
|
||||
return ranges
|
||||
|
||||
|
||||
def changed_files(ranges):
|
||||
files = set()
|
||||
for base, tip in ranges:
|
||||
files.update(path for path in git("diff", "--name-only", base, tip).splitlines() if path)
|
||||
return sorted(files)
|
||||
|
||||
|
||||
def plan(files):
|
||||
"""The checks for these changed files, in the order they run."""
|
||||
def any_of(*prefixes):
|
||||
return any(path.startswith(prefixes) for path in files)
|
||||
|
||||
def any_suffix(*suffixes):
|
||||
return any(path.endswith(suffixes) for path in files)
|
||||
|
||||
checks = [("version files agree", ["python3", "scripts/check-version.py"]),
|
||||
("package versions", ["make", "-s", "package-version-test"])]
|
||||
if any_of("locales/", "assets/fonts/") or any_suffix(".zi"):
|
||||
checks.append(("translations and fonts", ["make", "-s", "locale-check"]))
|
||||
if any_suffix(".zi"):
|
||||
checks.append(("canonical UI APIs", ["make", "-s", "clean-text-api-check", "button-api-check"]))
|
||||
if any_of("apps/", "src/cells/", "locales/"):
|
||||
checks.append(("app bundles", ["make", "-s", "cells-test"]))
|
||||
if any_of(".github/workflows/", "scripts/package-release.py", "scripts/release",
|
||||
"scripts/check-version.py", "tests/release_test.py", "tests/package_workflow_test.py"):
|
||||
checks.append(("release workflow", ["python3", "tests/release_test.py"]))
|
||||
checks.append(("package workflow", ["python3", "tests/package_workflow_test.py"]))
|
||||
python = [path for path in files if path.endswith(".py") and (ROOT / path).is_file()]
|
||||
if python:
|
||||
checks.append(("Python syntax", ["python3", "-m", "py_compile", *python]))
|
||||
scripts = [path for path in files if path.endswith((".js", ".mjs")) and (ROOT / path).is_file()]
|
||||
for path in scripts:
|
||||
checks.append((path + " syntax", ["node", "--check", path]))
|
||||
return checks
|
||||
|
||||
|
||||
def run(checks, ranges):
|
||||
failed = []
|
||||
for base, tip in ranges:
|
||||
result = subprocess.run(["git", "diff", "--check", base, tip], cwd=ROOT,
|
||||
capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
print(result.stdout + result.stderr, end="")
|
||||
failed.append("whitespace in the pushed commits")
|
||||
environment = dict(os.environ)
|
||||
# Python caches under the checkout would be swept into commits.
|
||||
environment["PYTHONDONTWRITEBYTECODE"] = "1"
|
||||
for name, command in checks:
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, env=environment)
|
||||
seconds = time.monotonic() - started
|
||||
if result.returncode == 0:
|
||||
print(f"pre-push: {name} passed ({seconds:.0f} s)")
|
||||
continue
|
||||
output = (result.stdout + result.stderr).strip().splitlines()
|
||||
print("\n".join(output[-25:]))
|
||||
print(f"pre-push: {name} FAILED: {' '.join(command)}")
|
||||
failed.append(name)
|
||||
return failed
|
||||
|
||||
|
||||
def main():
|
||||
if sys.argv[1:] == ["--unpushed"]:
|
||||
lines = [f"HEAD {git('rev-parse', 'HEAD').strip()} refs/heads/master "
|
||||
f"{git('rev-parse', 'origin/master').strip()}"]
|
||||
else:
|
||||
lines = sys.stdin.read().splitlines()
|
||||
ranges = pushed_ranges(lines)
|
||||
if not ranges:
|
||||
return 0
|
||||
files = changed_files(ranges)
|
||||
if not files:
|
||||
return 0
|
||||
failed = run(plan(files), ranges)
|
||||
if failed:
|
||||
print("pre-push: not pushed. Fix: " + ", ".join(failed) +
|
||||
" (git push --no-verify skips these checks)")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+7
-5
@@ -237,18 +237,20 @@ function runStorageSync(retryDelay) {
|
||||
Module.__inbeStorageSyncLastError = message;
|
||||
Module.__inbeStorageSyncPending = true;
|
||||
console.error('IDBFS save failed: ' + message);
|
||||
// A package can replace a temporary file while IDBFS enumerates it.
|
||||
// Start a fresh snapshot after that race; persistent failures still
|
||||
// resolve false, leaving the dirty state available to an explicit save.
|
||||
// A package can replace a temporary file while IDBFS enumerates it, and
|
||||
// practice packs keep installing for several seconds after start. Start
|
||||
// a fresh snapshot after that race, waiting longer each time (about ten
|
||||
// seconds in all); persistent failures still resolve false, leaving the
|
||||
// dirty state available to an explicit save.
|
||||
var transient = error && (error.errno === 44 || error.name === 'NotFoundError' || error.name === 'AbortError');
|
||||
var attempt = Module.__inbeStorageSyncRetry || 0;
|
||||
if (transient && attempt < 3) {
|
||||
if (transient && attempt < 8) {
|
||||
Module.__inbeStorageSyncRetry = attempt + 1;
|
||||
if (Module.__inbeStorageSyncTimer) clearTimeout(Module.__inbeStorageSyncTimer);
|
||||
Module.__inbeStorageSyncTimer = setTimeout(function() {
|
||||
Module.__inbeStorageSyncTimer = 0;
|
||||
runStorageSync(retryDelay);
|
||||
}, 100 * Math.pow(2, attempt));
|
||||
}, Math.min(2000, 100 * Math.pow(2, attempt)));
|
||||
return;
|
||||
}
|
||||
finishStorageSync(false);
|
||||
|
||||
@@ -22,6 +22,7 @@ import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
@@ -167,6 +168,20 @@ def published_release(origin, ledger, app_id, output):
|
||||
return bundle, metadata
|
||||
|
||||
|
||||
def published_release_retried(origin, ledger, app_id, output, attempts=6, delay=15):
|
||||
"""The node restarts for a minute or so whenever it is deployed; ask
|
||||
again for a while before deciding that it serves no release."""
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
return published_release(origin, ledger, app_id, output)
|
||||
except (OSError, ValueError) as error:
|
||||
if attempt == attempts - 1:
|
||||
raise
|
||||
print(f"forward {app_id}: the node did not answer ({error}); asking again")
|
||||
time.sleep(delay)
|
||||
return None
|
||||
|
||||
|
||||
def published_games(binary, origin, ledger, output):
|
||||
"""Install and play the node's games, independently of Arcade's checkout.
|
||||
|
||||
@@ -177,7 +192,7 @@ def published_games(binary, origin, ledger, output):
|
||||
environment["YUE_DESKTOP_RECOVERY"] = "0"
|
||||
checked = 0
|
||||
for app_id, test in (("inbe.arcade", "tests/arcade_ui_test.sh"), ("inbe.chess", "tests/chess_ui_test.sh")):
|
||||
published = published_release(origin, ledger, app_id, output)
|
||||
published = published_release_retried(origin, ledger, app_id, output)
|
||||
if published is None:
|
||||
print(f"forward {app_id}: the node serves no release")
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user