Leave the visual suites out of CI and the release checks

make visual-test still runs them locally.
This commit is contained in:
waozi committed 2026-10-23 02:39:36 -03:00
1 parent 7378b08576
commit b0e206ae13
7 files changed
+9 -161

No files matched your search

+2 -2
View File
@@ -47,11 +47,11 @@ jobs:
if run['head_sha'] != commit or run['path'] != '.github/workflows/release.yml':
raise SystemExit('The Release checks must belong to the requested source commit')
jobs = api(f'repos/{repository}/actions/runs/{run_id}/jobs?per_page=100')['jobs']
for name in ('Secret Scan', 'Test (0/3)', 'Test (1/3)', 'Test (2/3)', 'Visual Tests'):
for name in ('Secret Scan', 'Test (0/3)', 'Test (1/3)', 'Test (2/3)'):
checks = [job for job in jobs if job['name'] == name]
if len(checks) != 1 or checks[0]['conclusion'] != 'success':
raise SystemExit(f'{name} must have passed before signing')
print('Verified passing Secret Scan, all three Test shards and Visual Tests for ' + commit)
print('Verified passing Secret Scan and all three Test shards for ' + commit)
PY
- name: Checkout verified source
-73
View File
@@ -162,79 +162,6 @@ jobs:
make -C .test-server build
make sync-server-test DAOCHI_BIN="$GITHUB_WORKSPACE/.test-server/daochi"
visual-test:
name: Visual Tests
runs-on: ubuntu-latest
needs: guard-package-urls
timeout-minutes: 120
env:
APT_CACHE_DIR: /tmp/apt-cache
steps:
- name: Checkout Repository
uses: actions/checkout@v4
with:
fetch-depth: 1
lfs: true
- name: Fetch Ziran packages
uses: ./.github/actions/ziran-packages
- name: Set up Node.js
uses: actions/setup-node@v4
with:
node-version: 22
- name: Set up Java
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: '21'
- name: Cache Test apt packages
uses: actions/cache@v4
with:
path: ${{ env.APT_CACHE_DIR }}
key: test-apt-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/apt/test-packages.txt') }}
restore-keys: |
test-apt-${{ runner.os }}-${{ runner.arch }}-
- name: Install Test Dependencies
timeout-minutes: 15
run: .github/scripts/install-apt-packages.sh .github/apt/test-packages.txt
- name: Require Visual Tests
run: make visual-test
- name: Save Visual Test Evidence
if: always()
uses: actions/upload-artifact@v4
with:
name: native-visual-tests
path: |
build/visual-test/
build/native-visual-test/
build/entity-profile-ui-test/
build/native-zoom-test/*.png
build/native-zoom-test/app.log
build/launcher-ui-test/
build/locale-render-ui-test/
build/native-screenshot-test/
build/native-text-pixels-test/*.png
build/native-text-pixels-test/*.log
build/settings-sidebar-test/
build/startup-quit-test/
build/lumi-ui-test/
build/lumi-history-ui-test/
build/habits-table-ui-test/
build/lists-ui-test/
build/sync-review-ui-test/
build/data-back-ui-test/
build/leaderboard-ui-test/
build/practice-images-ui-test/
retention-days: 7
browser-addons:
# Builds the web app once, packages it for Chrome and Firefox, and runs
# the smoke test in both browsers. It runs beside the tests; a failure in
+2 -78
View File
@@ -119,7 +119,7 @@ jobs:
subprocess.run(['python3', 'scripts/check-phone-check.py', version,
'--revision', candidate], check=True)
required = {
'Secret Scan', 'Test (0/3)', 'Test (1/3)', 'Test (2/3)', 'Visual Tests',
'Secret Scan', 'Test (0/3)', 'Test (1/3)', 'Test (2/3)',
'F-Droid / F-Droid security and clean build', 'Verify F-Droid reproducibility',
'Build Web', 'Build Chrome Extension', 'Build Firefox Add-on', 'Build Windows',
'Build Android',
@@ -145,7 +145,7 @@ jobs:
raise SystemExit('The identical source must also pass CI, including browser add-on smoke tests')
with output.open('a') as stream:
stream.write('reuse_artifacts=true\n')
print('All source, test, visual, platform and F-Droid gates passed; reusing their immutable artifacts')
print('All source, test, platform and F-Droid gates passed; reusing their immutable artifacts')
PY
- name: Fetch Ziran packages
@@ -226,81 +226,6 @@ jobs:
GITHUB_TOKEN: ${{ github.token }}
visual-test:
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
name: Visual Tests
runs-on: ubuntu-latest
needs: secret-scan
timeout-minutes: 120
env:
APT_CACHE_DIR: /tmp/apt-cache
steps:
- name: Checkout Repository
uses: actions/checkout@v4
with:
ref: ${{ inputs.revision || github.sha }}
fetch-depth: 1
lfs: true
- name: Fetch Ziran packages
uses: ./.github/actions/ziran-packages
- name: Set up Node.js
uses: actions/setup-node@v4
with:
node-version: 22
# Robolectric's Android 16 (SDK 36) sandbox needs Java 21.
- name: Set up Java
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: '21'
- name: Cache Test apt packages
uses: actions/cache@v4
with:
path: ${{ env.APT_CACHE_DIR }}
key: test-apt-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/apt/test-packages.txt') }}
restore-keys: |
test-apt-${{ runner.os }}-${{ runner.arch }}-
- name: Install Test Dependencies
timeout-minutes: 15
run: .github/scripts/install-apt-packages.sh .github/apt/test-packages.txt
- name: Require Visual Tests
run: make visual-test
- name: Save Visual Test Evidence
if: always()
uses: actions/upload-artifact@v4
with:
name: release-visual-tests
path: |
build/visual-test/
build/native-visual-test/
build/entity-profile-ui-test/
build/native-zoom-test/*.png
build/native-zoom-test/app.log
build/launcher-ui-test/
build/locale-render-ui-test/
build/native-screenshot-test/
build/native-text-pixels-test/*.png
build/native-text-pixels-test/*.log
build/settings-sidebar-test/
build/startup-quit-test/
build/lumi-ui-test/
build/lumi-history-ui-test/
build/habits-table-ui-test/
build/lists-ui-test/
build/sync-review-ui-test/
build/data-back-ui-test/
build/leaderboard-ui-test/
build/practice-images-ui-test/
retention-days: 7
fdroid:
if: needs.secret-scan.outputs.reuse_artifacts != 'true'
name: F-Droid
@@ -1110,7 +1035,6 @@ jobs:
needs:
- secret-scan
- test
- visual-test
- build-web
- build-chrome-extension
- build-firefox-addons
+2 -2
View File
@@ -814,7 +814,7 @@ A release names one fixed commit while peers keep working on master:
another session's work.
2. `RELEASE_READY_REVISION=COMMIT make release-ready` runs the local CI suites
in the Ubuntu 24.04 container with the candidate's locked packages:
Secret Scan, package URLs, file sizes, all tests, visual tests, package
Secret Scan, package URLs, file sizes, all tests, package
compatibility, Chrome and muted Firefox smoke tests, and the pinned real
Daochi sync server. Platform release builds remain separate. The existing
repository overlay materializes every candidate file and recreates source
@@ -839,7 +839,7 @@ A release names one fixed commit while peers keep working on master:
Subsequent master commits cannot cancel its release. Only the workflow
creates the release tag after all required gates pass.
When an earlier Release run already passed every test, visual, platform and
When an earlier Release run already passed every test, platform and
F-Droid reproducibility job for the source, supply its run ID as `reuse_run`.
The identical source must also pass CI, including browser add-on smoke tests.
The workflow admits reuse only when the release commit adds that version's
+1 -2
View File
@@ -13,7 +13,7 @@
# Build outputs stay in build/release-ready/; a run takes about 90 minutes.
#
# Covered: ci.yml Secret Scan, Package URLs, large-file guard, all three test
# shards, visual tests, Chrome/Firefox add-ons and smoke, the real pinned Daochi
# shards, Chrome/Firefox add-ons and smoke, the real pinned Daochi
# sync server, and release.yml's package compatibility test. Platform release
# builds remain separate. The server comes from its canonical checkout through
# another private overlay; no temporary source checkout is created.
@@ -79,7 +79,6 @@ else
job "Package URLs are public HTTPS" package_urls
job "Files over 1 MB" python3 scripts/check-file-size.py --since origin/master
job "Test: make test" make -k test
job "Test: make visual-test" make visual-test
job "Test: make package-compat-test" make package-compat-test
job "Browser add-ons: Chrome and Firefox" sh .github/local-ci/check-browser.sh
job "Sync: pinned Daochi server" sh .github/local-ci/check-sync.sh
-2
View File
@@ -154,8 +154,6 @@ shutil.copytree(root / 'download/cmake', stage / 'cmake')
publication = workflow.split("\n release:\n", 1)[1].split("\n outputs:", 1)[0]
self.assertIn(" - test", publication.splitlines())
self.assertIn(" - verify-fdroid", publication.splitlines())
self.assertIn(" - visual-test", publication.splitlines())
self.assertIn("\n visual-test:\n", workflow)
android = workflow.split("\n build-android:\n", 1)[1].split("\n release:\n", 1)[0]
self.assertIn("needs: secret-scan", android)
self.assertLess(android.index("Verify and install Android SDK components"),
+2 -2
View File
@@ -227,7 +227,7 @@ class MotoWorkflowGateTest(unittest.TestCase):
def passing_jobs(self):
return [{"name": name, "conclusion": "success"} for name in
("Secret Scan", "Test (0/3)", "Test (1/3)", "Test (2/3)", "Visual Tests")]
("Secret Scan", "Test (0/3)", "Test (1/3)", "Test (2/3)")]
def test_all_three_shards_allow_the_exact_release(self):
self.check_run(self.passing_jobs())
@@ -412,7 +412,7 @@ class ReleaseArtifactReuseTest(unittest.TestCase):
builds.add(name)
self.assertIn("if: needs.secret-scan.outputs.reuse_artifacts != 'true'", body)
self.assertEqual({name for name in self.required if name.startswith("Build ")}, builds)
self.assertTrue({"Secret Scan", "Visual Tests", "Test (0/3)", "Test (1/3)", "Test (2/3)",
self.assertTrue({"Secret Scan", "Test (0/3)", "Test (1/3)", "Test (2/3)",
"F-Droid / F-Droid security and clean build", "Verify F-Droid reproducibility"}
<= self.required)
release_job = self.workflow.split("\n release:\n", 1)[1].split("\n notify-waozi-site:", 1)[0]