Files
breeze/tests/build_layout_test.py
T
waozi b8bfb7ff1c Reuse fixed release-ready outputs and one shared build cache
Custom RELEASE_READY_OUTPUT names now map to build/release-ready, and
only build/release-ready-phone keeps a separate purpose, so task and
version names no longer leave a new folder per run. Both share
build/release-ready/home under an exclusive cache lease held through
container exit. The release command uses the same mapping. Phone
installs take a lock, select only an APK this run built and reuse the
default signed path. Nothing prunes existing output.
2026-11-01 18:41:58 -03:00

297 lines
15 KiB
Python

"""Exercise the real build wrappers with a fake container, without a checkout or device."""
import importlib.util
import fcntl
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import time
import unittest
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location("build_layout", ROOT / "scripts/check-build-layout.py")
layout = importlib.util.module_from_spec(spec)
spec.loader.exec_module(layout)
class BuildLayoutTest(unittest.TestCase):
def setUp(self):
# Only fixtures and fake output are removed, never the project's build/.
parent = ROOT / "backup/build-layout-tests"
parent.mkdir(parents=True, exist_ok=True)
temporary = tempfile.TemporaryDirectory(dir=parent)
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name)
(self.root / "scripts").mkdir()
(self.root / "bin").mkdir()
(self.root / "server").mkdir()
for name in ("release-ready.sh", "check-build-layout.py", "phone-install.sh"):
shutil.copyfile(ROOT / "scripts" / name, self.root / "scripts" / name)
(self.root / "scripts/phone-check.py").write_text(
"ROOT = None\ndef release_content(root, commit):\n return commit\n")
(self.root / "scripts/check-release-overlay.py").write_text("# snapshot fixture\n")
self.executable("git", '''#!/usr/bin/env python3
import sys
args = sys.argv[1:]
if args[0] == '-C':
args = args[2:]
if args[0] == 'rev-parse':
print(args[1].split('^')[0])
elif args[0] == 'show':
print('fixed image inputs')
elif args[0] == 'log':
print('fixture revision')
elif args[0] != 'cat-file':
raise SystemExit('unexpected git: ' + repr(args))
''')
self.executable("podman", '''#!/usr/bin/env python3
import hashlib, json, os, pathlib, sys, time
args = sys.argv[1:]
if args[:2] == ['image', 'exists']:
sys.exit(0)
assert args[0] == 'run', args
mounts = [args[i+1] for i, value in enumerate(args) if value == '-v']
home = pathlib.Path(next(value.rsplit(':', 1)[0] for value in mounts
if value.endswith(':/home/ubuntu')))
output = pathlib.Path(next(value.rsplit(':', 1)[0] for value in mounts
if value.endswith(':/src/build')))
environment = dict(args[i+1].split('=', 1) for i, value in enumerate(args) if value == '-e')
commit = environment['RELEASE_READY_COMMIT']
cache = home / '.cache/ziran/sources'
cache.mkdir(parents=True, exist_ok=True)
# A shared source and a commit-specific source must coexist across runs.
shared = cache / 'shared'
reuse = shared.exists()
shared.write_text('same dependency')
identity = hashlib.sha256(('https://example.invalid/package\\n' + commit).encode()).hexdigest()
(cache / identity).write_text(commit)
record = dict(home=str(home), output=str(output), commit=commit, reuse=reuse,
mounts=mounts, environment=environment)
with open('runs.jsonl', 'a') as stream:
stream.write(json.dumps(record) + '\\n')
if os.environ.get('FIXTURE_BLOCK'):
pathlib.Path('entered').touch()
deadline = time.monotonic() + 15
while not pathlib.Path('continue').exists():
assert time.monotonic() < deadline, 'fixture lease timed out'
time.sleep(.02)
if not os.environ.get('FIXTURE_NO_APK'):
(output / 'android').mkdir(exist_ok=True)
(output / 'android/app-arm64-v8a-debug.apk').write_bytes(b'fixture apk')
sys.exit(int(os.environ.get('FIXTURE_STATUS', '0')))
''')
self.environment = dict(os.environ, PATH=str(self.root / "bin") + ":" + os.environ["PATH"],
RELEASE_READY_REVISION="1" * 40,
RELEASE_READY_DAOCHI_SOURCE=str(self.root / "server"),
PYTHONDONTWRITEBYTECODE="1")
for name in ("RELEASE_READY_OUTPUT", "RELEASE_READY_INCREMENTAL", "RELEASE_READY_DNS"):
self.environment.pop(name, None)
def executable(self, name, text):
path = self.root / "bin" / name
path.write_text(text)
path.chmod(0o755)
def run_build(self, output="build/release-ready", revision=None, status=0, targets=("android-debug",)):
environment = dict(self.environment, RELEASE_READY_OUTPUT=output, FIXTURE_STATUS=str(status))
if revision:
environment["RELEASE_READY_REVISION"] = revision
return subprocess.run(["sh", "scripts/release-ready.sh", *targets], cwd=self.root,
env=environment, capture_output=True, text=True, timeout=10)
def records(self):
path = self.root / "runs.jsonl"
return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
def test_two_names_and_commits_share_cache_and_one_output(self):
for name, revision in (("build/release-ready-61-apk", "1" * 40),
("build/release-ready-62b", "2" * 40)):
result = self.run_build(name, revision)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertFalse((self.root / name).exists())
first, second = self.records()
self.assertEqual(first["home"], second["home"])
self.assertEqual(first["output"], second["output"])
self.assertFalse(first["reuse"])
self.assertTrue(second["reuse"])
cache = Path(first["home"]) / ".cache/ziran/sources"
self.assertEqual(len(list(cache.iterdir())), 3)
self.assertEqual({p.name for p in (self.root / "build").iterdir()},
{"release-ready"})
self.assertEqual(Path(first["home"]), self.root / "build/release-ready/home")
self.assertTrue((self.root / "build/release-ready/build/android/app-arm64-v8a-debug.apk").is_file())
def test_phone_has_stable_output_and_the_same_home(self):
self.assertEqual(self.run_build().returncode, 0)
result = self.run_build("build/release-ready-phone", "2" * 40)
self.assertEqual(result.returncode, 0, result.stderr)
first, second = self.records()
self.assertNotEqual(first["output"], second["output"])
self.assertEqual(first["home"], second["home"])
self.assertTrue(second["reuse"])
self.assertEqual(second["environment"]["XDG_CACHE_HOME"], "/home/ubuntu/.cache")
self.assertEqual(second["environment"]["GRADLE_USER_HOME"], "/home/ubuntu/.gradle")
def test_full_run_stamp_uses_the_same_mapping_as_the_release_gate(self):
alias = "build/release-ready-release61"
result = self.run_build(alias, targets=())
self.assertEqual(result.returncode, 0, result.stderr)
stamp = self.root / layout.release_ready_output(alias) / "stamp.json"
self.assertTrue(json.loads(stamp.read_text())["passed"])
self.assertFalse((self.root / alias).exists())
environment = dict(self.environment, RELEASE_READY_OUTPUT=alias)
script = ("import importlib.util; "
"s=importlib.util.spec_from_file_location('release', 'scripts/release.py'); "
"m=importlib.util.module_from_spec(s); s.loader.exec_module(m); print(m.READY_OUTPUT)")
result = subprocess.run([sys.executable, "-c", script], cwd=ROOT, env=environment,
capture_output=True, text=True, timeout=10)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stdout.strip(), "build/release-ready")
def start_blocked(self):
process = subprocess.Popen(["sh", "scripts/release-ready.sh", "android-debug"], cwd=self.root,
env=dict(self.environment, FIXTURE_BLOCK="1"),
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
self.addCleanup(self.finish_process, process)
self.wait_for(lambda: (self.root / "entered").exists())
return process
def finish_process(self, process):
(self.root / "continue").touch()
process.communicate(timeout=20)
def wait_for(self, condition):
deadline = time.monotonic() + 5
while not condition():
self.assertLess(time.monotonic(), deadline, "fixture did not reach expected state")
time.sleep(.02)
def test_active_output_is_not_changed_and_cache_users_are_serialized(self):
first = self.start_blocked()
with (self.root / "build/release-ready/cache.lock").open("a") as lease:
with self.assertRaises(BlockingIOError):
fcntl.flock(lease, fcntl.LOCK_EX | fcntl.LOCK_NB)
stamp = self.root / "build/release-ready/stamp.json"
stamp.write_text("active run's file")
result = self.run_build("build/release-ready-another-task")
self.assertNotEqual(result.returncode, 0)
self.assertIn("another run is using", result.stderr)
self.assertEqual(stamp.read_text(), "active run's file")
second = subprocess.Popen(["sh", "scripts/release-ready.sh", "android-debug"], cwd=self.root,
env=dict(self.environment, RELEASE_READY_OUTPUT="build/release-ready-phone"),
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
self.addCleanup(self.finish_process, second)
# The second process owns its output lease before waiting on the cache.
self.wait_for(lambda: (self.root / "build/release-ready-phone/lock").exists())
self.assertIsNone(second.poll())
self.assertEqual(len(self.records()), 1)
(self.root / "continue").touch()
for process in (first, second):
stdout, stderr = process.communicate(timeout=10)
self.assertEqual(process.returncode, 0, stdout + stderr)
self.assertEqual(len(self.records()), 2)
self.assertTrue(self.records()[1]["reuse"])
def test_failed_build_releases_leases_and_invalidates_old_stamp(self):
self.assertEqual(self.run_build(targets=()).returncode, 0)
result = self.run_build(status=7)
self.assertEqual(result.returncode, 7)
self.assertFalse((self.root / "build/release-ready/stamp.json").exists())
self.assertEqual(self.run_build().returncode, 0)
def test_unrelated_outputs_and_old_folders_are_never_pruned(self):
paths = ("build/promo-video", "build/package-releases", "build/dist", "build/release-ready-old")
for name in paths:
path = self.root / name
path.mkdir(parents=True)
(path / "keep").write_text("owner data")
self.assertEqual(self.run_build().returncode, 0)
for name in paths:
self.assertEqual((self.root / name / "keep").read_text(), "owner data")
for name in ("build/promo-video", "build/package-releases", "build/dist",
"build/release-ready-../promo-video", "build/release-ready-foo/bar"):
result = self.run_build(name)
self.assertEqual(result.returncode, 2, result.stderr)
def test_symlink_output_is_refused_without_touching_target(self):
target = self.root / "build/promo-video"
target.mkdir(parents=True)
(target / "keep").write_text("owner data")
(self.root / "build/release-ready").symlink_to(target, target_is_directory=True)
result = self.run_build()
self.assertEqual(result.returncode, 2)
self.assertEqual(list(target.iterdir()), [target / "keep"])
self.assertEqual(self.records(), [])
def test_phone_default_build_and_versioned_apks_reuse_signing_output(self):
# Only these fixture executables are on the phone script's PATH.
self.executable("adb", '''#!/usr/bin/env python3
import hashlib, pathlib, sys
args = sys.argv[3:]
if args == ['get-state']:
print('device')
elif args == ['shell', 'getprop', 'ro.product.cpu.abi']:
print('arm64-v8a')
elif args == ['shell', 'getprop', 'ro.product.model']:
print('Fixture')
elif args[:3] == ['shell', 'pm', 'path']:
print('package:/fixture/base.apk')
elif args[:2] == ['shell', 'sha256sum']:
data = pathlib.Path('build/phone-install/app-arm64-v8a-debug-signed.apk').read_bytes()
print(hashlib.sha256(data).hexdigest(), '/fixture/base.apk')
elif args[:2] not in (['install', '-r'], ['shell', 'am'], ['shell', 'pidof']):
raise SystemExit('unexpected adb: ' + repr(args))
''')
sdk = self.root / "sdk/build-tools/1"
sdk.mkdir(parents=True)
for name, source in {
"zipalign": "import shutil, sys; shutil.copyfile(sys.argv[-2], sys.argv[-1])",
"apksigner": ("import shutil, sys; "
"sys.exit(0) if sys.argv[1] == 'verify' else "
"shutil.copyfile(sys.argv[-1], sys.argv[sys.argv.index('--out')+1])"),
}.items():
path = sdk / name
path.write_text("#!/usr/bin/env python3\n" + source + "\n")
path.chmod(0o755)
key = self.root / "debug.keystore"
key.touch()
environment = dict(self.environment, ANDROID_SDK_ROOT=str(self.root / "sdk"),
ANDROID_DEBUG_KEYSTORE=str(key))
def install(*arguments):
result = subprocess.run(["sh", "scripts/phone-install.sh", "--device", "fixture", *arguments],
cwd=self.root, env=environment, capture_output=True, text=True, timeout=10)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
install()
record = self.records()[0]
self.assertEqual(Path(record["output"]), self.root / "build/release-ready-phone/build")
for version in ("61", "62"):
apk = self.root / ("inbe-" + version + ".apk")
apk.write_bytes(version.encode())
install("--apk", str(apk))
self.assertEqual([p.name for p in (self.root / "build/phone-install").iterdir()],
["app-arm64-v8a-debug-signed.apk"])
signed = self.root / "build/phone-install/app-arm64-v8a-debug-signed.apk"
self.assertEqual(signed.read_bytes(), b"62")
failed = subprocess.run(["sh", "scripts/phone-install.sh", "--device", "fixture"],
cwd=self.root, env=dict(environment, FIXTURE_STATUS="9"),
capture_output=True, text=True, timeout=10)
self.assertEqual(failed.returncode, 9, failed.stdout + failed.stderr)
self.assertNotIn("phone-install: installed", failed.stdout)
self.assertEqual(signed.read_bytes(), b"62")
stale = subprocess.run(["sh", "scripts/phone-install.sh", "--device", "fixture"],
cwd=self.root, env=dict(environment, FIXTURE_NO_APK="1"),
capture_output=True, text=True, timeout=10)
self.assertEqual(stale.returncode, 1, stale.stdout + stale.stderr)
self.assertIn("the build left no", stale.stderr)
self.assertNotIn("phone-install: installed", stale.stdout)
self.assertEqual(signed.read_bytes(), b"62")
if __name__ == "__main__":
unittest.main()