Files
breeze/scripts/upload-practice-packs.py

258 lines
13 KiB
Python
Executable File

#!/usr/bin/env python3
"""Put every practice pack this build needs on Waozi's node.
The app downloads each practice and narration pack by its SHA-256 from the
node, so a build installed on a phone only works when its packs are there.
Pushed builds get theirs from the App packages workflow; a local build does
not. The owner allows uploading practice packs at any time (never app
releases or latest-v2.json): this signs the built packs with the Inbe
publisher key, uploads only those the node does not hold yet (a paid pack's
product marker first, so it is never served without a grant) and checks
that each one is served afterwards. Sun Salutation, the largest pack, is then
downloaded from the node in full and checked, so every phone build tests
that practices really download.
Run after `make` or a phone build, from the repository root:
python3 scripts/upload-practice-packs.py
Limit staging, signing and transport to exact built pack filenames:
python3 scripts/upload-practice-packs.py --pack astral_projection.pt --pack guide.pt
The existing --content option selects base content only.
"""
import argparse
from datetime import datetime
from hashlib import sha256
import importlib.util
import io
import json
from pathlib import Path
import shutil
import shlex
import subprocess
import sys
import tarfile
import tempfile
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
NODE = "https://api.waozi.xyz/api/v2/packages/inbe.practices/"
TARGET = "[email protected]"
STORE = "/var/lib/daochi/daochi.db.packages"
KEY = ROOT / "backup/package-publisher/2026-10/publisher.pem"
KEY_ID = "inbe-2026-10"
RECEIPT = ROOT / "backup/practice-packs/uploads.txt"
# The node's proxy refuses Python's default user agent with 403, which would
# read as a paid pack that is present.
HEADERS = {"User-Agent": "InnerBreeze-practice-packs/1"}
spec = importlib.util.spec_from_file_location("package_release", ROOT / "scripts/package-release.py")
package_release = importlib.util.module_from_spec(spec)
spec.loader.exec_module(package_release)
def served(digest, timeout=30):
"""The node's status for a pack: 200 served, 403 paid and present, 404 missing."""
request = urllib.request.Request(NODE + digest + ".zx", method="HEAD", headers=HEADERS)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
return response.status
except urllib.error.HTTPError as error:
return error.code
def matches(digest, size, timeout=120):
request = urllib.request.Request(NODE + digest + ".zx", headers=HEADERS)
with urllib.request.urlopen(request, timeout=timeout) as response:
checksum = sha256()
count = 0
while block := response.read(65536):
count += len(block)
if count > size:
return False
checksum.update(block)
return count == size and checksum.hexdigest() == digest
def pack_identifier(pack):
"""The built filename without .zx, including its language and voice."""
return (pack['practice'] + ('.' + pack['language'] if pack['language'] else '') +
('.' + pack['voice'] if pack.get('voice') else ''))
def select_packs(catalog, identifiers=None, content=None):
if identifiers is not None:
known = {pack_identifier(pack) for pack in catalog}
unknown = set(identifiers) - known
if not identifiers or unknown or not all(identifiers):
raise ValueError('unknown or empty pack selection: ' + ', '.join(sorted(unknown)))
return [pack for pack in catalog if pack_identifier(pack) in set(identifiers)]
return [pack for pack in catalog if not content or
(pack['practice'] in content and not pack['language'])]
def stage_selected_packs(store, packs):
"""Sign only the selected immutable content; never stage app releases."""
key = package_release.serialization.load_pem_private_key(KEY.read_bytes(), password=None)
if not isinstance(key, package_release.Ed25519PrivateKey):
raise ValueError('publisher key must be Ed25519')
publishers = json.loads((ROOT / 'apps/publishers.json').read_text())
public_key = key.public_key().public_bytes(package_release.serialization.Encoding.Raw,
package_release.serialization.PublicFormat.Raw).hex()
trusted = package_release.node_release.trusted_pin(publishers, KEY_ID, 'inbe.practices')
if trusted is None or trusted['public_key'] != public_key:
raise ValueError('signing key must match the public key shipped in apps/publishers.json')
versions = json.loads((ROOT / 'apps/versions.json').read_text())
component = versions['apps']['practices']
if component['id'] != 'inbe.practices':
raise ValueError('invalid Practices package identity')
directory = store / 'inbe.practices'
directory.mkdir(parents=True, exist_ok=True)
for pack in packs:
digest = pack['sha256']
target = directory / (digest + '.zx')
shutil.copyfile(ROOT / 'build/practice-content' / (pack_identifier(pack) + '.zx'), target)
with target.open('rb') as stream:
checksum = sha256()
while block := stream.read(65536):
checksum.update(block)
if target.stat().st_size != pack['size'] or checksum.hexdigest() != digest:
raise ValueError('practice content differs from its signed catalog')
value = dict(app_id=component['id'], sequence=component['sequence'],
version=component['version'], runtime='kryon-app-v1',
host_api=component.get('host_api', versions['host_api']),
module_api=versions['module_api'], data_schema=1, key_id=KEY_ID,
artifacts=[dict(variant='module', sha256=digest, size=pack['size'])],
dependencies=[])
value['signature'] = key.sign(package_release.signing_message(value)).hex()
package_release.validate_release(value, 'inbe.practices', publishers)
package_release.atomic_write(directory / (digest + '.zx.release-v2.json'),
(json.dumps(value, separators=(',', ':')) + chr(10)).encode())
if pack.get('product'):
package_release.atomic_write(directory / (digest + '.zx.product'), pack['product'].encode())
def sun_salutation_downloads(packs):
"""Downloads the Sun Salutation pack this build lists and checks its bytes."""
pack = next(pack for pack in packs if pack["practice"] == "sunsalutation" and not pack["language"])
started = time.monotonic()
if not matches(pack["sha256"], pack["size"]):
print("Sun Salutation does not download exactly from the node", file=sys.stderr)
return False
print(f"Sun Salutation downloads from the node: {pack['size'] / 1e6:.1f} MB "
f"in {time.monotonic() - started:.1f} s")
return True
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--check", action="store_true", help="only report packs missing on the node")
selection = parser.add_mutually_exclusive_group()
selection.add_argument("--content", action="append",
choices=json.loads((ROOT / "apps/practice/content.json").read_text())["practices"],
help="limit this attempt to the named practice's content pack; repeat for another practice")
selection.add_argument('--pack', action='append',
help='exact catalog filename without .zx, e.g. astral_projection.pt, guide.pt or taichi.en.male; repeat for another pack')
args = parser.parse_args()
practices = (ROOT / 'build/apps/practices.zx').read_bytes()
try:
listed = select_packs(package_release.catalog_from_bundle(practices), args.pack, args.content)
except ValueError as error:
parser.error(str(error))
sun_content = any(pack['practice'] == 'sunsalutation' and not pack['language'] for pack in listed)
with tempfile.TemporaryDirectory(prefix="inbe-practice-packs-", dir=ROOT / "build") as scratch:
store = Path(scratch)
# Availability checks need only the catalog, not a signing key or any
# unrelated app bundle. Installation must also work on build hosts
# that do not hold the private publisher key.
if args.check:
missing = []
for pack in listed:
status = served(pack['sha256'])
expected = 403 if pack.get('product') else 200
if status == 404:
missing.append(pack)
elif status != expected:
raise SystemExit(f"node answered {status} for practice pack {pack_identifier(pack)}")
print(f"{len(listed)} packs built, {len(missing)} missing on the node")
if missing:
for pack in missing:
print(f"missing: {pack_identifier(pack)}", file=sys.stderr)
return 1
if args.pack is not None:
for pack in listed:
if not pack.get('product') and not matches(pack['sha256'], pack['size']):
raise SystemExit(f"pack {pack['sha256']} is not served exactly")
return 0 if not sun_content or sun_salutation_downloads(listed) else 1
# Only content is being delivered. Requiring a complete, matching
# root and every unrelated app here made local content repair fail
# before it could upload the practice the installed build requests.
stage_selected_packs(store, listed)
directory = store / "inbe.practices"
# Only selected catalog content can reach the transport.
catalog = {pack["sha256"] for pack in listed}
packs = sorted(directory / (digest + ".zx") for digest in catalog)
missing = []
for path in packs:
digest = path.stem
gated = (directory / (digest + ".zx.product")).exists()
status = served(digest)
if status == 404:
missing.append(path)
elif status != (403 if gated else 200):
raise SystemExit(f"node answered {status} for {'paid' if gated else 'free'} pack {digest}")
print(f"{len(packs)} packs built, {len(missing)} missing on the node")
if not missing:
if args.pack is not None:
for path in packs:
if not (directory / (path.stem + '.zx.product')).exists() and not matches(path.stem, path.stat().st_size):
raise SystemExit(f'pack {path.stem} is not served exactly')
return 0 if not sun_content or sun_salutation_downloads(listed) else 1
archive = io.BytesIO()
members = []
with tarfile.open(fileobj=archive, mode="w") as tar:
for path in missing:
for suffix in (".zx.product", ".zx", ".zx.release-v2.json"):
member = directory / (path.stem + suffix)
if member.exists():
info = tar.gettarinfo(str(member), arcname="inbe.practices/" + member.name)
info.uid = info.gid = 0
info.uname = info.gname = "root"
info.mode = 0o600
with member.open("rb") as handle:
tar.addfile(info, handle)
members.append(info.name)
# Extract only into the practices folder, never over a file that is
# already there: packs are immutable once served.
permissions = ' '.join(shlex.quote(name) for name in members)
command = ("set -e; cd " + STORE + " && tar -x --keep-old-files --no-same-permissions "
"--wildcards 'inbe.practices/*' -f - && chmod 600 " + permissions)
subprocess.run(["ssh", "-o", "BatchMode=yes", TARGET, command],
input=archive.getvalue(), check=True)
lines = []
for path in (packs if args.pack is not None else missing):
digest = path.stem
gated = (directory / (digest + ".zx.product")).exists()
if gated:
if served(digest) != 403:
raise SystemExit(f"paid pack {digest} is not withheld by the node")
elif not matches(digest, path.stat().st_size):
raise SystemExit(f"pack {digest} is not served exactly")
if path in missing:
lines.append(f"{digest} {path.stat().st_size} {'paid' if gated else 'free'}")
RECEIPT.parent.mkdir(parents=True, exist_ok=True)
with RECEIPT.open("a") as receipt:
receipt.write(datetime.now().astimezone().isoformat(timespec="seconds") +
f" uploaded {len(lines)} practice packs\n")
receipt.writelines(line + "\n" for line in lines)
print("\n".join(lines))
print(f"Uploaded {len(lines)} practice packs; each is served")
if sun_content and not sun_salutation_downloads(listed):
return 1
return 0
if __name__ == "__main__":
sys.exit(main())