diff --git a/scripts/install-package-receiver.py b/scripts/install-package-receiver.py new file mode 100644 index 000000000..b64bfbe95 --- /dev/null +++ b/scripts/install-package-receiver.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Install the existing signed package receiver, without publishing any package.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import shlex +import subprocess + +ROOT = Path(__file__).resolve().parents[1] + + +def wrapper(store): + return '''#!/usr/bin/python3 +# Managed by Inbe package receiver. +import os, shlex, sys +store = STORE +original = os.environ.get("SSH_ORIGINAL_COMMAND", "") +arguments = shlex.split(original) if original else ["inbe-packages", *sys.argv[1:]] +if arguments != ["inbe-packages", store]: + raise SystemExit("Only the configured signed package upload is permitted") +os.environ["SSH_ORIGINAL_COMMAND"] = shlex.join(arguments) +os.execv(sys.executable, [sys.executable, "/opt/daochi/inbe-package-receiver/package-release.py", + "--receive-archive", "--store", store, + "--publishers", "/opt/daochi/inbe-package-receiver/publishers.json"]) +'''.replace('STORE', repr(store)) + + +def install(target, store): + if not re.fullmatch(r'[A-Za-z0-9_.@-]+', target) or target.startswith('-'): + raise ValueError('invalid SSH target') + if not store.startswith('/') or any(c in store for c in '\n\r\0'): + raise ValueError('package store must be absolute') + configuration = dict(store=store, files={ + 'package-release.py': (ROOT / 'scripts/package-release.py').read_text(), + 'practice_content.py': (ROOT / 'scripts/practice_content.py').read_text(), + 'publishers.json': (ROOT / 'apps/publishers.json').read_text(), + 'inbe-packages': wrapper(store)}) + remote = r''' +from pathlib import Path +import fcntl, hashlib, json, os, subprocess, sys, tempfile +configuration = json.load(sys.stdin) +store = Path(configuration['store']) +if not store.is_dir() or store.is_symlink(): + raise RuntimeError('Refusing an absent or symlinked package store') +lock = open('/var/lock/daochi-deploy.lock', 'a') +fcntl.flock(lock, fcntl.LOCK_EX) +directory = Path('/opt/daochi/inbe-package-receiver') +launcher = Path('/usr/local/bin/inbe-packages') +if launcher.exists() and 'Managed by Inbe package receiver.' not in launcher.read_text(): + raise RuntimeError('Refusing to replace an unrelated package launcher') +if directory.exists() and not (directory / 'managed.json').exists(): + raise RuntimeError('Refusing to replace an unmanaged receiver') +directory.mkdir(mode=0o755, exist_ok=True) +with tempfile.TemporaryDirectory(prefix='receiver-', dir=directory) as temporary: + stage = Path(temporary) + for name, text in configuration['files'].items(): + (stage / name).write_text(text) + subprocess.run([sys.executable, '-m', 'py_compile', str(stage / 'package-release.py'), + str(stage / 'practice_content.py'), str(stage / 'inbe-packages')], check=True) + subprocess.run([sys.executable, str(stage / 'package-release.py'), '--help'], + check=True, stdout=subprocess.DEVNULL) + hashes = {} + for name in ('package-release.py', 'practice_content.py', 'publishers.json'): + target = directory / name + (stage / name).chmod(0o644) + os.replace(stage / name, target) + hashes[name] = hashlib.sha256(target.read_bytes()).hexdigest() + # The small launcher is installed last, after its imports are ready. + (stage / 'inbe-packages').chmod(0o755) + os.replace(stage / 'inbe-packages', launcher) + hashes['inbe-packages'] = hashlib.sha256(launcher.read_bytes()).hexdigest() + receipt = dict(installed=True, package_store=str(store), files=hashes, + packages_published=False) + (directory / 'managed.json').write_text(json.dumps(receipt, indent=2) + '\n') +print(json.dumps(receipt)) +''' + options = ['ssh', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', + '-o', 'ConnectTimeout=8', '--', target, 'python3 -c ' + shlex.quote(remote)] + result = subprocess.run(options, input=json.dumps(configuration), text=True, + capture_output=True, check=True) + receipt = json.loads(result.stdout) + saved = ROOT / 'backup/practice-downloads/receiver-install.json' + saved.parent.mkdir(parents=True, exist_ok=True) + saved.write_text(json.dumps(receipt, indent=2) + '\n') + print('Signed package receiver installed; no packages published.') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--ssh-target', required=True) + parser.add_argument('--store', required=True) + arguments = parser.parse_args() + install(arguments.ssh_target, arguments.store) diff --git a/scripts/package-release.py b/scripts/package-release.py index d0bf222f5..91557b912 100644 --- a/scripts/package-release.py +++ b/scripts/package-release.py @@ -22,6 +22,8 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'scripts')) +from practice_content import catalog_from_bundle, bundle_assets NAMES = ("habits", "practices", "lumi", "lists", "diary", "leaderboard", "inbe") IDENTIFIER = re.compile(r"[a-z0-9][a-z0-9_.-]{0,62}\Z") VERSION = re.compile(r"(?:0|[1-9][0-9]{0,8})\.(?:0|[1-9][0-9]{0,8})\.(?:0|[1-9][0-9]{0,8})\Z") @@ -101,6 +103,17 @@ def stage(store, key_path, key_id, publishers_path=None): releases[name] = value payloads[name] = data store.mkdir(parents=True, exist_ok=True) + content_payloads = [] + for pack in catalog_from_bundle(payloads['practices']): + filename = pack['practice'] + ('.' + pack['language'] if pack['language'] else '') + '.zib' + data = (ROOT / 'build/practice-content' / filename).read_bytes() + if len(data) != pack['size'] or sha256(data).hexdigest() != pack['sha256']: + raise ValueError('practice content differs from its signed catalog') + value = dict(releases['practices']) + value['data_schema'] = 1 + value['artifacts'] = [dict(variant='module', sha256=pack['sha256'], size=len(data))] + value['signature'] = key.sign(signing_message(value)).hex() + content_payloads.append((pack['sha256'], data, value)) # Validate all old pointers before publishing any new one. Use the same # per-app lock name as Daochi's staging tool. with ExitStack() as stack: @@ -125,6 +138,15 @@ def stage(store, key_path, key_id, publishers_path=None): raise ValueError("immutable payload does not match its hash") if archive.exists() and archive.read_bytes() != encoded: raise ValueError("release sequence is already occupied") + # Content is installed before the harness advertises its hashes. + for digest, data, value in content_payloads: + directory = store / 'inbe.practices' + payload = directory / (digest + '.zib') + if payload.exists() and payload.read_bytes() != data: + raise ValueError('immutable practice payload differs') + atomic_write(payload, data) + atomic_write(directory / (digest + '.zib.release-v2.json'), + (json.dumps(value, separators=(',', ':')) + '\n').encode()) for name in NAMES: value = releases[name] directory = store / value["app_id"] @@ -151,6 +173,13 @@ def rollout_value(percent, hours): return dict(percent=percent, hours=hours) +def stored_practice_packs(store): + directory = store / 'inbe.practices' + value = json.loads((directory / 'latest-v2.json').read_bytes()) + data = (directory / (value['artifacts'][0]['sha256'] + '.zib')).read_bytes() + return catalog_from_bundle(data) + + def publish(store, target, remote_store, rollout=None): if not re.fullmatch(r"[a-zA-Z0-9_.@-]+", target) or target.startswith("-"): raise ValueError("invalid SSH target") @@ -172,6 +201,10 @@ def publish(store, target, remote_store, rollout=None): f"release-{value['sequence']}.json"): path = directory / filename tar.add(path, arcname=str(path.relative_to(store)), recursive=False) + for pack in stored_practice_packs(store): + for suffix in ('.zib', '.zib.release-v2.json'): + path = store / 'inbe.practices' / (pack['sha256'] + suffix) + tar.add(path, arcname=str(path.relative_to(store)), recursive=False) if rollout is not None: encoded = json.dumps(rollout_value(rollout["percent"], rollout["hours"])).encode() member = tarfile.TarInfo("rollout.json") @@ -226,7 +259,22 @@ def verify_origin(store, origin, open_url=open_public_url): checksum.update(block) if size != artifact["size"] or checksum.hexdigest() != digest: raise ValueError(f"{app_id}: public package hash or size differs from the signed publication") - print("Verified public signed metadata and package bytes for the root and all six cells") + for pack in stored_practice_packs(store): + route = origin + '/api/v2/packages/inbe.practices/' + pack['sha256'] + '.zib' + checksum = sha256() + size = 0 + with open_url(route, timeout=30) as response: + while True: + block = response.read(65536) + if not block: + break + size += len(block) + if size > pack['size']: + raise ValueError('public practice content exceeds its catalog size') + checksum.update(block) + if size != pack['size'] or checksum.hexdigest() != pack['sha256']: + raise ValueError('public practice content differs from its signed catalog') + print("Verified public signed metadata, app packages and practice content") def version_tuple(value): @@ -338,7 +386,7 @@ def import_archive(archive, store, publishers_path=None): if member.name in entries: raise ValueError("duplicate archive member") total += member.size - if len(entries) >= 28 or total > 7 * 64 * 1024 * 1024 + 1024 * 1024: + if len(entries) >= 158 or total > 1024 * 1024 * 1024: raise ValueError("archive exceeds the release size limit") entries[member.name] = tar.extractfile(member).read() # The node accepts archives built before and after Leaderboard became an @@ -377,12 +425,40 @@ def import_archive(archive, store, publishers_path=None): raise ValueError("release sequence is already occupied") releases[app_id] = (value, encoded, data, old != value) root = releases["inbe"] + content_payloads = [] + expected_entries = set() + for app_id, (value, encoded, data, _) in releases.items(): + digest = value['artifacts'][0]['sha256'] + expected_entries.update((f'{app_id}/latest-v2.json', f'{app_id}/{digest}.zib', + f'{app_id}/{digest}.zib.release-v2.json', f"{app_id}/release-{value['sequence']}.json")) + for pack in catalog_from_bundle(releases['inbe.practices'][2]): + path = 'inbe.practices/' + pack['sha256'] + '.zib' + expected_entries.update((path, path + '.release-v2.json')) + data = entries[path] + encoded = entries[path + '.release-v2.json'] + value = json.loads(encoded) + validate_release(value, 'inbe.practices', publishers) + if (value['data_schema'] != 1 or value['artifacts'][0]['sha256'] != pack['sha256'] or + value['artifacts'][0]['size'] != pack['size'] or len(data) != pack['size'] or + sha256(data).hexdigest() != pack['sha256']): + raise ValueError('practice content differs from its signed catalog') + bundle_assets(data) + content_payloads.append((path, data, encoded)) + if set(entries) - expected_entries: + raise ValueError('archive contains unreferenced content') for dependency in root[0]["dependencies"]: child, _, payload, _ = releases[dependency["app_id"]] artifact = child["artifacts"][0] if dependency != dict(app_id=child["app_id"], version=child["version"], sha256=artifact["sha256"], size=artifact["size"]) or root[2].count(payload) != 1: raise ValueError("root does not contain its exact signed cell dependency") + # Content precedes both the harness and the root that references it. + for path, data, encoded in content_payloads: + target = store / path + if target.exists() and target.read_bytes() != data: + raise ValueError('immutable practice content differs') + atomic_write(target, data) + atomic_write(store / (path + '.release-v2.json'), encoded) # Dependencies go first. Publishing the root is the final step. for app_id in sorted(present, key=lambda value: value == "inbe"): value, encoded, data, changed = releases[app_id] diff --git a/scripts/practice_content.py b/scripts/practice_content.py new file mode 100644 index 000000000..007ccf9ff --- /dev/null +++ b/scripts/practice_content.py @@ -0,0 +1,154 @@ +"""Small signed Practices catalog and immutable resource packs. + +The catalog is an asset of the signed Practices harness. Its exact hashes bind +each practice and language to a resource-only ZIB. All blobs use the existing +inbe.practices publisher and Daochi artifact endpoint, with no new registry IDs. +""" +import json +from hashlib import sha256 +from pathlib import Path +import struct +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +CATALOG = 'assets/practices/catalog.json' +NAMES = ('whm', 'meditation', 'sunsalutation', 'patterns', 'taichi') +MAXIMUM = 64 * 1024 * 1024 + + +def bundle_assets(data): + """Read the bounded ZIB v26 resource table; never execute a bundle.""" + if data[:8] != b'ZIB\0\x1a\0\0\0' or len(data) > MAXIMUM: + raise ValueError('unsupported resource bundle') + at = 8 + + def take(size): + nonlocal at + if size < 0 or at + size > len(data): + raise ValueError('truncated resource bundle') + value = data[at:at + size] + at += size + return value + + def number(): + return struct.unpack(' 10000: + raise ValueError('too many resource assets') + result = {} + for _ in range(count): + path = text().decode('utf-8') + if path in result: + raise ValueError('duplicate resource path') + result[path] = take(number()) + if at != len(data): + raise ValueError('trailing resource bundle bytes') + return result + + +def catalog_from_bundle(data): + # Older releases have no content catalog and remain installable. + if data[:8] != b'ZIB\0\x1a\0\0\0': + return [] + assets = bundle_assets(data) + if CATALOG not in assets: + return [] + catalog = json.loads(assets[CATALOG]) + packs = catalog.get('packs') + if catalog.get('schema') != 1 or not isinstance(packs, list) or len(packs) > 65: + raise ValueError('invalid practice catalog') + seen = set() + for pack in packs: + practice = pack.get('practice') + language = pack.get('language') + digest = pack.get('sha256') + size = pack.get('size') + if (practice not in NAMES or not isinstance(language, str) or + (language and (len(language) != 2 or not language.isascii() or not language.islower())) or + not isinstance(digest, str) or len(digest) != 64 or + any(c not in '0123456789abcdef' for c in digest) or + type(size) is not int or not 8 <= size <= MAXIMUM or + (practice, language) in seen): + raise ValueError('invalid or duplicate practice pack') + seen.add((practice, language)) + if {(name, '') for name in NAMES} - seen: + raise ValueError('practice catalog lacks base content') + return packs + + +def prepare(compiler): + configuration = json.loads((ROOT / 'apps/practice/content.json').read_text()) + if configuration['schema'] != 1 or tuple(configuration['practices']) != NAMES: + raise ValueError('unsupported practice content configuration') + output = ROOT / 'build/practice-content' + output.mkdir(parents=True, exist_ok=True) + packs = [] + for practice in NAMES: + directories = [ROOT / ('build/package-assets/practice-' + practice)] + directories += sorted((ROOT / 'build/package-assets').glob('practice-' + practice + '-voice-*')) + for directory in directories: + language = directory.name.split('-voice-', 1)[1] if '-voice-' in directory.name else '' + if language and not narration_current(configuration, practice, language): + print(f'{practice}/{language}: incomplete narration excluded from the catalog') + continue + assets = directory / 'assets' + assets.mkdir(parents=True, exist_ok=True) + # Even timer-only practices have a versioned content identity. + identity = assets / 'practices' / practice / ('voice/' + language if language else '') / 'pack.json' + identity.parent.mkdir(parents=True, exist_ok=True) + identity.write_text(json.dumps(dict(schema=1, practice=practice, language=language), + separators=(',', ':')) + '\n') + path = output / (practice + ('.' + language if language else '') + '.zib') + subprocess.run([str(compiler), 'bundle', '--root', str(ROOT), + '--entry', 'content:Main', '--asset-dir', 'assets=' + str(assets), + '-o', str(path), str(ROOT / 'apps/practice/content.zi')], check=True) + data = path.read_bytes() + if len(data) > min(MAXIMUM, configuration['maximum_pack_bytes']): + raise ValueError('practice pack exceeds its size budget') + packs.append(dict(practice=practice, language=language, + sha256=sha256(data).hexdigest(), size=len(data))) + target = ROOT / 'build/package-assets/practices' / CATALOG + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(dict(schema=1, packs=packs), separators=(',', ':')) + '\n') + print(f'Prepared {len(packs)} separate practice content packs') + + +def narration_current(configuration, practice, language): + """Partial or stale recordings never become an advertised language pack.""" + import importlib.util + settings = configuration['narration'].get(practice) + if not settings: + return False + receipt_path = ROOT / settings['receipt'] + if not receipt_path.is_file(): + return False + spec = importlib.util.spec_from_file_location('practice_speech', ROOT / 'scripts/generate-lumi-guide-narration.py') + speech = importlib.util.module_from_spec(spec) + spec.loader.exec_module(speech) + speech.LOCALES = ROOT / settings['text_directory'] + speech.VOICE = ROOT / settings['output_directory'] + config = json.loads((ROOT / settings['config']).read_text()) + keys = sorted(speech.entries(ROOT / config['keys_from'])) + receipt = json.loads(receipt_path.read_text()) + return all(speech.current(receipt, config, language, key, text) + for _, key, text in speech.wanted(config, [language], keys)) + + +if __name__ == '__main__': + import sys + prepare(Path(sys.argv[1]).resolve()) diff --git a/tests/package_receiver_test.py b/tests/package_receiver_test.py new file mode 100644 index 000000000..6c5f8c011 --- /dev/null +++ b/tests/package_receiver_test.py @@ -0,0 +1,41 @@ +"""The installed SSH launcher accepts only the configured upload command.""" +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('receiver_install', ROOT / 'scripts/install-package-receiver.py') +receiver = importlib.util.module_from_spec(spec) +spec.loader.exec_module(receiver) + + +class ReceiverLauncherTest(unittest.TestCase): + def test_direct_and_forced_ssh_commands_are_exact(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + effect = directory / 'effect.py' + effect.write_text('import os,sys,json\nprint(json.dumps({"arguments":sys.argv[1:],"command":os.environ["SSH_ORIGINAL_COMMAND"]}))\n') + launcher = directory / 'launcher.py' + launcher.write_text(receiver.wrapper('/packages').replace( + '/opt/daochi/inbe-package-receiver/package-release.py', str(effect))) + environment = {key: value for key, value in os.environ.items() if key != 'SSH_ORIGINAL_COMMAND'} + for arguments, original in ((['/packages'], ''), ([], 'inbe-packages /packages')): + result = subprocess.run(['python3', str(launcher), *arguments], + env=environment | {'SSH_ORIGINAL_COMMAND': original}, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + value = json.loads(result.stdout) + self.assertEqual(value['command'], 'inbe-packages /packages') + self.assertEqual(value['arguments'][:3], ['--receive-archive', '--store', '/packages']) + for original in ('inbe-packages /other', 'sh -c whoami', 'inbe-packages /packages extra'): + result = subprocess.run(['python3', str(launcher)], + env=environment | {'SSH_ORIGINAL_COMMAND': original}, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, '') + + +if __name__ == '__main__': + unittest.main()