Support signed practice content in the package publisher

This commit is contained in:
waozi committed 2026-10-15 02:38:13 -03:00
1 parent 9ea32baf78
commit 6c860ce7f9
4 files changed
+368 -2

No files matched your search

+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env python3
"""Install the existing signed package receiver, without publishing any package."""
import argparse
import hashlib
import json
from pathlib import Path
import re
import shlex
import subprocess
ROOT = Path(__file__).resolve().parents[1]
def wrapper(store):
return '''#!/usr/bin/python3
# Managed by Inbe package receiver.
import os, shlex, sys
store = STORE
original = os.environ.get("SSH_ORIGINAL_COMMAND", "")
arguments = shlex.split(original) if original else ["inbe-packages", *sys.argv[1:]]
if arguments != ["inbe-packages", store]:
raise SystemExit("Only the configured signed package upload is permitted")
os.environ["SSH_ORIGINAL_COMMAND"] = shlex.join(arguments)
os.execv(sys.executable, [sys.executable, "/opt/daochi/inbe-package-receiver/package-release.py",
"--receive-archive", "--store", store,
"--publishers", "/opt/daochi/inbe-package-receiver/publishers.json"])
'''.replace('STORE', repr(store))
def install(target, store):
if not re.fullmatch(r'[A-Za-z0-9_.@-]+', target) or target.startswith('-'):
raise ValueError('invalid SSH target')
if not store.startswith('/') or any(c in store for c in '\n\r\0'):
raise ValueError('package store must be absolute')
configuration = dict(store=store, files={
'package-release.py': (ROOT / 'scripts/package-release.py').read_text(),
'practice_content.py': (ROOT / 'scripts/practice_content.py').read_text(),
'publishers.json': (ROOT / 'apps/publishers.json').read_text(),
'inbe-packages': wrapper(store)})
remote = r'''
from pathlib import Path
import fcntl, hashlib, json, os, subprocess, sys, tempfile
configuration = json.load(sys.stdin)
store = Path(configuration['store'])
if not store.is_dir() or store.is_symlink():
raise RuntimeError('Refusing an absent or symlinked package store')
lock = open('/var/lock/daochi-deploy.lock', 'a')
fcntl.flock(lock, fcntl.LOCK_EX)
directory = Path('/opt/daochi/inbe-package-receiver')
launcher = Path('/usr/local/bin/inbe-packages')
if launcher.exists() and 'Managed by Inbe package receiver.' not in launcher.read_text():
raise RuntimeError('Refusing to replace an unrelated package launcher')
if directory.exists() and not (directory / 'managed.json').exists():
raise RuntimeError('Refusing to replace an unmanaged receiver')
directory.mkdir(mode=0o755, exist_ok=True)
with tempfile.TemporaryDirectory(prefix='receiver-', dir=directory) as temporary:
stage = Path(temporary)
for name, text in configuration['files'].items():
(stage / name).write_text(text)
subprocess.run([sys.executable, '-m', 'py_compile', str(stage / 'package-release.py'),
str(stage / 'practice_content.py'), str(stage / 'inbe-packages')], check=True)
subprocess.run([sys.executable, str(stage / 'package-release.py'), '--help'],
check=True, stdout=subprocess.DEVNULL)
hashes = {}
for name in ('package-release.py', 'practice_content.py', 'publishers.json'):
target = directory / name
(stage / name).chmod(0o644)
os.replace(stage / name, target)
hashes[name] = hashlib.sha256(target.read_bytes()).hexdigest()
# The small launcher is installed last, after its imports are ready.
(stage / 'inbe-packages').chmod(0o755)
os.replace(stage / 'inbe-packages', launcher)
hashes['inbe-packages'] = hashlib.sha256(launcher.read_bytes()).hexdigest()
receipt = dict(installed=True, package_store=str(store), files=hashes,
packages_published=False)
(directory / 'managed.json').write_text(json.dumps(receipt, indent=2) + '\n')
print(json.dumps(receipt))
'''
options = ['ssh', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes',
'-o', 'ConnectTimeout=8', '--', target, 'python3 -c ' + shlex.quote(remote)]
result = subprocess.run(options, input=json.dumps(configuration), text=True,
capture_output=True, check=True)
receipt = json.loads(result.stdout)
saved = ROOT / 'backup/practice-downloads/receiver-install.json'
saved.parent.mkdir(parents=True, exist_ok=True)
saved.write_text(json.dumps(receipt, indent=2) + '\n')
print('Signed package receiver installed; no packages published.')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--ssh-target', required=True)
parser.add_argument('--store', required=True)
arguments = parser.parse_args()
install(arguments.ssh_target, arguments.store)
+78 -2
View File
@@ -22,6 +22,8 @@ from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / 'scripts'))
from practice_content import catalog_from_bundle, bundle_assets
NAMES = ("habits", "practices", "lumi", "lists", "diary", "leaderboard", "inbe")
IDENTIFIER = re.compile(r"[a-z0-9][a-z0-9_.-]{0,62}\Z")
VERSION = re.compile(r"(?:0|[1-9][0-9]{0,8})\.(?:0|[1-9][0-9]{0,8})\.(?:0|[1-9][0-9]{0,8})\Z")
@@ -101,6 +103,17 @@ def stage(store, key_path, key_id, publishers_path=None):
releases[name] = value
payloads[name] = data
store.mkdir(parents=True, exist_ok=True)
content_payloads = []
for pack in catalog_from_bundle(payloads['practices']):
filename = pack['practice'] + ('.' + pack['language'] if pack['language'] else '') + '.zib'
data = (ROOT / 'build/practice-content' / filename).read_bytes()
if len(data) != pack['size'] or sha256(data).hexdigest() != pack['sha256']:
raise ValueError('practice content differs from its signed catalog')
value = dict(releases['practices'])
value['data_schema'] = 1
value['artifacts'] = [dict(variant='module', sha256=pack['sha256'], size=len(data))]
value['signature'] = key.sign(signing_message(value)).hex()
content_payloads.append((pack['sha256'], data, value))
# Validate all old pointers before publishing any new one. Use the same
# per-app lock name as Daochi's staging tool.
with ExitStack() as stack:
@@ -125,6 +138,15 @@ def stage(store, key_path, key_id, publishers_path=None):
raise ValueError("immutable payload does not match its hash")
if archive.exists() and archive.read_bytes() != encoded:
raise ValueError("release sequence is already occupied")
# Content is installed before the harness advertises its hashes.
for digest, data, value in content_payloads:
directory = store / 'inbe.practices'
payload = directory / (digest + '.zib')
if payload.exists() and payload.read_bytes() != data:
raise ValueError('immutable practice payload differs')
atomic_write(payload, data)
atomic_write(directory / (digest + '.zib.release-v2.json'),
(json.dumps(value, separators=(',', ':')) + '\n').encode())
for name in NAMES:
value = releases[name]
directory = store / value["app_id"]
@@ -151,6 +173,13 @@ def rollout_value(percent, hours):
return dict(percent=percent, hours=hours)
def stored_practice_packs(store):
directory = store / 'inbe.practices'
value = json.loads((directory / 'latest-v2.json').read_bytes())
data = (directory / (value['artifacts'][0]['sha256'] + '.zib')).read_bytes()
return catalog_from_bundle(data)
def publish(store, target, remote_store, rollout=None):
if not re.fullmatch(r"[a-zA-Z0-9_.@-]+", target) or target.startswith("-"):
raise ValueError("invalid SSH target")
@@ -172,6 +201,10 @@ def publish(store, target, remote_store, rollout=None):
f"release-{value['sequence']}.json"):
path = directory / filename
tar.add(path, arcname=str(path.relative_to(store)), recursive=False)
for pack in stored_practice_packs(store):
for suffix in ('.zib', '.zib.release-v2.json'):
path = store / 'inbe.practices' / (pack['sha256'] + suffix)
tar.add(path, arcname=str(path.relative_to(store)), recursive=False)
if rollout is not None:
encoded = json.dumps(rollout_value(rollout["percent"], rollout["hours"])).encode()
member = tarfile.TarInfo("rollout.json")
@@ -226,7 +259,22 @@ def verify_origin(store, origin, open_url=open_public_url):
checksum.update(block)
if size != artifact["size"] or checksum.hexdigest() != digest:
raise ValueError(f"{app_id}: public package hash or size differs from the signed publication")
print("Verified public signed metadata and package bytes for the root and all six cells")
for pack in stored_practice_packs(store):
route = origin + '/api/v2/packages/inbe.practices/' + pack['sha256'] + '.zib'
checksum = sha256()
size = 0
with open_url(route, timeout=30) as response:
while True:
block = response.read(65536)
if not block:
break
size += len(block)
if size > pack['size']:
raise ValueError('public practice content exceeds its catalog size')
checksum.update(block)
if size != pack['size'] or checksum.hexdigest() != pack['sha256']:
raise ValueError('public practice content differs from its signed catalog')
print("Verified public signed metadata, app packages and practice content")
def version_tuple(value):
@@ -338,7 +386,7 @@ def import_archive(archive, store, publishers_path=None):
if member.name in entries:
raise ValueError("duplicate archive member")
total += member.size
if len(entries) >= 28 or total > 7 * 64 * 1024 * 1024 + 1024 * 1024:
if len(entries) >= 158 or total > 1024 * 1024 * 1024:
raise ValueError("archive exceeds the release size limit")
entries[member.name] = tar.extractfile(member).read()
# The node accepts archives built before and after Leaderboard became an
@@ -377,12 +425,40 @@ def import_archive(archive, store, publishers_path=None):
raise ValueError("release sequence is already occupied")
releases[app_id] = (value, encoded, data, old != value)
root = releases["inbe"]
content_payloads = []
expected_entries = set()
for app_id, (value, encoded, data, _) in releases.items():
digest = value['artifacts'][0]['sha256']
expected_entries.update((f'{app_id}/latest-v2.json', f'{app_id}/{digest}.zib',
f'{app_id}/{digest}.zib.release-v2.json', f"{app_id}/release-{value['sequence']}.json"))
for pack in catalog_from_bundle(releases['inbe.practices'][2]):
path = 'inbe.practices/' + pack['sha256'] + '.zib'
expected_entries.update((path, path + '.release-v2.json'))
data = entries[path]
encoded = entries[path + '.release-v2.json']
value = json.loads(encoded)
validate_release(value, 'inbe.practices', publishers)
if (value['data_schema'] != 1 or value['artifacts'][0]['sha256'] != pack['sha256'] or
value['artifacts'][0]['size'] != pack['size'] or len(data) != pack['size'] or
sha256(data).hexdigest() != pack['sha256']):
raise ValueError('practice content differs from its signed catalog')
bundle_assets(data)
content_payloads.append((path, data, encoded))
if set(entries) - expected_entries:
raise ValueError('archive contains unreferenced content')
for dependency in root[0]["dependencies"]:
child, _, payload, _ = releases[dependency["app_id"]]
artifact = child["artifacts"][0]
if dependency != dict(app_id=child["app_id"], version=child["version"],
sha256=artifact["sha256"], size=artifact["size"]) or root[2].count(payload) != 1:
raise ValueError("root does not contain its exact signed cell dependency")
# Content precedes both the harness and the root that references it.
for path, data, encoded in content_payloads:
target = store / path
if target.exists() and target.read_bytes() != data:
raise ValueError('immutable practice content differs')
atomic_write(target, data)
atomic_write(store / (path + '.release-v2.json'), encoded)
# Dependencies go first. Publishing the root is the final step.
for app_id in sorted(present, key=lambda value: value == "inbe"):
value, encoded, data, changed = releases[app_id]
+154
View File
@@ -0,0 +1,154 @@
"""Small signed Practices catalog and immutable resource packs.
The catalog is an asset of the signed Practices harness. Its exact hashes bind
each practice and language to a resource-only ZIB. All blobs use the existing
inbe.practices publisher and Daochi artifact endpoint, with no new registry IDs.
"""
import json
from hashlib import sha256
from pathlib import Path
import struct
import subprocess
ROOT = Path(__file__).resolve().parents[1]
CATALOG = 'assets/practices/catalog.json'
NAMES = ('whm', 'meditation', 'sunsalutation', 'patterns', 'taichi')
MAXIMUM = 64 * 1024 * 1024
def bundle_assets(data):
"""Read the bounded ZIB v26 resource table; never execute a bundle."""
if data[:8] != b'ZIB\0\x1a\0\0\0' or len(data) > MAXIMUM:
raise ValueError('unsupported resource bundle')
at = 8
def take(size):
nonlocal at
if size < 0 or at + size > len(data):
raise ValueError('truncated resource bundle')
value = data[at:at + size]
at += size
return value
def number():
return struct.unpack('<I', take(4))[0]
def text():
return take(number())
text(); text()
for _ in range(number()):
text(); text()
for _ in range(number()):
for _ in range(7):
text()
take(8); text(); take(4)
for _ in range(number()):
for _ in range(3):
text()
take(number())
count = number()
if count > 10000:
raise ValueError('too many resource assets')
result = {}
for _ in range(count):
path = text().decode('utf-8')
if path in result:
raise ValueError('duplicate resource path')
result[path] = take(number())
if at != len(data):
raise ValueError('trailing resource bundle bytes')
return result
def catalog_from_bundle(data):
# Older releases have no content catalog and remain installable.
if data[:8] != b'ZIB\0\x1a\0\0\0':
return []
assets = bundle_assets(data)
if CATALOG not in assets:
return []
catalog = json.loads(assets[CATALOG])
packs = catalog.get('packs')
if catalog.get('schema') != 1 or not isinstance(packs, list) or len(packs) > 65:
raise ValueError('invalid practice catalog')
seen = set()
for pack in packs:
practice = pack.get('practice')
language = pack.get('language')
digest = pack.get('sha256')
size = pack.get('size')
if (practice not in NAMES or not isinstance(language, str) or
(language and (len(language) != 2 or not language.isascii() or not language.islower())) or
not isinstance(digest, str) or len(digest) != 64 or
any(c not in '0123456789abcdef' for c in digest) or
type(size) is not int or not 8 <= size <= MAXIMUM or
(practice, language) in seen):
raise ValueError('invalid or duplicate practice pack')
seen.add((practice, language))
if {(name, '') for name in NAMES} - seen:
raise ValueError('practice catalog lacks base content')
return packs
def prepare(compiler):
configuration = json.loads((ROOT / 'apps/practice/content.json').read_text())
if configuration['schema'] != 1 or tuple(configuration['practices']) != NAMES:
raise ValueError('unsupported practice content configuration')
output = ROOT / 'build/practice-content'
output.mkdir(parents=True, exist_ok=True)
packs = []
for practice in NAMES:
directories = [ROOT / ('build/package-assets/practice-' + practice)]
directories += sorted((ROOT / 'build/package-assets').glob('practice-' + practice + '-voice-*'))
for directory in directories:
language = directory.name.split('-voice-', 1)[1] if '-voice-' in directory.name else ''
if language and not narration_current(configuration, practice, language):
print(f'{practice}/{language}: incomplete narration excluded from the catalog')
continue
assets = directory / 'assets'
assets.mkdir(parents=True, exist_ok=True)
# Even timer-only practices have a versioned content identity.
identity = assets / 'practices' / practice / ('voice/' + language if language else '') / 'pack.json'
identity.parent.mkdir(parents=True, exist_ok=True)
identity.write_text(json.dumps(dict(schema=1, practice=practice, language=language),
separators=(',', ':')) + '\n')
path = output / (practice + ('.' + language if language else '') + '.zib')
subprocess.run([str(compiler), 'bundle', '--root', str(ROOT),
'--entry', 'content:Main', '--asset-dir', 'assets=' + str(assets),
'-o', str(path), str(ROOT / 'apps/practice/content.zi')], check=True)
data = path.read_bytes()
if len(data) > min(MAXIMUM, configuration['maximum_pack_bytes']):
raise ValueError('practice pack exceeds its size budget')
packs.append(dict(practice=practice, language=language,
sha256=sha256(data).hexdigest(), size=len(data)))
target = ROOT / 'build/package-assets/practices' / CATALOG
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(json.dumps(dict(schema=1, packs=packs), separators=(',', ':')) + '\n')
print(f'Prepared {len(packs)} separate practice content packs')
def narration_current(configuration, practice, language):
"""Partial or stale recordings never become an advertised language pack."""
import importlib.util
settings = configuration['narration'].get(practice)
if not settings:
return False
receipt_path = ROOT / settings['receipt']
if not receipt_path.is_file():
return False
spec = importlib.util.spec_from_file_location('practice_speech', ROOT / 'scripts/generate-lumi-guide-narration.py')
speech = importlib.util.module_from_spec(spec)
spec.loader.exec_module(speech)
speech.LOCALES = ROOT / settings['text_directory']
speech.VOICE = ROOT / settings['output_directory']
config = json.loads((ROOT / settings['config']).read_text())
keys = sorted(speech.entries(ROOT / config['keys_from']))
receipt = json.loads(receipt_path.read_text())
return all(speech.current(receipt, config, language, key, text)
for _, key, text in speech.wanted(config, [language], keys))
if __name__ == '__main__':
import sys
prepare(Path(sys.argv[1]).resolve())
+41
View File
@@ -0,0 +1,41 @@
"""The installed SSH launcher accepts only the configured upload command."""
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location('receiver_install', ROOT / 'scripts/install-package-receiver.py')
receiver = importlib.util.module_from_spec(spec)
spec.loader.exec_module(receiver)
class ReceiverLauncherTest(unittest.TestCase):
def test_direct_and_forced_ssh_commands_are_exact(self):
with tempfile.TemporaryDirectory() as temporary:
directory = Path(temporary)
effect = directory / 'effect.py'
effect.write_text('import os,sys,json\nprint(json.dumps({"arguments":sys.argv[1:],"command":os.environ["SSH_ORIGINAL_COMMAND"]}))\n')
launcher = directory / 'launcher.py'
launcher.write_text(receiver.wrapper('/packages').replace(
'/opt/daochi/inbe-package-receiver/package-release.py', str(effect)))
environment = {key: value for key, value in os.environ.items() if key != 'SSH_ORIGINAL_COMMAND'}
for arguments, original in ((['/packages'], ''), ([], 'inbe-packages /packages')):
result = subprocess.run(['python3', str(launcher), *arguments],
env=environment | {'SSH_ORIGINAL_COMMAND': original}, capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
value = json.loads(result.stdout)
self.assertEqual(value['command'], 'inbe-packages /packages')
self.assertEqual(value['arguments'][:3], ['--receive-archive', '--store', '/packages'])
for original in ('inbe-packages /other', 'sh -c whoami', 'inbe-packages /packages extra'):
result = subprocess.run(['python3', str(launcher)],
env=environment | {'SSH_ORIGINAL_COMMAND': original}, capture_output=True, text=True)
self.assertNotEqual(result.returncode, 0)
self.assertEqual(result.stdout, '')
if __name__ == '__main__':
unittest.main()