Support signed practice content in the package publisher
This commit is contained in:
1 parent
9ea32baf78
commit
6c860ce7f9
4 files changed
+368
-2
No files matched your search
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install the existing signed package receiver, without publishing any package."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def wrapper(store):
|
||||
return '''#!/usr/bin/python3
|
||||
# Managed by Inbe package receiver.
|
||||
import os, shlex, sys
|
||||
store = STORE
|
||||
original = os.environ.get("SSH_ORIGINAL_COMMAND", "")
|
||||
arguments = shlex.split(original) if original else ["inbe-packages", *sys.argv[1:]]
|
||||
if arguments != ["inbe-packages", store]:
|
||||
raise SystemExit("Only the configured signed package upload is permitted")
|
||||
os.environ["SSH_ORIGINAL_COMMAND"] = shlex.join(arguments)
|
||||
os.execv(sys.executable, [sys.executable, "/opt/daochi/inbe-package-receiver/package-release.py",
|
||||
"--receive-archive", "--store", store,
|
||||
"--publishers", "/opt/daochi/inbe-package-receiver/publishers.json"])
|
||||
'''.replace('STORE', repr(store))
|
||||
|
||||
|
||||
def install(target, store):
|
||||
if not re.fullmatch(r'[A-Za-z0-9_.@-]+', target) or target.startswith('-'):
|
||||
raise ValueError('invalid SSH target')
|
||||
if not store.startswith('/') or any(c in store for c in '\n\r\0'):
|
||||
raise ValueError('package store must be absolute')
|
||||
configuration = dict(store=store, files={
|
||||
'package-release.py': (ROOT / 'scripts/package-release.py').read_text(),
|
||||
'practice_content.py': (ROOT / 'scripts/practice_content.py').read_text(),
|
||||
'publishers.json': (ROOT / 'apps/publishers.json').read_text(),
|
||||
'inbe-packages': wrapper(store)})
|
||||
remote = r'''
|
||||
from pathlib import Path
|
||||
import fcntl, hashlib, json, os, subprocess, sys, tempfile
|
||||
configuration = json.load(sys.stdin)
|
||||
store = Path(configuration['store'])
|
||||
if not store.is_dir() or store.is_symlink():
|
||||
raise RuntimeError('Refusing an absent or symlinked package store')
|
||||
lock = open('/var/lock/daochi-deploy.lock', 'a')
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
directory = Path('/opt/daochi/inbe-package-receiver')
|
||||
launcher = Path('/usr/local/bin/inbe-packages')
|
||||
if launcher.exists() and 'Managed by Inbe package receiver.' not in launcher.read_text():
|
||||
raise RuntimeError('Refusing to replace an unrelated package launcher')
|
||||
if directory.exists() and not (directory / 'managed.json').exists():
|
||||
raise RuntimeError('Refusing to replace an unmanaged receiver')
|
||||
directory.mkdir(mode=0o755, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix='receiver-', dir=directory) as temporary:
|
||||
stage = Path(temporary)
|
||||
for name, text in configuration['files'].items():
|
||||
(stage / name).write_text(text)
|
||||
subprocess.run([sys.executable, '-m', 'py_compile', str(stage / 'package-release.py'),
|
||||
str(stage / 'practice_content.py'), str(stage / 'inbe-packages')], check=True)
|
||||
subprocess.run([sys.executable, str(stage / 'package-release.py'), '--help'],
|
||||
check=True, stdout=subprocess.DEVNULL)
|
||||
hashes = {}
|
||||
for name in ('package-release.py', 'practice_content.py', 'publishers.json'):
|
||||
target = directory / name
|
||||
(stage / name).chmod(0o644)
|
||||
os.replace(stage / name, target)
|
||||
hashes[name] = hashlib.sha256(target.read_bytes()).hexdigest()
|
||||
# The small launcher is installed last, after its imports are ready.
|
||||
(stage / 'inbe-packages').chmod(0o755)
|
||||
os.replace(stage / 'inbe-packages', launcher)
|
||||
hashes['inbe-packages'] = hashlib.sha256(launcher.read_bytes()).hexdigest()
|
||||
receipt = dict(installed=True, package_store=str(store), files=hashes,
|
||||
packages_published=False)
|
||||
(directory / 'managed.json').write_text(json.dumps(receipt, indent=2) + '\n')
|
||||
print(json.dumps(receipt))
|
||||
'''
|
||||
options = ['ssh', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes',
|
||||
'-o', 'ConnectTimeout=8', '--', target, 'python3 -c ' + shlex.quote(remote)]
|
||||
result = subprocess.run(options, input=json.dumps(configuration), text=True,
|
||||
capture_output=True, check=True)
|
||||
receipt = json.loads(result.stdout)
|
||||
saved = ROOT / 'backup/practice-downloads/receiver-install.json'
|
||||
saved.parent.mkdir(parents=True, exist_ok=True)
|
||||
saved.write_text(json.dumps(receipt, indent=2) + '\n')
|
||||
print('Signed package receiver installed; no packages published.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--ssh-target', required=True)
|
||||
parser.add_argument('--store', required=True)
|
||||
arguments = parser.parse_args()
|
||||
install(arguments.ssh_target, arguments.store)
|
||||
@@ -22,6 +22,8 @@ from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
from practice_content import catalog_from_bundle, bundle_assets
|
||||
NAMES = ("habits", "practices", "lumi", "lists", "diary", "leaderboard", "inbe")
|
||||
IDENTIFIER = re.compile(r"[a-z0-9][a-z0-9_.-]{0,62}\Z")
|
||||
VERSION = re.compile(r"(?:0|[1-9][0-9]{0,8})\.(?:0|[1-9][0-9]{0,8})\.(?:0|[1-9][0-9]{0,8})\Z")
|
||||
@@ -101,6 +103,17 @@ def stage(store, key_path, key_id, publishers_path=None):
|
||||
releases[name] = value
|
||||
payloads[name] = data
|
||||
store.mkdir(parents=True, exist_ok=True)
|
||||
content_payloads = []
|
||||
for pack in catalog_from_bundle(payloads['practices']):
|
||||
filename = pack['practice'] + ('.' + pack['language'] if pack['language'] else '') + '.zib'
|
||||
data = (ROOT / 'build/practice-content' / filename).read_bytes()
|
||||
if len(data) != pack['size'] or sha256(data).hexdigest() != pack['sha256']:
|
||||
raise ValueError('practice content differs from its signed catalog')
|
||||
value = dict(releases['practices'])
|
||||
value['data_schema'] = 1
|
||||
value['artifacts'] = [dict(variant='module', sha256=pack['sha256'], size=len(data))]
|
||||
value['signature'] = key.sign(signing_message(value)).hex()
|
||||
content_payloads.append((pack['sha256'], data, value))
|
||||
# Validate all old pointers before publishing any new one. Use the same
|
||||
# per-app lock name as Daochi's staging tool.
|
||||
with ExitStack() as stack:
|
||||
@@ -125,6 +138,15 @@ def stage(store, key_path, key_id, publishers_path=None):
|
||||
raise ValueError("immutable payload does not match its hash")
|
||||
if archive.exists() and archive.read_bytes() != encoded:
|
||||
raise ValueError("release sequence is already occupied")
|
||||
# Content is installed before the harness advertises its hashes.
|
||||
for digest, data, value in content_payloads:
|
||||
directory = store / 'inbe.practices'
|
||||
payload = directory / (digest + '.zib')
|
||||
if payload.exists() and payload.read_bytes() != data:
|
||||
raise ValueError('immutable practice payload differs')
|
||||
atomic_write(payload, data)
|
||||
atomic_write(directory / (digest + '.zib.release-v2.json'),
|
||||
(json.dumps(value, separators=(',', ':')) + '\n').encode())
|
||||
for name in NAMES:
|
||||
value = releases[name]
|
||||
directory = store / value["app_id"]
|
||||
@@ -151,6 +173,13 @@ def rollout_value(percent, hours):
|
||||
return dict(percent=percent, hours=hours)
|
||||
|
||||
|
||||
def stored_practice_packs(store):
|
||||
directory = store / 'inbe.practices'
|
||||
value = json.loads((directory / 'latest-v2.json').read_bytes())
|
||||
data = (directory / (value['artifacts'][0]['sha256'] + '.zib')).read_bytes()
|
||||
return catalog_from_bundle(data)
|
||||
|
||||
|
||||
def publish(store, target, remote_store, rollout=None):
|
||||
if not re.fullmatch(r"[a-zA-Z0-9_.@-]+", target) or target.startswith("-"):
|
||||
raise ValueError("invalid SSH target")
|
||||
@@ -172,6 +201,10 @@ def publish(store, target, remote_store, rollout=None):
|
||||
f"release-{value['sequence']}.json"):
|
||||
path = directory / filename
|
||||
tar.add(path, arcname=str(path.relative_to(store)), recursive=False)
|
||||
for pack in stored_practice_packs(store):
|
||||
for suffix in ('.zib', '.zib.release-v2.json'):
|
||||
path = store / 'inbe.practices' / (pack['sha256'] + suffix)
|
||||
tar.add(path, arcname=str(path.relative_to(store)), recursive=False)
|
||||
if rollout is not None:
|
||||
encoded = json.dumps(rollout_value(rollout["percent"], rollout["hours"])).encode()
|
||||
member = tarfile.TarInfo("rollout.json")
|
||||
@@ -226,7 +259,22 @@ def verify_origin(store, origin, open_url=open_public_url):
|
||||
checksum.update(block)
|
||||
if size != artifact["size"] or checksum.hexdigest() != digest:
|
||||
raise ValueError(f"{app_id}: public package hash or size differs from the signed publication")
|
||||
print("Verified public signed metadata and package bytes for the root and all six cells")
|
||||
for pack in stored_practice_packs(store):
|
||||
route = origin + '/api/v2/packages/inbe.practices/' + pack['sha256'] + '.zib'
|
||||
checksum = sha256()
|
||||
size = 0
|
||||
with open_url(route, timeout=30) as response:
|
||||
while True:
|
||||
block = response.read(65536)
|
||||
if not block:
|
||||
break
|
||||
size += len(block)
|
||||
if size > pack['size']:
|
||||
raise ValueError('public practice content exceeds its catalog size')
|
||||
checksum.update(block)
|
||||
if size != pack['size'] or checksum.hexdigest() != pack['sha256']:
|
||||
raise ValueError('public practice content differs from its signed catalog')
|
||||
print("Verified public signed metadata, app packages and practice content")
|
||||
|
||||
|
||||
def version_tuple(value):
|
||||
@@ -338,7 +386,7 @@ def import_archive(archive, store, publishers_path=None):
|
||||
if member.name in entries:
|
||||
raise ValueError("duplicate archive member")
|
||||
total += member.size
|
||||
if len(entries) >= 28 or total > 7 * 64 * 1024 * 1024 + 1024 * 1024:
|
||||
if len(entries) >= 158 or total > 1024 * 1024 * 1024:
|
||||
raise ValueError("archive exceeds the release size limit")
|
||||
entries[member.name] = tar.extractfile(member).read()
|
||||
# The node accepts archives built before and after Leaderboard became an
|
||||
@@ -377,12 +425,40 @@ def import_archive(archive, store, publishers_path=None):
|
||||
raise ValueError("release sequence is already occupied")
|
||||
releases[app_id] = (value, encoded, data, old != value)
|
||||
root = releases["inbe"]
|
||||
content_payloads = []
|
||||
expected_entries = set()
|
||||
for app_id, (value, encoded, data, _) in releases.items():
|
||||
digest = value['artifacts'][0]['sha256']
|
||||
expected_entries.update((f'{app_id}/latest-v2.json', f'{app_id}/{digest}.zib',
|
||||
f'{app_id}/{digest}.zib.release-v2.json', f"{app_id}/release-{value['sequence']}.json"))
|
||||
for pack in catalog_from_bundle(releases['inbe.practices'][2]):
|
||||
path = 'inbe.practices/' + pack['sha256'] + '.zib'
|
||||
expected_entries.update((path, path + '.release-v2.json'))
|
||||
data = entries[path]
|
||||
encoded = entries[path + '.release-v2.json']
|
||||
value = json.loads(encoded)
|
||||
validate_release(value, 'inbe.practices', publishers)
|
||||
if (value['data_schema'] != 1 or value['artifacts'][0]['sha256'] != pack['sha256'] or
|
||||
value['artifacts'][0]['size'] != pack['size'] or len(data) != pack['size'] or
|
||||
sha256(data).hexdigest() != pack['sha256']):
|
||||
raise ValueError('practice content differs from its signed catalog')
|
||||
bundle_assets(data)
|
||||
content_payloads.append((path, data, encoded))
|
||||
if set(entries) - expected_entries:
|
||||
raise ValueError('archive contains unreferenced content')
|
||||
for dependency in root[0]["dependencies"]:
|
||||
child, _, payload, _ = releases[dependency["app_id"]]
|
||||
artifact = child["artifacts"][0]
|
||||
if dependency != dict(app_id=child["app_id"], version=child["version"],
|
||||
sha256=artifact["sha256"], size=artifact["size"]) or root[2].count(payload) != 1:
|
||||
raise ValueError("root does not contain its exact signed cell dependency")
|
||||
# Content precedes both the harness and the root that references it.
|
||||
for path, data, encoded in content_payloads:
|
||||
target = store / path
|
||||
if target.exists() and target.read_bytes() != data:
|
||||
raise ValueError('immutable practice content differs')
|
||||
atomic_write(target, data)
|
||||
atomic_write(store / (path + '.release-v2.json'), encoded)
|
||||
# Dependencies go first. Publishing the root is the final step.
|
||||
for app_id in sorted(present, key=lambda value: value == "inbe"):
|
||||
value, encoded, data, changed = releases[app_id]
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Small signed Practices catalog and immutable resource packs.
|
||||
|
||||
The catalog is an asset of the signed Practices harness. Its exact hashes bind
|
||||
each practice and language to a resource-only ZIB. All blobs use the existing
|
||||
inbe.practices publisher and Daochi artifact endpoint, with no new registry IDs.
|
||||
"""
|
||||
import json
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CATALOG = 'assets/practices/catalog.json'
|
||||
NAMES = ('whm', 'meditation', 'sunsalutation', 'patterns', 'taichi')
|
||||
MAXIMUM = 64 * 1024 * 1024
|
||||
|
||||
|
||||
def bundle_assets(data):
|
||||
"""Read the bounded ZIB v26 resource table; never execute a bundle."""
|
||||
if data[:8] != b'ZIB\0\x1a\0\0\0' or len(data) > MAXIMUM:
|
||||
raise ValueError('unsupported resource bundle')
|
||||
at = 8
|
||||
|
||||
def take(size):
|
||||
nonlocal at
|
||||
if size < 0 or at + size > len(data):
|
||||
raise ValueError('truncated resource bundle')
|
||||
value = data[at:at + size]
|
||||
at += size
|
||||
return value
|
||||
|
||||
def number():
|
||||
return struct.unpack('<I', take(4))[0]
|
||||
|
||||
def text():
|
||||
return take(number())
|
||||
|
||||
text(); text()
|
||||
for _ in range(number()):
|
||||
text(); text()
|
||||
for _ in range(number()):
|
||||
for _ in range(7):
|
||||
text()
|
||||
take(8); text(); take(4)
|
||||
for _ in range(number()):
|
||||
for _ in range(3):
|
||||
text()
|
||||
take(number())
|
||||
count = number()
|
||||
if count > 10000:
|
||||
raise ValueError('too many resource assets')
|
||||
result = {}
|
||||
for _ in range(count):
|
||||
path = text().decode('utf-8')
|
||||
if path in result:
|
||||
raise ValueError('duplicate resource path')
|
||||
result[path] = take(number())
|
||||
if at != len(data):
|
||||
raise ValueError('trailing resource bundle bytes')
|
||||
return result
|
||||
|
||||
|
||||
def catalog_from_bundle(data):
|
||||
# Older releases have no content catalog and remain installable.
|
||||
if data[:8] != b'ZIB\0\x1a\0\0\0':
|
||||
return []
|
||||
assets = bundle_assets(data)
|
||||
if CATALOG not in assets:
|
||||
return []
|
||||
catalog = json.loads(assets[CATALOG])
|
||||
packs = catalog.get('packs')
|
||||
if catalog.get('schema') != 1 or not isinstance(packs, list) or len(packs) > 65:
|
||||
raise ValueError('invalid practice catalog')
|
||||
seen = set()
|
||||
for pack in packs:
|
||||
practice = pack.get('practice')
|
||||
language = pack.get('language')
|
||||
digest = pack.get('sha256')
|
||||
size = pack.get('size')
|
||||
if (practice not in NAMES or not isinstance(language, str) or
|
||||
(language and (len(language) != 2 or not language.isascii() or not language.islower())) or
|
||||
not isinstance(digest, str) or len(digest) != 64 or
|
||||
any(c not in '0123456789abcdef' for c in digest) or
|
||||
type(size) is not int or not 8 <= size <= MAXIMUM or
|
||||
(practice, language) in seen):
|
||||
raise ValueError('invalid or duplicate practice pack')
|
||||
seen.add((practice, language))
|
||||
if {(name, '') for name in NAMES} - seen:
|
||||
raise ValueError('practice catalog lacks base content')
|
||||
return packs
|
||||
|
||||
|
||||
def prepare(compiler):
|
||||
configuration = json.loads((ROOT / 'apps/practice/content.json').read_text())
|
||||
if configuration['schema'] != 1 or tuple(configuration['practices']) != NAMES:
|
||||
raise ValueError('unsupported practice content configuration')
|
||||
output = ROOT / 'build/practice-content'
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
packs = []
|
||||
for practice in NAMES:
|
||||
directories = [ROOT / ('build/package-assets/practice-' + practice)]
|
||||
directories += sorted((ROOT / 'build/package-assets').glob('practice-' + practice + '-voice-*'))
|
||||
for directory in directories:
|
||||
language = directory.name.split('-voice-', 1)[1] if '-voice-' in directory.name else ''
|
||||
if language and not narration_current(configuration, practice, language):
|
||||
print(f'{practice}/{language}: incomplete narration excluded from the catalog')
|
||||
continue
|
||||
assets = directory / 'assets'
|
||||
assets.mkdir(parents=True, exist_ok=True)
|
||||
# Even timer-only practices have a versioned content identity.
|
||||
identity = assets / 'practices' / practice / ('voice/' + language if language else '') / 'pack.json'
|
||||
identity.parent.mkdir(parents=True, exist_ok=True)
|
||||
identity.write_text(json.dumps(dict(schema=1, practice=practice, language=language),
|
||||
separators=(',', ':')) + '\n')
|
||||
path = output / (practice + ('.' + language if language else '') + '.zib')
|
||||
subprocess.run([str(compiler), 'bundle', '--root', str(ROOT),
|
||||
'--entry', 'content:Main', '--asset-dir', 'assets=' + str(assets),
|
||||
'-o', str(path), str(ROOT / 'apps/practice/content.zi')], check=True)
|
||||
data = path.read_bytes()
|
||||
if len(data) > min(MAXIMUM, configuration['maximum_pack_bytes']):
|
||||
raise ValueError('practice pack exceeds its size budget')
|
||||
packs.append(dict(practice=practice, language=language,
|
||||
sha256=sha256(data).hexdigest(), size=len(data)))
|
||||
target = ROOT / 'build/package-assets/practices' / CATALOG
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(json.dumps(dict(schema=1, packs=packs), separators=(',', ':')) + '\n')
|
||||
print(f'Prepared {len(packs)} separate practice content packs')
|
||||
|
||||
|
||||
def narration_current(configuration, practice, language):
|
||||
"""Partial or stale recordings never become an advertised language pack."""
|
||||
import importlib.util
|
||||
settings = configuration['narration'].get(practice)
|
||||
if not settings:
|
||||
return False
|
||||
receipt_path = ROOT / settings['receipt']
|
||||
if not receipt_path.is_file():
|
||||
return False
|
||||
spec = importlib.util.spec_from_file_location('practice_speech', ROOT / 'scripts/generate-lumi-guide-narration.py')
|
||||
speech = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(speech)
|
||||
speech.LOCALES = ROOT / settings['text_directory']
|
||||
speech.VOICE = ROOT / settings['output_directory']
|
||||
config = json.loads((ROOT / settings['config']).read_text())
|
||||
keys = sorted(speech.entries(ROOT / config['keys_from']))
|
||||
receipt = json.loads(receipt_path.read_text())
|
||||
return all(speech.current(receipt, config, language, key, text)
|
||||
for _, key, text in speech.wanted(config, [language], keys))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
prepare(Path(sys.argv[1]).resolve())
|
||||
@@ -0,0 +1,41 @@
|
||||
"""The installed SSH launcher accepts only the configured upload command."""
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location('receiver_install', ROOT / 'scripts/install-package-receiver.py')
|
||||
receiver = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(receiver)
|
||||
|
||||
|
||||
class ReceiverLauncherTest(unittest.TestCase):
|
||||
def test_direct_and_forced_ssh_commands_are_exact(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
directory = Path(temporary)
|
||||
effect = directory / 'effect.py'
|
||||
effect.write_text('import os,sys,json\nprint(json.dumps({"arguments":sys.argv[1:],"command":os.environ["SSH_ORIGINAL_COMMAND"]}))\n')
|
||||
launcher = directory / 'launcher.py'
|
||||
launcher.write_text(receiver.wrapper('/packages').replace(
|
||||
'/opt/daochi/inbe-package-receiver/package-release.py', str(effect)))
|
||||
environment = {key: value for key, value in os.environ.items() if key != 'SSH_ORIGINAL_COMMAND'}
|
||||
for arguments, original in ((['/packages'], ''), ([], 'inbe-packages /packages')):
|
||||
result = subprocess.run(['python3', str(launcher), *arguments],
|
||||
env=environment | {'SSH_ORIGINAL_COMMAND': original}, capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
value = json.loads(result.stdout)
|
||||
self.assertEqual(value['command'], 'inbe-packages /packages')
|
||||
self.assertEqual(value['arguments'][:3], ['--receive-archive', '--store', '/packages'])
|
||||
for original in ('inbe-packages /other', 'sh -c whoami', 'inbe-packages /packages extra'):
|
||||
result = subprocess.run(['python3', str(launcher)],
|
||||
env=environment | {'SSH_ORIGINAL_COMMAND': original}, capture_output=True, text=True)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, '')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user