Allow a signed Sun Salutation content repair without a package release
This commit is contained in:
1 parent
e03f04a88a
commit
85e3b71d08
3 files changed
+250
No files matched your search
@@ -3129,6 +3129,7 @@ test: package-manifest-test
|
||||
.PHONY: package-upload-test
|
||||
package-upload-test:
|
||||
python3 tests/package_upload_test.py
|
||||
python3 tests/package_content_repair_test.py
|
||||
|
||||
test: package-upload-test
|
||||
|
||||
|
||||
@@ -231,6 +231,101 @@ def open_public_url(url, timeout):
|
||||
return urlopen(Request(url, headers={"User-Agent": "Inbe-Publisher/1.0"}), timeout=timeout)
|
||||
|
||||
|
||||
def stage_content_repair(store, practice, key_path, key_id, release_path, publishers_path=None):
|
||||
"""Prepare an immutable resource without advancing any app release."""
|
||||
publishers = json.loads((publishers_path or ROOT / 'apps/publishers.json').read_text())
|
||||
value = json.loads(release_path.read_bytes())
|
||||
validate_release(value, 'inbe.practices', publishers)
|
||||
key = serialization.load_pem_private_key(key_path.read_bytes(), password=None)
|
||||
if not isinstance(key, Ed25519PrivateKey):
|
||||
raise ValueError('publisher key must be Ed25519')
|
||||
public = key.public_key().public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw).hex()
|
||||
if not any(p['key_id'] == key_id and p['public_key'] == public and
|
||||
'inbe.practices' in p['app_ids'] for p in publishers):
|
||||
raise ValueError('content repair key must match the shipped Practices publisher')
|
||||
catalog = json.loads((ROOT / 'build/package-assets/practices/assets/practices/catalog.json').read_bytes())
|
||||
matches = [p for p in catalog['packs'] if p['practice'] == practice and not p['language']]
|
||||
if len(matches) != 1 or matches[0].get('product'):
|
||||
raise ValueError('repair requires one free base practice pack')
|
||||
pack = matches[0]
|
||||
data = (ROOT / 'build/practice-content' / (practice + '.zib')).read_bytes()
|
||||
if len(data) != pack['size'] or sha256(data).hexdigest() != pack['sha256']:
|
||||
raise ValueError('repair content differs from the app catalog')
|
||||
validate_repair_assets(data)
|
||||
value = dict(value, data_schema=1, key_id=key_id,
|
||||
artifacts=[dict(variant='module', sha256=pack['sha256'], size=len(data))])
|
||||
value['signature'] = key.sign(signing_message(value)).hex()
|
||||
validate_release(value, 'inbe.practices', publishers)
|
||||
directory = store / 'inbe.practices'
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
atomic_write(directory / (pack['sha256'] + '.zib'), data)
|
||||
atomic_write(directory / (pack['sha256'] + '.zib.release-v2.json'),
|
||||
json.dumps(value, separators=(',', ':')).encode())
|
||||
return pack['sha256']
|
||||
|
||||
|
||||
def validate_repair_assets(data):
|
||||
"""This repair is limited to Sun Salutation pictures, never paid content."""
|
||||
assets = bundle_assets(data)
|
||||
prefix = 'assets/practices/sunsalutation/'
|
||||
if not assets or any(not path.startswith(prefix) or path.startswith(
|
||||
(prefix + 'voice/', prefix + 'voice-male/')) for path in assets):
|
||||
raise ValueError('repair may contain only Sun Salutation picture resources')
|
||||
|
||||
|
||||
def import_content_repair(entries, store, publishers):
|
||||
if len(entries) != 2:
|
||||
raise ValueError('repair must contain exactly one pack and its signature')
|
||||
payloads = [name for name in entries if re.fullmatch(r'inbe\.practices/[0-9a-f]{64}\.zib', name)]
|
||||
if len(payloads) != 1 or payloads[0] + '.release-v2.json' not in entries:
|
||||
raise ValueError('invalid content repair archive')
|
||||
name = payloads[0]
|
||||
data = entries[name]
|
||||
metadata = entries[name + '.release-v2.json']
|
||||
value = json.loads(metadata)
|
||||
validate_release(value, 'inbe.practices', publishers)
|
||||
artifact = value['artifacts'][0]
|
||||
digest = Path(name).stem
|
||||
if (value['data_schema'] != 1 or artifact['sha256'] != digest or
|
||||
artifact['size'] != len(data) or sha256(data).hexdigest() != digest):
|
||||
raise ValueError('repair content does not match its signed artifact')
|
||||
validate_repair_assets(data)
|
||||
directory = store / 'inbe.practices'
|
||||
directory.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
with (directory / '.publish.lock').open('a+b') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
if (store / (name + '.product')).exists():
|
||||
raise ValueError('repair must not replace paid content')
|
||||
for filename in (name, name + '.release-v2.json'):
|
||||
target = store / filename
|
||||
if target.exists() and target.read_bytes() != entries[filename]:
|
||||
raise ValueError('repair must not replace immutable content')
|
||||
atomic_write(store / name, data)
|
||||
atomic_write(store / (name + '.release-v2.json'), metadata)
|
||||
|
||||
|
||||
def publish_content_repair(store, digest, target, remote_store):
|
||||
if not re.fullmatch(r'[a-zA-Z0-9_.@-]+', target) or target.startswith('-'):
|
||||
raise ValueError('invalid SSH target')
|
||||
if not remote_store.startswith('/') or any(c in remote_store for c in '\n\r\0'):
|
||||
raise ValueError('invalid remote store')
|
||||
import tarfile
|
||||
with tempfile.TemporaryDirectory(prefix='inbe-content-repair-') as temporary:
|
||||
archive = Path(temporary) / 'repair.tar'
|
||||
with tarfile.open(archive, 'w') as tar:
|
||||
for suffix in ('.zib', '.zib.release-v2.json'):
|
||||
path = store / 'inbe.practices' / (digest + suffix)
|
||||
tar.add(path, arcname=str(path.relative_to(store)), recursive=False)
|
||||
options = ['-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes']
|
||||
if os.environ.get('INBE_SSH_IDENTITY'):
|
||||
options += ['-i', os.environ['INBE_SSH_IDENTITY']]
|
||||
if os.environ.get('INBE_SSH_KNOWN_HOSTS'):
|
||||
options += ['-o', 'UserKnownHostsFile=' + os.environ['INBE_SSH_KNOWN_HOSTS']]
|
||||
with archive.open('rb') as upload:
|
||||
subprocess.run(['ssh', *options, '--', target,
|
||||
shlex.join(['inbe-packages', remote_store])], stdin=upload, check=True)
|
||||
|
||||
|
||||
def verify_origin(store, origin, open_url=open_public_url):
|
||||
"""Require the public node to serve the exact signed files we published."""
|
||||
parsed = urlsplit(origin)
|
||||
@@ -401,6 +496,13 @@ def import_archive(archive, store, publishers_path=None):
|
||||
# The node accepts archives built before and after Leaderboard became an
|
||||
# app, so the importer and the app can be updated in either order.
|
||||
present = {name.split("/")[0] for name in entries}
|
||||
if present == {'inbe.practices'}:
|
||||
if rollout is not None:
|
||||
raise ValueError('content repair cannot change rollout')
|
||||
import_content_repair(entries, store, publishers)
|
||||
archive.unlink()
|
||||
archive.parent.rmdir()
|
||||
return
|
||||
if present != ids and present != ids - {"inbe.leaderboard"}:
|
||||
raise ValueError("archive must contain the root and every cell")
|
||||
with ExitStack() as stack:
|
||||
@@ -533,8 +635,30 @@ def main():
|
||||
parser.add_argument("--publishers", type=Path, help="node-owned public publisher pins")
|
||||
parser.add_argument("--rollout", type=int, help="publish changed releases to this share of devices first")
|
||||
parser.add_argument("--rollout-hours", type=int, default=24, help="then to every device after these hours")
|
||||
parser.add_argument('--repair-content', choices=['sunsalutation'],
|
||||
help='restore only the missing Sun Salutation picture pack')
|
||||
parser.add_argument('--repair-release', type=Path, help='existing signed Practices metadata for a content repair')
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.repair_content:
|
||||
if not args.key or not args.key_id or not args.repair_release or args.rollout is not None:
|
||||
parser.error('content repair needs --key, --key-id and --repair-release, without --rollout')
|
||||
digest = stage_content_repair(args.store, args.repair_content, args.key,
|
||||
args.key_id, args.repair_release, args.publishers)
|
||||
if args.ssh_target:
|
||||
if not args.remote_store or not args.verify_origin:
|
||||
parser.error('repair publication requires --remote-store and --verify-origin')
|
||||
origin = args.verify_origin.rstrip('/')
|
||||
parsed = urlsplit(origin)
|
||||
if parsed.scheme != 'https' or not parsed.hostname or parsed.path or parsed.query or parsed.fragment or parsed.username or parsed.password:
|
||||
raise ValueError('repair verification requires an HTTPS origin')
|
||||
publish_content_repair(args.store, digest, args.ssh_target, args.remote_store)
|
||||
expected = (args.store / 'inbe.practices' / (digest + '.zib')).read_bytes()
|
||||
with open_public_url(origin + '/api/v2/packages/inbe.practices/' + digest + '.zib', 30) as response:
|
||||
if response.read(len(expected) + 1) != expected:
|
||||
raise ValueError('public repair differs from the app catalog')
|
||||
print('Sun Salutation picture pack repaired' if args.ssh_target else 'Sun Salutation picture repair staged')
|
||||
return
|
||||
if args.receive_archive:
|
||||
if not args.publishers:
|
||||
parser.error("--publishers is required for the restricted upload receiver")
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Repair one signed picture pack while preserving every app release pointer."""
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location('content_repair', ROOT / 'scripts/package-release.py')
|
||||
publisher = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(publisher)
|
||||
|
||||
|
||||
def resource(path):
|
||||
def number(value):
|
||||
return struct.pack('<I', value)
|
||||
|
||||
def text(value):
|
||||
value = value.encode()
|
||||
return number(len(value)) + value
|
||||
|
||||
return (b'ZIB\0' + number(26) + text('content') + text('Main') + number(0) * 4 +
|
||||
number(1) + text(path) + number(3) + b'png')
|
||||
|
||||
|
||||
class ContentRepairTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory(prefix='inbe-content-repair-test-')
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.root = Path(self.temporary.name)
|
||||
self.signer = Ed25519PrivateKey.generate()
|
||||
self.key = self.root / 'publisher.pem'
|
||||
self.key.write_bytes(self.signer.private_bytes(serialization.Encoding.PEM,
|
||||
serialization.PrivateFormat.PKCS8, serialization.NoEncryption()))
|
||||
public = self.signer.public_key().public_bytes(serialization.Encoding.Raw,
|
||||
serialization.PublicFormat.Raw).hex()
|
||||
self.publishers = self.root / 'publishers.json'
|
||||
self.publishers.write_text(json.dumps([dict(key_id='test', public_key=public,
|
||||
app_ids=['inbe.practices'])]))
|
||||
self.data = resource('assets/practices/sunsalutation/characters/girl/01.png')
|
||||
self.digest = sha256(self.data).hexdigest()
|
||||
pack = dict(practice='sunsalutation', language='', size=len(self.data), sha256=self.digest)
|
||||
catalog = self.root / 'build/package-assets/practices/assets/practices/catalog.json'
|
||||
catalog.parent.mkdir(parents=True)
|
||||
catalog.write_text(json.dumps(dict(schema=1, packs=[pack])))
|
||||
source = self.root / 'build/practice-content/sunsalutation.zib'
|
||||
source.parent.mkdir(parents=True)
|
||||
source.write_bytes(self.data)
|
||||
value = dict(app_id='inbe.practices', sequence=7, version='1.1.5', runtime='kryon-app-v1',
|
||||
host_api=5, module_api=2, data_schema=0, key_id='test', dependencies=[],
|
||||
artifacts=[dict(variant='module', size=8, sha256='0' * 64)])
|
||||
value['signature'] = self.signer.sign(publisher.signing_message(value)).hex()
|
||||
self.release = self.root / 'live.json'
|
||||
self.release.write_text(json.dumps(value))
|
||||
self.store = self.root / 'staged'
|
||||
with patch.object(publisher, 'ROOT', self.root):
|
||||
digest = publisher.stage_content_repair(self.store, 'sunsalutation', self.key,
|
||||
'test', self.release, self.publishers)
|
||||
self.assertEqual(self.digest, digest)
|
||||
self.filename = 'inbe.practices/' + digest + '.zib'
|
||||
self.entries = {name: (self.store / name).read_bytes() for name in
|
||||
(self.filename, self.filename + '.release-v2.json')}
|
||||
self.remote = self.root / 'remote'
|
||||
self.latest = self.remote / 'inbe.practices/latest-v2.json'
|
||||
self.latest.parent.mkdir(parents=True)
|
||||
self.latest.write_bytes(b'original latest metadata')
|
||||
self.pins = json.loads(self.publishers.read_text())
|
||||
|
||||
def test_repair_is_idempotent_and_preserves_app_release(self):
|
||||
publisher.import_content_repair(self.entries, self.remote, self.pins)
|
||||
publisher.import_content_repair(self.entries, self.remote, self.pins)
|
||||
self.assertEqual(self.data, (self.remote / self.filename).read_bytes())
|
||||
self.assertEqual(b'original latest metadata', self.latest.read_bytes())
|
||||
self.assertFalse((self.store / 'inbe.practices/latest-v2.json').exists())
|
||||
|
||||
def test_existing_receiver_dispatches_repair_archive(self):
|
||||
upload = self.root / 'upload'
|
||||
upload.mkdir()
|
||||
archive = upload / 'repair.tar'
|
||||
with tarfile.open(archive, 'w') as tar:
|
||||
for name, data in self.entries.items():
|
||||
member = tarfile.TarInfo(name)
|
||||
member.size = len(data)
|
||||
tar.addfile(member, io.BytesIO(data))
|
||||
publisher.import_archive(archive, self.remote, self.publishers)
|
||||
self.assertEqual(self.data, (self.remote / self.filename).read_bytes())
|
||||
self.assertEqual(b'original latest metadata', self.latest.read_bytes())
|
||||
|
||||
def test_tampered_bytes_signature_and_extra_files_are_rejected(self):
|
||||
for entries in (dict(self.entries, **{self.filename: b'bad'}),
|
||||
dict(self.entries, **{self.filename + '.release-v2.json': b'{}'}),
|
||||
dict(self.entries, **{'inbe.practices/latest-v2.json': b'new'})):
|
||||
with self.assertRaises((ValueError, KeyError)):
|
||||
publisher.import_content_repair(entries, self.remote, self.pins)
|
||||
self.assertFalse((self.remote / self.filename).exists())
|
||||
self.assertEqual(b'original latest metadata', self.latest.read_bytes())
|
||||
|
||||
def test_paid_content_and_immutable_conflicts_are_preserved(self):
|
||||
marker = self.remote / (self.filename + '.product')
|
||||
marker.write_text('paid')
|
||||
with self.assertRaisesRegex(ValueError, 'paid content'):
|
||||
publisher.import_content_repair(self.entries, self.remote, self.pins)
|
||||
marker.unlink()
|
||||
target = self.remote / self.filename
|
||||
target.write_bytes(b'original')
|
||||
with self.assertRaisesRegex(ValueError, 'immutable'):
|
||||
publisher.import_content_repair(self.entries, self.remote, self.pins)
|
||||
self.assertEqual(b'original', target.read_bytes())
|
||||
|
||||
def test_other_practices_and_narration_are_rejected(self):
|
||||
for path in ('assets/practices/taichi/01.png', 'assets/practices/sunsalutation/voice/en/01.ogg'):
|
||||
with self.assertRaisesRegex(ValueError, 'Sun Salutation picture'):
|
||||
publisher.validate_repair_assets(resource(path))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user