Files
apps/scripts/node_release.py

768 lines
38 KiB
Python
Executable File

#!/usr/bin/env python3
"""Sign, check and publish the Mesh releases of apps the node offers.
The node serves each app as signed release v2 metadata beside its immutable
ZX portable executable. ZX hosts ask the node with format=zx and the ZX
revision they run; hosts released before the ZX format keep their .zib
releases through 2027-10-23. A release may also carry a signed WebAssembly
variant lowered from its .zx, which web hosts ask for with format=wasm.
Inbe and Arcade publish their node apps with this tool, and Inbe's
node receiver imports it to check every release it accepts. Each owning
project keeps its own version ledger, built bundles and publisher pins:
apps/versions.json, build/node-apps and apps/publishers.json in the working
directory unless named.
"""
import argparse
import fcntl
from hashlib import sha256
import json
import os
from pathlib import Path
import re
import shlex
import subprocess
import sys
import tarfile
import tempfile
from urllib.error import HTTPError
from urllib.request import Request, urlopen
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
IDENTIFIER = re.compile(r"[a-z0-9][a-z0-9_.-]{0,62}\Z")
# Apps the node offers: inbe.<name>, never one of the IDs Inbe bundles, or an
# exact reverse-domain ID such as xyz.waozi.wurm that a publisher pin names
# in its node_app_ids.
NODE_APP_ID = re.compile(r"inbe\.[a-z][a-z0-9]{1,23}\Z")
EXACT_NODE_APP_ID = re.compile(r"[a-z][a-z0-9]*(?:-[a-z0-9]+)*(?:\.[a-z][a-z0-9]*(?:-[a-z0-9]+)*){2,4}\Z")
BUNDLED_IDS = {"inbe", "inbe.habits", "inbe.practices", "inbe.lists", "inbe.diary", "inbe.lumi",
"inbe.leaderboard"}
VERSION = re.compile(r"(?:0|[1-9][0-9]{0,8})\.(?:0|[1-9][0-9]{0,8})\.(?:0|[1-9][0-9]{0,8})\Z")
# The node's history lists each app's newest ten releases and the newest of
# every compatibility group, so a device on an older host still finds one
# built for it; going back offers the newest three of them that still run on
# the device. The node reads at most HISTORY_LIMIT entries.
RETAINED_RELEASES = 10
HISTORY_LIMIT = 64
# Payload signatures and the suffixes the node stores them under: ZX portable
# executables, and legacy bundles of hosts released before the ZX format.
SUFFIXES = ((b"ZX\0\0", ".zx"), (b"ZIB\0", ".zib"))
# A web host's variant: the WebAssembly module (version 1) Ziran lowers from
# the release's own .zx, archived as release-SEQUENCE.wasm.json.
WASM_SIGNATURE = b"\0asm\x01\0\0\0"
WASM_SUFFIX = ".wasm"
def zx_revision(data):
"""The executable revision a ZX header names (the u32 after its magic), or 0."""
if len(data) < 8 or not data.startswith(b"ZX\0\0"):
return 0
return int.from_bytes(data[4:8], "little")
def same_release(variant, value):
"""Whether two descriptors sign the same release, whatever artifact each names."""
def fields(release):
return {field: item for field, item in release.items() if field not in ("artifacts", "signature")}
return fields(variant) == fields(value)
def payload_suffix(data):
""".zx or legacy .zib for payload bytes, or None for anything else."""
return next((suffix for signature, suffix in SUFFIXES if data.startswith(signature)), None)
def stored_payload(directory, value):
"""The node file of a release's module artifact, in whichever format it was stored."""
digest = value["artifacts"][0]["sha256"]
legacy = directory / (digest + ".zib")
if legacy.is_file() and not (directory / (digest + ".zx")).is_file():
return legacy
return directory / (digest + ".zx")
def stored_revision(directory, value):
"""The ZX revision of a release's stored payload; 0 for a legacy .zib."""
payload = stored_payload(directory, value)
if payload.suffix != ".zx":
return 0
with payload.open("rb") as source:
return zx_revision(source.read(8))
def wasm_variant_path(directory, value):
return directory / f"release-{value['sequence']}{WASM_SUFFIX}.json"
def stored_wasm_variant(directory, value, app_id, publishers):
"""The release's signed WebAssembly variant on the node, or None."""
path = wasm_variant_path(directory, value)
if path.is_symlink() or not path.is_file() or path.stat().st_size > 65536:
return None
try:
variant = json.loads(path.read_bytes())
validate_release(variant, app_id, publishers)
except (ValueError, KeyError, TypeError):
return None
module = directory / (variant["artifacts"][0]["sha256"] + WASM_SUFFIX)
if not same_release(variant, value) or not module.is_file():
return None
return variant
def compatibility_groups(directory, value, app_id, publishers):
"""The groups of hosts that run a release.
A host runs releases only in its own payload format and ZX revision, for
its own module interface and a host interface it covers. Web hosts run a
release's WebAssembly variant, a format of its own.
"""
found = stored_payload(directory, value).suffix
interfaces = (stored_revision(directory, value), value["host_api"], value["module_api"])
groups = [(found, *interfaces)]
if found == ".zx" and stored_wasm_variant(directory, value, app_id, publishers) is not None:
groups.append((WASM_SUFFIX, *interfaces))
return groups
def retained_releases(directory, releases, app_id, publishers):
"""The newest release of every compatibility group, then the newest others.
Keeping the newest of every group keeps a release for every host that
could run one, so hosts released before a new format, ZX revision or
interface keep their last release while newer hosts move on.
"""
releases = sorted(releases, key=lambda value: value["sequence"], reverse=True)
kept = []
groups = set()
for value in releases:
found = set(compatibility_groups(directory, value, app_id, publishers)) - groups
if found and len(kept) < HISTORY_LIMIT:
groups.update(found)
kept.append(value)
for value in releases:
if len(kept) >= RETAINED_RELEASES:
break
if value not in kept:
kept.append(value)
return sorted(kept, key=lambda value: value["sequence"], reverse=True)
def exact_node_app_id(app_id):
"""Whether app_id may be configured exactly as an app the node offers."""
return isinstance(app_id, str) and bool(IDENTIFIER.fullmatch(app_id)) and \
bool(EXACT_NODE_APP_ID.fullmatch(app_id)) and not app_id.startswith("inbe.") and \
app_id not in BUNDLED_IDS
def node_app_id(app_id, publishers=()):
"""An app the node offers: inbe.<name>, or an exact ID a publisher pin names."""
if NODE_APP_ID.fullmatch(app_id) and app_id not in BUNDLED_IDS:
return True
return exact_node_app_id(app_id) and \
any(app_id in pin.get("node_app_ids", []) for pin in publishers)
def trusted_pin(publishers, key_id, app_id):
"""The pin that may sign app_id: named, under its prefix, or an exact node app ID it names."""
return next((pin for pin in publishers if pin["key_id"] == key_id and
(app_id in pin["app_ids"] or
(node_app_id(app_id) and "inbe." in pin.get("app_prefixes", [])) or
(exact_node_app_id(app_id) and app_id in pin.get("node_app_ids", [])))), None)
def node_app_name_matches(name, app_id):
"""A ledger NAME publishes inbe.NAME or an exact ID whose last part is NAME."""
return app_id == "inbe." + name or (exact_node_app_id(app_id) and app_id.rsplit(".", 1)[1] == name)
def signing_message(value):
fields = ("app_id", "sequence", "version", "runtime", "host_api",
"module_api", "data_schema", "key_id")
lines = ["daochi-zib-release-v2", *(str(value[name]) for name in fields),
str(len(value["artifacts"]))]
for artifact in value["artifacts"]:
lines.extend(str(artifact[name]) for name in ("variant", "sha256", "size"))
lines.append(str(len(value["dependencies"])))
for dependency in value["dependencies"]:
lines.extend(str(dependency[name]) for name in ("app_id", "version", "sha256", "size"))
return ("\n".join(lines) + "\n").encode("ascii")
def atomic_write(path, data):
fd, name = tempfile.mkstemp(prefix=".inbe-", dir=path.parent)
try:
with os.fdopen(fd, "wb") as output:
output.write(data)
output.flush()
os.fsync(output.fileno())
os.replace(name, path)
directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(directory)
finally:
os.close(directory)
finally:
Path(name).unlink(missing_ok=True)
def version_tuple(value):
return tuple(map(int, value["version"].split(".")))
def validate_release(value, app_id, publishers):
fields = {"app_id", "sequence", "version", "runtime", "host_api", "module_api",
"data_schema", "key_id", "artifacts", "dependencies", "signature"}
# A beta app the node offers carries "channel":"beta"; apps hide it until
# a person shows beta apps. It changes who sees the app, not what runs.
present = set(value) - {"channel"} if isinstance(value, dict) else set()
if not isinstance(value, dict) or present != fields or value["app_id"] != app_id or \
value["runtime"] != "kryon-app-v1" or value.get("channel", "beta") != "beta" or \
("channel" in value and not node_app_id(app_id, publishers)):
raise ValueError("invalid release identity or fields")
if not isinstance(value["version"], str) or not VERSION.fullmatch(value["version"]):
raise ValueError("invalid numeric release version")
for field, minimum, maximum in (("sequence", 1, 2**31 - 1), ("host_api", 1, 65535),
("module_api", 1, 65535), ("data_schema", 0, 65535)):
if type(value[field]) is not int or not minimum <= value[field] <= maximum:
raise ValueError("invalid release sequence or API version")
if not isinstance(value["artifacts"], list) or len(value["artifacts"]) != 1:
raise ValueError("Inbe release must contain exactly one module artifact")
artifact = value["artifacts"][0]
if not isinstance(artifact, dict) or set(artifact) != {"variant", "sha256", "size"} or \
artifact["variant"] != "module" or not isinstance(artifact["sha256"], str) or \
not re.fullmatch(r"[0-9a-f]{64}", artifact["sha256"]) or \
type(artifact["size"]) is not int or not 8 <= artifact["size"] <= PACKAGE_MAXIMUM_BYTES:
raise ValueError("invalid module artifact")
dependencies = value["dependencies"]
# Inbe's root depends on each app it bundles; every other app on none.
expected = sorted(BUNDLED_IDS - {"inbe"}) if app_id == "inbe" else []
# Roots built before Leaderboard became an app have no dependency on it.
if app_id == "inbe" and isinstance(dependencies, list) and \
len(dependencies) == len(expected) - 1:
expected.remove("inbe.leaderboard")
if not isinstance(dependencies, list) or len(dependencies) != len(expected):
raise ValueError("invalid root dependencies")
for dependency, expected_id in zip(dependencies, expected):
if not isinstance(dependency, dict) or set(dependency) != {"app_id", "version", "sha256", "size"} or \
dependency["app_id"] != expected_id or not isinstance(dependency["version"], str) or \
not VERSION.fullmatch(dependency["version"]) or not isinstance(dependency["sha256"], str) or \
not re.fullmatch(r"[0-9a-f]{64}", dependency["sha256"]) or \
type(dependency["size"]) is not int or not 8 <= dependency["size"] <= PACKAGE_MAXIMUM_BYTES:
raise ValueError("invalid dependency descriptor")
pin = trusted_pin(publishers, value["key_id"], app_id)
if pin is None:
raise ValueError("publisher is not trusted for this app")
try:
if not isinstance(value["signature"], str) or not re.fullmatch(r"[0-9a-f]{128}", value["signature"]):
raise ValueError("invalid release signature")
Ed25519PublicKey.from_public_bytes(bytes.fromhex(pin["public_key"])).verify(
bytes.fromhex(value["signature"]), signing_message(value))
except Exception as error:
raise ValueError("release signature rejected") from error
def write_history(directory, app_id, publishers):
"""Lists the retained releases, newest first, in history-v2.json.
Every release stays archived. The history names the newest ten the node
still trusts and still has the payload for, always including the newest
release of each format, ZX revision and interface.
"""
releases = []
for path in directory.glob("release-*.json"):
if path.is_symlink() or not path.is_file() or path.stat().st_size > 65536:
continue
try:
value = json.loads(path.read_bytes())
validate_release(value, app_id, publishers)
except (ValueError, KeyError, TypeError):
continue
payload = stored_payload(directory, value)
if path.name == f"release-{value['sequence']}.json" and payload.is_file():
releases.append(value)
if not releases:
return
history = dict(versions=retained_releases(directory, releases, app_id, publishers))
atomic_write(directory / "history-v2.json", (json.dumps(history, separators=(",", ":")) + "\n").encode())
def write_histories(store, publishers_path):
"""Writes the history of every app in the store, under its publication lock."""
publishers = json.loads(publishers_path.read_text())
for directory in sorted(store.iterdir()):
app_id = directory.name
if directory.is_symlink() or not directory.is_dir() or \
(app_id not in BUNDLED_IDS and not node_app_id(app_id, publishers)) or \
not (directory / "latest-v2.json").is_file():
continue
with (directory / ".publish.lock").open("a+b") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
write_history(directory, app_id, publishers)
def open_public_url(url, timeout):
return urlopen(Request(url, headers={"User-Agent": "Inbe-Publisher/1.0"}), timeout=timeout)
def upload(archive, target, remote_store):
"""Streams archive to the node's one restricted upload command."""
options = ["-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes"]
if os.environ.get("INBE_SSH_IDENTITY"):
# Only the upload key, never another key from an agent or ~/.ssh.
options += ["-i", os.environ["INBE_SSH_IDENTITY"], "-o", "IdentitiesOnly=yes"]
if os.environ.get("INBE_SSH_KNOWN_HOSTS"):
options += ["-o", "UserKnownHostsFile=" + os.environ["INBE_SSH_KNOWN_HOSTS"]]
with archive.open("rb") as stream:
subprocess.run(["ssh", *options, "--", target, shlex.join(["inbe-packages", remote_store])],
stdin=stream, check=True)
def check_zx_revision(data, revision, name):
"""A ledger that names its hosts' ZX revision publishes only payloads of it."""
if revision is None:
return
if type(revision) is not int or not 1 <= revision <= 65535:
raise ValueError(f"invalid zx_revision for {name}")
if zx_revision(data) != revision:
raise ValueError(f"{name}.zx is ZX revision {zx_revision(data)}, but its hosts run {revision}")
def node_app_release(name, key, key_id, beta=False, *, ledger_path, bundle_directory, publishers=()):
"""The signed release of NAME.zx in bundle_directory from its ledger entry.
Its ID is inbe.NAME, or an exact ID ending in .NAME that publishers name.
"""
versions = json.loads(ledger_path.read_text())
component = versions.get("node_apps", {}).get(name)
if component is None or not isinstance(component.get("id"), str) or \
not node_app_id(component["id"], publishers) or not node_app_name_matches(name, component["id"]):
raise ValueError(f"{name} is not an app the node offers")
if not VERSION.fullmatch(component["version"]) or not 0 < component["sequence"] <= 2**31 - 1:
raise ValueError(f"invalid {name} version or sequence")
data = read_bundle(bundle_directory / f"{name}.zx")
check_zx_revision(data, component.get("zx_revision", versions.get("zx_revision")), name)
value = dict(app_id=component["id"], sequence=component["sequence"], version=component["version"],
runtime="kryon-app-v1", host_api=component.get("host_api", versions["host_api"]),
module_api=versions["module_api"], data_schema=0, key_id=key_id,
artifacts=[dict(variant="module", sha256=sha256(data).hexdigest(), size=len(data))],
dependencies=[])
value["signature"] = key.sign(signing_message(value)).hex()
if beta or component.get("beta") is True:
value["channel"] = "beta"
return value, data
# A collection such as Arcade carries no games: each is downloaded on its
# own. They sit beside its bundle in NAME.games/<sha256>.zx, and its catalog
# pins their digests. The node serves a file only for a signed release that
# names it, so each game gets a lookup: this same release with that game as
# its module. It is never a latest or archived release.
GAME_LIMIT = 32
PACKAGE_MAXIMUM_BYTES = 2**63 - 2
def read_bundle(path, signature=b"ZX\0\0"):
length = path.stat().st_size
if not 8 <= length <= min(PACKAGE_MAXIMUM_BYTES, sys.maxsize - 1):
raise ValueError(f"invalid portable executable size: {path}")
with path.open('rb') as source:
data = source.read(length + 1)
if len(data) != length or not data.startswith(signature):
raise ValueError(f"invalid portable executable: {path}")
return data
def verify_payload(response, artifact):
if type(artifact['size']) is not int or not 8 <= artifact['size'] <= PACKAGE_MAXIMUM_BYTES:
return False
received = 0
digest = sha256()
while block := response.read(65536):
received += len(block)
if received > artifact['size']:
return False
digest.update(block)
return received == artifact['size'] and digest.hexdigest() == artifact['sha256']
def node_app_games(value, key, bundle_directory, name):
"""Signed lookups and bytes of the games NAME downloads one by one."""
directory = bundle_directory / f"{name}.games"
games = []
for path in sorted(directory.glob("*.zx")) if directory.is_dir() else []:
data = read_bundle(path)
digest = sha256(data).hexdigest()
if path.name != digest + ".zx":
raise ValueError(f"invalid {name} game payload {path.name}")
lookup = {field: item for field, item in value.items() if field != "signature"}
lookup["artifacts"] = [dict(variant="module", sha256=digest, size=len(data))]
lookup["signature"] = key.sign(signing_message(lookup)).hex()
games.append((lookup, data))
if len(games) > GAME_LIMIT:
raise ValueError(f"{name} names more than {GAME_LIMIT} games")
return games
def node_app_wasm(value, key, bundle_directory, name):
"""The signed WebAssembly variant of NAME's release from NAME.wasm, or None.
Ziran's wasm target lowers NAME.zx into NAME.wasm beside it. The variant
is the same release with that module as its artifact, signed by the same
key, so web hosts run the same program without lowering it themselves.
"""
path = bundle_directory / f"{name}{WASM_SUFFIX}"
if not path.is_file():
return None
data = read_bundle(path, WASM_SIGNATURE)
variant = {field: item for field, item in value.items() if field != "signature"}
variant["artifacts"] = [dict(variant="module", sha256=sha256(data).hexdigest(), size=len(data))]
variant["signature"] = key.sign(signing_message(variant)).hex()
return variant, data, (json.dumps(variant, separators=(",", ":")) + "\n").encode()
def game_lookups(directory, value):
"""The stored lookups of a release's games."""
lookups = []
main = value["artifacts"][0]["sha256"]
for path in sorted(directory.glob("*.zx.release-v2.json")):
if path.is_symlink() or not path.is_file() or path.stat().st_size > 65536:
continue
lookup = json.loads(path.read_bytes())
if lookup.get("sequence") == value["sequence"] and lookup.get("version") == value["version"] and \
lookup["artifacts"][0]["sha256"] != main:
lookups.append(lookup)
return lookups
def stage_node_app(store, name, key_path, key_id, publishers_path, beta=False, *,
ledger_path, bundle_directory):
if not IDENTIFIER.fullmatch(key_id) or key_id in (".", ".."):
raise ValueError("invalid publisher key ID")
key = serialization.load_pem_private_key(key_path.read_bytes(), password=None)
if not isinstance(key, Ed25519PrivateKey):
raise ValueError("publisher key must be Ed25519")
publishers = json.loads(publishers_path.read_text())
value, data = node_app_release(name, key, key_id, beta, ledger_path=ledger_path,
bundle_directory=bundle_directory, publishers=publishers)
validate_release(value, value["app_id"], publishers)
games = node_app_games(value, key, bundle_directory, name)
for lookup, _ in games:
validate_release(lookup, value["app_id"], publishers)
wasm = node_app_wasm(value, key, bundle_directory, name)
if wasm is not None:
validate_release(wasm[0], value["app_id"], publishers)
directory = store / value["app_id"]
directory.mkdir(parents=True, exist_ok=True, mode=0o700)
encoded = (json.dumps(value, separators=(",", ":")) + "\n").encode()
with (directory / ".publish.lock").open("a+b") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
write_node_app(directory, value, encoded, data, publishers, games, wasm)
return value
def write_node_app(directory, value, encoded, data, publishers, games=(), wasm=None):
"""Publishes one release of an app the node offers, its games and WebAssembly variant."""
current = directory / "latest-v2.json"
if current.exists():
previous = json.loads(current.read_bytes())
if previous != value and (value["sequence"] <= previous["sequence"] or
version_tuple(value) <= version_tuple(previous)):
raise ValueError(f"{value['app_id']} changed: advance its version and sequence")
suffix = payload_suffix(data)
if suffix is None:
raise ValueError("payload is not a ZX portable executable")
payload = directory / (value["artifacts"][0]["sha256"] + suffix)
archive = directory / f"release-{value['sequence']}.json"
if payload.exists() and payload.read_bytes() != data:
raise ValueError("immutable payload does not match its hash")
if archive.exists() and archive.read_bytes() != encoded:
raise ValueError("release sequence is already occupied")
if wasm is not None:
variant, module, encoded_variant = wasm
stored_module = directory / (variant["artifacts"][0]["sha256"] + WASM_SUFFIX)
variant_archive = wasm_variant_path(directory, value)
if stored_module.exists() and stored_module.read_bytes() != module:
raise ValueError("immutable WebAssembly module does not match its hash")
if variant_archive.exists() and variant_archive.read_bytes() != encoded_variant:
raise ValueError("the release already has another WebAssembly variant")
for lookup, game in games:
stored = directory / (lookup["artifacts"][0]["sha256"] + ".zx")
if stored.exists() and stored.read_bytes() != game:
raise ValueError("immutable game payload does not match its hash")
atomic_write(stored, game)
# An unchanged game keeps its bytes; its lookup names the newest release.
atomic_write(stored.with_name(stored.name + ".release-v2.json"),
(json.dumps(lookup, separators=(",", ":")) + "\n").encode())
if wasm is not None:
# Before the release itself, so a host that finds it finds its variant.
atomic_write(stored_module, module)
atomic_write(stored_module.with_name(stored_module.name + ".release-v2.json"), encoded_variant)
atomic_write(variant_archive, encoded_variant)
atomic_write(payload, data)
atomic_write(payload.with_name(payload.name + ".release-v2.json"), encoded)
atomic_write(archive, encoded)
atomic_write(current, encoded)
write_history(directory, value["app_id"], publishers)
def import_wasm_variant(entries, publishers, app_id, value, expected):
"""The archive's signed WebAssembly variant of the release, or None.
It must sign this same release, name exactly the module beside it, and
accompany a ZX release: it is lowered from that .zx.
"""
archived = f"{app_id}/release-{value['sequence']}{WASM_SUFFIX}.json"
if archived not in entries:
return None
encoded = entries[archived]
if len(encoded) > 65536:
raise ValueError("oversized WebAssembly variant")
variant = json.loads(encoded)
validate_release(variant, app_id, publishers)
module = variant["artifacts"][0]
file = f"{app_id}/{module['sha256']}{WASM_SUFFIX}"
data = entries.get(file, b"")
if not same_release(variant, value) or entries.get(file + ".release-v2.json") != encoded or \
f"{app_id}/{value['artifacts'][0]['sha256']}.zx" not in entries or \
len(data) != module["size"] or sha256(data).hexdigest() != module["sha256"] or \
not data.startswith(WASM_SIGNATURE):
raise ValueError("WebAssembly variant does not match this release")
expected.update((archived, file, file + ".release-v2.json"))
return variant, data, encoded
def import_node_app(entries, store, publishers, app_id):
"""Node-side merge of one app the node offers, signed and without dependencies."""
if not node_app_id(app_id, publishers):
raise ValueError(f"{app_id} is not an app the node offers")
encoded = entries.get(f"{app_id}/latest-v2.json")
if encoded is None or len(encoded) > 65536:
raise ValueError("missing or oversized release metadata")
value = json.loads(encoded)
validate_release(value, app_id, publishers)
if value["dependencies"]:
raise ValueError("an app the node offers has no dependencies")
artifact = value["artifacts"][0]
names = [artifact["sha256"] + suffix for _, suffix in SUFFIXES
if f"{app_id}/{artifact['sha256']}{suffix}" in entries]
payload = names[0] if len(names) == 1 else ""
data = entries.get(f"{app_id}/{payload}", b"")
if artifact["size"] != len(data) or sha256(data).hexdigest() != artifact["sha256"] or \
payload_suffix(data) is None or not payload.endswith(payload_suffix(data)):
raise ValueError("payload hash mismatch")
expected = {f"{app_id}/latest-v2.json", f"{app_id}/{payload}",
f"{app_id}/{payload}.release-v2.json",
f"{app_id}/release-{value['sequence']}.json"}
# Its games: each a signed lookup of this release beside the game's bytes.
games = []
for name in sorted(set(entries) - expected):
if not name.endswith(".zx.release-v2.json"):
continue
encoded_lookup = entries[name]
if len(encoded_lookup) > 65536:
raise ValueError("oversized game lookup")
lookup = json.loads(encoded_lookup)
validate_release(lookup, app_id, publishers)
game = lookup["artifacts"][0]
file = f"{app_id}/{game['sha256']}.zx"
game_bytes = entries.get(file, b"")
if name != file + ".release-v2.json" or game["sha256"] == artifact["sha256"] or \
{field: item for field, item in lookup.items() if field not in ("artifacts", "signature")} != \
{field: item for field, item in value.items() if field not in ("artifacts", "signature")} or \
len(game_bytes) != game["size"] or sha256(game_bytes).hexdigest() != game["sha256"] or \
not game_bytes.startswith(b"ZX\0\0"):
raise ValueError("game lookup does not match this release")
expected.update((name, file))
games.append((lookup, game_bytes))
wasm = import_wasm_variant(entries, publishers, app_id, value, expected)
if len(games) > GAME_LIMIT or set(entries) != expected:
raise ValueError("archive contains unreferenced content")
directory = store / app_id
directory.mkdir(parents=True, exist_ok=True, mode=0o700)
with (directory / ".publish.lock").open("a+b") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
write_node_app(directory, value, encoded, data, publishers, games, wasm)
def publish_node_app(store, name, target, remote_store, app_id=None):
"""Uploads the staged release of NAME, stored under app_id (default inbe.NAME)."""
if not re.fullmatch(r"[a-zA-Z0-9_.@-]+", target) or target.startswith("-"):
raise ValueError("invalid SSH target")
if not remote_store.startswith("/") or "\n" in remote_store:
raise ValueError("Mesh package store must be an absolute path")
with tempfile.TemporaryDirectory(prefix="inbe-upload-") as staging:
archive = Path(staging) / "packages.tar"
directory = store / (app_id or "inbe." + name)
value = json.loads((directory / "latest-v2.json").read_bytes())
payload = stored_payload(directory, value).name
games = [lookup["artifacts"][0]["sha256"] + ".zx" for lookup in game_lookups(directory, value)]
wasm = []
if wasm_variant_path(directory, value).is_file():
variant = json.loads(wasm_variant_path(directory, value).read_bytes())
module = variant["artifacts"][0]["sha256"] + WASM_SUFFIX
wasm = [wasm_variant_path(directory, value).name, module, module + ".release-v2.json"]
with tarfile.open(archive, "w") as tar:
for filename in ("latest-v2.json", payload, payload + ".release-v2.json",
f"release-{value['sequence']}.json",
*(name for game in games for name in (game, game + ".release-v2.json")), *wasm):
path = directory / filename
tar.add(path, arcname=str(path.relative_to(store)), recursive=False)
upload(archive, target, remote_store)
def verify_node_app(store, name, origin, open_url=open_public_url, *, app_id=None):
"""Require the public node to serve the exact signed release of NAME, under app_id (default inbe.NAME)."""
app_id = app_id or "inbe." + name
expected = json.loads((store / app_id / "latest-v2.json").read_bytes())
route = origin.rstrip("/") + "/api/v2/packages/" + app_id
payload = stored_payload(store / app_id, expected).name
query = "/latest?bucket=0&format=zx" if payload.endswith(".zx") else "/latest?bucket=0"
with open_url(route + query, timeout=30) as response:
encoded = response.read(65537)
if len(encoded) > 65536 or json.loads(encoded) != expected:
raise ValueError(f"{app_id}: public release metadata differs from the signed publication")
artifact = expected["artifacts"][0]
with open_url(route + "/" + payload, timeout=30) as response:
verified = verify_payload(response, artifact)
if not verified:
raise ValueError(f"{app_id}: public package hash or size differs from the signed publication")
lookups = game_lookups(store / app_id, expected)
for lookup in lookups:
game = lookup["artifacts"][0]
with open_url(route + "/" + game["sha256"] + ".zx", timeout=60) as response:
verified = verify_payload(response, game)
if not verified:
raise ValueError(f"{app_id}: a public game differs from the signed publication")
variant_path = wasm_variant_path(store / app_id, expected)
if variant_path.is_file():
variant = json.loads(variant_path.read_bytes())
with open_url(route + "/latest?bucket=0&format=wasm", timeout=30) as response:
encoded = response.read(65537)
if len(encoded) > 65536 or json.loads(encoded) != variant:
raise ValueError(f"{app_id}: public WebAssembly metadata differs from the signed publication")
module = variant["artifacts"][0]
with open_url(route + "/" + module["sha256"] + WASM_SUFFIX, timeout=60) as response:
verified = verify_payload(response, module)
if not verified:
raise ValueError(f"{app_id}: the public WebAssembly module differs from the signed publication")
print(f"Verified the public release of {app_id}" + (f" and its {len(lookups)} games" if lookups else "") +
(" and its WebAssembly variant" if variant_path.is_file() else ""))
def registered_apps(origin, open_url=open_public_url):
"""The app IDs the node's registry lists as active."""
with open_url(origin.rstrip("/") + "/api/v1/apps", timeout=30) as response:
listing = json.loads(response.read(1048577))
return {app.get("app_id") for app in listing.get("apps", []) if app.get("status") == "active"}
def served_release(origin, app_id, open_url=open_public_url):
"""The newest release the node serves for app_id in either format, or None.
A new release must advance past it: the node keeps one sequence per app,
whether its newest release is a .zx one or a legacy .zib one.
"""
newest = None
for query in ("/latest?bucket=0&format=zx", "/latest?bucket=0"):
try:
with open_url(origin.rstrip("/") + "/api/v2/packages/" + app_id + query,
timeout=30) as response:
encoded = response.read(65537)
except HTTPError as error:
if error.code == 404:
continue
raise
if len(encoded) > 65536:
raise ValueError(f"{app_id}: public release metadata is too large")
value = json.loads(encoded)
if newest is None or value["sequence"] > newest["sequence"]:
newest = value
return newest
def publish_node_apps(store, key_path, key_id, target, remote_store, origin, publishers_path, *,
ledger_path, bundle_directory, open_url=open_public_url):
"""Publishes every app the node offers whose ledger names a newer release.
Registering an app takes the node operator's key, which stays with the
operator, so an app the node does not list yet is reported and skipped.
"""
ledger = json.loads(ledger_path.read_text())
registered = registered_apps(origin, open_url)
for name, component in sorted(ledger.get("node_apps", {}).items()):
app_id = component["id"]
if app_id not in registered:
print(f"::warning title=App packages::{app_id} is not registered on the node, so it was not published")
continue
served = served_release(origin, app_id, open_url)
if served is not None and served["sequence"] >= component["sequence"]:
print(f"{app_id} {served['version']} is already published")
continue
value = stage_node_app(store, name, key_path, key_id, publishers_path,
ledger_path=ledger_path, bundle_directory=bundle_directory)
publish_node_app(store, name, target, remote_store, value["app_id"])
verify_node_app(store, name, origin, open_url, app_id=value["app_id"])
def add_node_app_arguments(parser):
parser.add_argument("--node-app", help="stage, and with --ssh-target publish, only this app the node offers")
parser.add_argument("--node-apps", action="store_true",
help="publish every app the node offers whose ledger names a newer release")
parser.add_argument("--app-ledger", type=Path, help="owning project's node-app version ledger")
parser.add_argument("--bundle-directory", type=Path, help="owning project's built node-app bundles")
parser.add_argument("--beta", action="store_true",
help="with --node-app: publish it as beta, listed only where beta apps are shown")
def publish_from_arguments(args, parser, project):
"""Runs --node-app or --node-apps for the project; False when neither is given."""
publishers = args.publishers or project / "apps/publishers.json"
ledger = args.app_ledger or project / "apps/versions.json"
bundles = args.bundle_directory or project / "build/node-apps"
rollout = getattr(args, "rollout", None)
if args.node_apps:
if not (args.key and args.key_id and args.ssh_target and args.remote_store and args.verify_origin) or \
rollout is not None:
parser.error("--node-apps needs --key, --key-id, --ssh-target, --remote-store and "
"--verify-origin, without --rollout")
publish_node_apps(args.store, args.key, args.key_id, args.ssh_target, args.remote_store,
args.verify_origin, publishers, ledger_path=ledger, bundle_directory=bundles)
return True
if args.node_app:
if not args.key or not args.key_id or rollout is not None:
parser.error("--node-app needs --key and --key-id, without --rollout")
value = stage_node_app(args.store, args.node_app, args.key, args.key_id, publishers, args.beta,
ledger_path=ledger, bundle_directory=bundles)
if args.ssh_target:
if not args.remote_store or not args.verify_origin:
parser.error("publishing an app the node offers needs --remote-store and --verify-origin")
publish_node_app(args.store, args.node_app, args.ssh_target, args.remote_store, value["app_id"])
verify_node_app(args.store, args.node_app, args.verify_origin, app_id=value["app_id"])
print(f"{args.node_app} release " + ("published" if args.ssh_target else "staged"))
return True
return False
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--store", type=Path, required=True)
parser.add_argument("--key", type=Path)
parser.add_argument("--key-id")
parser.add_argument("--ssh-target")
parser.add_argument("--remote-store")
parser.add_argument("--verify-origin", help="verify served release metadata and bytes after publishing")
parser.add_argument("--publishers", type=Path, help="publisher pins the signing key must match")
add_node_app_arguments(parser)
args = parser.parse_args()
try:
if not publish_from_arguments(args, parser, Path.cwd()):
parser.error("give --node-app NAME or --node-apps")
except (ValueError, KeyError, OSError, subprocess.CalledProcessError) as error:
parser.exit(1, f"Package publication failed: {error}\n")
if __name__ == "__main__":
main()