Produce and verify source-free QR Code ZX libraries

This commit is contained in:
waozi committed 2026-11-20 19:09:59 -03:00
1 parent ddbff0676d
commit 70676e3ad4
7 files changed
+311 -11

No files matched your search

+39
View File
@@ -0,0 +1,39 @@
name: ZX library
on:
push:
branches: [master]
workflow_dispatch:
permissions:
contents: read
jobs:
library:
runs-on: build-native-ziran
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Run source and source-free tests
run: make check check-binary-production ZX_REVISION="$GITHUB_SHA"
- uses: actions/upload-artifact@v4
with:
name: qrcode-zx-library-${{ github.sha }}
path: build/zx-library/default.zx
if-no-files-found: error
- name: Publish the verified library to Gitea Packages
if: github.server_url == 'https://git.waozi.xyz' && github.ref == 'refs/heads/master'
env:
PACKAGE_USER: ${{ secrets.ZIRAN_TOOLCHAIN_PACKAGE_USER }}
PACKAGE_TOKEN: ${{ secrets.ZIRAN_TOOLCHAIN_PACKAGE_TOKEN }}
run: |
if [ -z "$PACKAGE_USER" ] || [ -z "$PACKAGE_TOKEN" ]; then
echo "Gitea package writer is unavailable. The library is in this run's artifacts."
exit 0
fi
umask 077
ZIRAN_PACKAGE_NETRC=$(mktemp "$RUNNER_TEMP/ziran-netrc.XXXXXX")
export ZIRAN_PACKAGE_NETRC
trap 'rm -f "$ZIRAN_PACKAGE_NETRC"' EXIT
printf 'machine git.waozi.xyz login %s password %s\n' "$PACKAGE_USER" "$PACKAGE_TOKEN" > "$ZIRAN_PACKAGE_NETRC"
make publish-zx ZX_REVISION="$GITHUB_SHA"
+32 -7
View File
@@ -1,24 +1,49 @@
.DEFAULT_GOAL := check
ZIRAN ?= ziran
LOCKED := $(if $(wildcard ziran.local.toml),,--locked)
CC ?= cc
BUILD := build
ZX_REPOSITORY := https://git.waozi.xyz/ziranlang/qrcode.git
ZX_REVISION ?= $(shell git rev-parse HEAD)
ZX_PROFILE := default
ZX_LIBRARY := $(BUILD)/zx-library/$(ZX_PROFILE).zx
.PHONY: check test clean
.PHONY: check test clean zx-library check-binary check-binary-production publish-zx
# Checks the package, then runs the tests as C and as a portable bundle.
check: test
test:
$(ZIRAN) check --project
$(ZIRAN) build --project --target=c --entry qrcode_test:main \
$(ZIRAN) check --project $(LOCKED)
$(ZIRAN) build --project $(LOCKED) --target=c --entry qrcode_test:main \
-o $(BUILD)/test-c tests/qrcode_test.zi
$(CC) -std=c99 -pedantic-errors -I$$($(ZIRAN) pkg path ziran)/include \
$(CC) -std=c99 -pedantic-errors -I$$($(ZIRAN) pkg path ziran $(LOCKED))/include \
-I$(BUILD)/test-c $$(find $(BUILD)/test-c -name '*.c') -o $(BUILD)/qrcode-test
$(BUILD)/qrcode-test
$(ZIRAN) bundle --project --entry qrcode_test:main \
-o $(BUILD)/qrcode-test.zib tests/qrcode_test.zi
test "$$($(ZIRAN) run --project $(BUILD)/qrcode-test.zib)" = 0
$(ZIRAN) bundle --project $(LOCKED) --entry qrcode_test:main \
-o $(BUILD)/qrcode-test.zx tests/qrcode_test.zi
test "$$($(ZIRAN) run --project $(LOCKED) $(BUILD)/qrcode-test.zx)" = 0
zx-library:
mkdir -p $(dir $(ZX_LIBRARY))
toolchain=$$($(ZIRAN) pkg path ziran --locked) && \
"$$toolchain/build/bin/ziran" pkg zx pack-ir $(ZX_REPOSITORY) $(ZX_REVISION) $(ZX_PROFILE) $(ZX_LIBRARY) --locked
check-binary:
mkdir -p $(dir $(ZX_LIBRARY))
python3 tests/check_binary.py dev "$(ZIRAN)" $(ZX_REPOSITORY) $(ZX_REVISION) $(ZX_PROFILE) \
$(ZX_LIBRARY) qrcode_test:main tests/qrcode_test.zi
check-binary-production: zx-library
python3 tests/check_binary.py production "$(ZIRAN)" $(ZX_REPOSITORY) $(ZX_REVISION) $(ZX_PROFILE) \
$(ZX_LIBRARY) qrcode_test:main tests/qrcode_test.zi
publish-zx:
test -n "$$ZIRAN_PACKAGE_NETRC"
toolchain=$$($(ZIRAN) pkg path ziran --locked --offline) && \
"$$toolchain/build/bin/ziran" pkg zx publish-ir $(ZX_REPOSITORY) $(ZX_REVISION) $(ZX_PROFILE) $(ZX_LIBRARY) \
--netrc "$$ZIRAN_PACKAGE_NETRC"
clean:
rm -rf $(BUILD)
+30
View File
@@ -45,3 +45,33 @@ on a small thread stack.
```sh
make
```
## ZX library
`make check-binary` packs a development library from the working tree and
runs the same matrix hashes and input validation cases in native C and the ZX
engine. The isolated consumer imports `qrcode/qrcode` with an empty source
cache. An ignored `ziran.local.toml` toolchain override can select a local
compiler for this test.
The `default` profile sets no feature defines. It supports the portable ZX
engine and uses no package-specific native libraries or host bindings. The
producer and consumer tests run on Linux.
The Gitea workflow runs `make check check-binary-production` before publishing
`default.zx` and its descriptor `default.zx.tsv` under the full Git revision
in the `qrcode-zx-library` Generic Package Registry. The production target packs
the clean committed revision with the exact toolchain pinned in `ziran.lock`
and verifies registry discovery before running the offline consumer.
Use the full published revision when adding the dependency. Ziran records the
library profile, digest and toolchain identity in the consumer lock. The import
and types in the example above are the same with either dependency source or a
ZX library. The compiler can use the library's checked modules during application
lowering.
```sh
ziran add https://git.waozi.xyz/ziranlang/qrcode.git --ref "$commit"
ziran bundle --project --locked --entry app:main -o app.zx app.zi
ziran bundle --project --locked --offline --entry app:main -o app.zx app.zi
```
+206
View File
@@ -0,0 +1,206 @@
"""Run the package's reference suite with only its ZX library available."""
import hashlib
from http.server import BaseHTTPRequestHandler, HTTPServer
import json
import os
from pathlib import Path
import re
import shlex
import shutil
import ssl
import subprocess
import sys
from tempfile import TemporaryDirectory
from threading import Thread
from urllib.parse import urlsplit
def package_id(url, revision):
return "p" + hashlib.sha256(f"{url}\n{revision}".encode()).hexdigest()[:16]
def discover_library(root, ziran, env, library, descriptor, repository, revision, profile):
url = urlsplit(repository)
owner, package = url.path.strip("/").removesuffix(".git").split("/")
artifact = f"/api/packages/{owner}/generic/{package}-zx-library/{revision}/{profile}.zx"
requests = []
class Registry(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_GET(self):
requests.append(self.path)
if self.path == artifact + ".tsv":
data = descriptor.encode()
self.send_response(200)
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
elif self.path == artifact:
with library.open("rb") as source:
self.send_response(200)
self.send_header("Content-Length", str(library.stat().st_size))
self.end_headers()
shutil.copyfileobj(source, self.wfile)
else:
self.send_error(404)
cert, key = root / "cert.pem", root / "key.pem"
result = subprocess.run(
["openssl", "req", "-new", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1",
"-subj", f"/CN={url.hostname}", "-addext", f"subjectAltName=DNS:{url.hostname}",
"-keyout", key, "-out", cert], capture_output=True, text=True,
)
if result.returncode:
raise SystemExit(result.stderr)
tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
tls.load_cert_chain(cert, key)
# Keep the canonical URL and certificate name for discovery and cache
# identity. Only the test registry's transport is redirected to loopback.
curl = shutil.which("curl")
if curl is None:
raise SystemExit("curl is required for the ZX library verification")
tools = root / "tools"
tools.mkdir()
proxy = tools / "curl"
proxy.write_text(
'#!/bin/sh\nexec "$ZX_TEST_CURL" --disable --connect-to "$ZX_TEST_ROUTE" "$@"\n'
)
proxy.chmod(0o700)
with HTTPServer(("127.0.0.1", 0), Registry) as server:
server.socket = tls.wrap_socket(server.socket, server_side=True)
env.update(
ZIRAN_ZX_REGISTRY=f"{url.scheme}://{url.netloc}",
ZIRAN_ZX_PROFILE=profile, CURL_CA_BUNDLE=str(cert),
NO_PROXY=url.hostname, no_proxy=url.hostname,
PATH=f"{tools}:{os.environ['PATH']}", ZX_TEST_CURL=curl,
ZX_TEST_ROUTE=f"{url.hostname}:{url.port or 443}:127.0.0.1:{server.server_port}",
)
thread = Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
subprocess.run([*ziran, "add", repository, "--ref", revision], cwd=root, env=env, check=True)
finally:
server.shutdown()
thread.join()
if requests != [artifact + ".tsv", artifact]:
raise SystemExit(f"Unexpected library registry requests {requests!r}")
def prepare_development(ziran, root, env, repository, revision, profile, identity, tool_identity):
dev = root / "libraries"
dev.mkdir()
artifact = dev / f"{identity}.zx"
producer_env = dict(os.environ)
producer_env.pop("ZIRAN_DEV_ZX_DIR", None)
subprocess.run(
[*ziran, "pkg", "zx", "pack-dev", repository, revision, profile, artifact, "--offline"],
env=producer_env, check=True,
)
descriptor = inspect_library(ziran, artifact)
if descriptor[:4] != ["dev", identity, tool_identity, profile] or not re.fullmatch(r"[0-9a-f]{64}", descriptor[4]):
raise SystemExit("Development library has an unexpected package, toolchain, profile or source fingerprint")
env["ZIRAN_DEV_ZX_DIR"] = str(dev)
def inspect_library(ziran, library):
with library.open("rb") as artifact:
digest = hashlib.file_digest(artifact, "sha256").hexdigest()
descriptor = subprocess.check_output(
[*ziran, "pkg", "zx", "inspect-ir", library], text=True
).rstrip("\n").split("\t")
if len(descriptor) != (6 if descriptor[0] == "dev" else 5) or descriptor[-1] != digest:
raise SystemExit("ZX library inspection did not match the artifact digest")
return descriptor
def main():
mode, tool, repository, revision, profile, filename, entry, *sources = sys.argv[1:]
if mode not in ("dev", "production"):
raise SystemExit(f"Unknown ZX library test mode {mode!r}")
production = mode == "production"
ziran = shlex.split(tool)
ziran[0] = str(Path(shutil.which(ziran[0]) or ziran[0]).absolute())
library = Path(filename).resolve()
package = repository.rsplit("/", 1)[1].removesuffix(".git")
app = f"{package}_binary_test"
pin = json.loads(Path("ziran.lock").read_text())["toolchain"]
toolchain = subprocess.check_output(
[*ziran, "pkg", "path", "ziran", "--offline", *(["--locked"] if production else [])], text=True
).strip()
ziran = [str(Path(toolchain) / "build/bin/ziran")]
identity = package_id(repository, revision)
tool_identity = package_id(pin["url"], pin["commit"])
with TemporaryDirectory(prefix="consumer-", dir=library.parent) as directory:
root = Path(directory).resolve()
cache = root / "cache"
hosts = root / "hosts"
hosts.mkdir()
env = {**os.environ, "XDG_CACHE_HOME": str(cache), "ZIRAN_HOST_PATH": str(hosts),
"GIT_ALLOW_PROTOCOL": ""}
for name in ("DISPLAY", "WAYLAND_DISPLAY", "CURL_SSL_BACKEND", "ZIRAN_DEV_ZX_DIR", "ZIRAN_LOCK_GIT_OBJECTS",
"ZIRAN_PACKAGE_MAP", "ZIRAN_PACKAGE_NETRC", "ZIRAN_PINNED_LAUNCHER", "ZIRAN_STD"):
env.pop(name, None)
for source in map(Path, sources):
shutil.copyfile(source, root / source.name)
manifest = (
f'[package]\nname = "{app}"\nmodule_roots = ["."]\n'
f'[toolchain]\ngit = "{pin["url"]}"\nref = "{pin["ref"]}"\n'
)
# Reuse the exact compiler and standard library without copying a
# toolchain checkout into the isolated dependency cache.
(root / "ziran.local.toml").write_text(f"[overrides]\nziran = {json.dumps(toolchain)}\n")
lock_path = root / "ziran.lock"
dependency = {"id": identity, "name": package, "url": repository,
"ref": revision, "commit": revision, "dependencies": {}}
lock = {"format": 1, "root": {"name": app, "dependencies": {package: identity}},
"toolchain": pin, "packages": [dependency]}
if production:
descriptor = inspect_library(ziran, library)
if descriptor[:4] != ["2", identity, tool_identity, profile]:
raise SystemExit("Production library has an unexpected package, toolchain or profile")
(root / "ziran.toml").write_text(manifest)
discover_library(root, ziran, env, library, "\t".join(descriptor) + "\n", repository, revision, profile)
dependency["checked"] = {"profile": profile, "sha256": descriptor[-1], "toolchain": tool_identity}
saved = lock_path.read_bytes()
if json.loads(saved) != lock:
raise SystemExit("Package discovery recorded an unexpected library pin")
subprocess.run([*ziran, "lock", "--offline"], cwd=root, env=env, check=True)
if lock_path.read_bytes() != saved:
raise SystemExit("Offline lock changed the verified library pin")
else:
(root / "ziran.toml").write_text(
manifest + f'[dependencies.{package}]\ngit = "{repository}"\nref = "{revision}"\n'
)
lock_path.write_text(json.dumps(lock))
prepare_development(ziran, root, env, repository, revision, profile, identity, tool_identity)
main_source = Path(sources[0]).name
project = ["--project", "--offline", "--entry", entry]
subprocess.run(
[*ziran, "bundle", *project, "-o", "consumer.zx", main_source],
cwd=root, env=env, check=True,
)
subprocess.run(
[*ziran, "build", *project, "--target=c", "--exe", "-o", "native", main_source],
cwd=root, env=env, check=True,
)
if (cache / "ziran" / "sources").exists():
raise SystemExit("The dependency source was materialized during the offline build")
output = subprocess.check_output(
[*ziran, "run", "--backend", "engine", "consumer.zx"], cwd=root, env=env, text=True
)
native = subprocess.check_output(
[root / "native" / Path(main_source).stem], cwd=root, env=env, text=True
)
print(output, end="")
if output.splitlines() != [*native.splitlines(), "0"]:
raise SystemExit("The native and ZX consumers did not agree")
print(f"{package} {profile} {mode} library passed native and ZX offline without sources or host bindings")
if __name__ == "__main__":
main()
+1 -1
View File
@@ -1,4 +1,4 @@
#import "qrcode"
#import "qrcode/qrcode"
using QrEcc;
+2 -2
View File
@@ -8,8 +8,8 @@
"name": "qrcode"
},
"toolchain": {
"commit": "55cc0ed26e94ea473d17215bb9eadae3c11a4e8d",
"ref": "55cc0ed26e94ea473d17215bb9eadae3c11a4e8d",
"commit": "5462fe2ebf2c3c42663249e47ef99d61f45ff39f",
"ref": "5462fe2ebf2c3c42663249e47ef99d61f45ff39f",
"url": "https://git.waozi.xyz/ziranlang/ziran.git"
}
}
+1 -1
View File
@@ -4,7 +4,7 @@ module_roots = ["src", "tests"]
[toolchain]
git = "https://git.waozi.xyz/ziranlang/ziran.git"
ref = "55cc0ed26e94ea473d17215bb9eadae3c11a4e8d"
ref = "5462fe2ebf2c3c42663249e47ef99d61f45ff39f"
[exports]
qrcode = "src/qrcode.zi"