Produce and verify source-free Noise ZX libraries

This commit is contained in:
waozi committed 2026-11-20 19:10:00 -03:00
1 parent 5569fe97b7
commit 329a66333e
7 files changed
+304 -5

No files matched your search

+39
View File
@@ -0,0 +1,39 @@
name: ZX library
on:
push:
branches: [master]
workflow_dispatch:
permissions:
contents: read
jobs:
library:
runs-on: build-native-ziran
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Run source and source-free tests
run: make test check-binary-production ZX_REVISION="$GITHUB_SHA"
- uses: actions/upload-artifact@v4
with:
name: noise-zx-library-${{ github.sha }}
path: build/zx-library/default.zx
if-no-files-found: error
- name: Publish the verified library to Gitea Packages
if: github.server_url == 'https://git.waozi.xyz' && github.ref == 'refs/heads/master'
env:
PACKAGE_USER: ${{ secrets.ZIRAN_TOOLCHAIN_PACKAGE_USER }}
PACKAGE_TOKEN: ${{ secrets.ZIRAN_TOOLCHAIN_PACKAGE_TOKEN }}
run: |
if [ -z "$PACKAGE_USER" ] || [ -z "$PACKAGE_TOKEN" ]; then
echo "Gitea package writer is unavailable. The library is in this run's artifacts."
exit 0
fi
umask 077
ZIRAN_PACKAGE_NETRC=$(mktemp "$RUNNER_TEMP/ziran-netrc.XXXXXX")
export ZIRAN_PACKAGE_NETRC
trap 'rm -f "$ZIRAN_PACKAGE_NETRC"' EXIT
printf 'machine git.waozi.xyz login %s password %s\n' "$PACKAGE_USER" "$PACKAGE_TOKEN" > "$ZIRAN_PACKAGE_NETRC"
make publish-zx ZX_REVISION="$GITHUB_SHA"
+26 -1
View File
@@ -1,6 +1,31 @@
ZIRAN ?= ziran
LOCKED := $(if $(wildcard ziran.local.toml),,--locked)
.PHONY: test
ZX_REPOSITORY := https://git.waozi.xyz/ziranlang/noise.git
ZX_REVISION ?= $(shell git rev-parse HEAD)
ZX_PROFILE := default
ZX_LIBRARY := build/zx-library/$(ZX_PROFILE).zx
.PHONY: test zx-library check-binary check-binary-production publish-zx
test:
$(ZIRAN) build --project $(LOCKED) --target=c --entry fast_noise_test:Main --exe -o build/tests tests/fast_noise_test.zi
env -u DISPLAY -u WAYLAND_DISPLAY build/tests/fast_noise_test
zx-library:
mkdir -p $(dir $(ZX_LIBRARY))
toolchain=$$($(ZIRAN) pkg path ziran --locked) && \
"$$toolchain/build/bin/ziran" pkg zx pack-ir $(ZX_REPOSITORY) $(ZX_REVISION) $(ZX_PROFILE) $(ZX_LIBRARY) --locked
check-binary:
mkdir -p $(dir $(ZX_LIBRARY))
python3 tests/check_binary.py dev "$(ZIRAN)" $(ZX_REPOSITORY) $(ZX_REVISION) $(ZX_PROFILE) \
$(ZX_LIBRARY) fast_noise_test:Main tests/fast_noise_test.zi tests/fast_noise_vectors.zi
check-binary-production: zx-library
python3 tests/check_binary.py production "$(ZIRAN)" $(ZX_REPOSITORY) $(ZX_REVISION) $(ZX_PROFILE) \
$(ZX_LIBRARY) fast_noise_test:Main tests/fast_noise_test.zi tests/fast_noise_vectors.zi
publish-zx:
test -n "$$ZIRAN_PACKAGE_NETRC"
toolchain=$$($(ZIRAN) pkg path ziran --locked --offline) && \
"$$toolchain/build/bin/ziran" pkg zx publish-ir $(ZX_REPOSITORY) $(ZX_REVISION) $(ZX_PROFILE) $(ZX_LIBRARY) \
--netrc "$$ZIRAN_PACKAGE_NETRC"
+29
View File
@@ -27,4 +27,33 @@ read the state, so one state can be shared between threads.
`make test` compares 2112 values captured from FastNoiseLite C.
## ZX library
`make check-binary` packs a development library from the working tree and
runs the same 2112 comparisons in native C and the ZX engine. The isolated
consumer imports `noise/fast_noise` with an empty source cache. An ignored
`ziran.local.toml` toolchain override can select a local compiler for this test.
The `default` profile sets no feature defines. It supports the portable ZX
engine and uses no package-specific native libraries or host bindings. The
producer and consumer tests run on Linux.
The Gitea workflow runs `make test check-binary-production` before publishing
`default.zx` and its descriptor `default.zx.tsv` under the full Git revision
in the `noise-zx-library` Generic Package Registry. The production target packs
the clean committed revision with the exact toolchain pinned in `ziran.lock`
and verifies registry discovery before running the offline consumer.
Use the full published revision when adding the dependency. Ziran records the
library profile, digest and toolchain identity in the consumer lock. The import
and types in the example above are the same with either dependency source or a
ZX library. The compiler can use the library's checked modules during application
lowering.
```sh
ziran add https://git.waozi.xyz/ziranlang/noise.git --ref "$commit"
ziran bundle --project --locked --entry app:main -o app.zx app.zi
ziran bundle --project --locked --offline --entry app:main -o app.zx app.zi
```
License: MIT, including the FastNoiseLite notice (see `LICENSE`).
+206
View File
@@ -0,0 +1,206 @@
"""Run the package's reference suite with only its ZX library available."""
import hashlib
from http.server import BaseHTTPRequestHandler, HTTPServer
import json
import os
from pathlib import Path
import re
import shlex
import shutil
import ssl
import subprocess
import sys
from tempfile import TemporaryDirectory
from threading import Thread
from urllib.parse import urlsplit
def package_id(url, revision):
return "p" + hashlib.sha256(f"{url}\n{revision}".encode()).hexdigest()[:16]
def discover_library(root, ziran, env, library, descriptor, repository, revision, profile):
url = urlsplit(repository)
owner, package = url.path.strip("/").removesuffix(".git").split("/")
artifact = f"/api/packages/{owner}/generic/{package}-zx-library/{revision}/{profile}.zx"
requests = []
class Registry(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_GET(self):
requests.append(self.path)
if self.path == artifact + ".tsv":
data = descriptor.encode()
self.send_response(200)
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
elif self.path == artifact:
with library.open("rb") as source:
self.send_response(200)
self.send_header("Content-Length", str(library.stat().st_size))
self.end_headers()
shutil.copyfileobj(source, self.wfile)
else:
self.send_error(404)
cert, key = root / "cert.pem", root / "key.pem"
result = subprocess.run(
["openssl", "req", "-new", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1",
"-subj", f"/CN={url.hostname}", "-addext", f"subjectAltName=DNS:{url.hostname}",
"-keyout", key, "-out", cert], capture_output=True, text=True,
)
if result.returncode:
raise SystemExit(result.stderr)
tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
tls.load_cert_chain(cert, key)
# Keep the canonical URL and certificate name for discovery and cache
# identity. Only the test registry's transport is redirected to loopback.
curl = shutil.which("curl")
if curl is None:
raise SystemExit("curl is required for the ZX library verification")
tools = root / "tools"
tools.mkdir()
proxy = tools / "curl"
proxy.write_text(
'#!/bin/sh\nexec "$ZX_TEST_CURL" --disable --connect-to "$ZX_TEST_ROUTE" "$@"\n'
)
proxy.chmod(0o700)
with HTTPServer(("127.0.0.1", 0), Registry) as server:
server.socket = tls.wrap_socket(server.socket, server_side=True)
env.update(
ZIRAN_ZX_REGISTRY=f"{url.scheme}://{url.netloc}",
ZIRAN_ZX_PROFILE=profile, CURL_CA_BUNDLE=str(cert),
NO_PROXY=url.hostname, no_proxy=url.hostname,
PATH=f"{tools}:{os.environ['PATH']}", ZX_TEST_CURL=curl,
ZX_TEST_ROUTE=f"{url.hostname}:{url.port or 443}:127.0.0.1:{server.server_port}",
)
thread = Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
subprocess.run([*ziran, "add", repository, "--ref", revision], cwd=root, env=env, check=True)
finally:
server.shutdown()
thread.join()
if requests != [artifact + ".tsv", artifact]:
raise SystemExit(f"Unexpected library registry requests {requests!r}")
def prepare_development(ziran, root, env, repository, revision, profile, identity, tool_identity):
dev = root / "libraries"
dev.mkdir()
artifact = dev / f"{identity}.zx"
producer_env = dict(os.environ)
producer_env.pop("ZIRAN_DEV_ZX_DIR", None)
subprocess.run(
[*ziran, "pkg", "zx", "pack-dev", repository, revision, profile, artifact, "--offline"],
env=producer_env, check=True,
)
descriptor = inspect_library(ziran, artifact)
if descriptor[:4] != ["dev", identity, tool_identity, profile] or not re.fullmatch(r"[0-9a-f]{64}", descriptor[4]):
raise SystemExit("Development library has an unexpected package, toolchain, profile or source fingerprint")
env["ZIRAN_DEV_ZX_DIR"] = str(dev)
def inspect_library(ziran, library):
with library.open("rb") as artifact:
digest = hashlib.file_digest(artifact, "sha256").hexdigest()
descriptor = subprocess.check_output(
[*ziran, "pkg", "zx", "inspect-ir", library], text=True
).rstrip("\n").split("\t")
if len(descriptor) != (6 if descriptor[0] == "dev" else 5) or descriptor[-1] != digest:
raise SystemExit("ZX library inspection did not match the artifact digest")
return descriptor
def main():
mode, tool, repository, revision, profile, filename, entry, *sources = sys.argv[1:]
if mode not in ("dev", "production"):
raise SystemExit(f"Unknown ZX library test mode {mode!r}")
production = mode == "production"
ziran = shlex.split(tool)
ziran[0] = str(Path(shutil.which(ziran[0]) or ziran[0]).absolute())
library = Path(filename).resolve()
package = repository.rsplit("/", 1)[1].removesuffix(".git")
app = f"{package}_binary_test"
pin = json.loads(Path("ziran.lock").read_text())["toolchain"]
toolchain = subprocess.check_output(
[*ziran, "pkg", "path", "ziran", "--offline", *(["--locked"] if production else [])], text=True
).strip()
ziran = [str(Path(toolchain) / "build/bin/ziran")]
identity = package_id(repository, revision)
tool_identity = package_id(pin["url"], pin["commit"])
with TemporaryDirectory(prefix="consumer-", dir=library.parent) as directory:
root = Path(directory).resolve()
cache = root / "cache"
hosts = root / "hosts"
hosts.mkdir()
env = {**os.environ, "XDG_CACHE_HOME": str(cache), "ZIRAN_HOST_PATH": str(hosts),
"GIT_ALLOW_PROTOCOL": ""}
for name in ("DISPLAY", "WAYLAND_DISPLAY", "CURL_SSL_BACKEND", "ZIRAN_DEV_ZX_DIR", "ZIRAN_LOCK_GIT_OBJECTS",
"ZIRAN_PACKAGE_MAP", "ZIRAN_PACKAGE_NETRC", "ZIRAN_PINNED_LAUNCHER", "ZIRAN_STD"):
env.pop(name, None)
for source in map(Path, sources):
shutil.copyfile(source, root / source.name)
manifest = (
f'[package]\nname = "{app}"\nmodule_roots = ["."]\n'
f'[toolchain]\ngit = "{pin["url"]}"\nref = "{pin["ref"]}"\n'
)
# Reuse the exact compiler and standard library without copying a
# toolchain checkout into the isolated dependency cache.
(root / "ziran.local.toml").write_text(f"[overrides]\nziran = {json.dumps(toolchain)}\n")
lock_path = root / "ziran.lock"
dependency = {"id": identity, "name": package, "url": repository,
"ref": revision, "commit": revision, "dependencies": {}}
lock = {"format": 1, "root": {"name": app, "dependencies": {package: identity}},
"toolchain": pin, "packages": [dependency]}
if production:
descriptor = inspect_library(ziran, library)
if descriptor[:4] != ["2", identity, tool_identity, profile]:
raise SystemExit("Production library has an unexpected package, toolchain or profile")
(root / "ziran.toml").write_text(manifest)
discover_library(root, ziran, env, library, "\t".join(descriptor) + "\n", repository, revision, profile)
dependency["checked"] = {"profile": profile, "sha256": descriptor[-1], "toolchain": tool_identity}
saved = lock_path.read_bytes()
if json.loads(saved) != lock:
raise SystemExit("Package discovery recorded an unexpected library pin")
subprocess.run([*ziran, "lock", "--offline"], cwd=root, env=env, check=True)
if lock_path.read_bytes() != saved:
raise SystemExit("Offline lock changed the verified library pin")
else:
(root / "ziran.toml").write_text(
manifest + f'[dependencies.{package}]\ngit = "{repository}"\nref = "{revision}"\n'
)
lock_path.write_text(json.dumps(lock))
prepare_development(ziran, root, env, repository, revision, profile, identity, tool_identity)
main_source = Path(sources[0]).name
project = ["--project", "--offline", "--entry", entry]
subprocess.run(
[*ziran, "bundle", *project, "-o", "consumer.zx", main_source],
cwd=root, env=env, check=True,
)
subprocess.run(
[*ziran, "build", *project, "--target=c", "--exe", "-o", "native", main_source],
cwd=root, env=env, check=True,
)
if (cache / "ziran" / "sources").exists():
raise SystemExit("The dependency source was materialized during the offline build")
output = subprocess.check_output(
[*ziran, "run", "--backend", "engine", "consumer.zx"], cwd=root, env=env, text=True
)
native = subprocess.check_output(
[root / "native" / Path(main_source).stem], cwd=root, env=env, text=True
)
print(output, end="")
if output.splitlines() != [*native.splitlines(), "0"]:
raise SystemExit("The native and ZX consumers did not agree")
print(f"{package} {profile} {mode} library passed native and ZX offline without sources or host bindings")
if __name__ == "__main__":
main()
+1 -1
View File
@@ -2,7 +2,7 @@
// OpenSimplex2S FBm (octaves 1-3, lacunarity 1 and 2, gain 0.5, 0.75 and 1),
// Ridged and PingPong, plus OpenSimplex2, Perlin, ValueCubic, Value, weighted
// strength, both 3D plane rotations and seed wraparound at 2^31 - 1.
FastNoise :: #import "fast_noise";
FastNoise :: #import "noise/fast_noise";
Vectors :: #import "fast_noise_vectors";
TOLERANCE :: 0.000001;
+2 -2
View File
@@ -8,8 +8,8 @@
"name": "noise"
},
"toolchain": {
"commit": "6dfa969c4764745cbfcbfbbf1153fb02cb9f1fa1",
"ref": "6dfa969c4764745cbfcbfbbf1153fb02cb9f1fa1",
"commit": "5462fe2ebf2c3c42663249e47ef99d61f45ff39f",
"ref": "5462fe2ebf2c3c42663249e47ef99d61f45ff39f",
"url": "https://git.waozi.xyz/ziranlang/ziran.git"
}
}
+1 -1
View File
@@ -4,7 +4,7 @@ module_roots = ["src", "tests"]
[toolchain]
git = "https://git.waozi.xyz/ziranlang/ziran.git"
ref = "6dfa969c4764745cbfcbfbbf1153fb02cb9f1fa1"
ref = "5462fe2ebf2c3c42663249e47ef99d61f45ff39f"
[exports]
fast_noise = "src/fast_noise.zi"