Build and verify versioned ZX library packages
Build check / check (push) Canceled after 6m6s

This commit is contained in:
waozi committed 2026-11-20 17:04:13 -03:00
1 parent 9a22b19d74
commit fc11f5cefb
16 files changed
+896 -177

No files matched your search

+35
View File
@@ -52,6 +52,41 @@ jobs:
run: make
- name: Run the language, backend, and portable runtime checks
run: env -u DISPLAY -u WAYLAND_DISPLAY WINDOWS_REQUIRED=1 ARM_REQUIRED=1 WASM_REQUIRED=1 ZIRAN_WASM_CC=clang-18 make check CHECK_JOBS=2
- name: Pack the tested Ziran toolchain
id: toolchain
if: github.server_url == 'https://git.waozi.xyz' && github.event_name == 'push' && github.ref == 'refs/heads/master' && (env.BUILD_NATIVE != 'true' || steps.native.outputs.current == 'true')
env:
SOURCE: ${{ steps.native.outputs.source || github.workspace }}
run: |
output="build/toolchains/$GITHUB_SHA"
if [ "$BUILD_NATIVE" = true ]; then
gitea-native-build run python3 scripts/toolchain_package.py pack "$GITHUB_SHA" "$output"
else
python3 scripts/toolchain_package.py pack "$GITHUB_SHA" "$output"
fi
printf 'directory=%s/%s\n' "$SOURCE" "$output" >> "$GITHUB_OUTPUT"
- name: Keep the verified toolchain archive
if: steps.toolchain.outputs.directory != ''
uses: actions/upload-artifact@v4
with:
name: toolchain-${{ github.sha }}
path: ${{ steps.toolchain.outputs.directory }}
if-no-files-found: error
- name: Publish the toolchain to Gitea Packages
if: steps.toolchain.outputs.directory != ''
env:
SOURCE: ${{ steps.native.outputs.source || github.workspace }}
OUTPUT: ${{ steps.toolchain.outputs.directory }}
ZIRAN_TOOLCHAIN_PACKAGE_USER: ${{ secrets.ZIRAN_TOOLCHAIN_PACKAGE_USER }}
ZIRAN_TOOLCHAIN_PACKAGE_TOKEN: ${{ secrets.ZIRAN_TOOLCHAIN_PACKAGE_TOKEN }}
run: |
if [ -z "$ZIRAN_TOOLCHAIN_PACKAGE_USER$ZIRAN_TOOLCHAIN_PACKAGE_TOKEN" ]; then
echo "Gitea package writer is unavailable. The toolchain is in this run's artifacts."
exit 0
fi
for descriptor in "$OUTPUT"/*.json; do
python3 "$SOURCE/scripts/toolchain_package.py" publish "$GITHUB_SHA" "$descriptor"
done
- name: Show how much the compile cache saved
if: always()
run: ccache --show-stats || true
+27
View File
@@ -0,0 +1,27 @@
name: Quick Ziran build
on:
workflow_dispatch:
permissions:
contents: read
jobs:
quick:
runs-on: build-ziran
timeout-minutes: 15
steps:
- uses: https://github.com/actions/checkout@v4
with:
persist-credentials: false
- name: Prepare the C compiler cache
run: sh scripts/ci_packages.sh ccache
- uses: ./.github/actions/compile-cache
with:
key: check
- name: Compile and start Ziran
run: |
make JOBS=2 build/bin/ziran
build/bin/ziran --version
- name: Report compile cache savings
if: always()
run: ccache --show-stats
+35
View File
@@ -47,6 +47,41 @@ jobs:
run: make
- name: Run the language, backend, and portable runtime checks
run: env -u DISPLAY -u WAYLAND_DISPLAY WINDOWS_REQUIRED=1 ARM_REQUIRED=1 WASM_REQUIRED=1 ZIRAN_WASM_CC=clang-18 make check CHECK_JOBS=4
- name: Pack the tested Ziran toolchain
id: toolchain
if: github.server_url == 'https://git.waozi.xyz' && github.event_name == 'push' && github.ref == 'refs/heads/master' && (env.BUILD_NATIVE != 'true' || steps.native.outputs.current == 'true')
env:
SOURCE: ${{ steps.native.outputs.source || github.workspace }}
run: |
output="build/toolchains/$GITHUB_SHA"
if [ "$BUILD_NATIVE" = true ]; then
gitea-native-build run python3 scripts/toolchain_package.py pack "$GITHUB_SHA" "$output"
else
python3 scripts/toolchain_package.py pack "$GITHUB_SHA" "$output"
fi
printf 'directory=%s/%s\n' "$SOURCE" "$output" >> "$GITHUB_OUTPUT"
- name: Keep the verified toolchain archive
if: steps.toolchain.outputs.directory != ''
uses: actions/upload-artifact@v4
with:
name: toolchain-${{ github.sha }}
path: ${{ steps.toolchain.outputs.directory }}
if-no-files-found: error
- name: Publish the toolchain to Gitea Packages
if: steps.toolchain.outputs.directory != ''
env:
SOURCE: ${{ steps.native.outputs.source || github.workspace }}
OUTPUT: ${{ steps.toolchain.outputs.directory }}
ZIRAN_TOOLCHAIN_PACKAGE_USER: ${{ secrets.ZIRAN_TOOLCHAIN_PACKAGE_USER }}
ZIRAN_TOOLCHAIN_PACKAGE_TOKEN: ${{ secrets.ZIRAN_TOOLCHAIN_PACKAGE_TOKEN }}
run: |
if [ -z "$ZIRAN_TOOLCHAIN_PACKAGE_USER$ZIRAN_TOOLCHAIN_PACKAGE_TOKEN" ]; then
echo "Gitea package writer is unavailable. The toolchain is in this run's artifacts."
exit 0
fi
for descriptor in "$OUTPUT"/*.json; do
python3 "$SOURCE/scripts/toolchain_package.py" publish "$GITHUB_SHA" "$descriptor"
done
- name: Show how much the compile cache saved
if: always()
run: ccache --show-stats || true
+2 -1
View File
@@ -1,9 +1,10 @@
# ZX executable instruction container — revision 28
This is the one `.zx` format. The format owner is `cmd/compiler_bundle.zi`;
This document specifies the `.zx` executable format. The format owner is `cmd/compiler_bundle.zi`;
`cmd/compiler_exec_link.zi` binds it to the deterministic LIR codec and
verifier (current payload schema 4). `zi2zx` writes it; `ziran run`, `zi2zx run`,
the embedding API (`include/ziran_host.h`), `zx2wasm`, and `zx2native` read it.
Checked packages use the [ZX library format](zx-library-format.md).
## Integers, limits and ordering
+45
View File
@@ -0,0 +1,45 @@
# ZX checked library format, revision 1
`ziran pkg zx pack-ir` writes `.zx` libraries for use by the package compiler.
The package driver verifies the library and supplies its checked modules to
the compiler by package identity. ZX runtime modules and applications use the
[executable format](zx-executable-format.md).
All integers are unsigned little-endian. The 16-byte header has four fields.
| Offset | Width | Field |
| --- | --- | --- |
| 0 | 4 | Bytes `ZXLB` |
| 4 | 4 | Format revision, `1` |
| 8 | 4 | Section count, 3 to 4,098 |
| 12 | 4 | Exact file length, up to 268,435,456 bytes |
Sections follow the header in increasing bytewise order by path. Each record
has a four-byte path length, a four-byte payload length, a 32-byte raw SHA-256
of the payload, and then the path and payload bytes. Records are contiguous.
Paths are unique, relative, and 1 to 1,024 bytes long. Paths cannot contain
empty, `.` or `..` components, control bytes below 32, a backslash or a colon.
Payloads are nonempty. The final payload ends at the declared file length.
The required `checked-package.tsv` section is the literal `2`, package
identity, toolchain identity, and profile, separated by tabs and terminated
by one newline. Both identities have the form `p` followed by 16 lowercase
hexadecimal digits. The profile is a registry path segment of up to 128
ASCII letters, digits, hyphens, underscores, or periods, excluding `.` and
`..`. The required `ziran.toml` section contains the provider manifest. All
remaining sections hold serialized checked modules with `.zir` paths derived
from the provider's module paths. At least one module is required.
The registry descriptor repeats the metadata fields and adds the full
lowercase hexadecimal SHA-256 of the entire `.zx` file as a fifth field.
The package driver matches these pins to the package and toolchain in the
resolved dependency graph. The compiler checks module identities against
that graph when it opens their checked payloads.
Development libraries use `ZXLD` as their magic. Their metadata starts with
`dev` and adds a fifth field containing a lowercase hexadecimal source
fingerprint. It covers the provider manifest and lock, scanned module and
export source bytes, and feature defines. The writer checks the fingerprint
again before installing the completed file. Production package and runtime
readers reject this magic. A local development reader must be selected
explicitly and still checks the package and toolchain identities.
+3
View File
@@ -2,6 +2,9 @@
## [Unreleased]
- Package tested native Ziran toolchains in Gitea by exact source revision and Linux host profile.
- Distribute checked packages as versioned `.zx` libraries for source-free imports in ZX and native builds.
- Publish and fetch verified `.zx` images and checked compiler companions through Gitea. Package locks can discover and pin compatible checked dependencies for source-free ZX builds.
- C embedders can open seekable ZX app images through `BundleOpenReadAt` without keeping a full input buffer. The callback is needed only during verification, and images with assets, module linkage, or additional attachments use the byte-based API.
+22 -4
View File
@@ -848,7 +848,9 @@ PackageMain :: (launcher: string, arguments: []string) -> s32 {
return 0
}
command: string = arguments[0]
pack_checked := command == "pkg" && arguments.count >= 3 && arguments[1] == "zx" && arguments[2] == "pack-ir"
package_zx := command == "pkg" && arguments.count >= 3 && arguments[1] == "zx"
pack_dev := package_zx && arguments[2] == "pack-dev"
pack_checked := pack_dev || (package_zx && arguments[2] == "pack-ir")
// A binary-only consumer need not have a project or check out its sources.
if !pack_checked && command == "pkg" && arguments.count >= 2 && arguments[1] == "zx" {
return PkgZx(launcher, arguments[2:])
@@ -928,6 +930,7 @@ PackageMain :: (launcher: string, arguments: []string) -> s32 {
project: bool = PkgHasArgument(flags, "--project") || implied
locked_mode: bool = PkgHasArgument(flags, "--locked")
offline: bool = PkgHasArgument(flags, "--offline")
if pack_dev && locked_mode { return PkgFail("pack-dev cannot be used with --locked") }
filtered := PkgStrings(arguments.count)
if filtered.count != arguments.count { return PkgFail("cannot allocate arguments") }
count: s32 = 0
@@ -967,12 +970,27 @@ PackageMain :: (launcher: string, arguments: []string) -> s32 {
}
selection := PkgCheckedSelection.{call = PkgSelectChecked, context = cast(*void)(*checked_pins)}
checked: *PkgCheckedSelection = null
if command == "bundle" || (command == "build" && PkgHasArgument(flags, "--target=zx")) ||
compile := command == "build" || command == "run" || command == "check"
target := PkgHasTarget(flags[1:])
if compile && lifecycle && !target { checked_pins.lifecycle_manifest = current }
if command == "bundle" || (compile && !target) ||
(compile && PkgHasArgument(flags, "--target=c")) ||
(command == "build" && PkgHasArgument(flags, "--target=zx")) ||
pack_checked || (command == "pkg" && count >= 1 && filtered[0] == "list") {
if !PkgReadCheckedPins(root, launcher, locked, *checked_pins) {
return PkgFail("invalid checked packages in ziran.zx.lock")
}
if checked_pins.count != 0 { checked = *selection }
if command == "bundle" || compile || pack_dev {
development := PkgGetenv("ZIRAN_DEV_ZX_DIR")
if development != "" {
if locked_mode { return PkgFail("ZIRAN_DEV_ZX_DIR cannot be used with --locked") }
checked_pins.development_directory = PkgCanonical(development)
if !DirectoryExists(checked_pins.development_directory) {
return PkgFail("ZIRAN_DEV_ZX_DIR must name a directory")
}
}
}
if checked_pins.count != 0 || checked_pins.development_directory != "" { checked = *selection }
}
if !PkgCheckoutGraph(root, locked, offline, locked_mode, current, checked) {
if pkg_failure_reported { return 2 }
@@ -1028,7 +1046,7 @@ PackageMain :: (launcher: string, arguments: []string) -> s32 {
if pack_checked {
directory := PkgToolDirectory(launcher, locked.tool_path, "zi2zir")
if directory == "" { return PkgFail("cannot find the pinned Ziran compiler") }
return PkgPackChecked(root, PkgJoin(directory, "/zi2zir"), locked, current, filtered[2:count])
return PkgPackChecked(root, PkgJoin(directory, "/zi2zir"), locked, current, filtered[2:count], pack_dev)
}
if command == "fetch" { return 0 }
if command == "compile-commands" {
+182 -48
View File
@@ -5,74 +5,192 @@
#import "package_manifest"
#import "package_map"
#import "package_script"
#import "binary"
#import "byte_text_linux"
#import "file_linux"
#import "mapped_file_linux"
#import "zip"
#import "zip_file_linux"
#scope_file
Metadata :: "checked-package.tsv"
Manifest :: "ziran.toml"
MaximumBytes :: 268435456
HeaderBytes :: 16
RecordBytes :: 40
MaximumNameBytes :: 1024
MaximumRecords :: 4098
ArchiveIdentity :: (id: string, tool_url: string, tool_commit: string, profile: string) -> string {
ArchiveIdentity :: (id: string, tool_url: string, tool_commit: string, profile: string,
source_digest: string) -> string {
tool := PkgLockId(tool_url, tool_commit, false)
return PkgJoin3(PkgJoin3("1\t", id, "\t"), PkgJoin3(tool, "\t", profile), "\n")
prefix := CheckedDescriptorPrefix(id, tool, profile)
if source_digest == "" { return PkgJoin(prefix, "\n") }
return PkgJoin3(PkgJoin("dev\t", prefix[2:]), "\t", PkgJoin(source_digest, "\n"))
}
ValidIdentity :: (identity: string, id: string, tool: string, profile: string, development: bool) -> bool {
prefix := ifx development then "dev\t" else "2\t"
if identity.count < prefix.count + 38 || identity[0:prefix.count] != prefix ||
identity[identity.count - 1:] != "\n" { return false }
fields := identity[prefix.count:identity.count - 1]
profile_end := cast(s64)fields.count
if development {
profile_end -= 65
if profile_end <= 36 || fields[profile_end:profile_end + 1] != "\t" ||
!PkgHex(fields[profile_end + 1:], 64) { return false }
}
if fields[17:18] != "\t" || fields[35:36] != "\t" ||
!PkgCheckedIdentity(fields[0:17]) || !PkgCheckedIdentity(fields[18:35]) { return false }
actual_profile := fields[36:profile_end]
return PkgRegistrySegment(actual_profile) && (id == "" || fields[0:17] == id) &&
(tool == "" || fields[18:35] == tool) &&
(profile == "" || actual_profile == profile)
}
SourceFingerprint :: (root: string, current: *PkgManifest, options: []string, modules: *PkgModuleSet) -> string {
modules.count = 0
if !PkgScanModules(root, current, modules) { return "" }
paths := PkgStrings(modules.count + current.export_count + 2)
if paths.count == 0 { return "" }
paths[0] = Manifest
paths[1] = "ziran.lock"
for index: 0..modules.count - 1 { paths[index + 2] = modules.items[index].path[root.count + 1:] }
for index: 0..current.export_count - 1 { paths[modules.count + 2 + index] = current.exports[index].value }
for index: 1..paths.count - 1 {
at := index
while at > 0 && PkgLexLess(paths[at], paths[at - 1]) {
previous := paths[at - 1]
paths[at - 1] = paths[at]
paths[at] = previous
at -= 1
}
}
digest := PkgDigest("ZX development provider source 1")
for option: options { PkgArtifactMix(*digest, option) }
previous := ""
for path: paths {
if path == previous { continue }
hash := PkgArtifactHashFile(PkgJoin3(root, "/", path))
if !PkgHex(hash, 64) { return "" }
PkgArtifactMix(*digest, path)
PkgArtifactMix(*digest, hash)
previous = path
}
return digest
}
StoreU32 :: (bytes: []u8, offset: s32, value: u32) {
for index: 0..3 { bytes[offset + index] = cast(u8)(value >> cast(u32)(index * 8)) }
}
WriteAll :: (file: FileHandle, bytes: []u8, offset: s64) -> bool {
written: s64 = 0
while written < cast(s64)bytes.count {
size := WriteAt(file, bytes[written:], offset + written)
if size <= 0 { return false }
written += size
}
return true
}
VerifyImageDigest :: (bytes: []u8, expected: string) -> bool {
if expected == "" { return true }
digest: [32]u8
if expected.count != 64 || !PkgSha256Bytes(bytes, digest[:]) {
return false
}
hex := "0123456789abcdef"
for index: 0..31 {
if expected[2 * index] != hex[cast(s32)(digest[index] >> cast(u8)4)] ||
expected[2 * index + 1] != hex[cast(s32)(digest[index] & cast(u8)15)] { return false }
}
return true
}
#scope_export
// The archive never supplies files whose kind or location is chosen by a ZIP
// extractor. Only stored module records and the two fixed metadata files are
// read. When staging, all outputs are regular files below a private directory.
PkgRegistrySegment :: (value: string) -> bool {
if value.count == 0 || value.count > 128 || value == "." || value == ".." { return false }
for ch: value {
if !(ch >= #char "a" && ch <= #char "z" || ch >= #char "A" && ch <= #char "Z" ||
ch >= #char "0" && ch <= #char "9" || ch == #char "-" || ch == #char "_" || ch == #char ".") {
return false
}
}
return true
}
CheckedDescriptorPrefix :: (id: string, tool: string, profile: string) -> string {
return PkgJoin3(PkgJoin3("2\t", id, "\t"), PkgJoin3(tool, "\t", profile), "")
}
// Staged sections are regular files in an empty private directory. The caller
// discards that directory if any section or the package identity fails verification.
PkgCheckedArchive :: (path: string, digest: string, stage: string, id: string,
tool_url: string, tool_commit: string, profile: string, identity: *string = null) -> bool {
tool_url: string, tool_commit: string, profile: string, identity: *string = null,
development: bool = false) -> bool {
mapped := MapPrivate(path)
if !mapped.valid { return false }
defer UnmapFile(mapped)
if mapped.length == 0 || mapped.length > MaximumBytes { return false }
if mapped.length < HeaderBytes || mapped.length > MaximumBytes { return false }
bytes := MappedBytes(mapped)
if digest != "" && PkgDigest(TextFromBytes(bytes)) != digest { return false }
scan := OpenZipDirectory(bytes)
if !scan.valid || scan.count < 3 || scan.count > 4098 { return false }
text := TextFromBytes(bytes)
limit := cast(s64)bytes.count
magic := ifx development then "ZXLD" else "ZXLB"
if text[0:4] != magic || ReadU32LE(text, 4) != 1 || ReadU32LE(text, 12) != cast(u32)limit ||
!VerifyImageDigest(bytes, digest) { return false }
count := ReadU32LE(text, 8)
if count < 3 || count > MaximumRecords { return false }
position: s64 = HeaderBytes
prior := ""
metadata := false
manifest := false
for index: 0..cast(s32)scan.count - 1 {
for index: 0..cast(s32)count - 1 {
unused index
named := NextZipEntry(bytes, scan)
if !named.valid || named.entry.method != 0 || named.entry.size == 0 ||
named.name_size > 1024 { return false }
scan = named.directory
name := TextFromBytes(bytes[named.name_start:named.name_start + named.name_size])
if limit - position < RecordBytes { return false }
name_size := cast(s64)ReadU32LE(text, cast(s32)position)
data_size := cast(s64)ReadU32LE(text, cast(s32)position + 4)
hash_at := position + 8
position += RecordBytes
if name_size < 1 || name_size > MaximumNameBytes || data_size < 1 ||
name_size + data_size > limit - position { return false }
name := text[position:position + name_size]
position += name_size
if !PkgArtifactPath(name) || (prior != "" && !PkgLexLess(prior, name)) { return false }
prior = name
data := bytes[named.entry.data_start:named.entry.data_start + named.entry.size]
if ZipCRC32(data) != named.entry.crc32 { return false }
hash: [32]u8
if !PkgSha256Bytes(bytes[position:position + data_size], hash[:]) { return false }
for part: 0..31 {
if hash[part] != bytes[hash_at + part] { return false }
}
data := text[position:position + data_size]
position += data_size
if name == Metadata {
if id != "" && TextFromBytes(data) != ArchiveIdentity(id, tool_url, tool_commit, profile) {
tool := ifx tool_commit == "" then "" else PkgLockId(tool_url, tool_commit, false)
if !ValidIdentity(data, id, tool, profile, development) {
return false
}
if identity != null {
if data.count > 512 { return false }
<<identity = PkgJoin("", TextFromBytes(data))
}
if identity != null { <<identity = PkgJoin("", data) }
metadata = true
} else if name == Manifest { manifest = true }
else if !PkgEndsWith(name, ".zir") { return false }
if stage != "" && name != Metadata {
file := PkgJoin3(stage, "/", name)
if !PkgMkdirs(ParentDirectory(file)) || !PkgWrite(file, TextFromBytes(data)) { return false }
if !PkgMkdirs(ParentDirectory(file)) || !PkgWrite(file, data) { return false }
}
}
return metadata && manifest && scan.cursor == scan.end
return metadata && manifest && position == limit
}
PkgCheckedPack :: (root: string, locked: *PkgLock, current: *PkgManifest,
compiler: string, repository: string, revision: string, profile: string, output: string,
options: []string) -> s32 {
options: []string, development: bool) -> s32 {
id := PkgLockId(repository, revision, false)
modules := cast(*PkgModuleSet)PkgAllocZero(cast(u64)size_of(PkgModuleSet))
if modules == null { return PkgFail("cannot allocate checked package modules") }
source_digest: string
if development {
source_digest = SourceFingerprint(root, current, options, modules)
if source_digest == "" { return PkgFail("cannot fingerprint development package sources") }
}
stage := PkgArtifactStage(PkgJoin(PkgCacheRoot(), "/zx"))
if stage == "" { return PkgFail("cannot stage checked package modules") }
defer PkgArtifactRemove(stage)
@@ -95,8 +213,8 @@ PkgCheckedPack :: (root: string, locked: *PkgLock, current: *PkgManifest,
values := string.[map]
if PkgRun(command, root, names[:], values[:]) != 0 { return 2 }
modules := cast(*PkgModuleSet)PkgAllocZero(cast(u64)size_of(PkgModuleSet))
if modules == null || !PkgScanModules(root, current, modules) { return PkgFail("cannot find checked package modules") }
modules.count = 0
if !PkgScanModules(root, current, modules) { return PkgFail("cannot find checked package modules") }
paths := PkgStrings(modules.count + 2)
sources := PkgStrings(modules.count + 2)
if paths.count == 0 || sources.count == 0 { return PkgFail("cannot allocate checked package entries") }
@@ -104,7 +222,7 @@ PkgCheckedPack :: (root: string, locked: *PkgLock, current: *PkgManifest,
sources[0] = PkgJoin3(root, "/", Manifest)
paths[1] = Metadata
sources[1] = PkgJoin3(stage, "/", Metadata)
if !PkgWrite(sources[1], ArchiveIdentity(id, locked.tool_url, locked.tool_commit, profile)) {
if !PkgWrite(sources[1], ArchiveIdentity(id, locked.tool_url, locked.tool_commit, profile, source_digest)) {
return PkgFail("cannot write checked package metadata")
}
count: s32 = 2
@@ -130,36 +248,52 @@ PkgCheckedPack :: (root: string, locked: *PkgLock, current: *PkgManifest,
at -= 1
}
}
temporary := PkgTemporarySibling(output, ".zir.zip")
if count > MaximumRecords { return PkgFail("too many checked package sections") }
temporary := PkgTemporarySibling(output, ".zx")
if temporary == "" { return PkgFail("cannot stage the checked package archive") }
defer RemoveFile(temporary)
file := OpenExclusivePrivate(temporary)
if !file.valid { return PkgFail("cannot create checked package archive") }
position: u64 = 0
entries := cast(*ZipFileRecord)PkgAlloc(cast(u64)count * cast(u64)size_of(ZipFileRecord))
valid := entries != null
position: s64 = HeaderBytes
valid := true
for index: 0..count - 1 {
if !valid { break }
name := paths[index]
if name.count > MaximumNameBytes { valid = false; break }
mapped := MapPrivate(sources[index])
if !mapped.valid || mapped.length == 0 || mapped.length > MaximumBytes - position {
record_size := RecordBytes + cast(s64)name.count
if !mapped.valid || mapped.length == 0 || mapped.length > MaximumBytes ||
record_size + cast(s64)mapped.length > MaximumBytes - position {
unused UnmapFile(mapped)
valid = false
break
}
entries[index] = WriteStoredZipFileEntry(file, position, paths[index], MappedBytes(mapped))
record: [RecordBytes + MaximumNameBytes]u8
StoreU32(record[:], 0, cast(u32)name.count)
StoreU32(record[:], 4, cast(u32)mapped.length)
data := MappedBytes(mapped)
valid = PkgSha256Bytes(data, record[8:RecordBytes]) &&
PkgCopy(record[:], RecordBytes, name) == record_size &&
WriteAll(file, record[0:record_size], position) &&
WriteAll(file, data, position + record_size)
position += record_size + cast(s64)mapped.length
unused UnmapFile(mapped)
position = StoredZipFileNext(entries[index])
valid = entries[index].valid && position < MaximumBytes
}
directory_start := position
for index: 0..count - 1 {
if !valid { break }
position = WriteStoredZipFileDirectoryEntry(file, position, paths[index], entries[index])
valid = position != 0 && position < MaximumBytes - 22
if valid {
header: [HeaderBytes]u8
magic := ifx development then "ZXLD" else "ZXLB"
unused PkgCopy(header[:], 0, magic)
StoreU32(header[:], 4, 1)
StoreU32(header[:], 8, cast(u32)count)
StoreU32(header[:], 12, cast(u32)position)
valid = WriteAll(file, header[:], 0) && SyncFile(file)
}
if valid { valid = FinishStoredZipFile(file, directory_start, position, cast(u32)count) && SyncFile(file) }
valid = CloseFile(file) && valid
if !valid || !PkgCheckedArchive(temporary, "", "", id, locked.tool_url, locked.tool_commit, profile) ||
!RenamePath(temporary, output) { return PkgFail("cannot finish checked package archive") }
if !valid || !PkgCheckedArchive(temporary, "", "", id, locked.tool_url, locked.tool_commit,
profile, null, development) { return PkgFail("cannot finish checked package archive") }
if development && SourceFingerprint(root, current, options, modules) != source_digest {
return PkgFail("development package sources changed during export")
}
if !RenamePath(temporary, output) { return PkgFail("cannot finish checked package archive") }
return 0
}
+5
View File
@@ -389,6 +389,11 @@ PkgShowToolFailure :: (output: string) {
else { PkgPrint(output, true); PkgPrint("\n", true) }
}
PkgSha256Bytes :: (bytes: []u8, digest: []u8) -> bool {
return bytes.count > 0 && digest.count == 32 &&
PkgSha256Raw(*bytes[0], cast(u64)bytes.count, *digest[0]) != null
}
PkgDigest :: (value: string) -> string {
digest: [32]u8
memory := PkgAlloc(cast(u64)value.count + 1)
+1 -1
View File
@@ -8,7 +8,7 @@ PkgExplanation :: struct {
}
#scope_file
DIAGNOSTIC_CODES: string = "assembly.codec\nassembly.container\nassembly.encode\nassembly.execute\nassembly.gap.cpu-feature\nassembly.gap.machine-host\nassembly.gap.machine-host-abi\nassembly.gap.machine-platform-service-unavailable\nassembly.gap.os-service\nassembly.gap.target-unimplemented\nassembly.inspect\nassembly.lower\nassembly.machine\nassembly.optimize\nassembly.output\nassembly.verify\ncheck.array_signature\ncheck.assembly\ncheck.assert\ncheck.cleanup\ncheck.compile_time\ncheck.constant\ncheck.constant_type\ncheck.consteval\ncheck.enum\ncheck.enum_scope\ncheck.file_scope\ncheck.foreign\ncheck.global\ncheck.if_case\ncheck.import\ncheck.inline\ncheck.jai_syntax\ncheck.record\ncheck.reserved_name\ncheck.signature\ncheck.slice_lifetime\ncheck.slice_signature\ncheck.slot_initializer\ncheck.slot_signature\ncheck.specialize\ncheck.storage\ncheck.template\ncheck.type\ncheck.type_application\ncheck.type_order\ncheck.using_scope\ncheck.vec_copy\ncommand.arguments\ncompiler.context\ncompiler.internal\ncompiler.memory\nemit.array_count\nemit.array_element\nemit.array_target\nemit.expression\nemit.loop_nesting\nemit.parallel\nformat.changed\nformat.input\nformat.output\ngpu.allocator\ngpu.device\ngpu.pointer\nhost.bind\nlaw.gate\nlaw.identity\nlaw.waiver\nlint.bool\nlint.cast\nlint.char\nmodule.ambiguous\nmodule.compile_cycle\nmodule.compile_depth\nmodule.duplicate\nmodule.extension\nmodule.identity\nmodule.input\nmodule.limit\nmodule.not_found\nmodule.parameter_conflict\nmodule.parameter_type\nmodule.parameter_unknown\nmodule.path\nmodule.root\npackage.ambiguous\npackage.command\npackage.identity\npackage.map\npackage.module\npackage.not_found\npackage.process\npackage.project\npackage.scope\npackage.tool\nparallel.effect\nparallel.leave\nparallel.memory\nparallel.nest\nparallel.region\nparse.caller_location\nparse.fatal\nparse.jai_char\nparse.jai_syntax\nparse.modifier\nparse.procedure_name\nparse.syntax\nparse.this\nproof.claim\nproof.graph\nproof.identity\nproof.syntax\nwasm.container\nwasm.gap.address\nwasm.gap.aggregate\nwasm.gap.allocate\nwasm.gap.call\nwasm.gap.compare\nwasm.gap.constant\nwasm.gap.control\nwasm.gap.convert\nwasm.gap.data-size\nwasm.gap.edge\nwasm.gap.entry\nwasm.gap.float-operator\nwasm.gap.float-width\nwasm.gap.foreign-service\nwasm.gap.frame\nwasm.gap.fuel\nwasm.gap.function\nwasm.gap.global\nwasm.gap.global-constant\nwasm.gap.global-layout\nwasm.gap.host\nwasm.gap.host-capability-abi\nwasm.gap.initializer\nwasm.gap.instruction\nwasm.gap.integer-width\nwasm.gap.load\nwasm.gap.offset\nwasm.gap.operand\nwasm.gap.operator\nwasm.gap.platform-service-abi\nwasm.gap.platform-service-unavailable\nwasm.gap.resize\nwasm.gap.result\nwasm.gap.signature\nwasm.gap.store\nwasm.gap.type\nwasm.input\nwasm.output\nwasm.verify\nzir.input\nzir.invalid\nzir.noncanonical\nzir.output\nzir.validation\nzir_c.body\nzir_c.entry\nzir_c.enum\nzir_c.global\nzir_c.record\nzir_c.toolchain\nzir_go.bind\nzir_go.body\nzir_go.callback\nzir_go.enum\nzir_go.exe\nzir_go.export\nzir_go.global\nzir_go.import\nzir_go.parameter\nzir_go.tests\nzir_go.type\nzir_go.union\nzx.assets\nzx.bind\nzx.call\nzx.capability\nzx.container\nzx.debug\nzx.entry\nzx.host_alias\nzx.input\nzx.module\nzx.module-export\nzx.module-import\nzx.module-run\nzx.native\nzx.native.pointers\nzx.native.relocation\nzx.output\nzx.runtime\nzx.state\nzx.statement\nzx.union\nzx.wasm\n";
DIAGNOSTIC_CODES: string = "assembly.codec\nassembly.container\nassembly.encode\nassembly.execute\nassembly.gap.cpu-feature\nassembly.gap.machine-host\nassembly.gap.machine-host-abi\nassembly.gap.machine-platform-service-unavailable\nassembly.gap.os-service\nassembly.gap.target-unimplemented\nassembly.inspect\nassembly.lower\nassembly.machine\nassembly.optimize\nassembly.output\nassembly.verify\ncheck.array_signature\ncheck.assembly\ncheck.assert\ncheck.cleanup\ncheck.compile_time\ncheck.constant\ncheck.constant_type\ncheck.consteval\ncheck.enum\ncheck.enum_scope\ncheck.file_scope\ncheck.foreign\ncheck.global\ncheck.if_case\ncheck.import\ncheck.inline\ncheck.jai_syntax\ncheck.record\ncheck.reserved_name\ncheck.signature\ncheck.slice_lifetime\ncheck.slice_signature\ncheck.slot_initializer\ncheck.slot_signature\ncheck.specialize\ncheck.storage\ncheck.template\ncheck.type\ncheck.type_application\ncheck.type_order\ncheck.using_scope\ncheck.vec_copy\ncommand.arguments\ncompiler.context\ncompiler.internal\ncompiler.memory\nemit.array_count\nemit.array_element\nemit.array_target\nemit.expression\nemit.loop_nesting\nemit.parallel\nformat.changed\nformat.input\nformat.output\ngpu.allocator\ngpu.device\ngpu.pointer\nhost.bind\nlaw.gate\nlaw.identity\nlaw.waiver\nlint.bool\nlint.cast\nlint.char\nmodule.ambiguous\nmodule.compile_cycle\nmodule.compile_depth\nmodule.duplicate\nmodule.extension\nmodule.identity\nmodule.input\nmodule.limit\nmodule.not_found\nmodule.parameter_conflict\nmodule.parameter_type\nmodule.parameter_unknown\nmodule.path\nmodule.root\npackage.ambiguous\npackage.command\npackage.identity\npackage.map\npackage.module\npackage.not_found\npackage.process\npackage.project\npackage.scope\npackage.tool\npackage.zx-lock\nparallel.effect\nparallel.leave\nparallel.memory\nparallel.nest\nparallel.region\nparse.caller_location\nparse.fatal\nparse.jai_char\nparse.jai_syntax\nparse.modifier\nparse.procedure_name\nparse.syntax\nparse.this\nproof.claim\nproof.graph\nproof.identity\nproof.syntax\nwasm.container\nwasm.gap.address\nwasm.gap.aggregate\nwasm.gap.allocate\nwasm.gap.call\nwasm.gap.compare\nwasm.gap.constant\nwasm.gap.control\nwasm.gap.convert\nwasm.gap.data-size\nwasm.gap.edge\nwasm.gap.entry\nwasm.gap.float-operator\nwasm.gap.float-width\nwasm.gap.foreign-service\nwasm.gap.frame\nwasm.gap.fuel\nwasm.gap.function\nwasm.gap.global\nwasm.gap.global-constant\nwasm.gap.global-layout\nwasm.gap.host\nwasm.gap.host-capability-abi\nwasm.gap.initializer\nwasm.gap.instruction\nwasm.gap.integer-width\nwasm.gap.load\nwasm.gap.offset\nwasm.gap.operand\nwasm.gap.operator\nwasm.gap.platform-service-abi\nwasm.gap.platform-service-unavailable\nwasm.gap.resize\nwasm.gap.result\nwasm.gap.signature\nwasm.gap.store\nwasm.gap.type\nwasm.input\nwasm.output\nwasm.verify\nzir.input\nzir.invalid\nzir.noncanonical\nzir.output\nzir.validation\nzir_c.body\nzir_c.entry\nzir_c.enum\nzir_c.global\nzir_c.record\nzir_c.toolchain\nzir_go.bind\nzir_go.body\nzir_go.callback\nzir_go.enum\nzir_go.exe\nzir_go.export\nzir_go.global\nzir_go.import\nzir_go.parameter\nzir_go.tests\nzir_go.type\nzir_go.union\nzx.assets\nzx.bind\nzx.call\nzx.capability\nzx.container\nzx.debug\nzx.entry\nzx.host_alias\nzx.input\nzx.module\nzx.module-export\nzx.module-import\nzx.module-run\nzx.native\nzx.native.pointers\nzx.native.relocation\nzx.output\nzx.runtime\nzx.state\nzx.statement\nzx.union\nzx.wasm\n";
PkgStartsWith :: (value: string, prefix: string) -> bool {
if prefix.count > value.count { return false }
+90 -64
View File
@@ -12,36 +12,25 @@
#scope_file
MaximumImageBytes :: 268435456
RegistrySegment :: (value: string) -> bool {
if value.count == 0 || value.count > 128 || value == "." || value == ".." { return false }
for ch: value {
if !(ch >= #char "a" && ch <= #char "z" || ch >= #char "A" && ch <= #char "Z" ||
ch >= #char "0" && ch <= #char "9" || ch == #char "-" || ch == #char "_" || ch == #char ".") {
return false
}
}
return true
}
ModuleName :: (value: string) -> bool {
if value.count > 512 { return false }
start: s64 = 0
for index: 0..cast(s64)value.count - 1 {
if value[index] == #char "/" {
if !RegistrySegment(value[start:index]) { return false }
if !PkgRegistrySegment(value[start:index]) { return false }
start = index + 1
}
}
return RegistrySegment(value[start:])
return PkgRegistrySegment(value[start:])
}
HexDigest :: (value: string) -> bool { return PkgHex(value, 64) }
// Accept the origin of a Gitea repository, not an arbitrary download URL.
// The commit and profile are separate checked path segments.
RegistryUrl :: (repository: string, revision: string, profile: string, extension: string = ".zx") -> string {
RegistryUrl :: (repository: string, revision: string, profile: string, checked: bool = false) -> string {
if repository.count > 576 || repository.count <= 8 || repository[0:8] != "https://" ||
!PkgValidCommit(revision) || !RegistrySegment(profile) { return "" }
!PkgValidCommit(revision) || !PkgRegistrySegment(profile) { return "" }
authority_end: s64 = 8
while authority_end < repository.count && repository[authority_end] != #char "/" { authority_end += 1 }
if authority_end == 8 || authority_end == repository.count { return "" }
@@ -67,11 +56,12 @@ RegistryUrl :: (repository: string, revision: string, profile: string, extension
owner := repository[authority_end + 1:owner_end]
repo := repository[owner_end + 1:]
if PkgEndsWith(repo, ".git") { repo = repo[0:repo.count - 4] }
if !RegistrySegment(owner) || !RegistrySegment(repo) { return "" }
if !PkgRegistrySegment(owner) || !PkgRegistrySegment(repo) { return "" }
namespace := PkgJoin3(repository[0:authority_end], "/api/packages/", owner)
package := PkgJoin3(namespace, "/generic/", repo)
version := PkgJoin3(package, "-zx/", revision)
return PkgJoin(PkgJoin3(version, "/", profile), extension)
channel := ifx checked then "-zx-library/" else "-zx/"
version := PkgJoin3(package, channel, revision)
return PkgJoin(PkgJoin3(version, "/", profile), ".zx")
}
// Validate the size before hashing or passing a remote image to the verifier.
@@ -92,8 +82,8 @@ ImageDigest :: (path: string) -> string {
}
}
ImageVerified :: (launcher: string, path: string, module: string) -> bool {
if PkgEndsWith(path, ".zir.zip") { return PkgCheckedArchive(path, "", "", "", "", "", "") }
ImageVerified :: (launcher: string, path: string, module: string, id: string = "", profile: string = "") -> bool {
if id != "" { return PkgCheckedArchive(path, "", "", id, "", "", profile) }
compiler := PkgJoin(PkgLauncherDirectory(launcher), "/zi2zx")
args := string.[compiler, "--emit", "linkage", "-o", "-", path]
result := PkgCapture(args[:], "")
@@ -139,7 +129,7 @@ TransferFailure :: (action: string, status: string) -> s32 {
}
CheckedDescriptorUrl :: (archive_url: string) -> string {
return PkgJoin(archive_url[0:archive_url.count - 4], ".tsv")
return PkgJoin(archive_url, ".tsv")
}
PublishFile :: (url: string, path: string, digest: string, netrc: string, stage: string) -> s32 {
@@ -157,15 +147,14 @@ PublishFile :: (url: string, path: string, digest: string, netrc: string, stage:
Publish :: (launcher: string, url: string, file: string, netrc: string,
id: string, profile: string) -> s32 {
extension := ifx PkgEndsWith(url, ".zir.zip") then ".zir.zip" else ".zx"
if !PkgEndsWith(file, extension) { return PkgFail(PkgJoin("expected an artifact ending in ", extension)) }
if !PkgEndsWith(file, ".zx") { return PkgFail("expected an artifact ending in .zx") }
source := PkgCanonical(file)
if source == "" { return PkgFail("cannot find the portable image") }
// Verify and send the same private copy even if a build replaces FILE.
stage := PkgArtifactStage(PkgJoin(PkgCacheRoot(), "/zx"))
if stage == "" { return PkgFail("cannot stage the portable image") }
defer PkgArtifactRemove(stage)
image := PkgJoin3(stage, "/upload", extension)
image := PkgJoin(stage, "/upload.zx")
digest := ImageDigest(source)
if !HexDigest(digest) { return PkgFail("expected a nonempty artifact within the size limit") }
empty: [0]string
@@ -178,15 +167,8 @@ Publish :: (launcher: string, url: string, file: string, netrc: string,
if !ImageVerified(launcher, image, "") { return PkgFail("cannot verify the portable image") }
} else {
identity: string
if !PkgCheckedArchive(image, "", "", "", "", "", "", *identity) {
return PkgFail("cannot verify the checked package archive")
}
prefix := PkgJoin3("1\t", id, "\t")
tool_end := prefix.count + 17
suffix := PkgJoin3("\t", profile, "\n")
if identity.count != tool_end + suffix.count || identity[0:prefix.count] != prefix ||
!PkgCheckedIdentity(identity[prefix.count:tool_end]) || identity[tool_end:] != suffix {
return PkgFail("archive does not match the declared repository revision or profile")
if !PkgCheckedArchive(image, "", "", id, "", "", profile, *identity) {
return PkgFail("checked package library failed format, revision or profile verification")
}
descriptor = PkgJoin3(identity[0:identity.count - 1], "\t", PkgJoin(digest, "\n"))
}
@@ -203,7 +185,7 @@ Publish :: (launcher: string, url: string, file: string, netrc: string,
}
Fetch :: (launcher: string, url: string, profile: string, digest: string,
module: string, netrc: string, offline: bool, output: *string) -> s32 {
module: string, netrc: string, offline: bool, output: *string, id: string = "") -> s32 {
if !HexDigest(digest) { return PkgFail("expected a full lowercase SHA-256 digest") }
cache := PkgJoin(PkgCacheRoot(), "/zx")
key := PkgDigest(PkgJoin3(url, PkgJoin("\n", digest), PkgJoin("\n", module)))
@@ -211,10 +193,9 @@ Fetch :: (launcher: string, url: string, profile: string, digest: string,
root := PkgJoin3(cache, "/", key)
name := module
if name == "" { name = profile }
extension := ifx PkgEndsWith(url, ".zir.zip") then ".zir.zip" else ".zx"
image := PkgJoin(PkgJoin3(root, "/", name), extension)
image := PkgJoin(PkgJoin3(root, "/", name), ".zx")
if PathExists(image) {
if ImageDigest(image) != digest || !ImageVerified(launcher, image, module) {
if ImageDigest(image) != digest || !ImageVerified(launcher, image, module, id, profile) {
return PkgFail("cached portable image failed digest, format or module-name verification")
}
} else {
@@ -222,10 +203,10 @@ Fetch :: (launcher: string, url: string, profile: string, digest: string,
stage := PkgArtifactStage(cache)
if stage == "" { return PkgFail("cannot stage the portable image") }
defer PkgArtifactRemove(stage)
temporary := PkgJoin3(stage, "/download", extension)
temporary := PkgJoin(stage, "/download.zx")
status := Transfer(url, temporary, netrc, false)
if status != "200" { return TransferFailure("cannot fetch the portable image from Gitea", status) }
if ImageDigest(temporary) != digest || !ImageVerified(launcher, temporary, module) {
if ImageDigest(temporary) != digest || !ImageVerified(launcher, temporary, module, id, profile) {
return PkgFail("downloaded portable image failed digest, format or module-name verification")
}
if !PkgMkdirs(ParentDirectory(image)) || !RenamePath(temporary, image) {
@@ -262,14 +243,16 @@ PkgZxUsage :: () -> string {
return PkgJoin3(
"usage: ziran pkg zx publish REPOSITORY REVISION PROFILE FILE.zx [--netrc FILE]\n",
" ziran pkg zx fetch REPOSITORY REVISION PROFILE SHA256 [--module LOGICAL_NAME] [--offline] [--netrc FILE]\n",
PkgJoin3(" ziran pkg zx pack-ir REPOSITORY REVISION PROFILE FILE.zir.zip [--define NAME]...\n",
" ziran pkg zx publish-ir REPOSITORY REVISION PROFILE FILE.zir.zip [--netrc FILE]\n",
" ziran pkg zx fetch-ir REPOSITORY REVISION PROFILE SHA256 [--offline] [--netrc FILE]\n"))
PkgJoin3(" ziran pkg zx pack-ir REPOSITORY REVISION PROFILE FILE.zx [--define NAME]... --locked [--offline]\n",
" ziran pkg zx publish-ir REPOSITORY REVISION PROFILE FILE.zx [--netrc FILE]\n",
" ziran pkg zx fetch-ir REPOSITORY REVISION PROFILE SHA256 [--offline] [--netrc FILE]\n ziran pkg zx pack-dev REPOSITORY REVISION PROFILE FILE.zx [--define NAME]... [--offline]\n ziran pkg zx inspect-ir FILE.zx\n"))
}
PkgCheckedPins :: struct {
launcher: string
tool_identity: string
lifecycle_manifest: *PkgManifest
development_directory: string
pins: [64]PinnedCheckedPackage
count: s32
}
@@ -279,7 +262,7 @@ PkgReadCheckedPins :: (root: string, launcher: string, locked: *PkgLock, pins: *
tool := PkgLockId(locked.tool_url, locked.tool_commit, false)
for item: locked.packages[0:locked.count] {
if item.checked_profile == "" { continue }
if !RegistrySegment(item.checked_profile) || item.checked_tool != tool { return false }
if !PkgRegistrySegment(item.checked_profile) || item.checked_tool != tool { return false }
pins.pins[pins.count] = .{package = item.id, profile = item.checked_profile, digest = item.checked_digest}
pins.count += 1
}
@@ -306,7 +289,7 @@ PkgReadCheckedPins :: (root: string, launcher: string, locked: *PkgLock, pins: *
profile := PkgJsonField(json, row, "profile")
digest := PkgJsonField(json, row, "sha256")
target := PkgFindById(locked, id)
if target < 0 || locked.packages[target].source || !RegistrySegment(profile) || !HexDigest(digest) { return false }
if target < 0 || locked.packages[target].source || !PkgRegistrySegment(profile) || !HexDigest(digest) { return false }
for previous: 0..index - 1 {
if PkgJsonField(json, ElementAt(json, items, previous), "package") == id { return false }
}
@@ -347,7 +330,7 @@ PackageNetrc :: (offline: bool, output: *string) -> bool {
OpenChecked :: (pins: *PkgCheckedPins, item: *PkgLocked, locked: *PkgLock,
pin: *PinnedCheckedPackage, offline: bool) -> bool {
url := RegistryUrl(CheckedRepository(item.url), item.commit, pin.profile, ".zir.zip")
url := RegistryUrl(CheckedRepository(item.url), item.commit, pin.profile, true)
if url == "" {
unused PkgFail("expected an HTTPS registry and exact checked package revision")
return false
@@ -355,7 +338,7 @@ OpenChecked :: (pins: *PkgCheckedPins, item: *PkgLocked, locked: *PkgLock,
netrc: string
if !PackageNetrc(offline, *netrc) { return false }
archive: string
if Fetch(pins.launcher, url, pin.profile, pin.digest, "", netrc, offline, *archive) != 0 { return false }
if Fetch(pins.launcher, url, pin.profile, pin.digest, "", netrc, offline, *archive, item.id) != 0 { return false }
pin.stage = PkgArtifactStage(PkgJoin(PkgCacheRoot(), "/zx"))
if pin.stage == "" || !PkgCheckedArchive(archive, pin.digest, pin.stage, item.id,
locked.tool_url, locked.tool_commit, pin.profile) {
@@ -370,10 +353,37 @@ OpenChecked :: (pins: *PkgCheckedPins, item: *PkgLocked, locked: *PkgLock,
PkgSelectChecked :: (context: *void, item: *PkgLocked, locked: *PkgLock, offline: bool) -> bool {
pins := cast(*PkgCheckedPins)context
if pins.lifecycle_manifest != null && pins.lifecycle_manifest.tool != "" { return true }
development: string
if pins.development_directory != "" {
path := PkgJoin(PkgJoin3(pins.development_directory, "/", item.id), ".zx")
if PathExists(path) { development = path }
}
for index: 0..pins.count - 1 {
pin := *pins.pins[index]
if pin.package == item.id { return OpenChecked(pins, item, locked, pin, offline) }
if pin.package != item.id { continue }
if development != "" { return OpenDevelopment(pins, item, locked, pin, development) }
return OpenChecked(pins, item, locked, pin, offline)
}
if development == "" { return true }
if pins.count >= 64 { return false }
pin := *pins.pins[pins.count]
pins.count += 1
pin.package = item.id
return OpenDevelopment(pins, item, locked, pin, development)
}
OpenDevelopment :: (pins: *PkgCheckedPins, item: *PkgLocked, locked: *PkgLock,
pin: *PinnedCheckedPackage, path: string) -> bool {
pin.stage = PkgArtifactStage(PkgJoin(PkgCacheRoot(), "/zx"))
if pin.stage == "" || !PkgCheckedArchive(path, "", pin.stage, item.id,
locked.tool_url, locked.tool_commit, "", null, true) {
unused PkgFail("development .zx library does not match its package or toolchain")
return false
}
item.path = pin.stage
item.checked = true
pins.tool_identity = PkgLockId(locked.tool_url, locked.tool_commit, false)
return true
}
@@ -387,7 +397,7 @@ PkgDiscoverChecked :: (context: *void, item: *PkgLocked, locked: *PkgLock, offli
saved := item.checked_profile != ""
replacing := saved && requested != "" && requested != item.checked_profile
if saved && !replacing {
if item.checked_tool != tool || !RegistrySegment(item.checked_profile) {
if item.checked_tool != tool || !PkgRegistrySegment(item.checked_profile) {
unused PkgFail("saved checked package requires a different toolchain or profile")
return false
}
@@ -403,7 +413,7 @@ PkgDiscoverChecked :: (context: *void, item: *PkgLocked, locked: *PkgLock, offli
prefix := PkgJoin(origin, "/")
if !replacing && (repository.count <= prefix.count || repository[0:prefix.count] != prefix) { return true }
profile := ifx requested == "" then "default" else requested
url := RegistryUrl(repository, item.commit, profile, ".zir.zip")
url := RegistryUrl(repository, item.commit, profile, true)
if url == "" {
unused PkgFail("invalid checked package registry address or profile")
return false
@@ -424,7 +434,7 @@ PkgDiscoverChecked :: (context: *void, item: *PkgLocked, locked: *PkgLock, offli
return false
}
metadata := PkgReadLimited(descriptor, 4096)
expected := PkgJoin3(PkgJoin3("1\t", item.id, "\t"), PkgJoin3(tool, "\t", profile), "\t")
expected := PkgJoin(CheckedDescriptorPrefix(item.id, tool, profile), "\t")
if !metadata.valid || metadata.text.count != expected.count + 65 ||
metadata.text[0:expected.count] != expected ||
metadata.text[metadata.text.count - 1:] != "\n" ||
@@ -454,10 +464,10 @@ PkgResolveZxLock :: (launcher: string, root: string, refresh: string, offline: b
}
PkgPackChecked :: (root: string, compiler: string, locked: *PkgLock, current: *PkgManifest,
arguments: []string) -> s32 {
arguments: []string, development: bool) -> s32 {
if arguments.count < 4 || (arguments.count - 4) % 2 != 0 ||
RegistryUrl(arguments[0], arguments[1], arguments[2], ".zir.zip") == "" ||
!PkgEndsWith(arguments[3], ".zir.zip") { return PkgFail(PkgTrim(PkgZxUsage())) }
RegistryUrl(arguments[0], arguments[1], arguments[2], true) == "" ||
!PkgEndsWith(arguments[3], ".zx") { return PkgFail(PkgTrim(PkgZxUsage())) }
for index: 4..arguments.count - 1 {
if (index - 4) % 2 == 0 {
if arguments[index] != "--define" { return PkgFail("expected --define NAME") }
@@ -467,14 +477,19 @@ PkgPackChecked :: (root: string, compiler: string, locked: *PkgLock, current: *P
revision := PkgCapture(revision_args[:], root)
origin_args := string.["git", "remote", "get-url", "origin"]
origin := PkgCapture(origin_args[:], root)
status_args := string.["git", "status", "--porcelain", "--untracked-files=normal"]
status := PkgCapture(status_args[:], root)
if revision.code != 0 || PkgTrim(revision.text) != arguments[1] || origin.code != 0 ||
!PkgSameSource(PkgTrim(origin.text), arguments[0]) || status.code != 0 || PkgTrim(status.text) != "" {
return PkgFail("checked package export requires the clean declared repository revision")
!PkgSameSource(PkgTrim(origin.text), arguments[0]) {
return PkgFail("checked package export requires the declared repository and HEAD revision")
}
if !development {
status_args := string.["git", "status", "--porcelain", "--untracked-files=normal"]
status := PkgCapture(status_args[:], root)
if status.code != 0 || PkgTrim(status.text) != "" || !PkgValidCommit(locked.tool_commit) {
return PkgFail("checked package export requires the clean declared repository revision")
}
}
return PkgCheckedPack(root, locked, current, compiler, arguments[0], arguments[1], arguments[2],
arguments[3], arguments[4:])
arguments[3], arguments[4:], development)
}
PkgZx :: (launcher: string, arguments: []string) -> s32 {
@@ -482,13 +497,24 @@ PkgZx :: (launcher: string, arguments: []string) -> s32 {
PkgPrint(PkgZxUsage(), false)
return 0
}
if arguments.count == 2 && arguments[0] == "inspect-ir" {
path := arguments[1]
digest := ImageDigest(path)
identity: string
if !PkgEndsWith(path, ".zx") || !HexDigest(digest) ||
(!PkgCheckedArchive(path, digest, "", "", "", "", "", *identity) &&
!PkgCheckedArchive(path, digest, "", "", "", "", "", *identity, true)) {
return PkgFail("cannot verify the checked .zx library")
}
PkgPrint(PkgJoin3(identity[0:identity.count - 1], "\t", PkgJoin(digest, "\n")), false)
return 0
}
checked := arguments.count != 0 && (arguments[0] == "publish-ir" || arguments[0] == "fetch-ir")
if arguments.count < 5 || (!checked && arguments[0] != "publish" && arguments[0] != "fetch") {
return PkgFail(PkgTrim(PkgZxUsage()))
}
publish := arguments[0] == "publish" || arguments[0] == "publish-ir"
extension := ifx checked then ".zir.zip" else ".zx"
url := RegistryUrl(arguments[1], arguments[2], arguments[3], extension)
url := RegistryUrl(arguments[1], arguments[2], arguments[3], checked)
if url == "" { return PkgFail("expected an HTTPS Gitea repository, full Git revision and portable profile") }
module: string
netrc: string
@@ -507,12 +533,12 @@ PkgZx :: (launcher: string, arguments: []string) -> s32 {
index += 1
} else { return PkgFail(PkgJoin("invalid portable package option: ", option)) }
}
id := ifx checked then PkgLockId(arguments[1], arguments[2], false) else ""
if publish {
id := ifx checked then PkgLockId(arguments[1], arguments[2], false) else ""
return Publish(launcher, url, arguments[4], netrc, id, arguments[3])
}
output: string
code := Fetch(launcher, url, arguments[3], arguments[4], module, netrc, offline, *output)
code := Fetch(launcher, url, arguments[3], arguments[4], module, netrc, offline, *output, id)
if code == 0 { PkgPrint(PkgJoin(output, "\n"), false) }
return code
}
@@ -559,7 +585,7 @@ PkgZxLockedArguments :: (launcher: string, arguments: []string, lock_at: s64, fl
profile := PkgJsonField(json, row, "profile")
digest := PkgJsonField(json, row, "sha256")
url := RegistryUrl(PkgJsonField(json, row, "repository"), PkgJsonField(json, row, "commit"), profile)
if !RegistrySegment(alias) || !ModuleName(name) || url == "" || !HexDigest(digest) {
if !PkgRegistrySegment(alias) || !ModuleName(name) || url == "" || !HexDigest(digest) {
return LockFailure("invalid pinned ZX module")
}
for prior: 0..index - 1 {
@@ -585,7 +611,7 @@ PkgZxLockedArguments :: (launcher: string, arguments: []string, lock_at: s64, fl
exported := PkgJsonField(json, row, "export")
found := false
for pinned: modules[0:module_count] { if pinned.alias == alias { found = true } }
if !RegistrySegment(module) || !RegistrySegment(function) || !RegistrySegment(exported) || !found {
if !PkgRegistrySegment(module) || !PkgRegistrySegment(function) || !PkgRegistrySegment(exported) || !found {
return LockFailure("invalid ZX import or unpinned interface alias")
}
if build {
+72 -14
View File
@@ -545,10 +545,39 @@ still checks typed imports and requires the host grants used by the image.
This workflow uses ZX runtime interfaces. Checked packages support source
imports and generics. Platform-native dependencies use native artifacts.
## Prebuilt Ziran toolchains
The canonical Gitea check job packages the Ziran executables, standard
library, headers, and native archive from the build it just tested. Each
artifact identifies the full Ziran commit and the build host's Linux
architecture and glibc version. The native job reuses its build directory.
Download `toolchain-<commit>` from the corresponding check run. When the
workflow has a package writer configured, the same two files are available
from Gitea Generic Packages under `ziranlang/ziran-toolchain`, using the full
Git commit as the package version. The `.json` descriptor and `.tar.gz`
archive share the profile name, for example `linux-x64-glibc-2.39`.
```sh
python3 scripts/toolchain_package.py verify "$commit" "$profile.json"
```
Compare `commit` to the toolchain commit in the consuming project's
`ziran.lock` and use a Linux host that supports the archive's profile.
The verification command checks the commit, archive length, and SHA-256.
The archive contains `build/bin/ziran` and its companion tools. Package
commands that require a source checkout still use their existing exact Git
source cache.
Configure `ZIRAN_TOOLCHAIN_PACKAGE_USER` and `ZIRAN_TOOLCHAIN_PACKAGE_TOKEN`
as Gitea Actions secrets for a package writer in the `ziranlang` organization.
The workflow uploads the archive first and its descriptor second. Existing
registry files are accepted on retry when their bytes match.
## Checked packages for source imports
A package with manifest exports can distribute its checked Ziran modules as
a `.zir.zip` compiler companion. Pack it from the clean provider revision
a `.zx` library. Pack it from the clean provider revision
using the exact, clean toolchain and dependencies pinned in `ziran.lock`.
Each profile names a fixed toolchain and set of feature defines at that
provider revision. Supply defines with `--define NAME` when packing.
@@ -556,24 +585,27 @@ provider revision. Supply defines with `--define NAME` when packing.
```sh
repo=https://git.waozi.xyz/example/counter.git
commit=$(git rev-parse HEAD)
ziran pkg zx pack-ir "$repo" "$commit" default build/counter.zir.zip --locked
sha256=$(ziran pkg zx publish-ir "$repo" "$commit" default build/counter.zir.zip --netrc "$NETRC_FILE")
ziran pkg zx pack-ir "$repo" "$commit" default build/counter.zx --locked
ziran pkg zx inspect-ir build/counter.zx
sha256=$(ziran pkg zx publish-ir "$repo" "$commit" default build/counter.zx --netrc "$NETRC_FILE")
ziran pkg zx fetch-ir "$repo" "$commit" default "$sha256"
```
`publish-ir` uploads the archive, then publishes the matching discovery
record at the same revision and profile. Either upload can be repeated with
identical bytes. Keep the archive and record together when operating a registry.
`publish-ir` uploads `<profile>.zx` and then `<profile>.zx.tsv` to the
`<repository>-zx-library` Generic package at the full Git revision. Either
upload can be repeated with identical bytes. `inspect-ir` verifies local
library bytes and prints the same tab-separated record with the package,
toolchain, profile, and digest.
`ziran add` and `ziran lock` discover companions on `https://git.waozi.xyz`
`ziran add` and `ziran lock` discover libraries on `https://git.waozi.xyz`
and save their exact profile, digest and toolchain identity in the package
entry in `ziran.lock`. For a different Gitea server, set `ZIRAN_ZX_REGISTRY`
to its HTTPS origin. Discovery uses the `default` profile unless
`ZIRAN_ZX_PROFILE` selects another. Set `ZIRAN_PACKAGE_NETRC`
for authenticated reads. A missing discovery record for an unpinned package
allows ordinary source resolution. Invalid metadata, an incompatible
toolchain, or a missing advertised archive stops resolution. A saved pin
continues to identify the same exact archive.
toolchain, or a missing advertised library stops resolution. A saved pin
continues to identify the same exact library.
```sh
ziran add "$repo" --ref "$commit"
@@ -582,7 +614,7 @@ ziran lock --offline
ziran bundle --project --offline --entry app:main -o app.zx app.zi
```
You can also supply a companion pin explicitly in `ziran.zx.lock`, using
You can also supply a library pin explicitly in `ziran.zx.lock`, using
the dependency's `id` from `ziran.lock` and the publisher's profile and digest.
An explicit pin for a package with a discovered pin must agree with it.
@@ -599,17 +631,43 @@ An explicit pin for a package with a discovered pin must agree with it.
}
```
`ziran bundle --project` and `ziran build --project --target=zx` select the
pinned checked archives before checking out those dependencies. Existing
source imports still resolve through the dependency aliases and manifest
Project `bundle`, `check`, and native `build` and `run` select pinned libraries
before checking out those dependencies. `build --target=zx` does the same.
Source imports resolve through the dependency aliases and manifest
exports. The compiler verifies the checked modules and uses their types,
bodies and specializations when it lowers the application. This skips
dependency source checking and keeps shared types and state in one program.
Each transitive package whose source will be absent needs a pin. `--offline`
requires the pinned archives in the local cache. Local source overrides
requires the pinned libraries in the local cache. Local source overrides
still take precedence unless `--locked` is set. Distribute the final `.zx`
with the portable image commands above.
### Check a library from a working tree
From the provider checkout, use its current `ziran.lock` and actual HEAD.
`pack-dev` includes the working source, manifest, lock and feature defines
in a source fingerprint. It allows local toolchain and source overrides.
```sh
ziran pkg zx pack-dev "$repo" "$(git rev-parse HEAD)" default build/counter-dev.zx --offline
ziran pkg zx inspect-ir build/counter-dev.zx
```
The inspection record starts with `dev`, followed by the package identity,
toolchain identity, profile, provider source fingerprint, and file SHA-256.
Copy the development library to a local directory as `<package-id>.zx`.
Point `ZIRAN_DEV_ZX_DIR` at that directory when compiling the consumer. Keep
the normal project lock and remove any source override for that provider
from the consumer. Several development libraries can share a directory.
```sh
ZIRAN_DEV_ZX_DIR="$libdir" ziran bundle --project --offline --entry app:main -o app.zx app.zi
```
The compiler checks each development library against the locked package and
toolchain. These files are for local verification. `publish-ir`, production
pins, and `--locked` commands reject development libraries.
## Dependency-owned native artifacts
A dependency can declare named artifacts alongside its existing project tool:
+198
View File
@@ -0,0 +1,198 @@
#!/usr/bin/env python3
"""Package the native toolchain from Ziran's completed Gitea check job."""
import argparse
import base64
import gzip
import hashlib
import http.client
import json
import os
import platform
import re
import ssl
import stat
import subprocess
import tarfile
import tempfile
from pathlib import Path
from urllib.parse import urlsplit
HOST = 'git.waozi.xyz'
REPOSITORY = f'https://{HOST}/ziranlang/ziran.git'
TOOLS = ('ziran', 'zi-fmt', 'zi-edit', 'zi2zir', 'zi-api', 'zi-inspect',
'zi2c', 'zi2go', 'zi2zx', 'zx2wasm', 'zx2native')
PROFILE = re.compile(r'linux-(?:x64|arm64)-glibc-[0-9]+\.[0-9]+')
COMMIT = re.compile(r'[0-9a-f]{40}')
def git(*args):
return subprocess.check_output(['git', *args])
def clean_revision(commit):
if not COMMIT.fullmatch(commit) or git('rev-parse', 'HEAD').strip().decode() != commit:
raise ValueError('toolchain source differs from the requested Git revision')
source = urlsplit(git('remote', 'get-url', 'origin').strip().decode())
if (source.scheme != 'https' or source.hostname != HOST or source.port is not None or
source.path not in ('/ziranlang/ziran', '/ziranlang/ziran.git')):
raise ValueError('toolchain source must use the canonical Gitea origin')
if git('status', '--porcelain', '--untracked-files=normal'):
raise ValueError('toolchain source must be clean')
def host_profile():
arch = {'x86_64': 'x64', 'aarch64': 'arm64'}.get(platform.machine())
libc = os.confstr('CS_GNU_LIBC_VERSION') if platform.system() == 'Linux' else None
if arch is None or libc is None or not re.fullmatch(r'glibc [0-9]+\.[0-9]+', libc):
raise ValueError('the toolchain registry supports Linux x64 and arm64 with glibc')
return f'linux-{arch}-{libc.replace(" ", "-")}'
def digest(file):
with file.open('rb') as stream:
return hash_stream(stream)
def hash_stream(stream):
checksum = hashlib.sha256()
for chunk in iter(lambda: stream.read(65536), b''):
checksum.update(chunk)
return checksum.hexdigest()
def write_archive(archive, files):
with gzip.GzipFile(filename='', fileobj=archive, mode='wb', mtime=0) as compressed:
with tarfile.open(fileobj=compressed, mode='w|', format=tarfile.PAX_FORMAT) as tar:
for file in sorted(files):
if not stat.S_ISREG(file.lstat().st_mode):
raise ValueError(f'expected a regular toolchain file: {file}')
with file.open('rb') as contents:
info = os.fstat(contents.fileno())
entry = tarfile.TarInfo(file.as_posix())
entry.size = info.st_size
entry.mode = 0o755 if info.st_mode & stat.S_IXUSR else 0o644
tar.addfile(entry, contents)
def pack(commit, output):
clean_revision(commit)
profile = host_profile()
binaries = [Path('build/bin', tool) for tool in TOOLS]
for binary in binaries:
if not binary.is_file() or not os.access(binary, os.X_OK):
raise ValueError(f'missing tested toolchain executable: {binary}')
subprocess.run([str(binaries[0]), 'version'], check=True)
sources = git('ls-files', '-z', '--', 'std', 'include', 'templates', 'scripts/ziran_artifacts.py')
files = binaries + [Path('build/libziran.a')]
files.extend(Path(os.fsdecode(name)) for name in sources.split(b'\0') if name)
output.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix='.pack-', dir=output) as stage:
archive = Path(stage, f'{profile}.tar.gz')
with archive.open('wb') as stream:
write_archive(stream, files)
descriptor = archive.with_name(f'{profile}.json')
metadata = {'version': 1, 'repository': REPOSITORY, 'commit': commit, 'profile': profile,
'file': archive.name, 'size': archive.stat().st_size, 'sha256': digest(archive)}
descriptor.write_text(json.dumps(metadata, sort_keys=True) + '\n')
clean_revision(commit)
archive.replace(output / archive.name)
descriptor.replace(output / descriptor.name)
print(f'Toolchain artifact {profile} for {commit}')
def read_artifact(descriptor):
with descriptor.open('rb') as stream:
contents = stream.read(4097)
if len(contents) > 4096:
raise ValueError('toolchain descriptor exceeds 4096 bytes')
row = json.loads(contents)
profile = row['profile']
archive = descriptor.with_name(f'{profile}.tar.gz') if PROFILE.fullmatch(profile) else None
if (row['version'] != 1 or row['repository'] != REPOSITORY or not COMMIT.fullmatch(row['commit']) or
archive is None or descriptor.name != f'{profile}.json' or row['file'] != archive.name or
row['size'] != archive.stat().st_size or row['sha256'] != digest(archive)):
raise ValueError('toolchain descriptor and archive did not verify')
return row['commit'], archive
def upload(connection, base, file, authorization):
path = f'{base}/{file.name}'
size = file.stat().st_size
headers = {'Authorization': authorization, 'Content-Length': str(size),
'Content-Type': 'application/octet-stream'}
interrupted = None
try:
with file.open('rb') as stream:
connection.request('PUT', path, body=stream, headers=headers)
with connection.getresponse() as response:
if response.status == 201:
return
if response.status != 409:
raise ValueError(f'Gitea rejected {file.name} (HTTP {response.status})')
except (BrokenPipeError, ConnectionResetError, http.client.RemoteDisconnected, ssl.SSLEOFError) as error:
# The server may answer a conflict before reading the complete upload.
interrupted = error
finally:
connection.close()
# A retry may reuse an immutable file only when the server has the same bytes.
try:
connection.request('GET', path, headers={'Authorization': authorization})
with connection.getresponse() as response:
if response.status == 404 and interrupted is not None:
raise interrupted
if response.status != 200 or int(response.getheader('Content-Length', -1)) != size:
raise ValueError(f'cannot verify existing Gitea file {file.name}')
if hash_stream(response) != digest(file):
raise ValueError(f'Gitea already has different bytes for {file.name}')
finally:
connection.close()
def verify(descriptor, expected_commit):
commit, archive = read_artifact(descriptor)
if commit != expected_commit:
raise ValueError('artifact revision differs from the workflow revision')
return archive
def publish(descriptor, expected_commit):
archive = verify(descriptor, expected_commit)
user = os.environ.get('ZIRAN_TOOLCHAIN_PACKAGE_USER')
token = os.environ.get('ZIRAN_TOOLCHAIN_PACKAGE_TOKEN')
if not user or not token or ':' in user:
raise ValueError('provide the Gitea package writer identity and token')
authorization = 'Basic ' + base64.b64encode(f'{user}:{token}'.encode()).decode()
base = f'/api/packages/ziranlang/generic/ziran-toolchain/{expected_commit}'
connection = http.client.HTTPSConnection(HOST, timeout=180)
upload(connection, base, archive, authorization)
upload(connection, base, descriptor, authorization)
print(f'Published {REPOSITORY} toolchain {archive.name} for {expected_commit}')
def main():
parser = argparse.ArgumentParser(description=__doc__)
subcommands = parser.add_subparsers(dest='command', required=True)
package = subcommands.add_parser('pack')
package.add_argument('commit')
package.add_argument('output', type=Path)
for command in ('verify', 'publish'):
artifact = subcommands.add_parser(command)
artifact.add_argument('commit')
artifact.add_argument('descriptor', type=Path)
try:
args = parser.parse_args()
if args.command == 'pack':
pack(args.commit, args.output)
elif args.command == 'publish':
publish(args.descriptor, args.commit)
else:
print(verify(args.descriptor, args.commit))
except (OSError, ValueError, KeyError, TypeError, subprocess.CalledProcessError,
http.client.HTTPException) as error:
parser.exit(1, f'toolchain package: {error}\n')
if __name__ == '__main__':
main()
+58 -15
View File
@@ -3,7 +3,7 @@
<head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><meta name="theme-color" content="#f7f3eb"><meta name="description" content="Packages and project imports: the Ziran guide, rendered from docs/PACKAGES.md."><title>Packages and project imports · Ziran guide</title><link rel="canonical" href="https://ziran-lang.org/guide/packages"><link rel="icon" href="../assets/logo-capybara.png" type="image/png"><link rel="stylesheet" href="../styles.css"><script defer src="../site.js"></script></head>
<body><a class="skip-link" href="#main">Skip to content</a><header class="site-header"><div class="site-header-inner"><a class="brand" href="../index.html"><img class="brand-icon" src="../assets/logo-capybara.png" alt="" width="36" height="36"><span>Ziran</span></a><button class="menu-button" type="button" aria-expanded="false" aria-controls="primary-nav">Menu</button><nav class="header-nav" id="primary-nav" aria-label="Primary"><a href="../get-started.html">Get started</a><a href="../examples.html">Examples</a><a href="../targets.html">Targets</a><a href="../modules.html">Modules</a><a href="../docs.html" aria-current="page">Docs</a><a href="../status.html">Status</a><a href="https://github.com/ziranlang/ziran">GitHub ↗</a><button class="theme-toggle" type="button" aria-label="Switch color theme"></button></nav></div></header>
<main id="main" class="article-layout"><div class="article-head"><p class="eyebrow"><a href="../docs.html#guide">Guide</a></p><h1>Packages and project imports</h1><p>Rendered from <a href="https://github.com/ziranlang/ziran/blob/master/docs/PACKAGES.md">docs/PACKAGES.md</a>. See the <a href="../docs.html#guide">documentation index</a> for the other guides.</p></div>
<div class="article-grid"><aside class="page-toc"><strong>On this page</strong><a href="#starting-a-project">Starting a project</a><a href="#building-and-running-a-program">Building and running a program</a><a href="#adding-a-dependency">Adding a dependency</a><a href="#importing-modules">Importing modules</a><a href="#publishing-a-package">Publishing a package</a><a href="#commands">Commands</a><a href="#project-tools-and-their-options">Project tools and their options</a><a href="#handing-a-project-to-its-tool">Handing a project to its tool</a><a href="#prebuilt-portable-images">Prebuilt portable images</a><a href="#checked-packages-for-source-imports">Checked packages for source imports</a><a href="#dependency-owned-native-artifacts">Dependency-owned native artifacts</a></aside><div class="article-body">
<div class="article-grid"><aside class="page-toc"><strong>On this page</strong><a href="#starting-a-project">Starting a project</a><a href="#building-and-running-a-program">Building and running a program</a><a href="#adding-a-dependency">Adding a dependency</a><a href="#importing-modules">Importing modules</a><a href="#publishing-a-package">Publishing a package</a><a href="#commands">Commands</a><a href="#project-tools-and-their-options">Project tools and their options</a><a href="#handing-a-project-to-its-tool">Handing a project to its tool</a><a href="#prebuilt-portable-images">Prebuilt portable images</a><a href="#prebuilt-ziran-toolchains">Prebuilt Ziran toolchains</a><a href="#checked-packages-for-source-imports">Checked packages for source imports</a><a href="#dependency-owned-native-artifacts">Dependency-owned native artifacts</a></aside><div class="article-body">
<section class="prose"><p>Ziran packages are Git repositories. There is no central registry: a
dependency is the URL of the repository that holds it, pinned to a full commit
in a committed <code>ziran.lock</code>. A Ziran application is itself a package with a
@@ -438,36 +438,61 @@ download, digest, image format or module name stops the command. The loader
still checks typed imports and requires the host grants used by the image.
This workflow uses ZX runtime interfaces. Checked packages support source
imports and generics. Platform-native dependencies use native artifacts.</p>
<h2 id="prebuilt-ziran-toolchains">Prebuilt Ziran toolchains</h2>
<p>The canonical Gitea check job packages the Ziran executables, standard
library, headers, and native archive from the build it just tested. Each
artifact identifies the full Ziran commit and the build host's Linux
architecture and glibc version. The native job reuses its build directory.</p>
<p>Download <code>toolchain-&lt;commit&gt;</code> from the corresponding check run. When the
workflow has a package writer configured, the same two files are available
from Gitea Generic Packages under <code>ziranlang/ziran-toolchain</code>, using the full
Git commit as the package version. The <code>.json</code> descriptor and <code>.tar.gz</code>
archive share the profile name, for example <code>linux-x64-glibc-2.39</code>.</p>
<pre><code class="language-sh">python3 scripts/toolchain_package.py verify &quot;$commit&quot; &quot;$profile.json&quot;
</code></pre>
<p>Compare <code>commit</code> to the toolchain commit in the consuming project's
<code>ziran.lock</code> and use a Linux host that supports the archive's profile.
The verification command checks the commit, archive length, and SHA-256.
The archive contains <code>build/bin/ziran</code> and its companion tools. Package
commands that require a source checkout still use their existing exact Git
source cache.</p>
<p>Configure <code>ZIRAN_TOOLCHAIN_PACKAGE_USER</code> and <code>ZIRAN_TOOLCHAIN_PACKAGE_TOKEN</code>
as Gitea Actions secrets for a package writer in the <code>ziranlang</code> organization.
The workflow uploads the archive first and its descriptor second. Existing
registry files are accepted on retry when their bytes match.</p>
<h2 id="checked-packages-for-source-imports">Checked packages for source imports</h2>
<p>A package with manifest exports can distribute its checked Ziran modules as
a <code>.zir.zip</code> compiler companion. Pack it from the clean provider revision
a <code>.zx</code> library. Pack it from the clean provider revision
using the exact, clean toolchain and dependencies pinned in <code>ziran.lock</code>.
Each profile names a fixed toolchain and set of feature defines at that
provider revision. Supply defines with <code>--define NAME</code> when packing.</p>
<pre><code class="language-sh">repo=https://git.waozi.xyz/example/counter.git
commit=$(git rev-parse HEAD)
ziran pkg zx pack-ir &quot;$repo&quot; &quot;$commit&quot; default build/counter.zir.zip --locked
sha256=$(ziran pkg zx publish-ir &quot;$repo&quot; &quot;$commit&quot; default build/counter.zir.zip --netrc &quot;$NETRC_FILE&quot;)
ziran pkg zx pack-ir &quot;$repo&quot; &quot;$commit&quot; default build/counter.zx --locked
ziran pkg zx inspect-ir build/counter.zx
sha256=$(ziran pkg zx publish-ir &quot;$repo&quot; &quot;$commit&quot; default build/counter.zx --netrc &quot;$NETRC_FILE&quot;)
ziran pkg zx fetch-ir &quot;$repo&quot; &quot;$commit&quot; default &quot;$sha256&quot;
</code></pre>
<p><code>publish-ir</code> uploads the archive, then publishes the matching discovery
record at the same revision and profile. Either upload can be repeated with
identical bytes. Keep the archive and record together when operating a registry.</p>
<p><code>ziran add</code> and <code>ziran lock</code> discover companions on <code>https://git.waozi.xyz</code>
<p><code>publish-ir</code> uploads <code>&lt;profile&gt;.zx</code> and then <code>&lt;profile&gt;.zx.tsv</code> to the
<code>&lt;repository&gt;-zx-library</code> Generic package at the full Git revision. Either
upload can be repeated with identical bytes. <code>inspect-ir</code> verifies local
library bytes and prints the same tab-separated record with the package,
toolchain, profile, and digest.</p>
<p><code>ziran add</code> and <code>ziran lock</code> discover libraries on <code>https://git.waozi.xyz</code>
and save their exact profile, digest and toolchain identity in the package
entry in <code>ziran.lock</code>. For a different Gitea server, set <code>ZIRAN_ZX_REGISTRY</code>
to its HTTPS origin. Discovery uses the <code>default</code> profile unless
<code>ZIRAN_ZX_PROFILE</code> selects another. Set <code>ZIRAN_PACKAGE_NETRC</code>
for authenticated reads. A missing discovery record for an unpinned package
allows ordinary source resolution. Invalid metadata, an incompatible
toolchain, or a missing advertised archive stops resolution. A saved pin
continues to identify the same exact archive.</p>
toolchain, or a missing advertised library stops resolution. A saved pin
continues to identify the same exact library.</p>
<pre><code class="language-sh">ziran add &quot;$repo&quot; --ref &quot;$commit&quot;
ziran bundle --project --entry app:main -o app.zx app.zi
ziran lock --offline
ziran bundle --project --offline --entry app:main -o app.zx app.zi
</code></pre>
<p>You can also supply a companion pin explicitly in <code>ziran.zx.lock</code>, using
<p>You can also supply a library pin explicitly in <code>ziran.zx.lock</code>, using
the dependency's <code>id</code> from <code>ziran.lock</code> and the publisher's profile and digest.
An explicit pin for a package with a discovered pin must agree with it.</p>
<pre><code class="language-json">{
@@ -481,16 +506,34 @@ An explicit pin for a package with a discovered pin must agree with it.</p>
]
}
</code></pre>
<p><code>ziran bundle --project</code> and <code>ziran build --project --target=zx</code> select the
pinned checked archives before checking out those dependencies. Existing
source imports still resolve through the dependency aliases and manifest
<p>Project <code>bundle</code>, <code>check</code>, and native <code>build</code> and <code>run</code> select pinned libraries
before checking out those dependencies. <code>build --target=zx</code> does the same.
Source imports resolve through the dependency aliases and manifest
exports. The compiler verifies the checked modules and uses their types,
bodies and specializations when it lowers the application. This skips
dependency source checking and keeps shared types and state in one program.
Each transitive package whose source will be absent needs a pin. <code>--offline</code>
requires the pinned archives in the local cache. Local source overrides
requires the pinned libraries in the local cache. Local source overrides
still take precedence unless <code>--locked</code> is set. Distribute the final <code>.zx</code>
with the portable image commands above.</p>
<h3 id="check-a-library-from-a-working-tree">Check a library from a working tree</h3>
<p>From the provider checkout, use its current <code>ziran.lock</code> and actual HEAD.
<code>pack-dev</code> includes the working source, manifest, lock and feature defines
in a source fingerprint. It allows local toolchain and source overrides.</p>
<pre><code class="language-sh">ziran pkg zx pack-dev &quot;$repo&quot; &quot;$(git rev-parse HEAD)&quot; default build/counter-dev.zx --offline
ziran pkg zx inspect-ir build/counter-dev.zx
</code></pre>
<p>The inspection record starts with <code>dev</code>, followed by the package identity,
toolchain identity, profile, provider source fingerprint, and file SHA-256.
Copy the development library to a local directory as <code>&lt;package-id&gt;.zx</code>.
Point <code>ZIRAN_DEV_ZX_DIR</code> at that directory when compiling the consumer. Keep
the normal project lock and remove any source override for that provider
from the consumer. Several development libraries can share a directory.</p>
<pre><code class="language-sh">ZIRAN_DEV_ZX_DIR=&quot;$libdir&quot; ziran bundle --project --offline --entry app:main -o app.zx app.zi
</code></pre>
<p>The compiler checks each development library against the locked package and
toolchain. These files are for local verification. <code>publish-ir</code>, production
pins, and <code>--locked</code> commands reject development libraries.</p>
<h2 id="dependency-owned-native-artifacts">Dependency-owned native artifacts</h2>
<p>A dependency can declare named artifacts alongside its existing project tool:</p>
<pre><code class="language-toml">[tool]
+5 -2
View File
@@ -20,7 +20,9 @@ Templates :: (test: *NativeTest) -> bool {
// explicitly select the compiler under test; the real source tree is never cloned.
if !FixtureInit(test, tool) || !NativeWrite(test, CheckPath(tool, "VERSION"), "template fixture\n") || !NativeAssert(test, CreateDirectories(work), "create template test folder") { return false }
unused FixtureCommit(test, tool, "template toolchain metadata")
prefixes := string.["https://github.com/ziranlang/ziran.git"]; targets := string.[FixtureUri(tool)]
tool_uri := FixtureUri(tool)
prefixes := string.["https://git.waozi.xyz/ziranlang/ziran.git", "https://github.com/ziranlang/ziran.git"]
targets := string.[tool_uri, tool_uri]
if !FixtureRedirects(test, prefixes[:], targets[:]) { return false }
create := string.["new", "greeter"]; created := PackageCommand(test, work, create[:])
greeter := CheckPath(work, "greeter"); manifest_path := CheckPath(greeter, "ziran.toml")
@@ -87,7 +89,8 @@ Templates :: (test: *NativeTest) -> bool {
!NativeWrite(test, CheckPath(kit, "templates/panel/ziran.toml"), "[package]\nname = \"{{name}}\"\n[toolchain]\ngit = \"https://github.com/ziranlang/ziran.git\"\nref = \"master\"\n") ||
!NativeWrite(test, CheckPath(kit, "templates/panel/panel.txt"), "panel for {{name}}\n") { return false }
unused FixtureCommit(test, kit, "template kit fixture")
kit_prefixes := string.[prefixes[0], "https://ziran-lang.org/owner/kit.git"]; kit_targets := string.[targets[0], FixtureUri(kit)]
kit_prefixes := string.[prefixes[0], prefixes[1], "https://ziran-lang.org/owner/kit.git"]
kit_targets := string.[tool_uri, tool_uri, FixtureUri(kit)]
if !FixtureRedirects(test, kit_prefixes[:], kit_targets[:]) { return false }
window_command := string.["new", "my-app", "--template", "ziran-lang.org/owner/kit"]
windowed := PackageCommand(test, work, window_command[:])
+116 -28
View File
@@ -4,17 +4,40 @@ exec python3 - "$1" <<'PY'
import base64
import hashlib
import http.server
import io
import json
import os
from pathlib import Path
import shutil
import ssl
import struct
import subprocess
import sys
import tempfile
import threading
import zipfile
def library_sections(image):
magic, version, count, size = struct.unpack_from('<4sIII', image)
assert magic == b'ZXLB' and version == 1 and size == len(image)
position = 16
sections = []
for _ in range(count):
name_length, length, sha = struct.unpack_from('<II32s', image, position)
position += 40
name = image[position:position + name_length]
position += name_length
contents = image[position:position + length]
position += length
assert len(contents) == length and hashlib.sha256(contents).digest() == sha
sections.append((name, contents))
assert position == size and [name for name, _ in sections] == sorted({name for name, _ in sections})
return sections
def make_library(sections):
body = b''.join(struct.pack('<II32s', len(name), len(data), hashlib.sha256(data).digest()) + name + data
for name, data in sections)
return struct.pack('<4sIII', b'ZXLB', 1, len(sections), 16 + len(body)) + body
class Registry(http.server.BaseHTTPRequestHandler):
@@ -229,8 +252,10 @@ main :: () -> s32 { return External(); }
source.mkdir()
(source / 'src').mkdir()
provider_module = source / 'src/state.zi'
provider_module.write_text('''State :: struct { value: s32; }
Advance :: (state: *State) { state.value += 2; }
provider_module.write_text('''Update :: #type (value: s32) -> s32;
State :: struct { value: s32; handler: Update; }
Advance :: (state: *State) { state.value = state.handler(state.value); }
Copy :: (value: $T) -> T { return value; }
''')
toolchain = Path(command('git', '-C', ziran.parent, 'rev-parse', '--show-toplevel'))
tool_url = 'https://git.waozi.xyz/ziranlang/ziran.git'
@@ -254,18 +279,62 @@ Advance :: (state: *State) { state.value += 2; }
(source / '.git/info/exclude').write_text('build/\nziran.local.toml\nziran.lock\n')
project(source, 'state', [], {})
command('git', '-C', source, 'config', 'remote.origin.url', repository)
archive = root / 'state.zir.zip'
archive = root / 'state.zx'
command(*package, 'pack-ir', repository, checked_revision, 'default', archive, cwd=source)
archive_digest = hashlib.sha256(archive.read_bytes()).hexdigest()
library = archive.read_bytes()
archive_digest = hashlib.sha256(library).hexdigest()
sections = library_sections(library)
assert [name for name, _ in sections] == [b'checked-package.tsv', b'src/state.zir', b'ziran.toml']
inspection = command(*package, 'inspect-ir', archive)
assert inspection == sections[0][1].decode().strip() + '\t' + archive_digest
repeated = root / 'repacked.zx'
command(*package, 'pack-ir', repository, checked_revision, 'default', repeated, cwd=source)
assert repeated.read_bytes() == library
malformed = root / 'malformed.zx'
for invalid in (b'PK\x03\x04' + library[4:],
library[:4] + struct.pack('<I', 2) + library[8:],
library[:8] + struct.pack('<I', 4099) + library[12:],
library[:16] + struct.pack('<I', 0xffffffff) + library[20:],
library[:-1], library + b'\0', library[:-1] + bytes([library[-1] ^ 1]),
make_library(sections[::-1]), make_library(sections[:1] + sections),
make_library([(b'../state.zir', sections[1][1]), *sections])):
malformed.write_bytes(invalid)
command(*package, 'inspect-ir', malformed, failure='cannot verify the checked .zx library')
assert command(*package, 'publish-ir', repository, checked_revision, 'default', archive,
'--netrc', netrc) == archive_digest
assert command(*package, 'publish-ir', repository, checked_revision, 'default', archive,
'--netrc', netrc) == archive_digest
command(*package, 'publish', repository, checked_revision, 'default', archive,
'--netrc', netrc, failure='cannot verify the portable image')
command(*package, 'publish-ir', repository, checked_revision, 'other', archive,
'--netrc', netrc, failure='archive does not match the declared repository revision or profile')
alternate = root / 'alternate.zir.zip'
'--netrc', netrc, failure='failed format, revision or profile verification')
alternate = root / 'alternate.zx'
command(*package, 'pack-ir', repository, checked_revision, 'other', alternate, cwd=source)
alternate_digest = command(*package, 'publish-ir', repository, checked_revision, 'other', alternate, '--netrc', netrc)
developer = root / 'developer.zx'
dev_pack = [*package, 'pack-dev', repository, checked_revision, 'default', developer, '--offline']
command(*dev_pack, cwd=source)
original_dev = command(*package, 'inspect-ir', developer).split('\t')
provider_module.write_text(provider_module.read_text().replace('state.handler(state.value);',
'state.handler(state.value) + 5;'))
command(*dev_pack, cwd=source)
current_dev = command(*package, 'inspect-ir', developer).split('\t')
assert developer.read_bytes()[:4] == b'ZXLD' and current_dev[0] == 'dev'
assert len(current_dev) == 6 and original_dev[:4] == current_dev[:4]
assert original_dev[4] != current_dev[4] and len(current_dev[4]) == 64
assert current_dev[5] == hashlib.sha256(developer.read_bytes()).hexdigest()
command(*package, 'pack-ir', repository, checked_revision, 'default', root / 'dirty.zx',
cwd=source, failure='requires the clean declared repository revision')
command(*dev_pack, '--locked', cwd=source, failure='pack-dev cannot be used with --locked')
before = len(server.requests)
command(*package, 'publish-ir', repository, checked_revision, 'default', developer,
'--netrc', netrc, failure='failed format, revision or profile verification')
assert len(server.requests) == before
dev_export = f'/api/packages/ziranlang/generic/fixture-zx-library/{checked_revision}/default.zx'
server.images[dev_export] = developer.read_bytes()
command(*package, 'fetch-ir', repository, checked_revision, 'default', current_dev[5],
failure='failed digest, format or module-name verification')
server.images[dev_export] = library
shutil.rmtree(source)
archive.unlink()
alternate.unlink()
@@ -278,40 +347,54 @@ Advance :: (state: *State) { state.value += 2; }
'commit': checked_revision, 'dependencies': {}}
project(consumer, 'consumer', [row], {'state': identity})
pins = consumer / 'ziran.zx.lock'
pins.write_text(json.dumps({'format': 1, 'packages': [{'package': identity,
'profile': 'default', 'sha256': archive_digest}]}))
app = consumer / 'app.zi'
app.write_text('''using Model :: #import "state/model"
Increment :: (value: s32) -> s32 { return value + 2; }
main :: () -> s32 {
state := Model.State.{value = 40};
state := Model.State.{value = Model.Copy(cast(s32)40), handler = Increment};
Model.Advance(*state);
return state.value;
}
''')
built = consumer / 'app.zx'
argv = [ziran, 'bundle', '--project', '--entry', 'app:main', '-o', built, app]
dev_dir = root / 'dev-libraries'
dev_dir.mkdir()
developer.rename(dev_dir / f'{identity}.zx')
env['ZIRAN_DEV_ZX_DIR'] = str(dev_dir)
before = len(server.requests)
command(*argv, '--offline', cwd=consumer)
assert command(ziran, 'run', '--backend', 'engine', built) == '47'
assert len(server.requests) == before and not pins.exists()
command(*argv, '--offline', '--locked', cwd=consumer, failure='ZIRAN_DEV_ZX_DIR cannot be used with --locked')
env.pop('ZIRAN_DEV_ZX_DIR')
pins.write_text(json.dumps({'format': 1, 'packages': [{'package': identity,
'profile': 'default', 'sha256': archive_digest}]}))
command(*argv, '--offline', failure='absent from the offline cache', cwd=consumer)
command(*argv, cwd=consumer)
assert command(ziran, 'run', '--backend', 'engine', built) == '42'
request_count = len(server.requests)
command(*argv, '--offline', cwd=consumer)
fetched_library = Path(command(*package, 'fetch-ir', repository, checked_revision, 'default',
archive_digest, '--offline'))
assert fetched_library.read_bytes() == library
assert len(server.requests) == request_count
# Ordinary add discovers the published descriptor and resolves its
# package graph from the archive before attempting any Git checkout.
automatic = root / 'automatic'
automatic.mkdir()
bare_manifest = '[package]\nname = "consumer"\nmodule_roots = ["."]\n' + tool_section
bare_manifest = '[package]\nname = "consumer"\nmodule_roots = ["."]\nentry = "app.zi"\n' + tool_section
automatic_manifest = automatic / 'ziran.toml'
automatic_manifest.write_text(bare_manifest)
(automatic / 'ziran.local.toml').write_text(f'[overrides]\nziran = "{toolchain}"\n')
auto_cache = root / 'automatic-cache'
env.update(XDG_CACHE_HOME=str(auto_cache), ZIRAN_ZX_REGISTRY=f'https://localhost:{server.server_port}')
checked_artifact = f'/api/packages/ziranlang/generic/fixture-zx/{checked_revision}/default.zir.zip'
descriptor = checked_artifact.removesuffix('.zip') + '.tsv'
checked_artifact = f'/api/packages/ziranlang/generic/fixture-zx-library/{checked_revision}/default.zx'
descriptor = checked_artifact + '.tsv'
good_metadata = server.images[descriptor]
expected_tool = 'p' + hashlib.sha256(f'{tool_url}\n{tool_revision}'.encode()).hexdigest()[:16]
assert good_metadata == f'1\t{identity}\t{expected_tool}\tdefault\t{archive_digest}\n'.encode()
assert good_metadata == f'2\t{identity}\t{expected_tool}\tdefault\t{archive_digest}\n'.encode()
add = [ziran, 'add', repository, '--name', 'state', '--ref', checked_revision]
server.images[descriptor] = good_metadata.replace(expected_tool.encode(), b'p0000000000000000')
command(*add, failure='metadata does not match its repository revision, toolchain or profile', cwd=automatic)
@@ -341,6 +424,16 @@ main :: () -> s32 {
assert automatic_lock.read_bytes() == saved
command(*auto_bundle, cwd=automatic)
assert command(ziran, 'run', '--backend', 'engine', auto_output) == '42'
command(ziran, 'check', '--offline', cwd=automatic)
native_output = automatic / 'native'
native_output.mkdir()
command(ziran, 'build', '--project', '--offline', '--target=c', '--exe', '--entry', 'app:main',
'-o', native_output, auto_app, cwd=automatic)
native = subprocess.run([native_output / 'app'], cwd=automatic, check=False, timeout=30)
assert native.returncode == 42
command(ziran, 'build', '--offline', cwd=automatic)
native = subprocess.run([automatic / 'build/consumer'], cwd=automatic, check=False, timeout=30)
assert native.returncode == 42
assert len(server.requests) == before and not (auto_cache / 'ziran/sources').exists()
env['ZIRAN_ZX_PROFILE'] = 'missing'
@@ -362,8 +455,10 @@ main :: () -> s32 {
(override / 'src').mkdir()
(override / 'ziran.toml').write_text('[package]\nname = "state"\nmodule_roots = ["src"]\n' +
tool_section + '[exports]\nmodel = "src/state.zi"\n')
(override / 'src/state.zi').write_text('''State :: struct { value: s32; }
Advance :: (state: *State) { state.value += 4; }
(override / 'src/state.zi').write_text('''Update :: #type (value: s32) -> s32;
State :: struct { value: s32; handler: Update; }
Advance :: (state: *State) { state.value = state.handler(state.value) + 2; }
Copy :: (value: $T) -> T { return value; }
''')
command('git', '-C', override, 'init', '-q')
(automatic / 'ziran.local.toml').write_text(f'[overrides]\nziran = "{toolchain}"\nstate = "{override}"\n')
@@ -376,18 +471,11 @@ Advance :: (state: *State) { state.value += 4; }
pins.write_text('{"format": 1,')
command(*argv, '--offline', failure='invalid checked packages in ziran.zx.lock', cwd=consumer)
wrong_tool = io.BytesIO()
with (zipfile.ZipFile(io.BytesIO(server.images[checked_artifact])) as original,
zipfile.ZipFile(wrong_tool, 'w') as invalid):
for name in original.namelist():
data = original.read(name)
if name == 'checked-package.tsv':
_, package_id, _, profile = data.decode().strip().split('\t')
data = f'1\t{package_id}\tp0000000000000000\t{profile}\n'.encode()
invalid.writestr(name, data)
server.images[checked_artifact] = wrong_tool.getvalue()
wrong_identity = f'2\t{identity}\tp0000000000000000\tdefault\n'.encode()
wrong_tool = make_library([(b'checked-package.tsv', wrong_identity), *sections[1:]])
server.images[checked_artifact] = wrong_tool
pins.write_text(json.dumps({'format': 1, 'packages': [{'package': identity, 'profile': 'default',
'sha256': hashlib.sha256(wrong_tool.getvalue()).hexdigest()}]}))
'sha256': hashlib.sha256(wrong_tool).hexdigest()}]}))
command(*argv, failure='archive does not match its pinned package, toolchain or profile', cwd=consumer)
finally:
server.shutdown()