This commit is contained in:
1 parent
9a22b19d74
commit
fc11f5cefb
16 files changed
+896
-177
No files matched your search
@@ -52,6 +52,41 @@ jobs:
|
||||
run: make
|
||||
- name: Run the language, backend, and portable runtime checks
|
||||
run: env -u DISPLAY -u WAYLAND_DISPLAY WINDOWS_REQUIRED=1 ARM_REQUIRED=1 WASM_REQUIRED=1 ZIRAN_WASM_CC=clang-18 make check CHECK_JOBS=2
|
||||
- name: Pack the tested Ziran toolchain
|
||||
id: toolchain
|
||||
if: github.server_url == 'https://git.waozi.xyz' && github.event_name == 'push' && github.ref == 'refs/heads/master' && (env.BUILD_NATIVE != 'true' || steps.native.outputs.current == 'true')
|
||||
env:
|
||||
SOURCE: ${{ steps.native.outputs.source || github.workspace }}
|
||||
run: |
|
||||
output="build/toolchains/$GITHUB_SHA"
|
||||
if [ "$BUILD_NATIVE" = true ]; then
|
||||
gitea-native-build run python3 scripts/toolchain_package.py pack "$GITHUB_SHA" "$output"
|
||||
else
|
||||
python3 scripts/toolchain_package.py pack "$GITHUB_SHA" "$output"
|
||||
fi
|
||||
printf 'directory=%s/%s\n' "$SOURCE" "$output" >> "$GITHUB_OUTPUT"
|
||||
- name: Keep the verified toolchain archive
|
||||
if: steps.toolchain.outputs.directory != ''
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: toolchain-${{ github.sha }}
|
||||
path: ${{ steps.toolchain.outputs.directory }}
|
||||
if-no-files-found: error
|
||||
- name: Publish the toolchain to Gitea Packages
|
||||
if: steps.toolchain.outputs.directory != ''
|
||||
env:
|
||||
SOURCE: ${{ steps.native.outputs.source || github.workspace }}
|
||||
OUTPUT: ${{ steps.toolchain.outputs.directory }}
|
||||
ZIRAN_TOOLCHAIN_PACKAGE_USER: ${{ secrets.ZIRAN_TOOLCHAIN_PACKAGE_USER }}
|
||||
ZIRAN_TOOLCHAIN_PACKAGE_TOKEN: ${{ secrets.ZIRAN_TOOLCHAIN_PACKAGE_TOKEN }}
|
||||
run: |
|
||||
if [ -z "$ZIRAN_TOOLCHAIN_PACKAGE_USER$ZIRAN_TOOLCHAIN_PACKAGE_TOKEN" ]; then
|
||||
echo "Gitea package writer is unavailable. The toolchain is in this run's artifacts."
|
||||
exit 0
|
||||
fi
|
||||
for descriptor in "$OUTPUT"/*.json; do
|
||||
python3 "$SOURCE/scripts/toolchain_package.py" publish "$GITHUB_SHA" "$descriptor"
|
||||
done
|
||||
- name: Show how much the compile cache saved
|
||||
if: always()
|
||||
run: ccache --show-stats || true
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
name: Quick Ziran build
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
quick:
|
||||
runs-on: build-ziran
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: https://github.com/actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Prepare the C compiler cache
|
||||
run: sh scripts/ci_packages.sh ccache
|
||||
- uses: ./.github/actions/compile-cache
|
||||
with:
|
||||
key: check
|
||||
- name: Compile and start Ziran
|
||||
run: |
|
||||
make JOBS=2 build/bin/ziran
|
||||
build/bin/ziran --version
|
||||
- name: Report compile cache savings
|
||||
if: always()
|
||||
run: ccache --show-stats
|
||||
@@ -47,6 +47,41 @@ jobs:
|
||||
run: make
|
||||
- name: Run the language, backend, and portable runtime checks
|
||||
run: env -u DISPLAY -u WAYLAND_DISPLAY WINDOWS_REQUIRED=1 ARM_REQUIRED=1 WASM_REQUIRED=1 ZIRAN_WASM_CC=clang-18 make check CHECK_JOBS=4
|
||||
- name: Pack the tested Ziran toolchain
|
||||
id: toolchain
|
||||
if: github.server_url == 'https://git.waozi.xyz' && github.event_name == 'push' && github.ref == 'refs/heads/master' && (env.BUILD_NATIVE != 'true' || steps.native.outputs.current == 'true')
|
||||
env:
|
||||
SOURCE: ${{ steps.native.outputs.source || github.workspace }}
|
||||
run: |
|
||||
output="build/toolchains/$GITHUB_SHA"
|
||||
if [ "$BUILD_NATIVE" = true ]; then
|
||||
gitea-native-build run python3 scripts/toolchain_package.py pack "$GITHUB_SHA" "$output"
|
||||
else
|
||||
python3 scripts/toolchain_package.py pack "$GITHUB_SHA" "$output"
|
||||
fi
|
||||
printf 'directory=%s/%s\n' "$SOURCE" "$output" >> "$GITHUB_OUTPUT"
|
||||
- name: Keep the verified toolchain archive
|
||||
if: steps.toolchain.outputs.directory != ''
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: toolchain-${{ github.sha }}
|
||||
path: ${{ steps.toolchain.outputs.directory }}
|
||||
if-no-files-found: error
|
||||
- name: Publish the toolchain to Gitea Packages
|
||||
if: steps.toolchain.outputs.directory != ''
|
||||
env:
|
||||
SOURCE: ${{ steps.native.outputs.source || github.workspace }}
|
||||
OUTPUT: ${{ steps.toolchain.outputs.directory }}
|
||||
ZIRAN_TOOLCHAIN_PACKAGE_USER: ${{ secrets.ZIRAN_TOOLCHAIN_PACKAGE_USER }}
|
||||
ZIRAN_TOOLCHAIN_PACKAGE_TOKEN: ${{ secrets.ZIRAN_TOOLCHAIN_PACKAGE_TOKEN }}
|
||||
run: |
|
||||
if [ -z "$ZIRAN_TOOLCHAIN_PACKAGE_USER$ZIRAN_TOOLCHAIN_PACKAGE_TOKEN" ]; then
|
||||
echo "Gitea package writer is unavailable. The toolchain is in this run's artifacts."
|
||||
exit 0
|
||||
fi
|
||||
for descriptor in "$OUTPUT"/*.json; do
|
||||
python3 "$SOURCE/scripts/toolchain_package.py" publish "$GITHUB_SHA" "$descriptor"
|
||||
done
|
||||
- name: Show how much the compile cache saved
|
||||
if: always()
|
||||
run: ccache --show-stats || true
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
# ZX executable instruction container — revision 28
|
||||
|
||||
This is the one `.zx` format. The format owner is `cmd/compiler_bundle.zi`;
|
||||
This document specifies the `.zx` executable format. The format owner is `cmd/compiler_bundle.zi`;
|
||||
`cmd/compiler_exec_link.zi` binds it to the deterministic LIR codec and
|
||||
verifier (current payload schema 4). `zi2zx` writes it; `ziran run`, `zi2zx run`,
|
||||
the embedding API (`include/ziran_host.h`), `zx2wasm`, and `zx2native` read it.
|
||||
Checked packages use the [ZX library format](zx-library-format.md).
|
||||
|
||||
## Integers, limits and ordering
|
||||
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# ZX checked library format, revision 1
|
||||
|
||||
`ziran pkg zx pack-ir` writes `.zx` libraries for use by the package compiler.
|
||||
The package driver verifies the library and supplies its checked modules to
|
||||
the compiler by package identity. ZX runtime modules and applications use the
|
||||
[executable format](zx-executable-format.md).
|
||||
|
||||
All integers are unsigned little-endian. The 16-byte header has four fields.
|
||||
|
||||
| Offset | Width | Field |
|
||||
| --- | --- | --- |
|
||||
| 0 | 4 | Bytes `ZXLB` |
|
||||
| 4 | 4 | Format revision, `1` |
|
||||
| 8 | 4 | Section count, 3 to 4,098 |
|
||||
| 12 | 4 | Exact file length, up to 268,435,456 bytes |
|
||||
|
||||
Sections follow the header in increasing bytewise order by path. Each record
|
||||
has a four-byte path length, a four-byte payload length, a 32-byte raw SHA-256
|
||||
of the payload, and then the path and payload bytes. Records are contiguous.
|
||||
Paths are unique, relative, and 1 to 1,024 bytes long. Paths cannot contain
|
||||
empty, `.` or `..` components, control bytes below 32, a backslash or a colon.
|
||||
Payloads are nonempty. The final payload ends at the declared file length.
|
||||
|
||||
The required `checked-package.tsv` section is the literal `2`, package
|
||||
identity, toolchain identity, and profile, separated by tabs and terminated
|
||||
by one newline. Both identities have the form `p` followed by 16 lowercase
|
||||
hexadecimal digits. The profile is a registry path segment of up to 128
|
||||
ASCII letters, digits, hyphens, underscores, or periods, excluding `.` and
|
||||
`..`. The required `ziran.toml` section contains the provider manifest. All
|
||||
remaining sections hold serialized checked modules with `.zir` paths derived
|
||||
from the provider's module paths. At least one module is required.
|
||||
|
||||
The registry descriptor repeats the metadata fields and adds the full
|
||||
lowercase hexadecimal SHA-256 of the entire `.zx` file as a fifth field.
|
||||
The package driver matches these pins to the package and toolchain in the
|
||||
resolved dependency graph. The compiler checks module identities against
|
||||
that graph when it opens their checked payloads.
|
||||
|
||||
Development libraries use `ZXLD` as their magic. Their metadata starts with
|
||||
`dev` and adds a fifth field containing a lowercase hexadecimal source
|
||||
fingerprint. It covers the provider manifest and lock, scanned module and
|
||||
export source bytes, and feature defines. The writer checks the fingerprint
|
||||
again before installing the completed file. Production package and runtime
|
||||
readers reject this magic. A local development reader must be selected
|
||||
explicitly and still checks the package and toolchain identities.
|
||||
@@ -2,6 +2,9 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
- Package tested native Ziran toolchains in Gitea by exact source revision and Linux host profile.
|
||||
- Distribute checked packages as versioned `.zx` libraries for source-free imports in ZX and native builds.
|
||||
|
||||
- Publish and fetch verified `.zx` images and checked compiler companions through Gitea. Package locks can discover and pin compatible checked dependencies for source-free ZX builds.
|
||||
|
||||
- C embedders can open seekable ZX app images through `BundleOpenReadAt` without keeping a full input buffer. The callback is needed only during verification, and images with assets, module linkage, or additional attachments use the byte-based API.
|
||||
|
||||
+22
-4
@@ -848,7 +848,9 @@ PackageMain :: (launcher: string, arguments: []string) -> s32 {
|
||||
return 0
|
||||
}
|
||||
command: string = arguments[0]
|
||||
pack_checked := command == "pkg" && arguments.count >= 3 && arguments[1] == "zx" && arguments[2] == "pack-ir"
|
||||
package_zx := command == "pkg" && arguments.count >= 3 && arguments[1] == "zx"
|
||||
pack_dev := package_zx && arguments[2] == "pack-dev"
|
||||
pack_checked := pack_dev || (package_zx && arguments[2] == "pack-ir")
|
||||
// A binary-only consumer need not have a project or check out its sources.
|
||||
if !pack_checked && command == "pkg" && arguments.count >= 2 && arguments[1] == "zx" {
|
||||
return PkgZx(launcher, arguments[2:])
|
||||
@@ -928,6 +930,7 @@ PackageMain :: (launcher: string, arguments: []string) -> s32 {
|
||||
project: bool = PkgHasArgument(flags, "--project") || implied
|
||||
locked_mode: bool = PkgHasArgument(flags, "--locked")
|
||||
offline: bool = PkgHasArgument(flags, "--offline")
|
||||
if pack_dev && locked_mode { return PkgFail("pack-dev cannot be used with --locked") }
|
||||
filtered := PkgStrings(arguments.count)
|
||||
if filtered.count != arguments.count { return PkgFail("cannot allocate arguments") }
|
||||
count: s32 = 0
|
||||
@@ -967,12 +970,27 @@ PackageMain :: (launcher: string, arguments: []string) -> s32 {
|
||||
}
|
||||
selection := PkgCheckedSelection.{call = PkgSelectChecked, context = cast(*void)(*checked_pins)}
|
||||
checked: *PkgCheckedSelection = null
|
||||
if command == "bundle" || (command == "build" && PkgHasArgument(flags, "--target=zx")) ||
|
||||
compile := command == "build" || command == "run" || command == "check"
|
||||
target := PkgHasTarget(flags[1:])
|
||||
if compile && lifecycle && !target { checked_pins.lifecycle_manifest = current }
|
||||
if command == "bundle" || (compile && !target) ||
|
||||
(compile && PkgHasArgument(flags, "--target=c")) ||
|
||||
(command == "build" && PkgHasArgument(flags, "--target=zx")) ||
|
||||
pack_checked || (command == "pkg" && count >= 1 && filtered[0] == "list") {
|
||||
if !PkgReadCheckedPins(root, launcher, locked, *checked_pins) {
|
||||
return PkgFail("invalid checked packages in ziran.zx.lock")
|
||||
}
|
||||
if checked_pins.count != 0 { checked = *selection }
|
||||
if command == "bundle" || compile || pack_dev {
|
||||
development := PkgGetenv("ZIRAN_DEV_ZX_DIR")
|
||||
if development != "" {
|
||||
if locked_mode { return PkgFail("ZIRAN_DEV_ZX_DIR cannot be used with --locked") }
|
||||
checked_pins.development_directory = PkgCanonical(development)
|
||||
if !DirectoryExists(checked_pins.development_directory) {
|
||||
return PkgFail("ZIRAN_DEV_ZX_DIR must name a directory")
|
||||
}
|
||||
}
|
||||
}
|
||||
if checked_pins.count != 0 || checked_pins.development_directory != "" { checked = *selection }
|
||||
}
|
||||
if !PkgCheckoutGraph(root, locked, offline, locked_mode, current, checked) {
|
||||
if pkg_failure_reported { return 2 }
|
||||
@@ -1028,7 +1046,7 @@ PackageMain :: (launcher: string, arguments: []string) -> s32 {
|
||||
if pack_checked {
|
||||
directory := PkgToolDirectory(launcher, locked.tool_path, "zi2zir")
|
||||
if directory == "" { return PkgFail("cannot find the pinned Ziran compiler") }
|
||||
return PkgPackChecked(root, PkgJoin(directory, "/zi2zir"), locked, current, filtered[2:count])
|
||||
return PkgPackChecked(root, PkgJoin(directory, "/zi2zir"), locked, current, filtered[2:count], pack_dev)
|
||||
}
|
||||
if command == "fetch" { return 0 }
|
||||
if command == "compile-commands" {
|
||||
|
||||
+182
-48
@@ -5,74 +5,192 @@
|
||||
#import "package_manifest"
|
||||
#import "package_map"
|
||||
#import "package_script"
|
||||
#import "binary"
|
||||
#import "byte_text_linux"
|
||||
#import "file_linux"
|
||||
#import "mapped_file_linux"
|
||||
#import "zip"
|
||||
#import "zip_file_linux"
|
||||
|
||||
#scope_file
|
||||
Metadata :: "checked-package.tsv"
|
||||
Manifest :: "ziran.toml"
|
||||
MaximumBytes :: 268435456
|
||||
HeaderBytes :: 16
|
||||
RecordBytes :: 40
|
||||
MaximumNameBytes :: 1024
|
||||
MaximumRecords :: 4098
|
||||
|
||||
ArchiveIdentity :: (id: string, tool_url: string, tool_commit: string, profile: string) -> string {
|
||||
ArchiveIdentity :: (id: string, tool_url: string, tool_commit: string, profile: string,
|
||||
source_digest: string) -> string {
|
||||
tool := PkgLockId(tool_url, tool_commit, false)
|
||||
return PkgJoin3(PkgJoin3("1\t", id, "\t"), PkgJoin3(tool, "\t", profile), "\n")
|
||||
prefix := CheckedDescriptorPrefix(id, tool, profile)
|
||||
if source_digest == "" { return PkgJoin(prefix, "\n") }
|
||||
return PkgJoin3(PkgJoin("dev\t", prefix[2:]), "\t", PkgJoin(source_digest, "\n"))
|
||||
}
|
||||
|
||||
ValidIdentity :: (identity: string, id: string, tool: string, profile: string, development: bool) -> bool {
|
||||
prefix := ifx development then "dev\t" else "2\t"
|
||||
if identity.count < prefix.count + 38 || identity[0:prefix.count] != prefix ||
|
||||
identity[identity.count - 1:] != "\n" { return false }
|
||||
fields := identity[prefix.count:identity.count - 1]
|
||||
profile_end := cast(s64)fields.count
|
||||
if development {
|
||||
profile_end -= 65
|
||||
if profile_end <= 36 || fields[profile_end:profile_end + 1] != "\t" ||
|
||||
!PkgHex(fields[profile_end + 1:], 64) { return false }
|
||||
}
|
||||
if fields[17:18] != "\t" || fields[35:36] != "\t" ||
|
||||
!PkgCheckedIdentity(fields[0:17]) || !PkgCheckedIdentity(fields[18:35]) { return false }
|
||||
actual_profile := fields[36:profile_end]
|
||||
return PkgRegistrySegment(actual_profile) && (id == "" || fields[0:17] == id) &&
|
||||
(tool == "" || fields[18:35] == tool) &&
|
||||
(profile == "" || actual_profile == profile)
|
||||
}
|
||||
|
||||
SourceFingerprint :: (root: string, current: *PkgManifest, options: []string, modules: *PkgModuleSet) -> string {
|
||||
modules.count = 0
|
||||
if !PkgScanModules(root, current, modules) { return "" }
|
||||
paths := PkgStrings(modules.count + current.export_count + 2)
|
||||
if paths.count == 0 { return "" }
|
||||
paths[0] = Manifest
|
||||
paths[1] = "ziran.lock"
|
||||
for index: 0..modules.count - 1 { paths[index + 2] = modules.items[index].path[root.count + 1:] }
|
||||
for index: 0..current.export_count - 1 { paths[modules.count + 2 + index] = current.exports[index].value }
|
||||
for index: 1..paths.count - 1 {
|
||||
at := index
|
||||
while at > 0 && PkgLexLess(paths[at], paths[at - 1]) {
|
||||
previous := paths[at - 1]
|
||||
paths[at - 1] = paths[at]
|
||||
paths[at] = previous
|
||||
at -= 1
|
||||
}
|
||||
}
|
||||
digest := PkgDigest("ZX development provider source 1")
|
||||
for option: options { PkgArtifactMix(*digest, option) }
|
||||
previous := ""
|
||||
for path: paths {
|
||||
if path == previous { continue }
|
||||
hash := PkgArtifactHashFile(PkgJoin3(root, "/", path))
|
||||
if !PkgHex(hash, 64) { return "" }
|
||||
PkgArtifactMix(*digest, path)
|
||||
PkgArtifactMix(*digest, hash)
|
||||
previous = path
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
StoreU32 :: (bytes: []u8, offset: s32, value: u32) {
|
||||
for index: 0..3 { bytes[offset + index] = cast(u8)(value >> cast(u32)(index * 8)) }
|
||||
}
|
||||
|
||||
WriteAll :: (file: FileHandle, bytes: []u8, offset: s64) -> bool {
|
||||
written: s64 = 0
|
||||
while written < cast(s64)bytes.count {
|
||||
size := WriteAt(file, bytes[written:], offset + written)
|
||||
if size <= 0 { return false }
|
||||
written += size
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
VerifyImageDigest :: (bytes: []u8, expected: string) -> bool {
|
||||
if expected == "" { return true }
|
||||
digest: [32]u8
|
||||
if expected.count != 64 || !PkgSha256Bytes(bytes, digest[:]) {
|
||||
return false
|
||||
}
|
||||
hex := "0123456789abcdef"
|
||||
for index: 0..31 {
|
||||
if expected[2 * index] != hex[cast(s32)(digest[index] >> cast(u8)4)] ||
|
||||
expected[2 * index + 1] != hex[cast(s32)(digest[index] & cast(u8)15)] { return false }
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
#scope_export
|
||||
// The archive never supplies files whose kind or location is chosen by a ZIP
|
||||
// extractor. Only stored module records and the two fixed metadata files are
|
||||
// read. When staging, all outputs are regular files below a private directory.
|
||||
PkgRegistrySegment :: (value: string) -> bool {
|
||||
if value.count == 0 || value.count > 128 || value == "." || value == ".." { return false }
|
||||
for ch: value {
|
||||
if !(ch >= #char "a" && ch <= #char "z" || ch >= #char "A" && ch <= #char "Z" ||
|
||||
ch >= #char "0" && ch <= #char "9" || ch == #char "-" || ch == #char "_" || ch == #char ".") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
CheckedDescriptorPrefix :: (id: string, tool: string, profile: string) -> string {
|
||||
return PkgJoin3(PkgJoin3("2\t", id, "\t"), PkgJoin3(tool, "\t", profile), "")
|
||||
}
|
||||
|
||||
// Staged sections are regular files in an empty private directory. The caller
|
||||
// discards that directory if any section or the package identity fails verification.
|
||||
PkgCheckedArchive :: (path: string, digest: string, stage: string, id: string,
|
||||
tool_url: string, tool_commit: string, profile: string, identity: *string = null) -> bool {
|
||||
tool_url: string, tool_commit: string, profile: string, identity: *string = null,
|
||||
development: bool = false) -> bool {
|
||||
mapped := MapPrivate(path)
|
||||
if !mapped.valid { return false }
|
||||
defer UnmapFile(mapped)
|
||||
if mapped.length == 0 || mapped.length > MaximumBytes { return false }
|
||||
if mapped.length < HeaderBytes || mapped.length > MaximumBytes { return false }
|
||||
bytes := MappedBytes(mapped)
|
||||
if digest != "" && PkgDigest(TextFromBytes(bytes)) != digest { return false }
|
||||
|
||||
scan := OpenZipDirectory(bytes)
|
||||
if !scan.valid || scan.count < 3 || scan.count > 4098 { return false }
|
||||
text := TextFromBytes(bytes)
|
||||
limit := cast(s64)bytes.count
|
||||
magic := ifx development then "ZXLD" else "ZXLB"
|
||||
if text[0:4] != magic || ReadU32LE(text, 4) != 1 || ReadU32LE(text, 12) != cast(u32)limit ||
|
||||
!VerifyImageDigest(bytes, digest) { return false }
|
||||
count := ReadU32LE(text, 8)
|
||||
if count < 3 || count > MaximumRecords { return false }
|
||||
position: s64 = HeaderBytes
|
||||
prior := ""
|
||||
metadata := false
|
||||
manifest := false
|
||||
for index: 0..cast(s32)scan.count - 1 {
|
||||
for index: 0..cast(s32)count - 1 {
|
||||
unused index
|
||||
named := NextZipEntry(bytes, scan)
|
||||
if !named.valid || named.entry.method != 0 || named.entry.size == 0 ||
|
||||
named.name_size > 1024 { return false }
|
||||
scan = named.directory
|
||||
name := TextFromBytes(bytes[named.name_start:named.name_start + named.name_size])
|
||||
if limit - position < RecordBytes { return false }
|
||||
name_size := cast(s64)ReadU32LE(text, cast(s32)position)
|
||||
data_size := cast(s64)ReadU32LE(text, cast(s32)position + 4)
|
||||
hash_at := position + 8
|
||||
position += RecordBytes
|
||||
if name_size < 1 || name_size > MaximumNameBytes || data_size < 1 ||
|
||||
name_size + data_size > limit - position { return false }
|
||||
name := text[position:position + name_size]
|
||||
position += name_size
|
||||
if !PkgArtifactPath(name) || (prior != "" && !PkgLexLess(prior, name)) { return false }
|
||||
prior = name
|
||||
data := bytes[named.entry.data_start:named.entry.data_start + named.entry.size]
|
||||
if ZipCRC32(data) != named.entry.crc32 { return false }
|
||||
hash: [32]u8
|
||||
if !PkgSha256Bytes(bytes[position:position + data_size], hash[:]) { return false }
|
||||
for part: 0..31 {
|
||||
if hash[part] != bytes[hash_at + part] { return false }
|
||||
}
|
||||
data := text[position:position + data_size]
|
||||
position += data_size
|
||||
if name == Metadata {
|
||||
if id != "" && TextFromBytes(data) != ArchiveIdentity(id, tool_url, tool_commit, profile) {
|
||||
tool := ifx tool_commit == "" then "" else PkgLockId(tool_url, tool_commit, false)
|
||||
if !ValidIdentity(data, id, tool, profile, development) {
|
||||
return false
|
||||
}
|
||||
if identity != null {
|
||||
if data.count > 512 { return false }
|
||||
<<identity = PkgJoin("", TextFromBytes(data))
|
||||
}
|
||||
if identity != null { <<identity = PkgJoin("", data) }
|
||||
metadata = true
|
||||
} else if name == Manifest { manifest = true }
|
||||
else if !PkgEndsWith(name, ".zir") { return false }
|
||||
if stage != "" && name != Metadata {
|
||||
file := PkgJoin3(stage, "/", name)
|
||||
if !PkgMkdirs(ParentDirectory(file)) || !PkgWrite(file, TextFromBytes(data)) { return false }
|
||||
if !PkgMkdirs(ParentDirectory(file)) || !PkgWrite(file, data) { return false }
|
||||
}
|
||||
}
|
||||
return metadata && manifest && scan.cursor == scan.end
|
||||
return metadata && manifest && position == limit
|
||||
}
|
||||
|
||||
PkgCheckedPack :: (root: string, locked: *PkgLock, current: *PkgManifest,
|
||||
compiler: string, repository: string, revision: string, profile: string, output: string,
|
||||
options: []string) -> s32 {
|
||||
options: []string, development: bool) -> s32 {
|
||||
id := PkgLockId(repository, revision, false)
|
||||
modules := cast(*PkgModuleSet)PkgAllocZero(cast(u64)size_of(PkgModuleSet))
|
||||
if modules == null { return PkgFail("cannot allocate checked package modules") }
|
||||
source_digest: string
|
||||
if development {
|
||||
source_digest = SourceFingerprint(root, current, options, modules)
|
||||
if source_digest == "" { return PkgFail("cannot fingerprint development package sources") }
|
||||
}
|
||||
stage := PkgArtifactStage(PkgJoin(PkgCacheRoot(), "/zx"))
|
||||
if stage == "" { return PkgFail("cannot stage checked package modules") }
|
||||
defer PkgArtifactRemove(stage)
|
||||
@@ -95,8 +213,8 @@ PkgCheckedPack :: (root: string, locked: *PkgLock, current: *PkgManifest,
|
||||
values := string.[map]
|
||||
if PkgRun(command, root, names[:], values[:]) != 0 { return 2 }
|
||||
|
||||
modules := cast(*PkgModuleSet)PkgAllocZero(cast(u64)size_of(PkgModuleSet))
|
||||
if modules == null || !PkgScanModules(root, current, modules) { return PkgFail("cannot find checked package modules") }
|
||||
modules.count = 0
|
||||
if !PkgScanModules(root, current, modules) { return PkgFail("cannot find checked package modules") }
|
||||
paths := PkgStrings(modules.count + 2)
|
||||
sources := PkgStrings(modules.count + 2)
|
||||
if paths.count == 0 || sources.count == 0 { return PkgFail("cannot allocate checked package entries") }
|
||||
@@ -104,7 +222,7 @@ PkgCheckedPack :: (root: string, locked: *PkgLock, current: *PkgManifest,
|
||||
sources[0] = PkgJoin3(root, "/", Manifest)
|
||||
paths[1] = Metadata
|
||||
sources[1] = PkgJoin3(stage, "/", Metadata)
|
||||
if !PkgWrite(sources[1], ArchiveIdentity(id, locked.tool_url, locked.tool_commit, profile)) {
|
||||
if !PkgWrite(sources[1], ArchiveIdentity(id, locked.tool_url, locked.tool_commit, profile, source_digest)) {
|
||||
return PkgFail("cannot write checked package metadata")
|
||||
}
|
||||
count: s32 = 2
|
||||
@@ -130,36 +248,52 @@ PkgCheckedPack :: (root: string, locked: *PkgLock, current: *PkgManifest,
|
||||
at -= 1
|
||||
}
|
||||
}
|
||||
temporary := PkgTemporarySibling(output, ".zir.zip")
|
||||
if count > MaximumRecords { return PkgFail("too many checked package sections") }
|
||||
temporary := PkgTemporarySibling(output, ".zx")
|
||||
if temporary == "" { return PkgFail("cannot stage the checked package archive") }
|
||||
defer RemoveFile(temporary)
|
||||
file := OpenExclusivePrivate(temporary)
|
||||
if !file.valid { return PkgFail("cannot create checked package archive") }
|
||||
position: u64 = 0
|
||||
entries := cast(*ZipFileRecord)PkgAlloc(cast(u64)count * cast(u64)size_of(ZipFileRecord))
|
||||
valid := entries != null
|
||||
position: s64 = HeaderBytes
|
||||
valid := true
|
||||
for index: 0..count - 1 {
|
||||
if !valid { break }
|
||||
name := paths[index]
|
||||
if name.count > MaximumNameBytes { valid = false; break }
|
||||
mapped := MapPrivate(sources[index])
|
||||
if !mapped.valid || mapped.length == 0 || mapped.length > MaximumBytes - position {
|
||||
record_size := RecordBytes + cast(s64)name.count
|
||||
if !mapped.valid || mapped.length == 0 || mapped.length > MaximumBytes ||
|
||||
record_size + cast(s64)mapped.length > MaximumBytes - position {
|
||||
unused UnmapFile(mapped)
|
||||
valid = false
|
||||
break
|
||||
}
|
||||
entries[index] = WriteStoredZipFileEntry(file, position, paths[index], MappedBytes(mapped))
|
||||
record: [RecordBytes + MaximumNameBytes]u8
|
||||
StoreU32(record[:], 0, cast(u32)name.count)
|
||||
StoreU32(record[:], 4, cast(u32)mapped.length)
|
||||
data := MappedBytes(mapped)
|
||||
valid = PkgSha256Bytes(data, record[8:RecordBytes]) &&
|
||||
PkgCopy(record[:], RecordBytes, name) == record_size &&
|
||||
WriteAll(file, record[0:record_size], position) &&
|
||||
WriteAll(file, data, position + record_size)
|
||||
position += record_size + cast(s64)mapped.length
|
||||
unused UnmapFile(mapped)
|
||||
position = StoredZipFileNext(entries[index])
|
||||
valid = entries[index].valid && position < MaximumBytes
|
||||
}
|
||||
directory_start := position
|
||||
for index: 0..count - 1 {
|
||||
if !valid { break }
|
||||
position = WriteStoredZipFileDirectoryEntry(file, position, paths[index], entries[index])
|
||||
valid = position != 0 && position < MaximumBytes - 22
|
||||
if valid {
|
||||
header: [HeaderBytes]u8
|
||||
magic := ifx development then "ZXLD" else "ZXLB"
|
||||
unused PkgCopy(header[:], 0, magic)
|
||||
StoreU32(header[:], 4, 1)
|
||||
StoreU32(header[:], 8, cast(u32)count)
|
||||
StoreU32(header[:], 12, cast(u32)position)
|
||||
valid = WriteAll(file, header[:], 0) && SyncFile(file)
|
||||
}
|
||||
if valid { valid = FinishStoredZipFile(file, directory_start, position, cast(u32)count) && SyncFile(file) }
|
||||
valid = CloseFile(file) && valid
|
||||
if !valid || !PkgCheckedArchive(temporary, "", "", id, locked.tool_url, locked.tool_commit, profile) ||
|
||||
!RenamePath(temporary, output) { return PkgFail("cannot finish checked package archive") }
|
||||
if !valid || !PkgCheckedArchive(temporary, "", "", id, locked.tool_url, locked.tool_commit,
|
||||
profile, null, development) { return PkgFail("cannot finish checked package archive") }
|
||||
if development && SourceFingerprint(root, current, options, modules) != source_digest {
|
||||
return PkgFail("development package sources changed during export")
|
||||
}
|
||||
if !RenamePath(temporary, output) { return PkgFail("cannot finish checked package archive") }
|
||||
return 0
|
||||
}
|
||||
@@ -389,6 +389,11 @@ PkgShowToolFailure :: (output: string) {
|
||||
else { PkgPrint(output, true); PkgPrint("\n", true) }
|
||||
}
|
||||
|
||||
PkgSha256Bytes :: (bytes: []u8, digest: []u8) -> bool {
|
||||
return bytes.count > 0 && digest.count == 32 &&
|
||||
PkgSha256Raw(*bytes[0], cast(u64)bytes.count, *digest[0]) != null
|
||||
}
|
||||
|
||||
PkgDigest :: (value: string) -> string {
|
||||
digest: [32]u8
|
||||
memory := PkgAlloc(cast(u64)value.count + 1)
|
||||
|
||||
@@ -8,7 +8,7 @@ PkgExplanation :: struct {
|
||||
}
|
||||
|
||||
#scope_file
|
||||
DIAGNOSTIC_CODES: string = "assembly.codec\nassembly.container\nassembly.encode\nassembly.execute\nassembly.gap.cpu-feature\nassembly.gap.machine-host\nassembly.gap.machine-host-abi\nassembly.gap.machine-platform-service-unavailable\nassembly.gap.os-service\nassembly.gap.target-unimplemented\nassembly.inspect\nassembly.lower\nassembly.machine\nassembly.optimize\nassembly.output\nassembly.verify\ncheck.array_signature\ncheck.assembly\ncheck.assert\ncheck.cleanup\ncheck.compile_time\ncheck.constant\ncheck.constant_type\ncheck.consteval\ncheck.enum\ncheck.enum_scope\ncheck.file_scope\ncheck.foreign\ncheck.global\ncheck.if_case\ncheck.import\ncheck.inline\ncheck.jai_syntax\ncheck.record\ncheck.reserved_name\ncheck.signature\ncheck.slice_lifetime\ncheck.slice_signature\ncheck.slot_initializer\ncheck.slot_signature\ncheck.specialize\ncheck.storage\ncheck.template\ncheck.type\ncheck.type_application\ncheck.type_order\ncheck.using_scope\ncheck.vec_copy\ncommand.arguments\ncompiler.context\ncompiler.internal\ncompiler.memory\nemit.array_count\nemit.array_element\nemit.array_target\nemit.expression\nemit.loop_nesting\nemit.parallel\nformat.changed\nformat.input\nformat.output\ngpu.allocator\ngpu.device\ngpu.pointer\nhost.bind\nlaw.gate\nlaw.identity\nlaw.waiver\nlint.bool\nlint.cast\nlint.char\nmodule.ambiguous\nmodule.compile_cycle\nmodule.compile_depth\nmodule.duplicate\nmodule.extension\nmodule.identity\nmodule.input\nmodule.limit\nmodule.not_found\nmodule.parameter_conflict\nmodule.parameter_type\nmodule.parameter_unknown\nmodule.path\nmodule.root\npackage.ambiguous\npackage.command\npackage.identity\npackage.map\npackage.module\npackage.not_found\npackage.process\npackage.project\npackage.scope\npackage.tool\nparallel.effect\nparallel.leave\nparallel.memory\nparallel.nest\nparallel.region\nparse.caller_location\nparse.fatal\nparse.jai_char\nparse.jai_syntax\nparse.modifier\nparse.procedure_name\nparse.syntax\nparse.this\nproof.claim\nproof.graph\nproof.identity\nproof.syntax\nwasm.container\nwasm.gap.address\nwasm.gap.aggregate\nwasm.gap.allocate\nwasm.gap.call\nwasm.gap.compare\nwasm.gap.constant\nwasm.gap.control\nwasm.gap.convert\nwasm.gap.data-size\nwasm.gap.edge\nwasm.gap.entry\nwasm.gap.float-operator\nwasm.gap.float-width\nwasm.gap.foreign-service\nwasm.gap.frame\nwasm.gap.fuel\nwasm.gap.function\nwasm.gap.global\nwasm.gap.global-constant\nwasm.gap.global-layout\nwasm.gap.host\nwasm.gap.host-capability-abi\nwasm.gap.initializer\nwasm.gap.instruction\nwasm.gap.integer-width\nwasm.gap.load\nwasm.gap.offset\nwasm.gap.operand\nwasm.gap.operator\nwasm.gap.platform-service-abi\nwasm.gap.platform-service-unavailable\nwasm.gap.resize\nwasm.gap.result\nwasm.gap.signature\nwasm.gap.store\nwasm.gap.type\nwasm.input\nwasm.output\nwasm.verify\nzir.input\nzir.invalid\nzir.noncanonical\nzir.output\nzir.validation\nzir_c.body\nzir_c.entry\nzir_c.enum\nzir_c.global\nzir_c.record\nzir_c.toolchain\nzir_go.bind\nzir_go.body\nzir_go.callback\nzir_go.enum\nzir_go.exe\nzir_go.export\nzir_go.global\nzir_go.import\nzir_go.parameter\nzir_go.tests\nzir_go.type\nzir_go.union\nzx.assets\nzx.bind\nzx.call\nzx.capability\nzx.container\nzx.debug\nzx.entry\nzx.host_alias\nzx.input\nzx.module\nzx.module-export\nzx.module-import\nzx.module-run\nzx.native\nzx.native.pointers\nzx.native.relocation\nzx.output\nzx.runtime\nzx.state\nzx.statement\nzx.union\nzx.wasm\n";
|
||||
DIAGNOSTIC_CODES: string = "assembly.codec\nassembly.container\nassembly.encode\nassembly.execute\nassembly.gap.cpu-feature\nassembly.gap.machine-host\nassembly.gap.machine-host-abi\nassembly.gap.machine-platform-service-unavailable\nassembly.gap.os-service\nassembly.gap.target-unimplemented\nassembly.inspect\nassembly.lower\nassembly.machine\nassembly.optimize\nassembly.output\nassembly.verify\ncheck.array_signature\ncheck.assembly\ncheck.assert\ncheck.cleanup\ncheck.compile_time\ncheck.constant\ncheck.constant_type\ncheck.consteval\ncheck.enum\ncheck.enum_scope\ncheck.file_scope\ncheck.foreign\ncheck.global\ncheck.if_case\ncheck.import\ncheck.inline\ncheck.jai_syntax\ncheck.record\ncheck.reserved_name\ncheck.signature\ncheck.slice_lifetime\ncheck.slice_signature\ncheck.slot_initializer\ncheck.slot_signature\ncheck.specialize\ncheck.storage\ncheck.template\ncheck.type\ncheck.type_application\ncheck.type_order\ncheck.using_scope\ncheck.vec_copy\ncommand.arguments\ncompiler.context\ncompiler.internal\ncompiler.memory\nemit.array_count\nemit.array_element\nemit.array_target\nemit.expression\nemit.loop_nesting\nemit.parallel\nformat.changed\nformat.input\nformat.output\ngpu.allocator\ngpu.device\ngpu.pointer\nhost.bind\nlaw.gate\nlaw.identity\nlaw.waiver\nlint.bool\nlint.cast\nlint.char\nmodule.ambiguous\nmodule.compile_cycle\nmodule.compile_depth\nmodule.duplicate\nmodule.extension\nmodule.identity\nmodule.input\nmodule.limit\nmodule.not_found\nmodule.parameter_conflict\nmodule.parameter_type\nmodule.parameter_unknown\nmodule.path\nmodule.root\npackage.ambiguous\npackage.command\npackage.identity\npackage.map\npackage.module\npackage.not_found\npackage.process\npackage.project\npackage.scope\npackage.tool\npackage.zx-lock\nparallel.effect\nparallel.leave\nparallel.memory\nparallel.nest\nparallel.region\nparse.caller_location\nparse.fatal\nparse.jai_char\nparse.jai_syntax\nparse.modifier\nparse.procedure_name\nparse.syntax\nparse.this\nproof.claim\nproof.graph\nproof.identity\nproof.syntax\nwasm.container\nwasm.gap.address\nwasm.gap.aggregate\nwasm.gap.allocate\nwasm.gap.call\nwasm.gap.compare\nwasm.gap.constant\nwasm.gap.control\nwasm.gap.convert\nwasm.gap.data-size\nwasm.gap.edge\nwasm.gap.entry\nwasm.gap.float-operator\nwasm.gap.float-width\nwasm.gap.foreign-service\nwasm.gap.frame\nwasm.gap.fuel\nwasm.gap.function\nwasm.gap.global\nwasm.gap.global-constant\nwasm.gap.global-layout\nwasm.gap.host\nwasm.gap.host-capability-abi\nwasm.gap.initializer\nwasm.gap.instruction\nwasm.gap.integer-width\nwasm.gap.load\nwasm.gap.offset\nwasm.gap.operand\nwasm.gap.operator\nwasm.gap.platform-service-abi\nwasm.gap.platform-service-unavailable\nwasm.gap.resize\nwasm.gap.result\nwasm.gap.signature\nwasm.gap.store\nwasm.gap.type\nwasm.input\nwasm.output\nwasm.verify\nzir.input\nzir.invalid\nzir.noncanonical\nzir.output\nzir.validation\nzir_c.body\nzir_c.entry\nzir_c.enum\nzir_c.global\nzir_c.record\nzir_c.toolchain\nzir_go.bind\nzir_go.body\nzir_go.callback\nzir_go.enum\nzir_go.exe\nzir_go.export\nzir_go.global\nzir_go.import\nzir_go.parameter\nzir_go.tests\nzir_go.type\nzir_go.union\nzx.assets\nzx.bind\nzx.call\nzx.capability\nzx.container\nzx.debug\nzx.entry\nzx.host_alias\nzx.input\nzx.module\nzx.module-export\nzx.module-import\nzx.module-run\nzx.native\nzx.native.pointers\nzx.native.relocation\nzx.output\nzx.runtime\nzx.state\nzx.statement\nzx.union\nzx.wasm\n";
|
||||
|
||||
PkgStartsWith :: (value: string, prefix: string) -> bool {
|
||||
if prefix.count > value.count { return false }
|
||||
|
||||
+90
-64
@@ -12,36 +12,25 @@
|
||||
#scope_file
|
||||
MaximumImageBytes :: 268435456
|
||||
|
||||
RegistrySegment :: (value: string) -> bool {
|
||||
if value.count == 0 || value.count > 128 || value == "." || value == ".." { return false }
|
||||
for ch: value {
|
||||
if !(ch >= #char "a" && ch <= #char "z" || ch >= #char "A" && ch <= #char "Z" ||
|
||||
ch >= #char "0" && ch <= #char "9" || ch == #char "-" || ch == #char "_" || ch == #char ".") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
ModuleName :: (value: string) -> bool {
|
||||
if value.count > 512 { return false }
|
||||
start: s64 = 0
|
||||
for index: 0..cast(s64)value.count - 1 {
|
||||
if value[index] == #char "/" {
|
||||
if !RegistrySegment(value[start:index]) { return false }
|
||||
if !PkgRegistrySegment(value[start:index]) { return false }
|
||||
start = index + 1
|
||||
}
|
||||
}
|
||||
return RegistrySegment(value[start:])
|
||||
return PkgRegistrySegment(value[start:])
|
||||
}
|
||||
|
||||
HexDigest :: (value: string) -> bool { return PkgHex(value, 64) }
|
||||
|
||||
// Accept the origin of a Gitea repository, not an arbitrary download URL.
|
||||
// The commit and profile are separate checked path segments.
|
||||
RegistryUrl :: (repository: string, revision: string, profile: string, extension: string = ".zx") -> string {
|
||||
RegistryUrl :: (repository: string, revision: string, profile: string, checked: bool = false) -> string {
|
||||
if repository.count > 576 || repository.count <= 8 || repository[0:8] != "https://" ||
|
||||
!PkgValidCommit(revision) || !RegistrySegment(profile) { return "" }
|
||||
!PkgValidCommit(revision) || !PkgRegistrySegment(profile) { return "" }
|
||||
authority_end: s64 = 8
|
||||
while authority_end < repository.count && repository[authority_end] != #char "/" { authority_end += 1 }
|
||||
if authority_end == 8 || authority_end == repository.count { return "" }
|
||||
@@ -67,11 +56,12 @@ RegistryUrl :: (repository: string, revision: string, profile: string, extension
|
||||
owner := repository[authority_end + 1:owner_end]
|
||||
repo := repository[owner_end + 1:]
|
||||
if PkgEndsWith(repo, ".git") { repo = repo[0:repo.count - 4] }
|
||||
if !RegistrySegment(owner) || !RegistrySegment(repo) { return "" }
|
||||
if !PkgRegistrySegment(owner) || !PkgRegistrySegment(repo) { return "" }
|
||||
namespace := PkgJoin3(repository[0:authority_end], "/api/packages/", owner)
|
||||
package := PkgJoin3(namespace, "/generic/", repo)
|
||||
version := PkgJoin3(package, "-zx/", revision)
|
||||
return PkgJoin(PkgJoin3(version, "/", profile), extension)
|
||||
channel := ifx checked then "-zx-library/" else "-zx/"
|
||||
version := PkgJoin3(package, channel, revision)
|
||||
return PkgJoin(PkgJoin3(version, "/", profile), ".zx")
|
||||
}
|
||||
|
||||
// Validate the size before hashing or passing a remote image to the verifier.
|
||||
@@ -92,8 +82,8 @@ ImageDigest :: (path: string) -> string {
|
||||
}
|
||||
}
|
||||
|
||||
ImageVerified :: (launcher: string, path: string, module: string) -> bool {
|
||||
if PkgEndsWith(path, ".zir.zip") { return PkgCheckedArchive(path, "", "", "", "", "", "") }
|
||||
ImageVerified :: (launcher: string, path: string, module: string, id: string = "", profile: string = "") -> bool {
|
||||
if id != "" { return PkgCheckedArchive(path, "", "", id, "", "", profile) }
|
||||
compiler := PkgJoin(PkgLauncherDirectory(launcher), "/zi2zx")
|
||||
args := string.[compiler, "--emit", "linkage", "-o", "-", path]
|
||||
result := PkgCapture(args[:], "")
|
||||
@@ -139,7 +129,7 @@ TransferFailure :: (action: string, status: string) -> s32 {
|
||||
}
|
||||
|
||||
CheckedDescriptorUrl :: (archive_url: string) -> string {
|
||||
return PkgJoin(archive_url[0:archive_url.count - 4], ".tsv")
|
||||
return PkgJoin(archive_url, ".tsv")
|
||||
}
|
||||
|
||||
PublishFile :: (url: string, path: string, digest: string, netrc: string, stage: string) -> s32 {
|
||||
@@ -157,15 +147,14 @@ PublishFile :: (url: string, path: string, digest: string, netrc: string, stage:
|
||||
|
||||
Publish :: (launcher: string, url: string, file: string, netrc: string,
|
||||
id: string, profile: string) -> s32 {
|
||||
extension := ifx PkgEndsWith(url, ".zir.zip") then ".zir.zip" else ".zx"
|
||||
if !PkgEndsWith(file, extension) { return PkgFail(PkgJoin("expected an artifact ending in ", extension)) }
|
||||
if !PkgEndsWith(file, ".zx") { return PkgFail("expected an artifact ending in .zx") }
|
||||
source := PkgCanonical(file)
|
||||
if source == "" { return PkgFail("cannot find the portable image") }
|
||||
// Verify and send the same private copy even if a build replaces FILE.
|
||||
stage := PkgArtifactStage(PkgJoin(PkgCacheRoot(), "/zx"))
|
||||
if stage == "" { return PkgFail("cannot stage the portable image") }
|
||||
defer PkgArtifactRemove(stage)
|
||||
image := PkgJoin3(stage, "/upload", extension)
|
||||
image := PkgJoin(stage, "/upload.zx")
|
||||
digest := ImageDigest(source)
|
||||
if !HexDigest(digest) { return PkgFail("expected a nonempty artifact within the size limit") }
|
||||
empty: [0]string
|
||||
@@ -178,15 +167,8 @@ Publish :: (launcher: string, url: string, file: string, netrc: string,
|
||||
if !ImageVerified(launcher, image, "") { return PkgFail("cannot verify the portable image") }
|
||||
} else {
|
||||
identity: string
|
||||
if !PkgCheckedArchive(image, "", "", "", "", "", "", *identity) {
|
||||
return PkgFail("cannot verify the checked package archive")
|
||||
}
|
||||
prefix := PkgJoin3("1\t", id, "\t")
|
||||
tool_end := prefix.count + 17
|
||||
suffix := PkgJoin3("\t", profile, "\n")
|
||||
if identity.count != tool_end + suffix.count || identity[0:prefix.count] != prefix ||
|
||||
!PkgCheckedIdentity(identity[prefix.count:tool_end]) || identity[tool_end:] != suffix {
|
||||
return PkgFail("archive does not match the declared repository revision or profile")
|
||||
if !PkgCheckedArchive(image, "", "", id, "", "", profile, *identity) {
|
||||
return PkgFail("checked package library failed format, revision or profile verification")
|
||||
}
|
||||
descriptor = PkgJoin3(identity[0:identity.count - 1], "\t", PkgJoin(digest, "\n"))
|
||||
}
|
||||
@@ -203,7 +185,7 @@ Publish :: (launcher: string, url: string, file: string, netrc: string,
|
||||
}
|
||||
|
||||
Fetch :: (launcher: string, url: string, profile: string, digest: string,
|
||||
module: string, netrc: string, offline: bool, output: *string) -> s32 {
|
||||
module: string, netrc: string, offline: bool, output: *string, id: string = "") -> s32 {
|
||||
if !HexDigest(digest) { return PkgFail("expected a full lowercase SHA-256 digest") }
|
||||
cache := PkgJoin(PkgCacheRoot(), "/zx")
|
||||
key := PkgDigest(PkgJoin3(url, PkgJoin("\n", digest), PkgJoin("\n", module)))
|
||||
@@ -211,10 +193,9 @@ Fetch :: (launcher: string, url: string, profile: string, digest: string,
|
||||
root := PkgJoin3(cache, "/", key)
|
||||
name := module
|
||||
if name == "" { name = profile }
|
||||
extension := ifx PkgEndsWith(url, ".zir.zip") then ".zir.zip" else ".zx"
|
||||
image := PkgJoin(PkgJoin3(root, "/", name), extension)
|
||||
image := PkgJoin(PkgJoin3(root, "/", name), ".zx")
|
||||
if PathExists(image) {
|
||||
if ImageDigest(image) != digest || !ImageVerified(launcher, image, module) {
|
||||
if ImageDigest(image) != digest || !ImageVerified(launcher, image, module, id, profile) {
|
||||
return PkgFail("cached portable image failed digest, format or module-name verification")
|
||||
}
|
||||
} else {
|
||||
@@ -222,10 +203,10 @@ Fetch :: (launcher: string, url: string, profile: string, digest: string,
|
||||
stage := PkgArtifactStage(cache)
|
||||
if stage == "" { return PkgFail("cannot stage the portable image") }
|
||||
defer PkgArtifactRemove(stage)
|
||||
temporary := PkgJoin3(stage, "/download", extension)
|
||||
temporary := PkgJoin(stage, "/download.zx")
|
||||
status := Transfer(url, temporary, netrc, false)
|
||||
if status != "200" { return TransferFailure("cannot fetch the portable image from Gitea", status) }
|
||||
if ImageDigest(temporary) != digest || !ImageVerified(launcher, temporary, module) {
|
||||
if ImageDigest(temporary) != digest || !ImageVerified(launcher, temporary, module, id, profile) {
|
||||
return PkgFail("downloaded portable image failed digest, format or module-name verification")
|
||||
}
|
||||
if !PkgMkdirs(ParentDirectory(image)) || !RenamePath(temporary, image) {
|
||||
@@ -262,14 +243,16 @@ PkgZxUsage :: () -> string {
|
||||
return PkgJoin3(
|
||||
"usage: ziran pkg zx publish REPOSITORY REVISION PROFILE FILE.zx [--netrc FILE]\n",
|
||||
" ziran pkg zx fetch REPOSITORY REVISION PROFILE SHA256 [--module LOGICAL_NAME] [--offline] [--netrc FILE]\n",
|
||||
PkgJoin3(" ziran pkg zx pack-ir REPOSITORY REVISION PROFILE FILE.zir.zip [--define NAME]...\n",
|
||||
" ziran pkg zx publish-ir REPOSITORY REVISION PROFILE FILE.zir.zip [--netrc FILE]\n",
|
||||
" ziran pkg zx fetch-ir REPOSITORY REVISION PROFILE SHA256 [--offline] [--netrc FILE]\n"))
|
||||
PkgJoin3(" ziran pkg zx pack-ir REPOSITORY REVISION PROFILE FILE.zx [--define NAME]... --locked [--offline]\n",
|
||||
" ziran pkg zx publish-ir REPOSITORY REVISION PROFILE FILE.zx [--netrc FILE]\n",
|
||||
" ziran pkg zx fetch-ir REPOSITORY REVISION PROFILE SHA256 [--offline] [--netrc FILE]\n ziran pkg zx pack-dev REPOSITORY REVISION PROFILE FILE.zx [--define NAME]... [--offline]\n ziran pkg zx inspect-ir FILE.zx\n"))
|
||||
}
|
||||
|
||||
PkgCheckedPins :: struct {
|
||||
launcher: string
|
||||
tool_identity: string
|
||||
lifecycle_manifest: *PkgManifest
|
||||
development_directory: string
|
||||
pins: [64]PinnedCheckedPackage
|
||||
count: s32
|
||||
}
|
||||
@@ -279,7 +262,7 @@ PkgReadCheckedPins :: (root: string, launcher: string, locked: *PkgLock, pins: *
|
||||
tool := PkgLockId(locked.tool_url, locked.tool_commit, false)
|
||||
for item: locked.packages[0:locked.count] {
|
||||
if item.checked_profile == "" { continue }
|
||||
if !RegistrySegment(item.checked_profile) || item.checked_tool != tool { return false }
|
||||
if !PkgRegistrySegment(item.checked_profile) || item.checked_tool != tool { return false }
|
||||
pins.pins[pins.count] = .{package = item.id, profile = item.checked_profile, digest = item.checked_digest}
|
||||
pins.count += 1
|
||||
}
|
||||
@@ -306,7 +289,7 @@ PkgReadCheckedPins :: (root: string, launcher: string, locked: *PkgLock, pins: *
|
||||
profile := PkgJsonField(json, row, "profile")
|
||||
digest := PkgJsonField(json, row, "sha256")
|
||||
target := PkgFindById(locked, id)
|
||||
if target < 0 || locked.packages[target].source || !RegistrySegment(profile) || !HexDigest(digest) { return false }
|
||||
if target < 0 || locked.packages[target].source || !PkgRegistrySegment(profile) || !HexDigest(digest) { return false }
|
||||
for previous: 0..index - 1 {
|
||||
if PkgJsonField(json, ElementAt(json, items, previous), "package") == id { return false }
|
||||
}
|
||||
@@ -347,7 +330,7 @@ PackageNetrc :: (offline: bool, output: *string) -> bool {
|
||||
|
||||
OpenChecked :: (pins: *PkgCheckedPins, item: *PkgLocked, locked: *PkgLock,
|
||||
pin: *PinnedCheckedPackage, offline: bool) -> bool {
|
||||
url := RegistryUrl(CheckedRepository(item.url), item.commit, pin.profile, ".zir.zip")
|
||||
url := RegistryUrl(CheckedRepository(item.url), item.commit, pin.profile, true)
|
||||
if url == "" {
|
||||
unused PkgFail("expected an HTTPS registry and exact checked package revision")
|
||||
return false
|
||||
@@ -355,7 +338,7 @@ OpenChecked :: (pins: *PkgCheckedPins, item: *PkgLocked, locked: *PkgLock,
|
||||
netrc: string
|
||||
if !PackageNetrc(offline, *netrc) { return false }
|
||||
archive: string
|
||||
if Fetch(pins.launcher, url, pin.profile, pin.digest, "", netrc, offline, *archive) != 0 { return false }
|
||||
if Fetch(pins.launcher, url, pin.profile, pin.digest, "", netrc, offline, *archive, item.id) != 0 { return false }
|
||||
pin.stage = PkgArtifactStage(PkgJoin(PkgCacheRoot(), "/zx"))
|
||||
if pin.stage == "" || !PkgCheckedArchive(archive, pin.digest, pin.stage, item.id,
|
||||
locked.tool_url, locked.tool_commit, pin.profile) {
|
||||
@@ -370,10 +353,37 @@ OpenChecked :: (pins: *PkgCheckedPins, item: *PkgLocked, locked: *PkgLock,
|
||||
|
||||
PkgSelectChecked :: (context: *void, item: *PkgLocked, locked: *PkgLock, offline: bool) -> bool {
|
||||
pins := cast(*PkgCheckedPins)context
|
||||
if pins.lifecycle_manifest != null && pins.lifecycle_manifest.tool != "" { return true }
|
||||
development: string
|
||||
if pins.development_directory != "" {
|
||||
path := PkgJoin(PkgJoin3(pins.development_directory, "/", item.id), ".zx")
|
||||
if PathExists(path) { development = path }
|
||||
}
|
||||
for index: 0..pins.count - 1 {
|
||||
pin := *pins.pins[index]
|
||||
if pin.package == item.id { return OpenChecked(pins, item, locked, pin, offline) }
|
||||
if pin.package != item.id { continue }
|
||||
if development != "" { return OpenDevelopment(pins, item, locked, pin, development) }
|
||||
return OpenChecked(pins, item, locked, pin, offline)
|
||||
}
|
||||
if development == "" { return true }
|
||||
if pins.count >= 64 { return false }
|
||||
pin := *pins.pins[pins.count]
|
||||
pins.count += 1
|
||||
pin.package = item.id
|
||||
return OpenDevelopment(pins, item, locked, pin, development)
|
||||
}
|
||||
|
||||
OpenDevelopment :: (pins: *PkgCheckedPins, item: *PkgLocked, locked: *PkgLock,
|
||||
pin: *PinnedCheckedPackage, path: string) -> bool {
|
||||
pin.stage = PkgArtifactStage(PkgJoin(PkgCacheRoot(), "/zx"))
|
||||
if pin.stage == "" || !PkgCheckedArchive(path, "", pin.stage, item.id,
|
||||
locked.tool_url, locked.tool_commit, "", null, true) {
|
||||
unused PkgFail("development .zx library does not match its package or toolchain")
|
||||
return false
|
||||
}
|
||||
item.path = pin.stage
|
||||
item.checked = true
|
||||
pins.tool_identity = PkgLockId(locked.tool_url, locked.tool_commit, false)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -387,7 +397,7 @@ PkgDiscoverChecked :: (context: *void, item: *PkgLocked, locked: *PkgLock, offli
|
||||
saved := item.checked_profile != ""
|
||||
replacing := saved && requested != "" && requested != item.checked_profile
|
||||
if saved && !replacing {
|
||||
if item.checked_tool != tool || !RegistrySegment(item.checked_profile) {
|
||||
if item.checked_tool != tool || !PkgRegistrySegment(item.checked_profile) {
|
||||
unused PkgFail("saved checked package requires a different toolchain or profile")
|
||||
return false
|
||||
}
|
||||
@@ -403,7 +413,7 @@ PkgDiscoverChecked :: (context: *void, item: *PkgLocked, locked: *PkgLock, offli
|
||||
prefix := PkgJoin(origin, "/")
|
||||
if !replacing && (repository.count <= prefix.count || repository[0:prefix.count] != prefix) { return true }
|
||||
profile := ifx requested == "" then "default" else requested
|
||||
url := RegistryUrl(repository, item.commit, profile, ".zir.zip")
|
||||
url := RegistryUrl(repository, item.commit, profile, true)
|
||||
if url == "" {
|
||||
unused PkgFail("invalid checked package registry address or profile")
|
||||
return false
|
||||
@@ -424,7 +434,7 @@ PkgDiscoverChecked :: (context: *void, item: *PkgLocked, locked: *PkgLock, offli
|
||||
return false
|
||||
}
|
||||
metadata := PkgReadLimited(descriptor, 4096)
|
||||
expected := PkgJoin3(PkgJoin3("1\t", item.id, "\t"), PkgJoin3(tool, "\t", profile), "\t")
|
||||
expected := PkgJoin(CheckedDescriptorPrefix(item.id, tool, profile), "\t")
|
||||
if !metadata.valid || metadata.text.count != expected.count + 65 ||
|
||||
metadata.text[0:expected.count] != expected ||
|
||||
metadata.text[metadata.text.count - 1:] != "\n" ||
|
||||
@@ -454,10 +464,10 @@ PkgResolveZxLock :: (launcher: string, root: string, refresh: string, offline: b
|
||||
}
|
||||
|
||||
PkgPackChecked :: (root: string, compiler: string, locked: *PkgLock, current: *PkgManifest,
|
||||
arguments: []string) -> s32 {
|
||||
arguments: []string, development: bool) -> s32 {
|
||||
if arguments.count < 4 || (arguments.count - 4) % 2 != 0 ||
|
||||
RegistryUrl(arguments[0], arguments[1], arguments[2], ".zir.zip") == "" ||
|
||||
!PkgEndsWith(arguments[3], ".zir.zip") { return PkgFail(PkgTrim(PkgZxUsage())) }
|
||||
RegistryUrl(arguments[0], arguments[1], arguments[2], true) == "" ||
|
||||
!PkgEndsWith(arguments[3], ".zx") { return PkgFail(PkgTrim(PkgZxUsage())) }
|
||||
for index: 4..arguments.count - 1 {
|
||||
if (index - 4) % 2 == 0 {
|
||||
if arguments[index] != "--define" { return PkgFail("expected --define NAME") }
|
||||
@@ -467,14 +477,19 @@ PkgPackChecked :: (root: string, compiler: string, locked: *PkgLock, current: *P
|
||||
revision := PkgCapture(revision_args[:], root)
|
||||
origin_args := string.["git", "remote", "get-url", "origin"]
|
||||
origin := PkgCapture(origin_args[:], root)
|
||||
status_args := string.["git", "status", "--porcelain", "--untracked-files=normal"]
|
||||
status := PkgCapture(status_args[:], root)
|
||||
if revision.code != 0 || PkgTrim(revision.text) != arguments[1] || origin.code != 0 ||
|
||||
!PkgSameSource(PkgTrim(origin.text), arguments[0]) || status.code != 0 || PkgTrim(status.text) != "" {
|
||||
return PkgFail("checked package export requires the clean declared repository revision")
|
||||
!PkgSameSource(PkgTrim(origin.text), arguments[0]) {
|
||||
return PkgFail("checked package export requires the declared repository and HEAD revision")
|
||||
}
|
||||
if !development {
|
||||
status_args := string.["git", "status", "--porcelain", "--untracked-files=normal"]
|
||||
status := PkgCapture(status_args[:], root)
|
||||
if status.code != 0 || PkgTrim(status.text) != "" || !PkgValidCommit(locked.tool_commit) {
|
||||
return PkgFail("checked package export requires the clean declared repository revision")
|
||||
}
|
||||
}
|
||||
return PkgCheckedPack(root, locked, current, compiler, arguments[0], arguments[1], arguments[2],
|
||||
arguments[3], arguments[4:])
|
||||
arguments[3], arguments[4:], development)
|
||||
}
|
||||
|
||||
PkgZx :: (launcher: string, arguments: []string) -> s32 {
|
||||
@@ -482,13 +497,24 @@ PkgZx :: (launcher: string, arguments: []string) -> s32 {
|
||||
PkgPrint(PkgZxUsage(), false)
|
||||
return 0
|
||||
}
|
||||
if arguments.count == 2 && arguments[0] == "inspect-ir" {
|
||||
path := arguments[1]
|
||||
digest := ImageDigest(path)
|
||||
identity: string
|
||||
if !PkgEndsWith(path, ".zx") || !HexDigest(digest) ||
|
||||
(!PkgCheckedArchive(path, digest, "", "", "", "", "", *identity) &&
|
||||
!PkgCheckedArchive(path, digest, "", "", "", "", "", *identity, true)) {
|
||||
return PkgFail("cannot verify the checked .zx library")
|
||||
}
|
||||
PkgPrint(PkgJoin3(identity[0:identity.count - 1], "\t", PkgJoin(digest, "\n")), false)
|
||||
return 0
|
||||
}
|
||||
checked := arguments.count != 0 && (arguments[0] == "publish-ir" || arguments[0] == "fetch-ir")
|
||||
if arguments.count < 5 || (!checked && arguments[0] != "publish" && arguments[0] != "fetch") {
|
||||
return PkgFail(PkgTrim(PkgZxUsage()))
|
||||
}
|
||||
publish := arguments[0] == "publish" || arguments[0] == "publish-ir"
|
||||
extension := ifx checked then ".zir.zip" else ".zx"
|
||||
url := RegistryUrl(arguments[1], arguments[2], arguments[3], extension)
|
||||
url := RegistryUrl(arguments[1], arguments[2], arguments[3], checked)
|
||||
if url == "" { return PkgFail("expected an HTTPS Gitea repository, full Git revision and portable profile") }
|
||||
module: string
|
||||
netrc: string
|
||||
@@ -507,12 +533,12 @@ PkgZx :: (launcher: string, arguments: []string) -> s32 {
|
||||
index += 1
|
||||
} else { return PkgFail(PkgJoin("invalid portable package option: ", option)) }
|
||||
}
|
||||
id := ifx checked then PkgLockId(arguments[1], arguments[2], false) else ""
|
||||
if publish {
|
||||
id := ifx checked then PkgLockId(arguments[1], arguments[2], false) else ""
|
||||
return Publish(launcher, url, arguments[4], netrc, id, arguments[3])
|
||||
}
|
||||
output: string
|
||||
code := Fetch(launcher, url, arguments[3], arguments[4], module, netrc, offline, *output)
|
||||
code := Fetch(launcher, url, arguments[3], arguments[4], module, netrc, offline, *output, id)
|
||||
if code == 0 { PkgPrint(PkgJoin(output, "\n"), false) }
|
||||
return code
|
||||
}
|
||||
@@ -559,7 +585,7 @@ PkgZxLockedArguments :: (launcher: string, arguments: []string, lock_at: s64, fl
|
||||
profile := PkgJsonField(json, row, "profile")
|
||||
digest := PkgJsonField(json, row, "sha256")
|
||||
url := RegistryUrl(PkgJsonField(json, row, "repository"), PkgJsonField(json, row, "commit"), profile)
|
||||
if !RegistrySegment(alias) || !ModuleName(name) || url == "" || !HexDigest(digest) {
|
||||
if !PkgRegistrySegment(alias) || !ModuleName(name) || url == "" || !HexDigest(digest) {
|
||||
return LockFailure("invalid pinned ZX module")
|
||||
}
|
||||
for prior: 0..index - 1 {
|
||||
@@ -585,7 +611,7 @@ PkgZxLockedArguments :: (launcher: string, arguments: []string, lock_at: s64, fl
|
||||
exported := PkgJsonField(json, row, "export")
|
||||
found := false
|
||||
for pinned: modules[0:module_count] { if pinned.alias == alias { found = true } }
|
||||
if !RegistrySegment(module) || !RegistrySegment(function) || !RegistrySegment(exported) || !found {
|
||||
if !PkgRegistrySegment(module) || !PkgRegistrySegment(function) || !PkgRegistrySegment(exported) || !found {
|
||||
return LockFailure("invalid ZX import or unpinned interface alias")
|
||||
}
|
||||
if build {
|
||||
|
||||
+72
-14
@@ -545,10 +545,39 @@ still checks typed imports and requires the host grants used by the image.
|
||||
This workflow uses ZX runtime interfaces. Checked packages support source
|
||||
imports and generics. Platform-native dependencies use native artifacts.
|
||||
|
||||
## Prebuilt Ziran toolchains
|
||||
|
||||
The canonical Gitea check job packages the Ziran executables, standard
|
||||
library, headers, and native archive from the build it just tested. Each
|
||||
artifact identifies the full Ziran commit and the build host's Linux
|
||||
architecture and glibc version. The native job reuses its build directory.
|
||||
|
||||
Download `toolchain-<commit>` from the corresponding check run. When the
|
||||
workflow has a package writer configured, the same two files are available
|
||||
from Gitea Generic Packages under `ziranlang/ziran-toolchain`, using the full
|
||||
Git commit as the package version. The `.json` descriptor and `.tar.gz`
|
||||
archive share the profile name, for example `linux-x64-glibc-2.39`.
|
||||
|
||||
```sh
|
||||
python3 scripts/toolchain_package.py verify "$commit" "$profile.json"
|
||||
```
|
||||
|
||||
Compare `commit` to the toolchain commit in the consuming project's
|
||||
`ziran.lock` and use a Linux host that supports the archive's profile.
|
||||
The verification command checks the commit, archive length, and SHA-256.
|
||||
The archive contains `build/bin/ziran` and its companion tools. Package
|
||||
commands that require a source checkout still use their existing exact Git
|
||||
source cache.
|
||||
|
||||
Configure `ZIRAN_TOOLCHAIN_PACKAGE_USER` and `ZIRAN_TOOLCHAIN_PACKAGE_TOKEN`
|
||||
as Gitea Actions secrets for a package writer in the `ziranlang` organization.
|
||||
The workflow uploads the archive first and its descriptor second. Existing
|
||||
registry files are accepted on retry when their bytes match.
|
||||
|
||||
## Checked packages for source imports
|
||||
|
||||
A package with manifest exports can distribute its checked Ziran modules as
|
||||
a `.zir.zip` compiler companion. Pack it from the clean provider revision
|
||||
a `.zx` library. Pack it from the clean provider revision
|
||||
using the exact, clean toolchain and dependencies pinned in `ziran.lock`.
|
||||
Each profile names a fixed toolchain and set of feature defines at that
|
||||
provider revision. Supply defines with `--define NAME` when packing.
|
||||
@@ -556,24 +585,27 @@ provider revision. Supply defines with `--define NAME` when packing.
|
||||
```sh
|
||||
repo=https://git.waozi.xyz/example/counter.git
|
||||
commit=$(git rev-parse HEAD)
|
||||
ziran pkg zx pack-ir "$repo" "$commit" default build/counter.zir.zip --locked
|
||||
sha256=$(ziran pkg zx publish-ir "$repo" "$commit" default build/counter.zir.zip --netrc "$NETRC_FILE")
|
||||
ziran pkg zx pack-ir "$repo" "$commit" default build/counter.zx --locked
|
||||
ziran pkg zx inspect-ir build/counter.zx
|
||||
sha256=$(ziran pkg zx publish-ir "$repo" "$commit" default build/counter.zx --netrc "$NETRC_FILE")
|
||||
ziran pkg zx fetch-ir "$repo" "$commit" default "$sha256"
|
||||
```
|
||||
|
||||
`publish-ir` uploads the archive, then publishes the matching discovery
|
||||
record at the same revision and profile. Either upload can be repeated with
|
||||
identical bytes. Keep the archive and record together when operating a registry.
|
||||
`publish-ir` uploads `<profile>.zx` and then `<profile>.zx.tsv` to the
|
||||
`<repository>-zx-library` Generic package at the full Git revision. Either
|
||||
upload can be repeated with identical bytes. `inspect-ir` verifies local
|
||||
library bytes and prints the same tab-separated record with the package,
|
||||
toolchain, profile, and digest.
|
||||
|
||||
`ziran add` and `ziran lock` discover companions on `https://git.waozi.xyz`
|
||||
`ziran add` and `ziran lock` discover libraries on `https://git.waozi.xyz`
|
||||
and save their exact profile, digest and toolchain identity in the package
|
||||
entry in `ziran.lock`. For a different Gitea server, set `ZIRAN_ZX_REGISTRY`
|
||||
to its HTTPS origin. Discovery uses the `default` profile unless
|
||||
`ZIRAN_ZX_PROFILE` selects another. Set `ZIRAN_PACKAGE_NETRC`
|
||||
for authenticated reads. A missing discovery record for an unpinned package
|
||||
allows ordinary source resolution. Invalid metadata, an incompatible
|
||||
toolchain, or a missing advertised archive stops resolution. A saved pin
|
||||
continues to identify the same exact archive.
|
||||
toolchain, or a missing advertised library stops resolution. A saved pin
|
||||
continues to identify the same exact library.
|
||||
|
||||
```sh
|
||||
ziran add "$repo" --ref "$commit"
|
||||
@@ -582,7 +614,7 @@ ziran lock --offline
|
||||
ziran bundle --project --offline --entry app:main -o app.zx app.zi
|
||||
```
|
||||
|
||||
You can also supply a companion pin explicitly in `ziran.zx.lock`, using
|
||||
You can also supply a library pin explicitly in `ziran.zx.lock`, using
|
||||
the dependency's `id` from `ziran.lock` and the publisher's profile and digest.
|
||||
An explicit pin for a package with a discovered pin must agree with it.
|
||||
|
||||
@@ -599,17 +631,43 @@ An explicit pin for a package with a discovered pin must agree with it.
|
||||
}
|
||||
```
|
||||
|
||||
`ziran bundle --project` and `ziran build --project --target=zx` select the
|
||||
pinned checked archives before checking out those dependencies. Existing
|
||||
source imports still resolve through the dependency aliases and manifest
|
||||
Project `bundle`, `check`, and native `build` and `run` select pinned libraries
|
||||
before checking out those dependencies. `build --target=zx` does the same.
|
||||
Source imports resolve through the dependency aliases and manifest
|
||||
exports. The compiler verifies the checked modules and uses their types,
|
||||
bodies and specializations when it lowers the application. This skips
|
||||
dependency source checking and keeps shared types and state in one program.
|
||||
Each transitive package whose source will be absent needs a pin. `--offline`
|
||||
requires the pinned archives in the local cache. Local source overrides
|
||||
requires the pinned libraries in the local cache. Local source overrides
|
||||
still take precedence unless `--locked` is set. Distribute the final `.zx`
|
||||
with the portable image commands above.
|
||||
|
||||
### Check a library from a working tree
|
||||
|
||||
From the provider checkout, use its current `ziran.lock` and actual HEAD.
|
||||
`pack-dev` includes the working source, manifest, lock and feature defines
|
||||
in a source fingerprint. It allows local toolchain and source overrides.
|
||||
|
||||
```sh
|
||||
ziran pkg zx pack-dev "$repo" "$(git rev-parse HEAD)" default build/counter-dev.zx --offline
|
||||
ziran pkg zx inspect-ir build/counter-dev.zx
|
||||
```
|
||||
|
||||
The inspection record starts with `dev`, followed by the package identity,
|
||||
toolchain identity, profile, provider source fingerprint, and file SHA-256.
|
||||
Copy the development library to a local directory as `<package-id>.zx`.
|
||||
Point `ZIRAN_DEV_ZX_DIR` at that directory when compiling the consumer. Keep
|
||||
the normal project lock and remove any source override for that provider
|
||||
from the consumer. Several development libraries can share a directory.
|
||||
|
||||
```sh
|
||||
ZIRAN_DEV_ZX_DIR="$libdir" ziran bundle --project --offline --entry app:main -o app.zx app.zi
|
||||
```
|
||||
|
||||
The compiler checks each development library against the locked package and
|
||||
toolchain. These files are for local verification. `publish-ir`, production
|
||||
pins, and `--locked` commands reject development libraries.
|
||||
|
||||
## Dependency-owned native artifacts
|
||||
|
||||
A dependency can declare named artifacts alongside its existing project tool:
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Package the native toolchain from Ziran's completed Gitea check job."""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import gzip
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import ssl
|
||||
import stat
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
HOST = 'git.waozi.xyz'
|
||||
REPOSITORY = f'https://{HOST}/ziranlang/ziran.git'
|
||||
TOOLS = ('ziran', 'zi-fmt', 'zi-edit', 'zi2zir', 'zi-api', 'zi-inspect',
|
||||
'zi2c', 'zi2go', 'zi2zx', 'zx2wasm', 'zx2native')
|
||||
PROFILE = re.compile(r'linux-(?:x64|arm64)-glibc-[0-9]+\.[0-9]+')
|
||||
COMMIT = re.compile(r'[0-9a-f]{40}')
|
||||
|
||||
|
||||
def git(*args):
|
||||
return subprocess.check_output(['git', *args])
|
||||
|
||||
|
||||
def clean_revision(commit):
|
||||
if not COMMIT.fullmatch(commit) or git('rev-parse', 'HEAD').strip().decode() != commit:
|
||||
raise ValueError('toolchain source differs from the requested Git revision')
|
||||
source = urlsplit(git('remote', 'get-url', 'origin').strip().decode())
|
||||
if (source.scheme != 'https' or source.hostname != HOST or source.port is not None or
|
||||
source.path not in ('/ziranlang/ziran', '/ziranlang/ziran.git')):
|
||||
raise ValueError('toolchain source must use the canonical Gitea origin')
|
||||
if git('status', '--porcelain', '--untracked-files=normal'):
|
||||
raise ValueError('toolchain source must be clean')
|
||||
|
||||
|
||||
def host_profile():
|
||||
arch = {'x86_64': 'x64', 'aarch64': 'arm64'}.get(platform.machine())
|
||||
libc = os.confstr('CS_GNU_LIBC_VERSION') if platform.system() == 'Linux' else None
|
||||
if arch is None or libc is None or not re.fullmatch(r'glibc [0-9]+\.[0-9]+', libc):
|
||||
raise ValueError('the toolchain registry supports Linux x64 and arm64 with glibc')
|
||||
return f'linux-{arch}-{libc.replace(" ", "-")}'
|
||||
|
||||
|
||||
def digest(file):
|
||||
with file.open('rb') as stream:
|
||||
return hash_stream(stream)
|
||||
|
||||
|
||||
def hash_stream(stream):
|
||||
checksum = hashlib.sha256()
|
||||
for chunk in iter(lambda: stream.read(65536), b''):
|
||||
checksum.update(chunk)
|
||||
return checksum.hexdigest()
|
||||
|
||||
|
||||
def write_archive(archive, files):
|
||||
with gzip.GzipFile(filename='', fileobj=archive, mode='wb', mtime=0) as compressed:
|
||||
with tarfile.open(fileobj=compressed, mode='w|', format=tarfile.PAX_FORMAT) as tar:
|
||||
for file in sorted(files):
|
||||
if not stat.S_ISREG(file.lstat().st_mode):
|
||||
raise ValueError(f'expected a regular toolchain file: {file}')
|
||||
with file.open('rb') as contents:
|
||||
info = os.fstat(contents.fileno())
|
||||
entry = tarfile.TarInfo(file.as_posix())
|
||||
entry.size = info.st_size
|
||||
entry.mode = 0o755 if info.st_mode & stat.S_IXUSR else 0o644
|
||||
tar.addfile(entry, contents)
|
||||
|
||||
|
||||
def pack(commit, output):
|
||||
clean_revision(commit)
|
||||
profile = host_profile()
|
||||
binaries = [Path('build/bin', tool) for tool in TOOLS]
|
||||
for binary in binaries:
|
||||
if not binary.is_file() or not os.access(binary, os.X_OK):
|
||||
raise ValueError(f'missing tested toolchain executable: {binary}')
|
||||
subprocess.run([str(binaries[0]), 'version'], check=True)
|
||||
sources = git('ls-files', '-z', '--', 'std', 'include', 'templates', 'scripts/ziran_artifacts.py')
|
||||
files = binaries + [Path('build/libziran.a')]
|
||||
files.extend(Path(os.fsdecode(name)) for name in sources.split(b'\0') if name)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix='.pack-', dir=output) as stage:
|
||||
archive = Path(stage, f'{profile}.tar.gz')
|
||||
with archive.open('wb') as stream:
|
||||
write_archive(stream, files)
|
||||
descriptor = archive.with_name(f'{profile}.json')
|
||||
metadata = {'version': 1, 'repository': REPOSITORY, 'commit': commit, 'profile': profile,
|
||||
'file': archive.name, 'size': archive.stat().st_size, 'sha256': digest(archive)}
|
||||
descriptor.write_text(json.dumps(metadata, sort_keys=True) + '\n')
|
||||
clean_revision(commit)
|
||||
archive.replace(output / archive.name)
|
||||
descriptor.replace(output / descriptor.name)
|
||||
print(f'Toolchain artifact {profile} for {commit}')
|
||||
|
||||
|
||||
def read_artifact(descriptor):
|
||||
with descriptor.open('rb') as stream:
|
||||
contents = stream.read(4097)
|
||||
if len(contents) > 4096:
|
||||
raise ValueError('toolchain descriptor exceeds 4096 bytes')
|
||||
row = json.loads(contents)
|
||||
profile = row['profile']
|
||||
archive = descriptor.with_name(f'{profile}.tar.gz') if PROFILE.fullmatch(profile) else None
|
||||
if (row['version'] != 1 or row['repository'] != REPOSITORY or not COMMIT.fullmatch(row['commit']) or
|
||||
archive is None or descriptor.name != f'{profile}.json' or row['file'] != archive.name or
|
||||
row['size'] != archive.stat().st_size or row['sha256'] != digest(archive)):
|
||||
raise ValueError('toolchain descriptor and archive did not verify')
|
||||
return row['commit'], archive
|
||||
|
||||
|
||||
def upload(connection, base, file, authorization):
|
||||
path = f'{base}/{file.name}'
|
||||
size = file.stat().st_size
|
||||
headers = {'Authorization': authorization, 'Content-Length': str(size),
|
||||
'Content-Type': 'application/octet-stream'}
|
||||
interrupted = None
|
||||
try:
|
||||
with file.open('rb') as stream:
|
||||
connection.request('PUT', path, body=stream, headers=headers)
|
||||
with connection.getresponse() as response:
|
||||
if response.status == 201:
|
||||
return
|
||||
if response.status != 409:
|
||||
raise ValueError(f'Gitea rejected {file.name} (HTTP {response.status})')
|
||||
except (BrokenPipeError, ConnectionResetError, http.client.RemoteDisconnected, ssl.SSLEOFError) as error:
|
||||
# The server may answer a conflict before reading the complete upload.
|
||||
interrupted = error
|
||||
finally:
|
||||
connection.close()
|
||||
# A retry may reuse an immutable file only when the server has the same bytes.
|
||||
try:
|
||||
connection.request('GET', path, headers={'Authorization': authorization})
|
||||
with connection.getresponse() as response:
|
||||
if response.status == 404 and interrupted is not None:
|
||||
raise interrupted
|
||||
if response.status != 200 or int(response.getheader('Content-Length', -1)) != size:
|
||||
raise ValueError(f'cannot verify existing Gitea file {file.name}')
|
||||
if hash_stream(response) != digest(file):
|
||||
raise ValueError(f'Gitea already has different bytes for {file.name}')
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def verify(descriptor, expected_commit):
|
||||
commit, archive = read_artifact(descriptor)
|
||||
if commit != expected_commit:
|
||||
raise ValueError('artifact revision differs from the workflow revision')
|
||||
return archive
|
||||
|
||||
|
||||
def publish(descriptor, expected_commit):
|
||||
archive = verify(descriptor, expected_commit)
|
||||
user = os.environ.get('ZIRAN_TOOLCHAIN_PACKAGE_USER')
|
||||
token = os.environ.get('ZIRAN_TOOLCHAIN_PACKAGE_TOKEN')
|
||||
if not user or not token or ':' in user:
|
||||
raise ValueError('provide the Gitea package writer identity and token')
|
||||
authorization = 'Basic ' + base64.b64encode(f'{user}:{token}'.encode()).decode()
|
||||
base = f'/api/packages/ziranlang/generic/ziran-toolchain/{expected_commit}'
|
||||
connection = http.client.HTTPSConnection(HOST, timeout=180)
|
||||
upload(connection, base, archive, authorization)
|
||||
upload(connection, base, descriptor, authorization)
|
||||
print(f'Published {REPOSITORY} toolchain {archive.name} for {expected_commit}')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
subcommands = parser.add_subparsers(dest='command', required=True)
|
||||
package = subcommands.add_parser('pack')
|
||||
package.add_argument('commit')
|
||||
package.add_argument('output', type=Path)
|
||||
for command in ('verify', 'publish'):
|
||||
artifact = subcommands.add_parser(command)
|
||||
artifact.add_argument('commit')
|
||||
artifact.add_argument('descriptor', type=Path)
|
||||
try:
|
||||
args = parser.parse_args()
|
||||
if args.command == 'pack':
|
||||
pack(args.commit, args.output)
|
||||
elif args.command == 'publish':
|
||||
publish(args.descriptor, args.commit)
|
||||
else:
|
||||
print(verify(args.descriptor, args.commit))
|
||||
except (OSError, ValueError, KeyError, TypeError, subprocess.CalledProcessError,
|
||||
http.client.HTTPException) as error:
|
||||
parser.exit(1, f'toolchain package: {error}\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+58
-15
@@ -3,7 +3,7 @@
|
||||
<head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><meta name="theme-color" content="#f7f3eb"><meta name="description" content="Packages and project imports: the Ziran guide, rendered from docs/PACKAGES.md."><title>Packages and project imports · Ziran guide</title><link rel="canonical" href="https://ziran-lang.org/guide/packages"><link rel="icon" href="../assets/logo-capybara.png" type="image/png"><link rel="stylesheet" href="../styles.css"><script defer src="../site.js"></script></head>
|
||||
<body><a class="skip-link" href="#main">Skip to content</a><header class="site-header"><div class="site-header-inner"><a class="brand" href="../index.html"><img class="brand-icon" src="../assets/logo-capybara.png" alt="" width="36" height="36"><span>Ziran</span></a><button class="menu-button" type="button" aria-expanded="false" aria-controls="primary-nav">Menu</button><nav class="header-nav" id="primary-nav" aria-label="Primary"><a href="../get-started.html">Get started</a><a href="../examples.html">Examples</a><a href="../targets.html">Targets</a><a href="../modules.html">Modules</a><a href="../docs.html" aria-current="page">Docs</a><a href="../status.html">Status</a><a href="https://github.com/ziranlang/ziran">GitHub ↗</a><button class="theme-toggle" type="button" aria-label="Switch color theme"></button></nav></div></header>
|
||||
<main id="main" class="article-layout"><div class="article-head"><p class="eyebrow"><a href="../docs.html#guide">Guide</a></p><h1>Packages and project imports</h1><p>Rendered from <a href="https://github.com/ziranlang/ziran/blob/master/docs/PACKAGES.md">docs/PACKAGES.md</a>. See the <a href="../docs.html#guide">documentation index</a> for the other guides.</p></div>
|
||||
<div class="article-grid"><aside class="page-toc"><strong>On this page</strong><a href="#starting-a-project">Starting a project</a><a href="#building-and-running-a-program">Building and running a program</a><a href="#adding-a-dependency">Adding a dependency</a><a href="#importing-modules">Importing modules</a><a href="#publishing-a-package">Publishing a package</a><a href="#commands">Commands</a><a href="#project-tools-and-their-options">Project tools and their options</a><a href="#handing-a-project-to-its-tool">Handing a project to its tool</a><a href="#prebuilt-portable-images">Prebuilt portable images</a><a href="#checked-packages-for-source-imports">Checked packages for source imports</a><a href="#dependency-owned-native-artifacts">Dependency-owned native artifacts</a></aside><div class="article-body">
|
||||
<div class="article-grid"><aside class="page-toc"><strong>On this page</strong><a href="#starting-a-project">Starting a project</a><a href="#building-and-running-a-program">Building and running a program</a><a href="#adding-a-dependency">Adding a dependency</a><a href="#importing-modules">Importing modules</a><a href="#publishing-a-package">Publishing a package</a><a href="#commands">Commands</a><a href="#project-tools-and-their-options">Project tools and their options</a><a href="#handing-a-project-to-its-tool">Handing a project to its tool</a><a href="#prebuilt-portable-images">Prebuilt portable images</a><a href="#prebuilt-ziran-toolchains">Prebuilt Ziran toolchains</a><a href="#checked-packages-for-source-imports">Checked packages for source imports</a><a href="#dependency-owned-native-artifacts">Dependency-owned native artifacts</a></aside><div class="article-body">
|
||||
<section class="prose"><p>Ziran packages are Git repositories. There is no central registry: a
|
||||
dependency is the URL of the repository that holds it, pinned to a full commit
|
||||
in a committed <code>ziran.lock</code>. A Ziran application is itself a package with a
|
||||
@@ -438,36 +438,61 @@ download, digest, image format or module name stops the command. The loader
|
||||
still checks typed imports and requires the host grants used by the image.
|
||||
This workflow uses ZX runtime interfaces. Checked packages support source
|
||||
imports and generics. Platform-native dependencies use native artifacts.</p>
|
||||
<h2 id="prebuilt-ziran-toolchains">Prebuilt Ziran toolchains</h2>
|
||||
<p>The canonical Gitea check job packages the Ziran executables, standard
|
||||
library, headers, and native archive from the build it just tested. Each
|
||||
artifact identifies the full Ziran commit and the build host's Linux
|
||||
architecture and glibc version. The native job reuses its build directory.</p>
|
||||
<p>Download <code>toolchain-<commit></code> from the corresponding check run. When the
|
||||
workflow has a package writer configured, the same two files are available
|
||||
from Gitea Generic Packages under <code>ziranlang/ziran-toolchain</code>, using the full
|
||||
Git commit as the package version. The <code>.json</code> descriptor and <code>.tar.gz</code>
|
||||
archive share the profile name, for example <code>linux-x64-glibc-2.39</code>.</p>
|
||||
<pre><code class="language-sh">python3 scripts/toolchain_package.py verify "$commit" "$profile.json"
|
||||
</code></pre>
|
||||
<p>Compare <code>commit</code> to the toolchain commit in the consuming project's
|
||||
<code>ziran.lock</code> and use a Linux host that supports the archive's profile.
|
||||
The verification command checks the commit, archive length, and SHA-256.
|
||||
The archive contains <code>build/bin/ziran</code> and its companion tools. Package
|
||||
commands that require a source checkout still use their existing exact Git
|
||||
source cache.</p>
|
||||
<p>Configure <code>ZIRAN_TOOLCHAIN_PACKAGE_USER</code> and <code>ZIRAN_TOOLCHAIN_PACKAGE_TOKEN</code>
|
||||
as Gitea Actions secrets for a package writer in the <code>ziranlang</code> organization.
|
||||
The workflow uploads the archive first and its descriptor second. Existing
|
||||
registry files are accepted on retry when their bytes match.</p>
|
||||
<h2 id="checked-packages-for-source-imports">Checked packages for source imports</h2>
|
||||
<p>A package with manifest exports can distribute its checked Ziran modules as
|
||||
a <code>.zir.zip</code> compiler companion. Pack it from the clean provider revision
|
||||
a <code>.zx</code> library. Pack it from the clean provider revision
|
||||
using the exact, clean toolchain and dependencies pinned in <code>ziran.lock</code>.
|
||||
Each profile names a fixed toolchain and set of feature defines at that
|
||||
provider revision. Supply defines with <code>--define NAME</code> when packing.</p>
|
||||
<pre><code class="language-sh">repo=https://git.waozi.xyz/example/counter.git
|
||||
commit=$(git rev-parse HEAD)
|
||||
ziran pkg zx pack-ir "$repo" "$commit" default build/counter.zir.zip --locked
|
||||
sha256=$(ziran pkg zx publish-ir "$repo" "$commit" default build/counter.zir.zip --netrc "$NETRC_FILE")
|
||||
ziran pkg zx pack-ir "$repo" "$commit" default build/counter.zx --locked
|
||||
ziran pkg zx inspect-ir build/counter.zx
|
||||
sha256=$(ziran pkg zx publish-ir "$repo" "$commit" default build/counter.zx --netrc "$NETRC_FILE")
|
||||
ziran pkg zx fetch-ir "$repo" "$commit" default "$sha256"
|
||||
</code></pre>
|
||||
<p><code>publish-ir</code> uploads the archive, then publishes the matching discovery
|
||||
record at the same revision and profile. Either upload can be repeated with
|
||||
identical bytes. Keep the archive and record together when operating a registry.</p>
|
||||
<p><code>ziran add</code> and <code>ziran lock</code> discover companions on <code>https://git.waozi.xyz</code>
|
||||
<p><code>publish-ir</code> uploads <code><profile>.zx</code> and then <code><profile>.zx.tsv</code> to the
|
||||
<code><repository>-zx-library</code> Generic package at the full Git revision. Either
|
||||
upload can be repeated with identical bytes. <code>inspect-ir</code> verifies local
|
||||
library bytes and prints the same tab-separated record with the package,
|
||||
toolchain, profile, and digest.</p>
|
||||
<p><code>ziran add</code> and <code>ziran lock</code> discover libraries on <code>https://git.waozi.xyz</code>
|
||||
and save their exact profile, digest and toolchain identity in the package
|
||||
entry in <code>ziran.lock</code>. For a different Gitea server, set <code>ZIRAN_ZX_REGISTRY</code>
|
||||
to its HTTPS origin. Discovery uses the <code>default</code> profile unless
|
||||
<code>ZIRAN_ZX_PROFILE</code> selects another. Set <code>ZIRAN_PACKAGE_NETRC</code>
|
||||
for authenticated reads. A missing discovery record for an unpinned package
|
||||
allows ordinary source resolution. Invalid metadata, an incompatible
|
||||
toolchain, or a missing advertised archive stops resolution. A saved pin
|
||||
continues to identify the same exact archive.</p>
|
||||
toolchain, or a missing advertised library stops resolution. A saved pin
|
||||
continues to identify the same exact library.</p>
|
||||
<pre><code class="language-sh">ziran add "$repo" --ref "$commit"
|
||||
ziran bundle --project --entry app:main -o app.zx app.zi
|
||||
ziran lock --offline
|
||||
ziran bundle --project --offline --entry app:main -o app.zx app.zi
|
||||
</code></pre>
|
||||
<p>You can also supply a companion pin explicitly in <code>ziran.zx.lock</code>, using
|
||||
<p>You can also supply a library pin explicitly in <code>ziran.zx.lock</code>, using
|
||||
the dependency's <code>id</code> from <code>ziran.lock</code> and the publisher's profile and digest.
|
||||
An explicit pin for a package with a discovered pin must agree with it.</p>
|
||||
<pre><code class="language-json">{
|
||||
@@ -481,16 +506,34 @@ An explicit pin for a package with a discovered pin must agree with it.</p>
|
||||
]
|
||||
}
|
||||
</code></pre>
|
||||
<p><code>ziran bundle --project</code> and <code>ziran build --project --target=zx</code> select the
|
||||
pinned checked archives before checking out those dependencies. Existing
|
||||
source imports still resolve through the dependency aliases and manifest
|
||||
<p>Project <code>bundle</code>, <code>check</code>, and native <code>build</code> and <code>run</code> select pinned libraries
|
||||
before checking out those dependencies. <code>build --target=zx</code> does the same.
|
||||
Source imports resolve through the dependency aliases and manifest
|
||||
exports. The compiler verifies the checked modules and uses their types,
|
||||
bodies and specializations when it lowers the application. This skips
|
||||
dependency source checking and keeps shared types and state in one program.
|
||||
Each transitive package whose source will be absent needs a pin. <code>--offline</code>
|
||||
requires the pinned archives in the local cache. Local source overrides
|
||||
requires the pinned libraries in the local cache. Local source overrides
|
||||
still take precedence unless <code>--locked</code> is set. Distribute the final <code>.zx</code>
|
||||
with the portable image commands above.</p>
|
||||
<h3 id="check-a-library-from-a-working-tree">Check a library from a working tree</h3>
|
||||
<p>From the provider checkout, use its current <code>ziran.lock</code> and actual HEAD.
|
||||
<code>pack-dev</code> includes the working source, manifest, lock and feature defines
|
||||
in a source fingerprint. It allows local toolchain and source overrides.</p>
|
||||
<pre><code class="language-sh">ziran pkg zx pack-dev "$repo" "$(git rev-parse HEAD)" default build/counter-dev.zx --offline
|
||||
ziran pkg zx inspect-ir build/counter-dev.zx
|
||||
</code></pre>
|
||||
<p>The inspection record starts with <code>dev</code>, followed by the package identity,
|
||||
toolchain identity, profile, provider source fingerprint, and file SHA-256.
|
||||
Copy the development library to a local directory as <code><package-id>.zx</code>.
|
||||
Point <code>ZIRAN_DEV_ZX_DIR</code> at that directory when compiling the consumer. Keep
|
||||
the normal project lock and remove any source override for that provider
|
||||
from the consumer. Several development libraries can share a directory.</p>
|
||||
<pre><code class="language-sh">ZIRAN_DEV_ZX_DIR="$libdir" ziran bundle --project --offline --entry app:main -o app.zx app.zi
|
||||
</code></pre>
|
||||
<p>The compiler checks each development library against the locked package and
|
||||
toolchain. These files are for local verification. <code>publish-ir</code>, production
|
||||
pins, and <code>--locked</code> commands reject development libraries.</p>
|
||||
<h2 id="dependency-owned-native-artifacts">Dependency-owned native artifacts</h2>
|
||||
<p>A dependency can declare named artifacts alongside its existing project tool:</p>
|
||||
<pre><code class="language-toml">[tool]
|
||||
|
||||
@@ -20,7 +20,9 @@ Templates :: (test: *NativeTest) -> bool {
|
||||
// explicitly select the compiler under test; the real source tree is never cloned.
|
||||
if !FixtureInit(test, tool) || !NativeWrite(test, CheckPath(tool, "VERSION"), "template fixture\n") || !NativeAssert(test, CreateDirectories(work), "create template test folder") { return false }
|
||||
unused FixtureCommit(test, tool, "template toolchain metadata")
|
||||
prefixes := string.["https://github.com/ziranlang/ziran.git"]; targets := string.[FixtureUri(tool)]
|
||||
tool_uri := FixtureUri(tool)
|
||||
prefixes := string.["https://git.waozi.xyz/ziranlang/ziran.git", "https://github.com/ziranlang/ziran.git"]
|
||||
targets := string.[tool_uri, tool_uri]
|
||||
if !FixtureRedirects(test, prefixes[:], targets[:]) { return false }
|
||||
create := string.["new", "greeter"]; created := PackageCommand(test, work, create[:])
|
||||
greeter := CheckPath(work, "greeter"); manifest_path := CheckPath(greeter, "ziran.toml")
|
||||
@@ -87,7 +89,8 @@ Templates :: (test: *NativeTest) -> bool {
|
||||
!NativeWrite(test, CheckPath(kit, "templates/panel/ziran.toml"), "[package]\nname = \"{{name}}\"\n[toolchain]\ngit = \"https://github.com/ziranlang/ziran.git\"\nref = \"master\"\n") ||
|
||||
!NativeWrite(test, CheckPath(kit, "templates/panel/panel.txt"), "panel for {{name}}\n") { return false }
|
||||
unused FixtureCommit(test, kit, "template kit fixture")
|
||||
kit_prefixes := string.[prefixes[0], "https://ziran-lang.org/owner/kit.git"]; kit_targets := string.[targets[0], FixtureUri(kit)]
|
||||
kit_prefixes := string.[prefixes[0], prefixes[1], "https://ziran-lang.org/owner/kit.git"]
|
||||
kit_targets := string.[tool_uri, tool_uri, FixtureUri(kit)]
|
||||
if !FixtureRedirects(test, kit_prefixes[:], kit_targets[:]) { return false }
|
||||
window_command := string.["new", "my-app", "--template", "ziran-lang.org/owner/kit"]
|
||||
windowed := PackageCommand(test, work, window_command[:])
|
||||
|
||||
+116
-28
@@ -4,17 +4,40 @@ exec python3 - "$1" <<'PY'
|
||||
import base64
|
||||
import hashlib
|
||||
import http.server
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import ssl
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import zipfile
|
||||
|
||||
|
||||
def library_sections(image):
|
||||
magic, version, count, size = struct.unpack_from('<4sIII', image)
|
||||
assert magic == b'ZXLB' and version == 1 and size == len(image)
|
||||
position = 16
|
||||
sections = []
|
||||
for _ in range(count):
|
||||
name_length, length, sha = struct.unpack_from('<II32s', image, position)
|
||||
position += 40
|
||||
name = image[position:position + name_length]
|
||||
position += name_length
|
||||
contents = image[position:position + length]
|
||||
position += length
|
||||
assert len(contents) == length and hashlib.sha256(contents).digest() == sha
|
||||
sections.append((name, contents))
|
||||
assert position == size and [name for name, _ in sections] == sorted({name for name, _ in sections})
|
||||
return sections
|
||||
|
||||
|
||||
def make_library(sections):
|
||||
body = b''.join(struct.pack('<II32s', len(name), len(data), hashlib.sha256(data).digest()) + name + data
|
||||
for name, data in sections)
|
||||
return struct.pack('<4sIII', b'ZXLB', 1, len(sections), 16 + len(body)) + body
|
||||
|
||||
|
||||
class Registry(http.server.BaseHTTPRequestHandler):
|
||||
@@ -229,8 +252,10 @@ main :: () -> s32 { return External(); }
|
||||
source.mkdir()
|
||||
(source / 'src').mkdir()
|
||||
provider_module = source / 'src/state.zi'
|
||||
provider_module.write_text('''State :: struct { value: s32; }
|
||||
Advance :: (state: *State) { state.value += 2; }
|
||||
provider_module.write_text('''Update :: #type (value: s32) -> s32;
|
||||
State :: struct { value: s32; handler: Update; }
|
||||
Advance :: (state: *State) { state.value = state.handler(state.value); }
|
||||
Copy :: (value: $T) -> T { return value; }
|
||||
''')
|
||||
toolchain = Path(command('git', '-C', ziran.parent, 'rev-parse', '--show-toplevel'))
|
||||
tool_url = 'https://git.waozi.xyz/ziranlang/ziran.git'
|
||||
@@ -254,18 +279,62 @@ Advance :: (state: *State) { state.value += 2; }
|
||||
(source / '.git/info/exclude').write_text('build/\nziran.local.toml\nziran.lock\n')
|
||||
project(source, 'state', [], {})
|
||||
command('git', '-C', source, 'config', 'remote.origin.url', repository)
|
||||
archive = root / 'state.zir.zip'
|
||||
archive = root / 'state.zx'
|
||||
command(*package, 'pack-ir', repository, checked_revision, 'default', archive, cwd=source)
|
||||
archive_digest = hashlib.sha256(archive.read_bytes()).hexdigest()
|
||||
library = archive.read_bytes()
|
||||
archive_digest = hashlib.sha256(library).hexdigest()
|
||||
sections = library_sections(library)
|
||||
assert [name for name, _ in sections] == [b'checked-package.tsv', b'src/state.zir', b'ziran.toml']
|
||||
inspection = command(*package, 'inspect-ir', archive)
|
||||
assert inspection == sections[0][1].decode().strip() + '\t' + archive_digest
|
||||
repeated = root / 'repacked.zx'
|
||||
command(*package, 'pack-ir', repository, checked_revision, 'default', repeated, cwd=source)
|
||||
assert repeated.read_bytes() == library
|
||||
malformed = root / 'malformed.zx'
|
||||
for invalid in (b'PK\x03\x04' + library[4:],
|
||||
library[:4] + struct.pack('<I', 2) + library[8:],
|
||||
library[:8] + struct.pack('<I', 4099) + library[12:],
|
||||
library[:16] + struct.pack('<I', 0xffffffff) + library[20:],
|
||||
library[:-1], library + b'\0', library[:-1] + bytes([library[-1] ^ 1]),
|
||||
make_library(sections[::-1]), make_library(sections[:1] + sections),
|
||||
make_library([(b'../state.zir', sections[1][1]), *sections])):
|
||||
malformed.write_bytes(invalid)
|
||||
command(*package, 'inspect-ir', malformed, failure='cannot verify the checked .zx library')
|
||||
assert command(*package, 'publish-ir', repository, checked_revision, 'default', archive,
|
||||
'--netrc', netrc) == archive_digest
|
||||
assert command(*package, 'publish-ir', repository, checked_revision, 'default', archive,
|
||||
'--netrc', netrc) == archive_digest
|
||||
command(*package, 'publish', repository, checked_revision, 'default', archive,
|
||||
'--netrc', netrc, failure='cannot verify the portable image')
|
||||
command(*package, 'publish-ir', repository, checked_revision, 'other', archive,
|
||||
'--netrc', netrc, failure='archive does not match the declared repository revision or profile')
|
||||
alternate = root / 'alternate.zir.zip'
|
||||
'--netrc', netrc, failure='failed format, revision or profile verification')
|
||||
alternate = root / 'alternate.zx'
|
||||
command(*package, 'pack-ir', repository, checked_revision, 'other', alternate, cwd=source)
|
||||
alternate_digest = command(*package, 'publish-ir', repository, checked_revision, 'other', alternate, '--netrc', netrc)
|
||||
developer = root / 'developer.zx'
|
||||
dev_pack = [*package, 'pack-dev', repository, checked_revision, 'default', developer, '--offline']
|
||||
command(*dev_pack, cwd=source)
|
||||
original_dev = command(*package, 'inspect-ir', developer).split('\t')
|
||||
provider_module.write_text(provider_module.read_text().replace('state.handler(state.value);',
|
||||
'state.handler(state.value) + 5;'))
|
||||
command(*dev_pack, cwd=source)
|
||||
current_dev = command(*package, 'inspect-ir', developer).split('\t')
|
||||
assert developer.read_bytes()[:4] == b'ZXLD' and current_dev[0] == 'dev'
|
||||
assert len(current_dev) == 6 and original_dev[:4] == current_dev[:4]
|
||||
assert original_dev[4] != current_dev[4] and len(current_dev[4]) == 64
|
||||
assert current_dev[5] == hashlib.sha256(developer.read_bytes()).hexdigest()
|
||||
command(*package, 'pack-ir', repository, checked_revision, 'default', root / 'dirty.zx',
|
||||
cwd=source, failure='requires the clean declared repository revision')
|
||||
command(*dev_pack, '--locked', cwd=source, failure='pack-dev cannot be used with --locked')
|
||||
before = len(server.requests)
|
||||
command(*package, 'publish-ir', repository, checked_revision, 'default', developer,
|
||||
'--netrc', netrc, failure='failed format, revision or profile verification')
|
||||
assert len(server.requests) == before
|
||||
dev_export = f'/api/packages/ziranlang/generic/fixture-zx-library/{checked_revision}/default.zx'
|
||||
server.images[dev_export] = developer.read_bytes()
|
||||
command(*package, 'fetch-ir', repository, checked_revision, 'default', current_dev[5],
|
||||
failure='failed digest, format or module-name verification')
|
||||
server.images[dev_export] = library
|
||||
shutil.rmtree(source)
|
||||
archive.unlink()
|
||||
alternate.unlink()
|
||||
@@ -278,40 +347,54 @@ Advance :: (state: *State) { state.value += 2; }
|
||||
'commit': checked_revision, 'dependencies': {}}
|
||||
project(consumer, 'consumer', [row], {'state': identity})
|
||||
pins = consumer / 'ziran.zx.lock'
|
||||
pins.write_text(json.dumps({'format': 1, 'packages': [{'package': identity,
|
||||
'profile': 'default', 'sha256': archive_digest}]}))
|
||||
app = consumer / 'app.zi'
|
||||
app.write_text('''using Model :: #import "state/model"
|
||||
Increment :: (value: s32) -> s32 { return value + 2; }
|
||||
main :: () -> s32 {
|
||||
state := Model.State.{value = 40};
|
||||
state := Model.State.{value = Model.Copy(cast(s32)40), handler = Increment};
|
||||
Model.Advance(*state);
|
||||
return state.value;
|
||||
}
|
||||
''')
|
||||
built = consumer / 'app.zx'
|
||||
argv = [ziran, 'bundle', '--project', '--entry', 'app:main', '-o', built, app]
|
||||
dev_dir = root / 'dev-libraries'
|
||||
dev_dir.mkdir()
|
||||
developer.rename(dev_dir / f'{identity}.zx')
|
||||
env['ZIRAN_DEV_ZX_DIR'] = str(dev_dir)
|
||||
before = len(server.requests)
|
||||
command(*argv, '--offline', cwd=consumer)
|
||||
assert command(ziran, 'run', '--backend', 'engine', built) == '47'
|
||||
assert len(server.requests) == before and not pins.exists()
|
||||
command(*argv, '--offline', '--locked', cwd=consumer, failure='ZIRAN_DEV_ZX_DIR cannot be used with --locked')
|
||||
env.pop('ZIRAN_DEV_ZX_DIR')
|
||||
pins.write_text(json.dumps({'format': 1, 'packages': [{'package': identity,
|
||||
'profile': 'default', 'sha256': archive_digest}]}))
|
||||
command(*argv, '--offline', failure='absent from the offline cache', cwd=consumer)
|
||||
command(*argv, cwd=consumer)
|
||||
assert command(ziran, 'run', '--backend', 'engine', built) == '42'
|
||||
request_count = len(server.requests)
|
||||
command(*argv, '--offline', cwd=consumer)
|
||||
fetched_library = Path(command(*package, 'fetch-ir', repository, checked_revision, 'default',
|
||||
archive_digest, '--offline'))
|
||||
assert fetched_library.read_bytes() == library
|
||||
assert len(server.requests) == request_count
|
||||
|
||||
# Ordinary add discovers the published descriptor and resolves its
|
||||
# package graph from the archive before attempting any Git checkout.
|
||||
automatic = root / 'automatic'
|
||||
automatic.mkdir()
|
||||
bare_manifest = '[package]\nname = "consumer"\nmodule_roots = ["."]\n' + tool_section
|
||||
bare_manifest = '[package]\nname = "consumer"\nmodule_roots = ["."]\nentry = "app.zi"\n' + tool_section
|
||||
automatic_manifest = automatic / 'ziran.toml'
|
||||
automatic_manifest.write_text(bare_manifest)
|
||||
(automatic / 'ziran.local.toml').write_text(f'[overrides]\nziran = "{toolchain}"\n')
|
||||
auto_cache = root / 'automatic-cache'
|
||||
env.update(XDG_CACHE_HOME=str(auto_cache), ZIRAN_ZX_REGISTRY=f'https://localhost:{server.server_port}')
|
||||
checked_artifact = f'/api/packages/ziranlang/generic/fixture-zx/{checked_revision}/default.zir.zip'
|
||||
descriptor = checked_artifact.removesuffix('.zip') + '.tsv'
|
||||
checked_artifact = f'/api/packages/ziranlang/generic/fixture-zx-library/{checked_revision}/default.zx'
|
||||
descriptor = checked_artifact + '.tsv'
|
||||
good_metadata = server.images[descriptor]
|
||||
expected_tool = 'p' + hashlib.sha256(f'{tool_url}\n{tool_revision}'.encode()).hexdigest()[:16]
|
||||
assert good_metadata == f'1\t{identity}\t{expected_tool}\tdefault\t{archive_digest}\n'.encode()
|
||||
assert good_metadata == f'2\t{identity}\t{expected_tool}\tdefault\t{archive_digest}\n'.encode()
|
||||
add = [ziran, 'add', repository, '--name', 'state', '--ref', checked_revision]
|
||||
server.images[descriptor] = good_metadata.replace(expected_tool.encode(), b'p0000000000000000')
|
||||
command(*add, failure='metadata does not match its repository revision, toolchain or profile', cwd=automatic)
|
||||
@@ -341,6 +424,16 @@ main :: () -> s32 {
|
||||
assert automatic_lock.read_bytes() == saved
|
||||
command(*auto_bundle, cwd=automatic)
|
||||
assert command(ziran, 'run', '--backend', 'engine', auto_output) == '42'
|
||||
command(ziran, 'check', '--offline', cwd=automatic)
|
||||
native_output = automatic / 'native'
|
||||
native_output.mkdir()
|
||||
command(ziran, 'build', '--project', '--offline', '--target=c', '--exe', '--entry', 'app:main',
|
||||
'-o', native_output, auto_app, cwd=automatic)
|
||||
native = subprocess.run([native_output / 'app'], cwd=automatic, check=False, timeout=30)
|
||||
assert native.returncode == 42
|
||||
command(ziran, 'build', '--offline', cwd=automatic)
|
||||
native = subprocess.run([automatic / 'build/consumer'], cwd=automatic, check=False, timeout=30)
|
||||
assert native.returncode == 42
|
||||
assert len(server.requests) == before and not (auto_cache / 'ziran/sources').exists()
|
||||
|
||||
env['ZIRAN_ZX_PROFILE'] = 'missing'
|
||||
@@ -362,8 +455,10 @@ main :: () -> s32 {
|
||||
(override / 'src').mkdir()
|
||||
(override / 'ziran.toml').write_text('[package]\nname = "state"\nmodule_roots = ["src"]\n' +
|
||||
tool_section + '[exports]\nmodel = "src/state.zi"\n')
|
||||
(override / 'src/state.zi').write_text('''State :: struct { value: s32; }
|
||||
Advance :: (state: *State) { state.value += 4; }
|
||||
(override / 'src/state.zi').write_text('''Update :: #type (value: s32) -> s32;
|
||||
State :: struct { value: s32; handler: Update; }
|
||||
Advance :: (state: *State) { state.value = state.handler(state.value) + 2; }
|
||||
Copy :: (value: $T) -> T { return value; }
|
||||
''')
|
||||
command('git', '-C', override, 'init', '-q')
|
||||
(automatic / 'ziran.local.toml').write_text(f'[overrides]\nziran = "{toolchain}"\nstate = "{override}"\n')
|
||||
@@ -376,18 +471,11 @@ Advance :: (state: *State) { state.value += 4; }
|
||||
pins.write_text('{"format": 1,')
|
||||
command(*argv, '--offline', failure='invalid checked packages in ziran.zx.lock', cwd=consumer)
|
||||
|
||||
wrong_tool = io.BytesIO()
|
||||
with (zipfile.ZipFile(io.BytesIO(server.images[checked_artifact])) as original,
|
||||
zipfile.ZipFile(wrong_tool, 'w') as invalid):
|
||||
for name in original.namelist():
|
||||
data = original.read(name)
|
||||
if name == 'checked-package.tsv':
|
||||
_, package_id, _, profile = data.decode().strip().split('\t')
|
||||
data = f'1\t{package_id}\tp0000000000000000\t{profile}\n'.encode()
|
||||
invalid.writestr(name, data)
|
||||
server.images[checked_artifact] = wrong_tool.getvalue()
|
||||
wrong_identity = f'2\t{identity}\tp0000000000000000\tdefault\n'.encode()
|
||||
wrong_tool = make_library([(b'checked-package.tsv', wrong_identity), *sections[1:]])
|
||||
server.images[checked_artifact] = wrong_tool
|
||||
pins.write_text(json.dumps({'format': 1, 'packages': [{'package': identity, 'profile': 'default',
|
||||
'sha256': hashlib.sha256(wrong_tool.getvalue()).hexdigest()}]}))
|
||||
'sha256': hashlib.sha256(wrong_tool).hexdigest()}]}))
|
||||
command(*argv, failure='archive does not match its pinned package, toolchain or profile', cwd=consumer)
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
Reference in new issue
Block a user