Build dependencies from local sources before the network
scripts/packages.sh now seeds the Ziran package cache from Git checkouts that already contain the locked commits, found in $ZIRAN_PACKAGE_SOURCES or F-Droid's ../srclib, including submodules such as Kryon's raylib. Builders that supply every source never fetch; anything missing is still downloaded.
This commit is contained in:
1 parent
4102724840
commit
42e5bab48d
3 files changed
+169
-4
No files matched your search
@@ -89,6 +89,10 @@ written under `build/dist/`.
|
||||
Dependencies, including Kryon and the Ziran toolchain, are pinned in
|
||||
`ziran.lock`. `make` fetches them into the Ziran package cache and links them
|
||||
under `build/packages/`; `sh scripts/packages.sh` does the same by hand.
|
||||
Builders that supply the sources themselves need no network: put Git checkouts
|
||||
containing the locked commits (including Kryon's `raylib` submodule) in
|
||||
`$ZIRAN_PACKAGE_SOURCES`, or declare them as F-Droid srclibs, which are found
|
||||
in `../srclib` automatically.
|
||||
|
||||
To debug Inbe against the root Kryon checkout, map it in an ignored
|
||||
`ziran.local.toml`:
|
||||
|
||||
+17
-4
@@ -3,6 +3,11 @@
|
||||
# build/packages/<name>. The Ziran package tool comes from the toolchain commit
|
||||
# in the same lock, so a fresh clone needs only git, make, and a C compiler.
|
||||
#
|
||||
# Sources already on disk are used before the network: set
|
||||
# ZIRAN_PACKAGE_SOURCES to a directory of checkouts, or build from F-Droid,
|
||||
# whose srclibs in ../srclib are found automatically. Each must contain the
|
||||
# locked commit; see scripts/seed-package-sources.py.
|
||||
#
|
||||
# sh scripts/packages.sh # use ziran.local.toml overrides if present
|
||||
# sh scripts/packages.sh --locked # release/CI: exact lock, no overrides
|
||||
# sh scripts/packages.sh --offline # use only the package cache
|
||||
@@ -28,6 +33,9 @@ tool_commit=$(toolchain_field commit)
|
||||
bootstrap="$root/build/ziran-bootstrap"
|
||||
ziran=${ZIRAN:-"$bootstrap/build/bin/ziran"}
|
||||
|
||||
python3 scripts/seed-package-sources.py
|
||||
tool_cache="${XDG_CACHE_HOME:-$HOME/.cache}/ziran/sources/$(python3 -c 'import hashlib, sys; print("p" + hashlib.sha256(f"{sys.argv[1]}\n{sys.argv[2]}".encode()).hexdigest()[:16])' "$tool_url" "$tool_commit")"
|
||||
|
||||
if [ -z "${ZIRAN:-}" ]; then
|
||||
current=$(git -C "$bootstrap" rev-parse HEAD 2>/dev/null || true)
|
||||
if [ "$current" != "$tool_commit" ]; then
|
||||
@@ -37,10 +45,15 @@ if [ -z "${ZIRAN:-}" ]; then
|
||||
exit 1 ;;
|
||||
esac
|
||||
rm -rf "$bootstrap"
|
||||
mkdir -p "$bootstrap"
|
||||
git -C "$bootstrap" init -q
|
||||
git -C "$bootstrap" fetch -q --depth 1 "$tool_url" "$tool_commit"
|
||||
git -C "$bootstrap" checkout -q --detach FETCH_HEAD
|
||||
if [ -d "$tool_cache" ]; then
|
||||
git clone -q --no-checkout "$tool_cache" "$bootstrap"
|
||||
git -C "$bootstrap" checkout -q --detach "$tool_commit"
|
||||
else
|
||||
mkdir -p "$bootstrap"
|
||||
git -C "$bootstrap" init -q
|
||||
git -C "$bootstrap" fetch -q --depth 1 "$tool_url" "$tool_commit"
|
||||
git -C "$bootstrap" checkout -q --detach FETCH_HEAD
|
||||
fi
|
||||
fi
|
||||
if [ ! -x "$ziran" ]; then
|
||||
env -u DISPLAY -u WAYLAND_DISPLAY make -C "$bootstrap" -s all >&2
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Seed the Ziran package cache from source checkouts that are already on disk.
|
||||
|
||||
A builder that supplies dependency sources itself (F-Droid srclibs, a release
|
||||
source tree, an offline mirror) should not need the network. For every entry
|
||||
in ziran.lock, and for the locked toolchain, this looks for a local Git
|
||||
checkout that contains the locked commit and clones it into the cache
|
||||
directory `ziran fetch` would use. A checkout is found by its repository or
|
||||
package name in:
|
||||
|
||||
$ZIRAN_PACKAGE_SOURCES when set
|
||||
../srclib F-Droid's srclib layout, when present
|
||||
|
||||
Submodules of a seeded package (Kryon's vendor/raylib) are filled from a local
|
||||
source with the submodule's name the same way. Anything not found is left for
|
||||
`ziran fetch` to download, and a checkout without the locked commit is never
|
||||
used.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
root = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def git(*args: str, cwd: Path | None = None) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["git", "-c", "protocol.file.allow=always", *args],
|
||||
cwd=cwd, text=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
|
||||
|
||||
def cache_dir() -> Path:
|
||||
base = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
|
||||
return Path(base) / "ziran" / "sources"
|
||||
|
||||
|
||||
def source_id(url: str, commit: str) -> str:
|
||||
# Matches PkgSourceId in Ziran's package tool.
|
||||
return "p" + hashlib.sha256(f"{url}\n{commit}".encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def key(name: str) -> str:
|
||||
name = name.lower()
|
||||
if name.endswith(".git"):
|
||||
name = name[:-4]
|
||||
return name.replace("_", "-")
|
||||
|
||||
|
||||
def source_dirs() -> list[Path]:
|
||||
dirs = []
|
||||
explicit = os.environ.get("ZIRAN_PACKAGE_SOURCES")
|
||||
if explicit:
|
||||
dirs.append(Path(explicit))
|
||||
srclib = root.parent / "srclib"
|
||||
if srclib.is_dir():
|
||||
dirs.append(srclib)
|
||||
return [d.resolve() for d in dirs if d.is_dir()]
|
||||
|
||||
|
||||
def index_sources(dirs: list[Path]) -> dict[str, Path]:
|
||||
found: dict[str, Path] = {}
|
||||
for directory in dirs:
|
||||
for child in sorted(directory.iterdir()):
|
||||
if child.is_dir():
|
||||
found.setdefault(key(child.name), child)
|
||||
return found
|
||||
|
||||
|
||||
def has_commit(repo: Path, commit: str) -> bool:
|
||||
return git("-C", str(repo), "cat-file", "-e",
|
||||
f"{commit}^{{commit}}").returncode == 0
|
||||
|
||||
|
||||
def find(sources: dict[str, Path], commit: str, *names: str) -> Path | None:
|
||||
for name in names:
|
||||
candidate = sources.get(key(name))
|
||||
if candidate is not None and has_commit(candidate, commit):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def seed_submodules(checkout: Path, sources: dict[str, Path]) -> None:
|
||||
listed = git("-C", str(checkout), "config", "-f", ".gitmodules",
|
||||
"--get-regexp", r"^submodule\..*\.url$")
|
||||
if listed.returncode != 0:
|
||||
return
|
||||
for line in listed.stdout.splitlines():
|
||||
setting, url = line.split(" ", 1)
|
||||
name = setting[len("submodule."):-len(".url")]
|
||||
path = git("-C", str(checkout), "config", "-f", ".gitmodules",
|
||||
f"submodule.{name}.path").stdout.strip()
|
||||
tree = git("-C", str(checkout), "ls-tree", "HEAD", path).stdout.split()
|
||||
if len(tree) < 3 or tree[1] != "commit":
|
||||
continue
|
||||
local = find(sources, tree[2], Path(path).name, url.rstrip("/").rsplit("/", 1)[-1])
|
||||
if local is None:
|
||||
continue
|
||||
result = git("-C", str(checkout), "-c", f"url.{local}.insteadOf={url}",
|
||||
"submodule", "update", "--init", "--quiet", "--", path)
|
||||
if result.returncode != 0:
|
||||
sys.stderr.write(result.stderr)
|
||||
|
||||
|
||||
def seed(url: str, commit: str, sources: dict[str, Path], *names: str) -> bool:
|
||||
destination = cache_dir() / source_id(url, commit)
|
||||
if destination.exists():
|
||||
return False
|
||||
local = find(sources, commit, url.rstrip("/").rsplit("/", 1)[-1], *names)
|
||||
if local is None:
|
||||
return False
|
||||
staging = destination.with_name(destination.name + ".seed")
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
staging.parent.mkdir(parents=True, exist_ok=True)
|
||||
for step in (("clone", "--quiet", "--no-checkout", str(local), str(staging)),
|
||||
("-C", str(staging), "checkout", "--quiet", "--detach", commit)):
|
||||
result = git(*step)
|
||||
if result.returncode != 0:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
sys.stderr.write(result.stderr)
|
||||
return False
|
||||
# Ziran's cache keeps the upstream URL; the local path was only the source.
|
||||
git("-C", str(staging), "remote", "set-url", "origin", url)
|
||||
seed_submodules(staging, sources)
|
||||
staging.rename(destination)
|
||||
print(f"packages: using local source {local} for {names[0] if names else url}")
|
||||
return True
|
||||
|
||||
|
||||
def main() -> None:
|
||||
sources = index_sources(source_dirs())
|
||||
if not sources:
|
||||
return
|
||||
lock = json.loads((root / "ziran.lock").read_text())
|
||||
edges = {value: name for name, value in lock["root"]["dependencies"].items()}
|
||||
for package in lock["packages"]:
|
||||
seed(package["url"], package["commit"], sources,
|
||||
edges.get(package["id"], package["name"]), package["name"])
|
||||
tool = lock["toolchain"]
|
||||
seed(tool["url"], tool["commit"], sources, "ziran")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user