Files
breeze/scripts/seed-package-sources.py
waozi 250938e668 Drop paths into the removed vendor trees
Kryon no longer vendors utf8proc, Monocypher, liboqs, or raylib; each is
its own Ziran package. The native build stops passing include paths and
a sync crypto list that point into Kryon's old vendor tree, the
secondary window test reads raylib's headers from its package, and the
Firefox add-on source zip leaves out liboqs's and raylib's docs, tests,
and examples from their package folders.
2026-09-29 21:44:41 -03:00

149 lines
5.3 KiB
Python

#!/usr/bin/env python3
"""Seed the Ziran package cache from source checkouts that are already on disk.
A builder that supplies dependency sources itself (F-Droid srclibs, a release
source tree, an offline mirror) should not need the network. For every entry
in ziran.lock, and for the locked toolchain, this looks for a local Git
checkout that contains the locked commit and clones it into the cache
directory `ziran fetch` would use. A checkout is found by its repository or
package name in:
$ZIRAN_PACKAGE_SOURCES when set
../srclib F-Droid's srclib layout, when present
Submodules of a seeded package are filled from a local source with the
submodule's name the same way. Anything not found is left for
`ziran fetch` to download, and a checkout without the locked commit is never
used.
"""
from pathlib import Path
import hashlib
import json
import os
import shutil
import subprocess
import sys
root = Path(__file__).resolve().parent.parent
def git(*args: str, cwd: Path | None = None) -> subprocess.CompletedProcess:
return subprocess.run(["git", "-c", "protocol.file.allow=always", *args],
cwd=cwd, text=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
def cache_dir() -> Path:
base = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
return Path(base) / "ziran" / "sources"
def source_id(url: str, commit: str) -> str:
# Matches PkgSourceId in Ziran's package tool.
return "p" + hashlib.sha256(f"{url}\n{commit}".encode()).hexdigest()[:16]
def key(name: str) -> str:
name = name.lower()
if name.endswith(".git"):
name = name[:-4]
return name.replace("_", "-")
def source_dirs() -> list[Path]:
dirs = []
explicit = os.environ.get("ZIRAN_PACKAGE_SOURCES")
if explicit:
dirs.append(Path(explicit))
srclib = root.parent / "srclib"
if srclib.is_dir():
dirs.append(srclib)
return [d.resolve() for d in dirs if d.is_dir()]
def index_sources(dirs: list[Path]) -> dict[str, Path]:
found: dict[str, Path] = {}
for directory in dirs:
for child in sorted(directory.iterdir()):
if child.is_dir():
found.setdefault(key(child.name), child)
return found
def has_commit(repo: Path, commit: str) -> bool:
return git("-C", str(repo), "cat-file", "-e",
f"{commit}^{{commit}}").returncode == 0
def find(sources: dict[str, Path], commit: str, *names: str) -> Path | None:
for name in names:
candidate = sources.get(key(name))
if candidate is not None and has_commit(candidate, commit):
return candidate
return None
def seed_submodules(checkout: Path, sources: dict[str, Path]) -> None:
listed = git("-C", str(checkout), "config", "-f", ".gitmodules",
"--get-regexp", r"^submodule\..*\.url$")
if listed.returncode != 0:
return
for line in listed.stdout.splitlines():
setting, url = line.split(" ", 1)
name = setting[len("submodule."):-len(".url")]
path = git("-C", str(checkout), "config", "-f", ".gitmodules",
f"submodule.{name}.path").stdout.strip()
tree = git("-C", str(checkout), "ls-tree", "HEAD", path).stdout.split()
if len(tree) < 3 or tree[1] != "commit":
continue
local = find(sources, tree[2], Path(path).name, url.rstrip("/").rsplit("/", 1)[-1])
if local is None:
continue
result = git("-C", str(checkout), "-c", f"url.{local}.insteadOf={url}",
"submodule", "update", "--init", "--quiet", "--", path)
if result.returncode != 0:
sys.stderr.write(result.stderr)
def seed(url: str, commit: str, sources: dict[str, Path], *names: str) -> bool:
destination = cache_dir() / source_id(url, commit)
if destination.exists():
return False
local = find(sources, commit, url.rstrip("/").rsplit("/", 1)[-1], *names)
if local is None:
return False
staging = destination.with_name(destination.name + ".seed")
shutil.rmtree(staging, ignore_errors=True)
staging.parent.mkdir(parents=True, exist_ok=True)
for step in (("clone", "--quiet", "--no-checkout", str(local), str(staging)),
("-C", str(staging), "checkout", "--quiet", "--detach", commit)):
result = git(*step)
if result.returncode != 0:
shutil.rmtree(staging, ignore_errors=True)
sys.stderr.write(result.stderr)
return False
# Ziran's cache keeps the upstream URL; the local path was only the source.
git("-C", str(staging), "remote", "set-url", "origin", url)
seed_submodules(staging, sources)
staging.rename(destination)
print(f"packages: using local source {local} for {names[0] if names else url}")
return True
def main() -> None:
sources = index_sources(source_dirs())
if not sources:
return
lock = json.loads((root / "ziran.lock").read_text())
edges = {value: name for name, value in lock["root"]["dependencies"].items()}
for package in lock["packages"]:
seed(package["url"], package["commit"], sources,
edges.get(package["id"], package["name"]), package["name"])
tool = lock["toolchain"]
seed(tool["url"], tool["commit"], sources, "ziran")
if __name__ == "__main__":
main()