diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 10dac30f4..e706bf45a 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -2,3 +2,4 @@ set -eu scripts/check-secrets.py --staged +scripts/check-file-size.py --staged diff --git a/.github/large-files.txt b/.github/large-files.txt new file mode 100644 index 000000000..ec3b05770 --- /dev/null +++ b/.github/large-files.txt @@ -0,0 +1,15 @@ +# Files over 1 MB that may be committed (scripts/check-file-size.py runs in +# the pre-commit hook and in CI). Only files the app ships or its build reads +# belong here; drafts, renders and design explorations go in the ignored +# backup/ folder, and large media can go in Git LFS (see .gitattributes). + +# Sun Salutation practice videos +assets/practices/sunsalutation/animations/*/sun-salutation-*.mp4 +# Lumi and the default profile picture +assets/lumi/mascot.png +assets/profile-pictures/flowers.png +# Built-in music as WAV for Plan 9; the other builds use the LFS OGG files +unpackaged_assets/audio/Elijah_K/*.wav +# Narration receipts the practice packages are built from +design/tai-chi/narration-receipt.json +design/tai-chi/narration-male-receipt.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b4d540a2..52db4ad4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,22 @@ jobs: python3 scripts/check-secrets.py --working-tree python3 scripts/check-secrets.py --history + # The pre-commit hook runs the same check on staged files. + - name: Refuse new files over 1 MB + env: + BEFORE: ${{ github.event.before }} + BASE_REF: ${{ github.base_ref }} + run: | + set -eu + if [ -n "$BASE_REF" ]; then + base=$(git merge-base "origin/$BASE_REF" HEAD) + elif [ -n "$BEFORE" ] && git cat-file -e "$BEFORE^{commit}" 2>/dev/null; then + base=$BEFORE + else + base=HEAD~1 + fi + python3 scripts/check-file-size.py --since "$base" + guard-package-urls: name: Package URLs are public HTTPS runs-on: ubuntu-latest diff --git a/scripts/check-file-size.py b/scripts/check-file-size.py new file mode 100755 index 000000000..60fb27881 --- /dev/null +++ b/scripts/check-file-size.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Refuse new or changed files over 1 MB unless .github/large-files.txt lists them. + +Git keeps every version of a file forever, so drafts, renders and design +explorations belong in the ignored backup/ folder. Files the app ships may be +listed in .github/large-files.txt (one path or glob per line). Files stored +in Git LFS are small pointers in Git and always pass. + + check-file-size.py --staged the pre-commit hook: what is staged + check-file-size.py --since REV CI: files changed from REV to HEAD + check-file-size.py --tracked every tracked file, for maintenance +""" +import argparse +import fnmatch +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +ALLOWLIST = ".github/large-files.txt" +LIMIT = 1024 * 1024 + + +def git(root, *argv, data=None): + return subprocess.run(["git", "--no-optional-locks", "-C", str(root), *argv], input=data, + check=True, capture_output=True, text=True).stdout + + +def allowlist_text(root, revision): + """The allowlist as staged (""), as of a commit, or on disk (None).""" + if revision is not None: + try: + return git(root, "show", f"{revision}:{ALLOWLIST}") + except subprocess.CalledProcessError: + if revision != "": + return "" + path = root / ALLOWLIST + return path.read_text() if path.is_file() else "" + + +def allowed_patterns(root, revision): + text = allowlist_text(root, revision) + patterns = [] + for line in text.splitlines(): + line = line.split("#", 1)[0].strip() + if line: + patterns.append(line) + return patterns + + +def sizes(root, revision, paths): + """Blob size of each path in the index (revision "") or a commit.""" + if not paths: + return {} + request = "".join(f"{revision}:{path}\n" for path in paths) + answer = git(root, "cat-file", "--batch-check=%(objecttype) %(objectsize)", data=request) + found = {} + for path, line in zip(paths, answer.splitlines()): + fields = line.split() + if len(fields) == 2 and fields[0] == "blob": + found[path] = int(fields[1]) + return found + + +def changed_paths(root, mode, since): + if mode == "staged": + output = git(root, "diff", "--cached", "--name-only", "--diff-filter=AM", "--no-renames", "-z") + elif mode == "since": + output = git(root, "diff", "--name-only", "--diff-filter=AM", "--no-renames", "-z", since, "HEAD") + else: + output = git(root, "ls-files", "-z") + return [path for path in output.split("\0") if path] + + +def oversized(root, mode, since=""): + revision = "" if mode == "staged" else "HEAD" + patterns = allowed_patterns(root, None if mode == "tracked" else revision) + found = [] + for path, size in sizes(root, revision, changed_paths(root, mode, since)).items(): + if size > LIMIT and not any(fnmatch.fnmatchcase(path, pattern) for pattern in patterns): + found.append((path, size)) + return sorted(found) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--staged", action="store_true") + mode.add_argument("--since", metavar="REV") + mode.add_argument("--tracked", action="store_true") + parser.add_argument("--root", type=Path, default=ROOT) + args = parser.parse_args() + chosen = "staged" if args.staged else "since" if args.since else "tracked" + found = oversized(args.root, chosen, args.since or "") + if not found: + return 0 + print("Files over 1 MB:", file=sys.stderr) + for path, size in found: + print(f" {path} ({size / 1024 / 1024:.1f} MB)", file=sys.stderr) + print("Keep drafts, renders and explorations in the ignored backup/ folder. If the app ships the " + f"file, list it in {ALLOWLIST} in the same commit.", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/file_size_test.py b/tests/file_size_test.py new file mode 100644 index 000000000..9e0ef96e3 --- /dev/null +++ b/tests/file_size_test.py @@ -0,0 +1,83 @@ +"""Files over 1 MB enter Git only when .github/large-files.txt lists them.""" +import importlib.util +from pathlib import Path +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("check_file_size", ROOT / "scripts/check-file-size.py") +guard = importlib.util.module_from_spec(spec) +spec.loader.exec_module(guard) + +LARGE = b"\0" * (guard.LIMIT + 1) + + +class FileSizeTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix="inbe-file-size-") + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.git("init", "-q") + self.git("config", "user.email", "test@example.invalid") + self.git("config", "user.name", "Test") + self.stage({"README.md": b"small\n", guard.ALLOWLIST: b"# shipped\nassets/*.mp4\n"}) + self.git("commit", "-q", "-m", "start") + self.base = self.git("rev-parse", "HEAD").strip() + + def git(self, *argv): + return subprocess.run(["git", "-C", str(self.root), *argv], check=True, capture_output=True, + text=True).stdout + + def stage(self, files): + for name, data in files.items(): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + self.git("add", name) + + def oversized(self, mode="staged", since=""): + return [path for path, _ in guard.oversized(self.root, mode, since)] + + def test_small_and_listed_files_pass(self): + self.stage({"src/main.zi": b"x" * guard.LIMIT, "assets/practice.mp4": LARGE}) + self.assertEqual(self.oversized(), []) + + def test_large_unlisted_files_are_refused(self): + self.stage({"design/render.png": LARGE, "assets/practice.png": LARGE}) + self.assertEqual(self.oversized(), ["assets/practice.png", "design/render.png"]) + + def test_listing_the_file_in_the_same_commit(self): + self.stage({"assets/banner.png": LARGE, + guard.ALLOWLIST: b"assets/*.mp4\nassets/banner.png\n"}) + self.assertEqual(self.oversized(), []) + + def test_an_unstaged_allowlist_does_not_count(self): + self.stage({"assets/banner.png": LARGE}) + (self.root / guard.ALLOWLIST).write_text("assets/banner.png\n") + self.assertEqual(self.oversized(), ["assets/banner.png"]) + + def test_changed_files_since_a_commit(self): + self.stage({"design/render.png": LARGE, "assets/practice.mp4": LARGE}) + self.git("commit", "-q", "-m", "media") + self.assertEqual(self.oversized("since", self.base), ["design/render.png"]) + self.stage({"docs/notes.md": b"later\n"}) + self.git("commit", "-q", "-m", "docs") + self.assertEqual(self.oversized("since", "HEAD~1"), []) + + def test_changing_an_existing_large_file(self): + self.stage({"design/render.png": LARGE}) + self.git("commit", "-q", "--no-verify", "-m", "old render") + self.stage({"design/render.png": LARGE + b"\1"}) + self.assertEqual(self.oversized(), ["design/render.png"]) + self.assertEqual(self.oversized("tracked"), ["design/render.png"]) + + def test_deleting_a_large_file_passes(self): + self.stage({"design/render.png": LARGE}) + self.git("commit", "-q", "--no-verify", "-m", "old render") + self.git("rm", "-q", "design/render.png") + self.assertEqual(self.oversized(), []) + + +if __name__ == "__main__": + unittest.main()