Refuse new files over 1 MB on commit and in CI

Git keeps every version of a file, and retired media made the history
317 MB. The pre-commit hook and CI's Secret Scan job now refuse added or
changed files over 1 MB unless .github/large-files.txt lists them. The list
holds what the app ships or builds from today: the Sun Salutation videos,
Lumi, the default profile picture, the Plan 9 WAV music and the Tai Chi
narration receipts. Drafts and explorations belong in the ignored backup/
folder; Git LFS files are small pointers and pass.
This commit is contained in:
waozi committed 2026-10-17 17:47:09 -03:00
1 parent e5785c1a4f
commit bc2c059b84
5 files changed
+221

No files matched your search

+1
View File
@@ -2,3 +2,4 @@
set -eu
scripts/check-secrets.py --staged
scripts/check-file-size.py --staged
+15
View File
@@ -0,0 +1,15 @@
# Files over 1 MB that may be committed (scripts/check-file-size.py runs in
# the pre-commit hook and in CI). Only files the app ships or its build reads
# belong here; drafts, renders and design explorations go in the ignored
# backup/ folder, and large media can go in Git LFS (see .gitattributes).
# Sun Salutation practice videos
assets/practices/sunsalutation/animations/*/sun-salutation-*.mp4
# Lumi and the default profile picture
assets/lumi/mascot.png
assets/profile-pictures/flowers.png
# Built-in music as WAV for Plan 9; the other builds use the LFS OGG files
unpackaged_assets/audio/Elijah_K/*.wav
# Narration receipts the practice packages are built from
design/tai-chi/narration-receipt.json
design/tai-chi/narration-male-receipt.json
+16
View File
@@ -42,6 +42,22 @@ jobs:
python3 scripts/check-secrets.py --working-tree
python3 scripts/check-secrets.py --history
# The pre-commit hook runs the same check on staged files.
- name: Refuse new files over 1 MB
env:
BEFORE: ${{ github.event.before }}
BASE_REF: ${{ github.base_ref }}
run: |
set -eu
if [ -n "$BASE_REF" ]; then
base=$(git merge-base "origin/$BASE_REF" HEAD)
elif [ -n "$BEFORE" ] && git cat-file -e "$BEFORE^{commit}" 2>/dev/null; then
base=$BEFORE
else
base=HEAD~1
fi
python3 scripts/check-file-size.py --since "$base"
guard-package-urls:
name: Package URLs are public HTTPS
runs-on: ubuntu-latest
+106
View File
@@ -0,0 +1,106 @@
#!/usr/bin/env python3
"""Refuse new or changed files over 1 MB unless .github/large-files.txt lists them.
Git keeps every version of a file forever, so drafts, renders and design
explorations belong in the ignored backup/ folder. Files the app ships may be
listed in .github/large-files.txt (one path or glob per line). Files stored
in Git LFS are small pointers in Git and always pass.
check-file-size.py --staged the pre-commit hook: what is staged
check-file-size.py --since REV CI: files changed from REV to HEAD
check-file-size.py --tracked every tracked file, for maintenance
"""
import argparse
import fnmatch
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
ALLOWLIST = ".github/large-files.txt"
LIMIT = 1024 * 1024
def git(root, *argv, data=None):
return subprocess.run(["git", "--no-optional-locks", "-C", str(root), *argv], input=data,
check=True, capture_output=True, text=True).stdout
def allowlist_text(root, revision):
"""The allowlist as staged (""), as of a commit, or on disk (None)."""
if revision is not None:
try:
return git(root, "show", f"{revision}:{ALLOWLIST}")
except subprocess.CalledProcessError:
if revision != "":
return ""
path = root / ALLOWLIST
return path.read_text() if path.is_file() else ""
def allowed_patterns(root, revision):
text = allowlist_text(root, revision)
patterns = []
for line in text.splitlines():
line = line.split("#", 1)[0].strip()
if line:
patterns.append(line)
return patterns
def sizes(root, revision, paths):
"""Blob size of each path in the index (revision "") or a commit."""
if not paths:
return {}
request = "".join(f"{revision}:{path}\n" for path in paths)
answer = git(root, "cat-file", "--batch-check=%(objecttype) %(objectsize)", data=request)
found = {}
for path, line in zip(paths, answer.splitlines()):
fields = line.split()
if len(fields) == 2 and fields[0] == "blob":
found[path] = int(fields[1])
return found
def changed_paths(root, mode, since):
if mode == "staged":
output = git(root, "diff", "--cached", "--name-only", "--diff-filter=AM", "--no-renames", "-z")
elif mode == "since":
output = git(root, "diff", "--name-only", "--diff-filter=AM", "--no-renames", "-z", since, "HEAD")
else:
output = git(root, "ls-files", "-z")
return [path for path in output.split("\0") if path]
def oversized(root, mode, since=""):
revision = "" if mode == "staged" else "HEAD"
patterns = allowed_patterns(root, None if mode == "tracked" else revision)
found = []
for path, size in sizes(root, revision, changed_paths(root, mode, since)).items():
if size > LIMIT and not any(fnmatch.fnmatchcase(path, pattern) for pattern in patterns):
found.append((path, size))
return sorted(found)
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--staged", action="store_true")
mode.add_argument("--since", metavar="REV")
mode.add_argument("--tracked", action="store_true")
parser.add_argument("--root", type=Path, default=ROOT)
args = parser.parse_args()
chosen = "staged" if args.staged else "since" if args.since else "tracked"
found = oversized(args.root, chosen, args.since or "")
if not found:
return 0
print("Files over 1 MB:", file=sys.stderr)
for path, size in found:
print(f" {path} ({size / 1024 / 1024:.1f} MB)", file=sys.stderr)
print("Keep drafts, renders and explorations in the ignored backup/ folder. If the app ships the "
f"file, list it in {ALLOWLIST} in the same commit.", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())
+83
View File
@@ -0,0 +1,83 @@
"""Files over 1 MB enter Git only when .github/large-files.txt lists them."""
import importlib.util
from pathlib import Path
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location("check_file_size", ROOT / "scripts/check-file-size.py")
guard = importlib.util.module_from_spec(spec)
spec.loader.exec_module(guard)
LARGE = b"\0" * (guard.LIMIT + 1)
class FileSizeTest(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory(prefix="inbe-file-size-")
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name)
self.git("init", "-q")
self.git("config", "user.email", "[email protected]")
self.git("config", "user.name", "Test")
self.stage({"README.md": b"small\n", guard.ALLOWLIST: b"# shipped\nassets/*.mp4\n"})
self.git("commit", "-q", "-m", "start")
self.base = self.git("rev-parse", "HEAD").strip()
def git(self, *argv):
return subprocess.run(["git", "-C", str(self.root), *argv], check=True, capture_output=True,
text=True).stdout
def stage(self, files):
for name, data in files.items():
path = self.root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
self.git("add", name)
def oversized(self, mode="staged", since=""):
return [path for path, _ in guard.oversized(self.root, mode, since)]
def test_small_and_listed_files_pass(self):
self.stage({"src/main.zi": b"x" * guard.LIMIT, "assets/practice.mp4": LARGE})
self.assertEqual(self.oversized(), [])
def test_large_unlisted_files_are_refused(self):
self.stage({"design/render.png": LARGE, "assets/practice.png": LARGE})
self.assertEqual(self.oversized(), ["assets/practice.png", "design/render.png"])
def test_listing_the_file_in_the_same_commit(self):
self.stage({"assets/banner.png": LARGE,
guard.ALLOWLIST: b"assets/*.mp4\nassets/banner.png\n"})
self.assertEqual(self.oversized(), [])
def test_an_unstaged_allowlist_does_not_count(self):
self.stage({"assets/banner.png": LARGE})
(self.root / guard.ALLOWLIST).write_text("assets/banner.png\n")
self.assertEqual(self.oversized(), ["assets/banner.png"])
def test_changed_files_since_a_commit(self):
self.stage({"design/render.png": LARGE, "assets/practice.mp4": LARGE})
self.git("commit", "-q", "-m", "media")
self.assertEqual(self.oversized("since", self.base), ["design/render.png"])
self.stage({"docs/notes.md": b"later\n"})
self.git("commit", "-q", "-m", "docs")
self.assertEqual(self.oversized("since", "HEAD~1"), [])
def test_changing_an_existing_large_file(self):
self.stage({"design/render.png": LARGE})
self.git("commit", "-q", "--no-verify", "-m", "old render")
self.stage({"design/render.png": LARGE + b"\1"})
self.assertEqual(self.oversized(), ["design/render.png"])
self.assertEqual(self.oversized("tracked"), ["design/render.png"])
def test_deleting_a_large_file_passes(self):
self.stage({"design/render.png": LARGE})
self.git("commit", "-q", "--no-verify", "-m", "old render")
self.git("rm", "-q", "design/render.png")
self.assertEqual(self.oversized(), [])
if __name__ == "__main__":
unittest.main()