Refuse new files over 1 MB on commit and in CI
Git keeps every version of a file, and retired media made the history 317 MB. The pre-commit hook and CI's Secret Scan job now refuse added or changed files over 1 MB unless .github/large-files.txt lists them. The list holds what the app ships or builds from today: the Sun Salutation videos, Lumi, the default profile picture, the Plan 9 WAV music and the Tai Chi narration receipts. Drafts and explorations belong in the ignored backup/ folder; Git LFS files are small pointers and pass.
This commit is contained in:
1 parent
e5785c1a4f
commit
bc2c059b84
5 files changed
+221
No files matched your search
@@ -2,3 +2,4 @@
|
||||
set -eu
|
||||
|
||||
scripts/check-secrets.py --staged
|
||||
scripts/check-file-size.py --staged
|
||||
@@ -0,0 +1,15 @@
|
||||
# Files over 1 MB that may be committed (scripts/check-file-size.py runs in
|
||||
# the pre-commit hook and in CI). Only files the app ships or its build reads
|
||||
# belong here; drafts, renders and design explorations go in the ignored
|
||||
# backup/ folder, and large media can go in Git LFS (see .gitattributes).
|
||||
|
||||
# Sun Salutation practice videos
|
||||
assets/practices/sunsalutation/animations/*/sun-salutation-*.mp4
|
||||
# Lumi and the default profile picture
|
||||
assets/lumi/mascot.png
|
||||
assets/profile-pictures/flowers.png
|
||||
# Built-in music as WAV for Plan 9; the other builds use the LFS OGG files
|
||||
unpackaged_assets/audio/Elijah_K/*.wav
|
||||
# Narration receipts the practice packages are built from
|
||||
design/tai-chi/narration-receipt.json
|
||||
design/tai-chi/narration-male-receipt.json
|
||||
@@ -42,6 +42,22 @@ jobs:
|
||||
python3 scripts/check-secrets.py --working-tree
|
||||
python3 scripts/check-secrets.py --history
|
||||
|
||||
# The pre-commit hook runs the same check on staged files.
|
||||
- name: Refuse new files over 1 MB
|
||||
env:
|
||||
BEFORE: ${{ github.event.before }}
|
||||
BASE_REF: ${{ github.base_ref }}
|
||||
run: |
|
||||
set -eu
|
||||
if [ -n "$BASE_REF" ]; then
|
||||
base=$(git merge-base "origin/$BASE_REF" HEAD)
|
||||
elif [ -n "$BEFORE" ] && git cat-file -e "$BEFORE^{commit}" 2>/dev/null; then
|
||||
base=$BEFORE
|
||||
else
|
||||
base=HEAD~1
|
||||
fi
|
||||
python3 scripts/check-file-size.py --since "$base"
|
||||
|
||||
guard-package-urls:
|
||||
name: Package URLs are public HTTPS
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Refuse new or changed files over 1 MB unless .github/large-files.txt lists them.
|
||||
|
||||
Git keeps every version of a file forever, so drafts, renders and design
|
||||
explorations belong in the ignored backup/ folder. Files the app ships may be
|
||||
listed in .github/large-files.txt (one path or glob per line). Files stored
|
||||
in Git LFS are small pointers in Git and always pass.
|
||||
|
||||
check-file-size.py --staged the pre-commit hook: what is staged
|
||||
check-file-size.py --since REV CI: files changed from REV to HEAD
|
||||
check-file-size.py --tracked every tracked file, for maintenance
|
||||
"""
|
||||
import argparse
|
||||
import fnmatch
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
ALLOWLIST = ".github/large-files.txt"
|
||||
LIMIT = 1024 * 1024
|
||||
|
||||
|
||||
def git(root, *argv, data=None):
|
||||
return subprocess.run(["git", "--no-optional-locks", "-C", str(root), *argv], input=data,
|
||||
check=True, capture_output=True, text=True).stdout
|
||||
|
||||
|
||||
def allowlist_text(root, revision):
|
||||
"""The allowlist as staged (""), as of a commit, or on disk (None)."""
|
||||
if revision is not None:
|
||||
try:
|
||||
return git(root, "show", f"{revision}:{ALLOWLIST}")
|
||||
except subprocess.CalledProcessError:
|
||||
if revision != "":
|
||||
return ""
|
||||
path = root / ALLOWLIST
|
||||
return path.read_text() if path.is_file() else ""
|
||||
|
||||
|
||||
def allowed_patterns(root, revision):
|
||||
text = allowlist_text(root, revision)
|
||||
patterns = []
|
||||
for line in text.splitlines():
|
||||
line = line.split("#", 1)[0].strip()
|
||||
if line:
|
||||
patterns.append(line)
|
||||
return patterns
|
||||
|
||||
|
||||
def sizes(root, revision, paths):
|
||||
"""Blob size of each path in the index (revision "") or a commit."""
|
||||
if not paths:
|
||||
return {}
|
||||
request = "".join(f"{revision}:{path}\n" for path in paths)
|
||||
answer = git(root, "cat-file", "--batch-check=%(objecttype) %(objectsize)", data=request)
|
||||
found = {}
|
||||
for path, line in zip(paths, answer.splitlines()):
|
||||
fields = line.split()
|
||||
if len(fields) == 2 and fields[0] == "blob":
|
||||
found[path] = int(fields[1])
|
||||
return found
|
||||
|
||||
|
||||
def changed_paths(root, mode, since):
|
||||
if mode == "staged":
|
||||
output = git(root, "diff", "--cached", "--name-only", "--diff-filter=AM", "--no-renames", "-z")
|
||||
elif mode == "since":
|
||||
output = git(root, "diff", "--name-only", "--diff-filter=AM", "--no-renames", "-z", since, "HEAD")
|
||||
else:
|
||||
output = git(root, "ls-files", "-z")
|
||||
return [path for path in output.split("\0") if path]
|
||||
|
||||
|
||||
def oversized(root, mode, since=""):
|
||||
revision = "" if mode == "staged" else "HEAD"
|
||||
patterns = allowed_patterns(root, None if mode == "tracked" else revision)
|
||||
found = []
|
||||
for path, size in sizes(root, revision, changed_paths(root, mode, since)).items():
|
||||
if size > LIMIT and not any(fnmatch.fnmatchcase(path, pattern) for pattern in patterns):
|
||||
found.append((path, size))
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--staged", action="store_true")
|
||||
mode.add_argument("--since", metavar="REV")
|
||||
mode.add_argument("--tracked", action="store_true")
|
||||
parser.add_argument("--root", type=Path, default=ROOT)
|
||||
args = parser.parse_args()
|
||||
chosen = "staged" if args.staged else "since" if args.since else "tracked"
|
||||
found = oversized(args.root, chosen, args.since or "")
|
||||
if not found:
|
||||
return 0
|
||||
print("Files over 1 MB:", file=sys.stderr)
|
||||
for path, size in found:
|
||||
print(f" {path} ({size / 1024 / 1024:.1f} MB)", file=sys.stderr)
|
||||
print("Keep drafts, renders and explorations in the ignored backup/ folder. If the app ships the "
|
||||
f"file, list it in {ALLOWLIST} in the same commit.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Files over 1 MB enter Git only when .github/large-files.txt lists them."""
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location("check_file_size", ROOT / "scripts/check-file-size.py")
|
||||
guard = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(guard)
|
||||
|
||||
LARGE = b"\0" * (guard.LIMIT + 1)
|
||||
|
||||
|
||||
class FileSizeTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
temporary = tempfile.TemporaryDirectory(prefix="inbe-file-size-")
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.root = Path(temporary.name)
|
||||
self.git("init", "-q")
|
||||
self.git("config", "user.email", "[email protected]")
|
||||
self.git("config", "user.name", "Test")
|
||||
self.stage({"README.md": b"small\n", guard.ALLOWLIST: b"# shipped\nassets/*.mp4\n"})
|
||||
self.git("commit", "-q", "-m", "start")
|
||||
self.base = self.git("rev-parse", "HEAD").strip()
|
||||
|
||||
def git(self, *argv):
|
||||
return subprocess.run(["git", "-C", str(self.root), *argv], check=True, capture_output=True,
|
||||
text=True).stdout
|
||||
|
||||
def stage(self, files):
|
||||
for name, data in files.items():
|
||||
path = self.root / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
self.git("add", name)
|
||||
|
||||
def oversized(self, mode="staged", since=""):
|
||||
return [path for path, _ in guard.oversized(self.root, mode, since)]
|
||||
|
||||
def test_small_and_listed_files_pass(self):
|
||||
self.stage({"src/main.zi": b"x" * guard.LIMIT, "assets/practice.mp4": LARGE})
|
||||
self.assertEqual(self.oversized(), [])
|
||||
|
||||
def test_large_unlisted_files_are_refused(self):
|
||||
self.stage({"design/render.png": LARGE, "assets/practice.png": LARGE})
|
||||
self.assertEqual(self.oversized(), ["assets/practice.png", "design/render.png"])
|
||||
|
||||
def test_listing_the_file_in_the_same_commit(self):
|
||||
self.stage({"assets/banner.png": LARGE,
|
||||
guard.ALLOWLIST: b"assets/*.mp4\nassets/banner.png\n"})
|
||||
self.assertEqual(self.oversized(), [])
|
||||
|
||||
def test_an_unstaged_allowlist_does_not_count(self):
|
||||
self.stage({"assets/banner.png": LARGE})
|
||||
(self.root / guard.ALLOWLIST).write_text("assets/banner.png\n")
|
||||
self.assertEqual(self.oversized(), ["assets/banner.png"])
|
||||
|
||||
def test_changed_files_since_a_commit(self):
|
||||
self.stage({"design/render.png": LARGE, "assets/practice.mp4": LARGE})
|
||||
self.git("commit", "-q", "-m", "media")
|
||||
self.assertEqual(self.oversized("since", self.base), ["design/render.png"])
|
||||
self.stage({"docs/notes.md": b"later\n"})
|
||||
self.git("commit", "-q", "-m", "docs")
|
||||
self.assertEqual(self.oversized("since", "HEAD~1"), [])
|
||||
|
||||
def test_changing_an_existing_large_file(self):
|
||||
self.stage({"design/render.png": LARGE})
|
||||
self.git("commit", "-q", "--no-verify", "-m", "old render")
|
||||
self.stage({"design/render.png": LARGE + b"\1"})
|
||||
self.assertEqual(self.oversized(), ["design/render.png"])
|
||||
self.assertEqual(self.oversized("tracked"), ["design/render.png"])
|
||||
|
||||
def test_deleting_a_large_file_passes(self):
|
||||
self.stage({"design/render.png": LARGE})
|
||||
self.git("commit", "-q", "--no-verify", "-m", "old render")
|
||||
self.git("rm", "-q", "design/render.png")
|
||||
self.assertEqual(self.oversized(), [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user