#!/usr/bin/env python3 """Scan tracked content for secrets without echoing secret values.""" from __future__ import annotations import argparse import dataclasses import hashlib import math import os import re import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[1] ZERO_SHA = "0" * 40 SKIP_PREFIXES = ( ".git/", ".gradle/", ".local/", ".secrets/", "build/", "node_modules/", "tmp/", "vendor/", "vendor-builds/", ) SKIP_SUFFIXES = ( ".a", ".aab", ".apk", ".AppImage", ".class", ".dll", ".dylib", ".exe", ".ico", ".jar", ".jpg", ".jpeg", ".keystore", ".mp3", ".ogg", ".otf", ".png", ".so", ".snap", ".svgz", ".ttf", ".wasm", ".wav", ".webp", ".zip", ) PLACEHOLDER_WORDS = ( "changeme", "dummy", "example", "fake", "placeholder", "redacted", "replace", "sample", "test", "your", ) @dataclasses.dataclass(frozen=True) class Rule: name: str pattern: re.Pattern[str] min_entropy: float = 0.0 @dataclasses.dataclass(frozen=True) class Finding: rule: str source: str path: str line: int fingerprint: str commit: str = "" blob: str = "" RULES = ( Rule("private-key-block", re.compile(r"-----BEGIN (?:[A-Z0-9]+ )?PRIVATE KEY-----")), Rule("aws-access-key", re.compile(r"(?P(?:A3T[A-Z0-9]|AKIA|ASIA)[A-Z0-9]{16})")), Rule("github-token", re.compile(r"(?Pgh[pousr]_[A-Za-z0-9_]{36,255}|github_pat_[A-Za-z0-9_]{22}_[A-Za-z0-9_]{59})")), Rule("google-api-key", re.compile(r"(?PAIza[0-9A-Za-z_-]{35})")), Rule("openai-api-key", re.compile(r"(?Psk-(?:proj-)?[A-Za-z0-9_-]{32,})")), Rule("slack-token", re.compile(r"(?Pxox[baprs]-[A-Za-z0-9-]{20,})")), Rule("stripe-secret-key", re.compile(r"(?Psk_(?:live|test)_[A-Za-z0-9]{16,})")), Rule( "credential-assignment", re.compile( r"(?i)\b(?:api[_-]?key|access[_-]?token|auth[_-]?token|client[_-]?secret|" r"password|passwd|private[_-]?key|secret)\b\s*[:=]\s*[\"']?" r"(?P[A-Za-z0-9+/_=.\-]{16,})" ), min_entropy=3.2, ), ) ALLOWED_FINDINGS = { # Historical code/build strings that match the generic credential-assignment # heuristic but are not committed credentials. ("droid/app/build.gradle", "credential-assignment", "dd9958e19a3251bf"), ("src/storage/sync_account.c", "credential-assignment", "cb92a7d64e014e24"), ("src/storage/sync_account.zi", "credential-assignment", "c08d2d4080bf45d1"), } def git(*args: str, input_bytes: bytes | None = None) -> bytes: try: return subprocess.check_output( ["git", *args], cwd=ROOT, input=input_bytes, stderr=subprocess.DEVNULL, ) except subprocess.CalledProcessError as exc: raise SystemExit(f"check-secrets: git {' '.join(args)} failed") from exc def is_skipped_path(path: str) -> bool: normalized = path.replace("\\", "/").lstrip("./") return normalized.startswith(SKIP_PREFIXES) or normalized.endswith(SKIP_SUFFIXES) def is_binary(data: bytes) -> bool: if b"\0" in data: return True sample = data[:4096] if not sample: return False control = sum(1 for byte in sample if byte < 9 or (13 < byte < 32)) return control / len(sample) > 0.08 def entropy(value: str) -> float: if not value: return 0.0 counts = {char: value.count(char) for char in set(value)} length = len(value) return -sum((count / length) * math.log2(count / length) for count in counts.values()) def looks_like_placeholder(value: str) -> bool: lowered = value.lower() if value.startswith("$") or "${{" in value: return True if any(word in lowered for word in PLACEHOLDER_WORDS): return True if re.fullmatch(r"[A-Z0-9_]+", value) and "_" in value: return True if len(set(value)) <= 3: return True return False def fingerprint(value: str) -> str: return hashlib.sha256(value.encode("utf-8", "replace")).hexdigest()[:16] def scan_text(data: bytes, *, source: str, path: str, commit: str = "", blob: str = "") -> list[Finding]: if is_skipped_path(path) or is_binary(data): return [] text = data.decode("utf-8", "replace") findings: list[Finding] = [] seen: set[tuple[str, int, str]] = set() for line_number, line in enumerate(text.splitlines(), 1): if len(line) > 20000: continue for rule in RULES: for match in rule.pattern.finditer(line): value = match.groupdict().get("secret") or match.group(0) value = value.strip("\"'") if looks_like_placeholder(value): continue if rule.min_entropy and entropy(value) < rule.min_entropy: continue key = (rule.name, line_number, fingerprint(value)) if (path, rule.name, key[2]) in ALLOWED_FINDINGS: continue if key in seen: continue seen.add(key) findings.append( Finding( rule=rule.name, source=source, path=path, line=line_number, fingerprint=key[2], commit=commit, blob=blob, ) ) return findings def staged_files() -> list[str]: data = git("diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z") return [path.decode("utf-8", "replace") for path in data.split(b"\0") if path] def tracked_files() -> list[str]: data = git("ls-files", "-z") return [path.decode("utf-8", "replace") for path in data.split(b"\0") if path] def scan_staged() -> list[Finding]: findings: list[Finding] = [] for path in staged_files(): if is_skipped_path(path): continue try: data = git("show", f":{path}") except SystemExit: continue findings.extend(scan_text(data, source="staged", path=path)) return findings def scan_working_tree() -> list[Finding]: findings: list[Finding] = [] for path in tracked_files(): if is_skipped_path(path): continue full_path = ROOT / path if not full_path.is_file(): continue findings.extend(scan_text(full_path.read_bytes(), source="working-tree", path=path)) return findings def scan_git_objects(rev_args: list[str], *, source: str) -> list[Finding]: data = git("rev-list", "--objects", *rev_args) findings: list[Finding] = [] seen_objects: set[tuple[str, str]] = set() for raw_line in data.splitlines(): parts = raw_line.split(maxsplit=1) if len(parts) != 2: continue oid = parts[0].decode("ascii", "replace") path = parts[1].decode("utf-8", "replace") if (oid, path) in seen_objects or is_skipped_path(path): continue seen_objects.add((oid, path)) if git("cat-file", "-t", oid).strip() != b"blob": continue blob = git("cat-file", "blob", oid) findings.extend(scan_text(blob, source=source, path=path, blob=oid)) return findings def commits_for_args(rev_args: list[str]) -> list[str]: if not rev_args: return [] data = git("rev-list", *rev_args) return [line.decode("ascii", "replace") for line in data.splitlines() if line] def commits_for_blob(blob: str) -> list[str]: if not blob: return [] try: data = git("log", "--all", f"--find-object={blob}", "--format=%H") except SystemExit: return [] commits: list[str] = [] for line in data.splitlines(): commit = line.decode("ascii", "replace") if commit and commit not in commits: commits.append(commit) return commits def scan_commit(commit: str) -> list[Finding]: findings = scan_text(git("log", "-1", "--format=%B", commit), source="commit-message", path="", commit=commit) tree = git("ls-tree", "-r", "-z", "--full-tree", commit) for entry in tree.split(b"\0"): if not entry: continue meta, raw_path = entry.split(b"\t", 1) mode, kind, oid = meta.decode("ascii", "replace").split() if kind != "blob": continue path = raw_path.decode("utf-8", "replace") if is_skipped_path(path): continue findings.extend(scan_text(git("cat-file", "blob", oid), source="commit", path=path, commit=commit, blob=oid)) return findings def scan_commits(rev_args: list[str]) -> list[Finding]: findings: list[Finding] = [] for commit in commits_for_args(rev_args): findings.extend(scan_commit(commit)) return findings def pre_push_rev_args(remote: str, stdin: bytes) -> list[list[str]]: ranges: list[list[str]] = [] for raw_line in stdin.splitlines(): parts = raw_line.decode("utf-8", "replace").split() if len(parts) != 4: continue _local_ref, local_sha, _remote_ref, remote_sha = parts if local_sha == ZERO_SHA: continue if remote_sha == ZERO_SHA: ranges.append([local_sha, "--not", f"--remotes={remote}"]) else: ranges.append([f"{remote_sha}..{local_sha}"]) return ranges def report(findings: list[Finding]) -> int: if not findings: print("check-secrets: no candidate secrets found") return 0 print(f"check-secrets: found {len(findings)} candidate secret(s); values are redacted", file=sys.stderr) for finding in findings: location = f"{finding.path}:{finding.line}" details = [f"rule={finding.rule}", f"fingerprint={finding.fingerprint}"] if finding.commit: details.append(f"commit={finding.commit[:12]}") if finding.blob: details.append(f"blob={finding.blob[:12]}") if finding.blob and not finding.commit and finding.source == "history": commits = commits_for_blob(finding.blob) if commits: details.append("commits=" + ",".join(commit[:12] for commit in commits[:5])) if len(commits) > 5: details.append(f"commit_count={len(commits)}") print(f" {location} ({', '.join(details)})", file=sys.stderr) return 1 def main() -> int: parser = argparse.ArgumentParser(description=__doc__) mode = parser.add_mutually_exclusive_group(required=True) mode.add_argument("--staged", action="store_true", help="scan files staged for commit") mode.add_argument("--working-tree", action="store_true", help="scan tracked files in the working tree") mode.add_argument("--history", action="store_true", help="scan every reachable historical blob") mode.add_argument("--commits", nargs="+", metavar="REV", help="scan every commit in the given rev-list arguments") mode.add_argument("--pre-push", action="store_true", help="scan commits read from git pre-push stdin") parser.add_argument("hook_args", nargs="*", help=argparse.SUPPRESS) args = parser.parse_args() os.chdir(ROOT) if args.staged: findings = scan_staged() elif args.working_tree: findings = scan_working_tree() elif args.history: findings = scan_git_objects(["--all"], source="history") elif args.commits: findings = scan_commits(args.commits) else: remote = args.hook_args[0] if args.hook_args else "origin" ranges = pre_push_rev_args(remote, sys.stdin.buffer.read()) findings = [] for rev_args in ranges: findings.extend(scan_commits(rev_args)) return report(findings) if __name__ == "__main__": raise SystemExit(main())