Files
breeze/scripts/check-secrets.py
T
waozi b5356aaac7 Port Inbe screens, storage, and platform hosts to Ziran
Replace the remaining .kry and C application sources with Ziran modules,
move vendored dependencies to their current pointers, and add Ziran tests for
the ported hosts and policies.
2026-09-27 15:40:52 -03:00

381 lines
12 KiB
Python
Executable File

#!/usr/bin/env python3
"""Scan tracked content for secrets without echoing secret values."""
from __future__ import annotations
import argparse
import dataclasses
import hashlib
import math
import os
import re
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
ZERO_SHA = "0" * 40
SKIP_PREFIXES = (
".git/",
".gradle/",
".local/",
".secrets/",
"build/",
"node_modules/",
"tmp/",
"vendor/",
"vendor-builds/",
)
SKIP_SUFFIXES = (
".a",
".aab",
".apk",
".AppImage",
".class",
".dll",
".dylib",
".exe",
".ico",
".jar",
".jpg",
".jpeg",
".keystore",
".mp3",
".ogg",
".otf",
".png",
".so",
".snap",
".svgz",
".ttf",
".wasm",
".wav",
".webp",
".zip",
)
PLACEHOLDER_WORDS = (
"changeme",
"dummy",
"example",
"fake",
"placeholder",
"redacted",
"replace",
"sample",
"test",
"your",
)
@dataclasses.dataclass(frozen=True)
class Rule:
name: str
pattern: re.Pattern[str]
min_entropy: float = 0.0
@dataclasses.dataclass(frozen=True)
class Finding:
rule: str
source: str
path: str
line: int
fingerprint: str
commit: str = ""
blob: str = ""
RULES = (
Rule("private-key-block", re.compile(r"-----BEGIN (?:[A-Z0-9]+ )?PRIVATE KEY-----")),
Rule("aws-access-key", re.compile(r"(?P<secret>(?:A3T[A-Z0-9]|AKIA|ASIA)[A-Z0-9]{16})")),
Rule("github-token", re.compile(r"(?P<secret>gh[pousr]_[A-Za-z0-9_]{36,255}|github_pat_[A-Za-z0-9_]{22}_[A-Za-z0-9_]{59})")),
Rule("google-api-key", re.compile(r"(?P<secret>AIza[0-9A-Za-z_-]{35})")),
Rule("openai-api-key", re.compile(r"(?P<secret>sk-(?:proj-)?[A-Za-z0-9_-]{32,})")),
Rule("slack-token", re.compile(r"(?P<secret>xox[baprs]-[A-Za-z0-9-]{20,})")),
Rule("stripe-secret-key", re.compile(r"(?P<secret>sk_(?:live|test)_[A-Za-z0-9]{16,})")),
Rule(
"credential-assignment",
re.compile(
r"(?i)\b(?:api[_-]?key|access[_-]?token|auth[_-]?token|client[_-]?secret|"
r"password|passwd|private[_-]?key|secret)\b\s*[:=]\s*[\"']?"
r"(?P<secret>[A-Za-z0-9+/_=.\-]{16,})"
),
min_entropy=3.2,
),
)
ALLOWED_FINDINGS = {
# Historical code/build strings that match the generic credential-assignment
# heuristic but are not committed credentials.
("droid/app/build.gradle", "credential-assignment", "dd9958e19a3251bf"),
("src/storage/sync_account.c", "credential-assignment", "cb92a7d64e014e24"),
("src/storage/sync_account.zi", "credential-assignment", "c08d2d4080bf45d1"),
}
def git(*args: str, input_bytes: bytes | None = None) -> bytes:
try:
return subprocess.check_output(
["git", *args],
cwd=ROOT,
input=input_bytes,
stderr=subprocess.DEVNULL,
)
except subprocess.CalledProcessError as exc:
raise SystemExit(f"check-secrets: git {' '.join(args)} failed") from exc
def is_skipped_path(path: str) -> bool:
normalized = path.replace("\\", "/").lstrip("./")
return normalized.startswith(SKIP_PREFIXES) or normalized.endswith(SKIP_SUFFIXES)
def is_binary(data: bytes) -> bool:
if b"\0" in data:
return True
sample = data[:4096]
if not sample:
return False
control = sum(1 for byte in sample if byte < 9 or (13 < byte < 32))
return control / len(sample) > 0.08
def entropy(value: str) -> float:
if not value:
return 0.0
counts = {char: value.count(char) for char in set(value)}
length = len(value)
return -sum((count / length) * math.log2(count / length) for count in counts.values())
def looks_like_placeholder(value: str) -> bool:
lowered = value.lower()
if value.startswith("$") or "${{" in value:
return True
if any(word in lowered for word in PLACEHOLDER_WORDS):
return True
if re.fullmatch(r"[A-Z0-9_]+", value) and "_" in value:
return True
if len(set(value)) <= 3:
return True
return False
def fingerprint(value: str) -> str:
return hashlib.sha256(value.encode("utf-8", "replace")).hexdigest()[:16]
def scan_text(data: bytes, *, source: str, path: str, commit: str = "", blob: str = "") -> list[Finding]:
if is_skipped_path(path) or is_binary(data):
return []
text = data.decode("utf-8", "replace")
findings: list[Finding] = []
seen: set[tuple[str, int, str]] = set()
for line_number, line in enumerate(text.splitlines(), 1):
if len(line) > 20000:
continue
for rule in RULES:
for match in rule.pattern.finditer(line):
value = match.groupdict().get("secret") or match.group(0)
value = value.strip("\"'")
if looks_like_placeholder(value):
continue
if rule.min_entropy and entropy(value) < rule.min_entropy:
continue
key = (rule.name, line_number, fingerprint(value))
if (path, rule.name, key[2]) in ALLOWED_FINDINGS:
continue
if key in seen:
continue
seen.add(key)
findings.append(
Finding(
rule=rule.name,
source=source,
path=path,
line=line_number,
fingerprint=key[2],
commit=commit,
blob=blob,
)
)
return findings
def staged_files() -> list[str]:
data = git("diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z")
return [path.decode("utf-8", "replace") for path in data.split(b"\0") if path]
def tracked_files() -> list[str]:
data = git("ls-files", "-z")
return [path.decode("utf-8", "replace") for path in data.split(b"\0") if path]
def scan_staged() -> list[Finding]:
findings: list[Finding] = []
for path in staged_files():
if is_skipped_path(path):
continue
try:
data = git("show", f":{path}")
except SystemExit:
continue
findings.extend(scan_text(data, source="staged", path=path))
return findings
def scan_working_tree() -> list[Finding]:
findings: list[Finding] = []
for path in tracked_files():
if is_skipped_path(path):
continue
full_path = ROOT / path
if not full_path.is_file():
continue
findings.extend(scan_text(full_path.read_bytes(), source="working-tree", path=path))
return findings
def scan_git_objects(rev_args: list[str], *, source: str) -> list[Finding]:
data = git("rev-list", "--objects", *rev_args)
findings: list[Finding] = []
seen_objects: set[tuple[str, str]] = set()
for raw_line in data.splitlines():
parts = raw_line.split(maxsplit=1)
if len(parts) != 2:
continue
oid = parts[0].decode("ascii", "replace")
path = parts[1].decode("utf-8", "replace")
if (oid, path) in seen_objects or is_skipped_path(path):
continue
seen_objects.add((oid, path))
if git("cat-file", "-t", oid).strip() != b"blob":
continue
blob = git("cat-file", "blob", oid)
findings.extend(scan_text(blob, source=source, path=path, blob=oid))
return findings
def commits_for_args(rev_args: list[str]) -> list[str]:
if not rev_args:
return []
data = git("rev-list", *rev_args)
return [line.decode("ascii", "replace") for line in data.splitlines() if line]
def commits_for_blob(blob: str) -> list[str]:
if not blob:
return []
try:
data = git("log", "--all", f"--find-object={blob}", "--format=%H")
except SystemExit:
return []
commits: list[str] = []
for line in data.splitlines():
commit = line.decode("ascii", "replace")
if commit and commit not in commits:
commits.append(commit)
return commits
def scan_commit(commit: str) -> list[Finding]:
findings = scan_text(git("log", "-1", "--format=%B", commit), source="commit-message", path="<commit-message>", commit=commit)
tree = git("ls-tree", "-r", "-z", "--full-tree", commit)
for entry in tree.split(b"\0"):
if not entry:
continue
meta, raw_path = entry.split(b"\t", 1)
mode, kind, oid = meta.decode("ascii", "replace").split()
if kind != "blob":
continue
path = raw_path.decode("utf-8", "replace")
if is_skipped_path(path):
continue
findings.extend(scan_text(git("cat-file", "blob", oid), source="commit", path=path, commit=commit, blob=oid))
return findings
def scan_commits(rev_args: list[str]) -> list[Finding]:
findings: list[Finding] = []
for commit in commits_for_args(rev_args):
findings.extend(scan_commit(commit))
return findings
def pre_push_rev_args(remote: str, stdin: bytes) -> list[list[str]]:
ranges: list[list[str]] = []
for raw_line in stdin.splitlines():
parts = raw_line.decode("utf-8", "replace").split()
if len(parts) != 4:
continue
_local_ref, local_sha, _remote_ref, remote_sha = parts
if local_sha == ZERO_SHA:
continue
if remote_sha == ZERO_SHA:
ranges.append([local_sha, "--not", f"--remotes={remote}"])
else:
ranges.append([f"{remote_sha}..{local_sha}"])
return ranges
def report(findings: list[Finding]) -> int:
if not findings:
print("check-secrets: no candidate secrets found")
return 0
print(f"check-secrets: found {len(findings)} candidate secret(s); values are redacted", file=sys.stderr)
for finding in findings:
location = f"{finding.path}:{finding.line}"
details = [f"rule={finding.rule}", f"fingerprint={finding.fingerprint}"]
if finding.commit:
details.append(f"commit={finding.commit[:12]}")
if finding.blob:
details.append(f"blob={finding.blob[:12]}")
if finding.blob and not finding.commit and finding.source == "history":
commits = commits_for_blob(finding.blob)
if commits:
details.append("commits=" + ",".join(commit[:12] for commit in commits[:5]))
if len(commits) > 5:
details.append(f"commit_count={len(commits)}")
print(f" {location} ({', '.join(details)})", file=sys.stderr)
return 1
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--staged", action="store_true", help="scan files staged for commit")
mode.add_argument("--working-tree", action="store_true", help="scan tracked files in the working tree")
mode.add_argument("--history", action="store_true", help="scan every reachable historical blob")
mode.add_argument("--commits", nargs="+", metavar="REV", help="scan every commit in the given rev-list arguments")
mode.add_argument("--pre-push", action="store_true", help="scan commits read from git pre-push stdin")
parser.add_argument("hook_args", nargs="*", help=argparse.SUPPRESS)
args = parser.parse_args()
os.chdir(ROOT)
if args.staged:
findings = scan_staged()
elif args.working_tree:
findings = scan_working_tree()
elif args.history:
findings = scan_git_objects(["--all"], source="history")
elif args.commits:
findings = scan_commits(args.commits)
else:
remote = args.hook_args[0] if args.hook_args else "origin"
ranges = pre_push_rev_args(remote, sys.stdin.buffer.read())
findings = []
for rev_args in ranges:
findings.extend(scan_commits(rev_args))
return report(findings)
if __name__ == "__main__":
raise SystemExit(main())