From 88d538cc888cc71bef36a8c4de9cb04fa2e5c14c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 05:19:41 +0000 Subject: [PATCH] Extend boundary scan with history mode and five more credential markers The tracked-file scan missed JWTs, test payment keys, Azure storage keys, credentials in URLs and *_SECRET/*_TOKEN assignments, and never looked at history, so a credential committed then deleted passed CI. Add those markers, a --history mode over every blob reachable from HEAD, a full-depth CI checkout and two regressions (named assignments, deleted secret in history). Still standard library only. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01ARKFEnTCNfYUYyr8gCqdNZ --- .github/workflows/boundary.yml | 4 ++ docs/ADOPTING-AEOS.md | 2 +- scripts/check_public_boundary.py | 60 ++++++++++++++++++++++++--- scripts/test_check_public_boundary.py | 29 ++++++++++++- 4 files changed, 88 insertions(+), 7 deletions(-) diff --git a/.github/workflows/boundary.yml b/.github/workflows/boundary.yml index 83b8e4f..46ca8b4 100644 --- a/.github/workflows/boundary.yml +++ b/.github/workflows/boundary.yml @@ -16,8 +16,12 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # full history for the --history scan # Standard library only; the runner's own Python 3 is enough. - name: Scanner regressions run: python3 -B -m unittest discover -s scripts -p "test_check_public_boundary.py" - name: Scan every tracked file and archive member run: python3 -B scripts/check_public_boundary.py + - name: Scan every file version reachable from this commit + run: python3 -B scripts/check_public_boundary.py --history diff --git a/docs/ADOPTING-AEOS.md b/docs/ADOPTING-AEOS.md index 343a0f2..2ed9e4e 100644 --- a/docs/ADOPTING-AEOS.md +++ b/docs/ADOPTING-AEOS.md @@ -43,7 +43,7 @@ The repository's [boundary scanner](../scripts/check_public_boundary.py) is sepa It matches selected credential values, private-path text and suspicious basenames. Non-UTF-8 content is skipped apart from filename checks; untracked files (including ignored files that are not tracked), Git history, unknown secret formats and classification of personal/business data are outside its coverage. A clean result does not establish that a repository is safe to disclose. `.gitignore` only helps prevent accidental additions; it does not remove tracked secrets or enforce access control. -- [Boundary CI](../.github/workflows/boundary.yml) runs the seven scanner tests and the tracked-file scan on Ubuntu. The tests cover synthetic marker matches, neutral placeholders, selected filenames, ZIP contents, malformed ZIPs, tracked/untracked selection and the current repository. Archive size/member limits, nested archives and binary-content skipping have no dedicated regressions in that suite. Cross-platform source compatibility is not a tested Windows/macOS scanner matrix. +- [Boundary CI](../.github/workflows/boundary.yml) runs the nine scanner tests, the tracked-file scan and a full-history scan on Ubuntu. The history scan reads every file version reachable from the checked-out commit, so a credential committed and later deleted still fails. The tests cover synthetic marker matches, named secret assignments, neutral placeholders, selected filenames, ZIP contents, malformed ZIPs, tracked/untracked selection, a deleted secret in history and the current repository. Archive size/member limits, nested archives and binary-content skipping have no dedicated regressions in that suite. Cross-platform source compatibility is not a tested Windows/macOS scanner matrix. - [Windows CI](../.github/workflows/validate.yml) runs `validate_candidate.py` on CPython 3.14.5. It executes the older unpublished repair candidate's journey and 86 package regressions plus repository tests. Its link/privacy checks also inspect current public.4 files, but it does **not** execute current public.4's first run or test a Python 3.11 matrix. Current public.4's Windows 3.11.15/3.14.5 results are the separate local receipt cited above. Neither successful workflow is an exhaustive security audit or an independent H3 review. Require a separate review of the final change and record its scope, verdict and remaining checks before owner-controlled integration. diff --git a/scripts/check_public_boundary.py b/scripts/check_public_boundary.py index 9cc4e47..5247e38 100644 --- a/scripts/check_public_boundary.py +++ b/scripts/check_public_boundary.py @@ -3,7 +3,9 @@ Complements validate_candidate.py (Windows, candidate-scoped) with a fast check that runs on any OS: credential markers, private paths and secret-bearing file names anywhere in the repository, including inside committed ZIP archives. -Bounded pattern scan, not an exhaustive secret audit. +With --history it scans every file version reachable from HEAD, so a secret +committed and later deleted is still found. Bounded pattern scan, not an +exhaustive secret audit. """ from pathlib import Path, PurePosixPath import argparse @@ -24,7 +26,11 @@ "model provider API key": r"\bsk-(?:proj-|ant-[a-z0-9]+-)?[A-Za-z0-9_-]{20,}", "Google API key": r"\bAIza[0-9A-Za-z_-]{35}\b", "live payment key": r"\b[rs]k_live_[0-9A-Za-z]{20,}", - "credential assignment": r"(?im)^\s*(?:api[_-]?key|access[_-]?token|client[_-]?secret|password)\s*[:=]\s*['\"]?[^\s'\"<{$]{8,}", + "test payment key": r"\b[rs]k_test_[0-9A-Za-z]{20,}", + "JSON web token": r"\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}", + "Azure storage key": r"(?i)\bAccountKey=[A-Za-z0-9+/]{40,}", + "credential in URL": r"(?i)\b[a-z][a-z0-9+.-]*://[^\s/:@'\"<>]+:[^\s/@'\"<>{$%]{6,}@", + "credential assignment": r"(?im)^\s*(?:export\s+)?(?:api[_-]?key|access[_-]?token|client[_-]?secret|password|[a-z0-9_]*_(?:secret|token|password|api_key))\s*[:=]\s*['\"]?[^\s'\"<{$]{8,}", "Windows user path": r"(?i)\b[a-z]:[/\\]+users[/\\]+(?!<|%|\{|\$|public\b|default\b)[^/\\\s]+", "macOS user path": r"/Users/(?!<|\{|\$|Shared\b)[A-Za-z0-9._-]+/", "synced personal folder": r"(?i)onedrive|dropbox[/\\]|google drive[/\\]", @@ -83,14 +89,55 @@ def tracked_files(root): return [p for p in out.decode("utf8").split("\0") if p] +def history_blobs(root): + """Yield (label, bytes) for every blob reachable from HEAD, once per blob.""" + listing = subprocess.run(["git", "rev-list", "--objects", "HEAD"], cwd=root, + capture_output=True, check=True).stdout.decode("utf8").splitlines() + paths = {} + for line in listing: + sha, _, rel = line.partition(" ") + if rel: + paths.setdefault(sha, rel) + shas = list(paths) + kinds = subprocess.run(["git", "cat-file", "--batch-check=%(objectname) %(objecttype)"], cwd=root, + input="\n".join(shas).encode(), capture_output=True, check=True).stdout.decode().split() + blobs = [kinds[i] for i in range(0, len(kinds), 2) if kinds[i + 1] == "blob"] + out = subprocess.run(["git", "cat-file", "--batch"], cwd=root, input="\n".join(blobs).encode(), + capture_output=True, check=True).stdout + pos = 0 + for sha in blobs: + end = out.index(b"\n", pos) + size = int(out[pos:end].split()[2]) + yield paths[sha], sha[:12], out[end + 1:end + 1 + size] + pos = end + 1 + size + 1 + + +def is_shallow(root): + out = subprocess.run(["git", "rev-parse", "--is-shallow-repository"], cwd=root, capture_output=True, check=True) + return out.stdout.strip() == b"true" + + +def exempt(rel, finding): + return rel in PATTERN_SOURCES and finding["marker"] in PATTERN_SOURCE_MARKERS + + +def scan_history(root=ROOT): + findings, count = [], 0 + for rel, short, raw in history_blobs(root): + count += 1 + findings += [dict(f, blob=short) for f in scan_bytes(rel, raw) if not exempt(rel, f)] + return {"blobs_scanned": count, "shallow": is_shallow(root), "findings": findings, + "limit": "Bounded marker/name scan of every blob reachable from HEAD (one path shown per blob), " + "not an exhaustive secret audit; a shallow clone covers only its fetched history"} + + def scan(root=ROOT, paths=None): paths = tracked_files(root) if paths is None else paths findings = [] for rel in paths: path = root / rel if path.is_file(): - findings += [f for f in scan_bytes(rel, path.read_bytes()) - if not (rel in PATTERN_SOURCES and f["marker"] in PATTERN_SOURCE_MARKERS)] + findings += [f for f in scan_bytes(rel, path.read_bytes()) if not exempt(rel, f)] return {"files_scanned": len(paths), "findings": findings, "limit": "Bounded marker/name scan of tracked files and ZIP members, not an exhaustive secret audit"} @@ -98,7 +145,10 @@ def scan(root=ROOT, paths=None): if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("paths", nargs="*", help="Repository-relative files (default: all tracked files)") + parser.add_argument("--history", action="store_true", help="Scan every file version reachable from HEAD") args = parser.parse_args() - result = scan(paths=args.paths or None) + if args.history and args.paths: + parser.error("--history scans all reachable history; it takes no paths") + result = scan_history() if args.history else scan(paths=args.paths or None) print(json.dumps(result, indent=2)) sys.exit(1 if result["findings"] else 0) diff --git a/scripts/test_check_public_boundary.py b/scripts/test_check_public_boundary.py index d9abe99..1600c79 100644 --- a/scripts/test_check_public_boundary.py +++ b/scripts/test_check_public_boundary.py @@ -11,7 +11,7 @@ import zipfile sys.path.insert(0, str(Path(__file__).resolve().parent)) -from check_public_boundary import scan, scan_bytes, scan_name +from check_public_boundary import scan, scan_bytes, scan_history, scan_name FAKE = { "private key block": "-----BEGIN " + "RSA PRIVATE KEY-----", @@ -21,6 +21,10 @@ "model provider API key": "sk-" + "ant-api03-" + "x" * 24, "Google API key": "AIza" + "B" * 35, "live payment key": "sk_" + "live_" + "c" * 24, + "test payment key": "sk_" + "test_" + "d" * 24, + "JSON web token": "eyJ" + "hbGciOiJIUzI1NiJ9" + ".eyJ" + "zdWIiOiIxMjM0In0" + "." + "e" * 43, + "Azure storage key": "Account" + "Key=" + "f" * 86 + "==", + "credential in URL": "postgres" + "://admin:" + "g" * 16 + "@db.example.com/app", "credential assignment": "api_key" + " = " + "s3cr3tvalue99", "Windows user path": "C:" + "\\Users\\" + "someone\\vault", "macOS user path": "/Users/" + "someone/vault", @@ -39,6 +43,11 @@ def test_each_marker_detected(self): with self.subTest(marker=marker): self.assertIn(marker, markers("doc.md", "context\n" + sample + " context")) + def test_named_secret_assignments(self): + for line in ("DB_" + "SECRET = 'h1ddenvalue99'", "export GITHUB_" + "TOKEN=abcdefgh1234"): + with self.subTest(line=line): + self.assertIn("credential assignment", markers("app.py", line)) + def test_placeholders_and_neutral_text_pass(self): neutral = "\n".join([ "vault_root: ", @@ -46,6 +55,8 @@ def test_placeholders_and_neutral_text_pass(self): "api_key = ${API_KEY}", "C:\\Users\\\\Workspace and C:\\Users\\Public", "/Users//vault", + "postgres://:@localhost/app and https://example.com/a:b@c", + "DB_SECRET = ${DB_SECRET}", "Do not store credentials in this file.", ]) self.assertEqual(markers("README.md", neutral), set()) @@ -84,6 +95,22 @@ def test_tracked_files_in_a_synthetic_repository(self): self.assertEqual(result["files_scanned"], 2) self.assertEqual(result["findings"], [{"path": "leak.md", "marker": "AWS access key id"}]) + def test_history_finds_deleted_secret(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + git = ["git", "-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "commit.gpgsign=false"] + subprocess.run(["git", "init", "-q"], cwd=root, check=True) + (root / "notes.md").write_text("token " + FAKE["GitHub token"], encoding="utf8") + subprocess.run(git + ["add", "notes.md"], cwd=root, check=True) + subprocess.run(git + ["commit", "-qm", "add"], cwd=root, check=True) + (root / "notes.md").write_text("clean", encoding="utf8") + subprocess.run(git + ["commit", "-qam", "remove"], cwd=root, check=True) + self.assertEqual(scan(root)["findings"], []) + result = scan_history(root) + self.assertEqual(result["blobs_scanned"], 2) + self.assertFalse(result["shallow"]) + self.assertEqual([(f["path"], f["marker"]) for f in result["findings"]], [("notes.md", "GitHub token")]) + def test_current_repository_is_clean(self): self.assertEqual(scan()["findings"], [])