diff --git a/docs/ADOPTING-AEOS.md b/docs/ADOPTING-AEOS.md index 2ed9e4e..31ff0f7 100644 --- a/docs/ADOPTING-AEOS.md +++ b/docs/ADOPTING-AEOS.md @@ -41,9 +41,9 @@ The rules that workers must not approve their own work, reviewers must be indepe The repository's [boundary scanner](../scripts/check_public_boundary.py) is separate from the installed package. From this repository root, run `python -B scripts/check_public_boundary.py` (use your available Python 3 interpreter). By default it reads the working-tree bytes of Git-tracked files, including staged additions, and recursively reads ZIP members within each archive's bounds (500 non-directory members and 50,000,000 declared uncompressed bytes). Findings cause a non-zero exit. Optional positional paths narrow the scan; it is not automatically a Git hook or a scan of an arbitrary team repository. -It matches selected credential values, private-path text and suspicious basenames. Non-UTF-8 content is skipped apart from filename checks; untracked files (including ignored files that are not tracked), Git history, unknown secret formats and classification of personal/business data are outside its coverage. A clean result does not establish that a repository is safe to disclose. `.gitignore` only helps prevent accidental additions; it does not remove tracked secrets or enforce access control. +It matches selected credential values, private-path text and suspicious basenames. Non-UTF-8 content is skipped apart from filename checks; untracked files (including ignored files that are not tracked), unknown secret formats and classification of personal/business data are outside its coverage. Git history is excluded by default. Run `python -B scripts/check_public_boundary.py --history` to check every blob/path association reachable from `HEAD`, including deleted filenames and unchanged-content renames. Other refs and unreachable objects are not scanned; the JSON `shallow` field reports whether only fetched history is available. Each unique blob is read once, filename checks and exemptions apply per path, and content is scanned once per interpretation (ZIP or ordinary text). ZIP member findings are labelled under each historical archive path. Git metadata and findings remain in memory; blob payloads are read one at a time, with no hard total-repository or individual-blob size cap. Archive limits remain per archive, including nested archives, rather than a global recursion budget. A clean result does not establish that a repository is safe to disclose. `.gitignore` only helps prevent accidental additions; it does not remove tracked secrets or enforce access control. -- [Boundary CI](../.github/workflows/boundary.yml) runs the nine scanner tests, the tracked-file scan and a full-history scan on Ubuntu. The history scan reads every file version reachable from the checked-out commit, so a credential committed and later deleted still fails. The tests cover synthetic marker matches, named secret assignments, neutral placeholders, selected filenames, ZIP contents, malformed ZIPs, tracked/untracked selection, a deleted secret in history and the current repository. Archive size/member limits, nested archives and binary-content skipping have no dedicated regressions in that suite. Cross-platform source compatibility is not a tested Windows/macOS scanner matrix. +- [Boundary CI](../.github/workflows/boundary.yml) runs the 20 scanner tests, the tracked-file scan and a full-history scan on Ubuntu. The history scan reads every file version reachable from the checked-out commit, so a credential committed and later deleted still fails. The tests cover synthetic marker matches, named secret assignments, neutral placeholders, selected filenames, ZIP contents, malformed ZIPs, tracked/untracked selection, deleted credentials and sensitive filenames, duplicate content, renames, path-specific exemptions, Unicode/space-containing paths, merged side-branch history, ZIP aliases and their size/member limits, shallow clones, empty history trees and the current repository. Nested archives and general binary-content skipping have no dedicated regressions in that suite. Cross-platform source compatibility is not a tested Windows/macOS scanner matrix. - [Windows CI](../.github/workflows/validate.yml) runs `validate_candidate.py` on CPython 3.14.5. It executes the older unpublished repair candidate's journey and 86 package regressions plus repository tests. Its link/privacy checks also inspect current public.4 files, but it does **not** execute current public.4's first run or test a Python 3.11 matrix. Current public.4's Windows 3.11.15/3.14.5 results are the separate local receipt cited above. Neither successful workflow is an exhaustive security audit or an independent H3 review. Require a separate review of the final change and record its scope, verdict and remaining checks before owner-controlled integration. diff --git a/scripts/check_public_boundary.py b/scripts/check_public_boundary.py index 5247e38..66719f7 100644 --- a/scripts/check_public_boundary.py +++ b/scripts/check_public_boundary.py @@ -60,14 +60,17 @@ def scan_name(label): def scan_bytes(label, raw): - findings = scan_name(label) + return scan_name(label) + scan_content(label, raw) + + +def scan_content(label, raw): if label.lower().endswith(".zip"): - return findings + scan_archive(label, raw) + return scan_archive(label, raw) try: text = raw.decode("utf8") except UnicodeDecodeError: - return findings # Binary payloads are out of scope for this text scan. - return findings + scan_text(label, text) + return [] # Binary payloads are out of scope for this text scan. + return scan_text(label, text) def scan_archive(label, raw): @@ -78,7 +81,9 @@ def scan_archive(label, raw): return [{"path": label, "marker": "archive exceeds scan bounds"}] findings = [] for info in infos: - findings += scan_bytes(label + "!" + info.filename, z.read(info)) + # Check the member's own basename before adding its display prefix. + findings += [dict(f, path=label + "!" + f["path"]) + for f in scan_bytes(info.filename, z.read(info))] return findings except zipfile.BadZipFile: return [{"path": label, "marker": "unreadable archive"}] @@ -90,26 +95,41 @@ def tracked_files(root): def history_blobs(root): - """Yield (label, bytes) for every blob reachable from HEAD, once per blob.""" - listing = subprocess.run(["git", "rev-list", "--objects", "HEAD"], cwd=root, - capture_output=True, check=True).stdout.decode("utf8").splitlines() + """Yield (paths, short SHA, bytes), reading each reachable blob once. + + Raw, NUL-delimited diffs retain every blob/path association, including + deleted names, merge parents and renames (represented as delete/add). + rev-list --objects supplies only one name for a shared object. + """ + listing = subprocess.run( + ["git", "log", "--format=", "--raw", "-r", "-z", "--no-abbrev", + "--no-renames", "--root", "-m", "HEAD"], cwd=root, + capture_output=True, check=True).stdout.split(b"\0") paths = {} - for line in listing: - sha, _, rel = line.partition(" ") - if rel: - paths.setdefault(sha, rel) - shas = list(paths) - kinds = subprocess.run(["git", "cat-file", "--batch-check=%(objectname) %(objecttype)"], cwd=root, - input="\n".join(shas).encode(), capture_output=True, check=True).stdout.decode().split() - blobs = [kinds[i] for i in range(0, len(kinds), 2) if kinds[i + 1] == "blob"] - out = subprocess.run(["git", "cat-file", "--batch"], cwd=root, input="\n".join(blobs).encode(), - capture_output=True, check=True).stdout - pos = 0 - for sha in blobs: - end = out.index(b"\n", pos) - size = int(out[pos:end].split()[2]) - yield paths[sha], sha[:12], out[end + 1:end + 1 + size] - pos = end + 1 + size + 1 + for index in range(0, len(listing) - 1, 2): + old_mode, new_mode, old_sha, new_sha, _ = listing[index].lstrip(b"\n:").split() + rel = listing[index + 1].decode("utf8", errors="surrogateescape") + for mode, sha in ((old_mode, old_sha), (new_mode, new_sha)): + # Regular files and symlinks are blobs; gitlinks are commits. + if mode.startswith((b"100", b"120")): + paths.setdefault(sha.decode("ascii"), {})[rel] = None + # Keep one blob's bytes at a time, rather than buffering all payloads. + with subprocess.Popen(["git", "cat-file", "--batch"], cwd=root, + stdin=subprocess.PIPE, stdout=subprocess.PIPE) as batch: + for sha in paths: + batch.stdin.write((sha + "\n").encode("ascii")) + batch.stdin.flush() + header = batch.stdout.readline().split() + if len(header) != 3 or header[:2] != [sha.encode("ascii"), b"blob"]: + raise ValueError("Unexpected Git blob response") + size = int(header[2]) + raw = batch.stdout.read(size) + if len(raw) != size or batch.stdout.read(1) != b"\n": + raise ValueError("Incomplete Git blob response") + yield list(paths[sha]), sha[:12], raw + batch.stdin.close() + if batch.wait(): + raise subprocess.CalledProcessError(batch.returncode, batch.args) def is_shallow(root): @@ -123,11 +143,23 @@ def exempt(rel, finding): def scan_history(root=ROOT): findings, count = [], 0 - for rel, short, raw in history_blobs(root): + for paths, short, raw in history_blobs(root): count += 1 - findings += [dict(f, blob=short) for f in scan_bytes(rel, raw) if not exempt(rel, f)] + content = {} + for rel in paths: + findings += [dict(f, blob=short) for f in scan_name(rel)] + # A blob can occur under both ZIP and ordinary names. Scan each + # interpretation once, then apply labels and exemptions per path. + archive = rel.lower().endswith(".zip") + if archive not in content: + content[archive] = (rel, scan_content(rel, raw)) + source, matches = content[archive] + for match in matches: + finding = dict(match, path=rel + match["path"][len(source):], blob=short) + if not exempt(rel, finding): + findings.append(finding) return {"blobs_scanned": count, "shallow": is_shallow(root), "findings": findings, - "limit": "Bounded marker/name scan of every blob reachable from HEAD (one path shown per blob), " + "limit": "Bounded marker/name scan of every blob/path association reachable from HEAD, " "not an exhaustive secret audit; a shallow clone covers only its fetched history"} diff --git a/scripts/test_check_public_boundary.py b/scripts/test_check_public_boundary.py index 1600c79..e13cfaa 100644 --- a/scripts/test_check_public_boundary.py +++ b/scripts/test_check_public_boundary.py @@ -8,9 +8,11 @@ import sys import tempfile import unittest +from unittest.mock import patch import zipfile sys.path.insert(0, str(Path(__file__).resolve().parent)) +import check_public_boundary as boundary from check_public_boundary import scan, scan_bytes, scan_history, scan_name FAKE = { @@ -115,5 +117,142 @@ def test_current_repository_is_clean(self): self.assertEqual(scan()["findings"], []) +class HistoryFilenameTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.git("init", "-q") + + def git(self, *args): + return subprocess.run( + ["git", "-c", "user.name=Synthetic Test", "-c", + "user.email=test@example.invalid", "-c", "commit.gpgsign=false", *args], + cwd=self.root, check=True, capture_output=True).stdout + + def commit(self, files): + for name, raw in files.items(): + path = self.root / name + if raw is None: + path.unlink() + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(raw) + self.git("add", "-A") + self.git("commit", "-qm", "synthetic fixture") + + def names(self, result): + return {f["path"] for f in result["findings"] + if f["marker"] == "secret-bearing file name"} + + def test_deleted_sensitive_filename_with_harmless_contents(self): + self.commit({".env": b"harmless"}) + self.commit({".env": None, "README.md": b"ordinary"}) + self.assertEqual(scan(self.root)["findings"], []) + self.assertEqual(self.names(scan_history(self.root)), {".env"}) + + def test_deleted_sensitive_names_sharing_an_ordinary_blob(self): + self.commit({"docs/notes.txt": b"harmless", ".env": b"harmless", + "old/credentials.json": b"harmless"}) + self.commit({".env": None, "old/credentials.json": None}) + self.assertEqual(scan(self.root)["findings"], []) + result = scan_history(self.root) + self.assertEqual(result["blobs_scanned"], 1) + self.assertEqual(self.names(result), {".env", "old/credentials.json"}) + self.assertEqual(len(result["findings"]), 2) + + def test_sensitive_name_renamed_twice_without_content_changes(self): + self.commit({"notes.txt": b"harmless"}) + self.git("mv", "notes.txt", ".env") + self.git("commit", "-qm", "sensitive name") + self.git("mv", ".env", "ordinary.txt") + self.git("commit", "-qm", "ordinary name") + result = scan_history(self.root) + self.assertEqual(result["blobs_scanned"], 1) + self.assertEqual(self.names(result), {".env"}) + + def test_ordinary_duplicate_names_pass(self): + self.commit({"docs/notes.txt": b"harmless", ".env.example": b"harmless", + "credentials.md": b"harmless"}) + self.commit({"docs/notes.txt": None}) + result = scan_history(self.root) + self.assertEqual(result["blobs_scanned"], 1) + self.assertEqual(result["findings"], []) + + def test_content_detection_is_deduplicated_but_labels_are_preserved(self): + raw = FAKE["GitHub token"].encode() + self.commit({"one.txt": raw, "two.txt": raw}) + with patch.object(boundary, "scan_text", wraps=boundary.scan_text) as text_scan: + result = scan_history(self.root) + self.assertEqual(text_scan.call_count, 1) + self.assertEqual(result["blobs_scanned"], 1) + self.assertEqual({(f["path"], f["marker"]) for f in result["findings"]}, + {("one.txt", "GitHub token"), ("two.txt", "GitHub token")}) + + def test_pattern_source_exemptions_are_applied_per_path(self): + self.commit({"scripts/check_public_boundary.py": b"One" + b"Drive/notes", + "ordinary.txt": b"One" + b"Drive/notes"}) + result = scan_history(self.root) + self.assertEqual([(f["path"], f["marker"]) for f in result["findings"]], + [("ordinary.txt", "synced personal folder")]) + + def test_zip_aliases_and_archive_rename_keep_member_checks_and_bounds(self): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr(".env", "harmless") + archive.writestr("token.txt", FAKE["GitHub token"]) + raw = buffer.getvalue() + self.commit({"data.bin": raw, "one.zip": raw, "two.ZIP": raw}) + self.commit({"one.zip": None, "two.ZIP": None}) + with patch.object(boundary, "scan_archive", wraps=boundary.scan_archive) as archive_scan: + result = scan_history(self.root) + self.assertEqual(archive_scan.call_count, 1) + self.assertEqual(result["blobs_scanned"], 1) + self.assertEqual(self.names(result), {"one.zip!.env", "two.ZIP!.env"}) + self.assertEqual({f["path"] for f in result["findings"] if f["marker"] == "GitHub token"}, + {"one.zip!token.txt", "two.ZIP!token.txt"}) + for bound in ("MAX_ARCHIVE_MEMBERS", "MAX_ARCHIVE_BYTES"): + with self.subTest(bound=bound), patch.object(boundary, bound, 1): + findings = scan_history(self.root)["findings"] + self.assertEqual({(f["path"], f["marker"]) for f in findings}, + {("one.zip", "archive exceeds scan bounds"), + ("two.ZIP", "archive exceeds scan bounds")}) + + def test_quoted_and_unicode_historical_paths(self): + name = 'folder with spaces/caf\u00e9/.env' + self.commit({name: b"harmless", "notes.txt": b"harmless"}) + self.commit({name: None}) + self.assertEqual(self.names(scan_history(self.root)), {name}) + + def test_sensitive_name_on_merged_side_branch(self): + self.commit({"notes.txt": b"harmless"}) + base = self.git("rev-parse", "HEAD").decode().strip() + self.git("checkout", "-qb", "side") + self.commit({".env": b"harmless"}) + self.commit({".env": None}) + self.git("checkout", "-qb", "other", base) + self.commit({"other.txt": b"ordinary"}) + self.git("merge", "--no-ff", "-qm", "synthetic merge", "side") + self.assertEqual(self.names(scan_history(self.root)), {".env"}) + + def test_shallow_clone_reports_only_fetched_history(self): + self.commit({".env": b"harmless", "notes.txt": b"harmless"}) + self.commit({".env": None}) + with tempfile.TemporaryDirectory() as temp: + clone = Path(temp) / "clone" + self.git("clone", "-q", "--depth=1", self.root.as_uri(), str(clone)) + result = scan_history(clone) + self.assertTrue(result["shallow"]) + self.assertEqual(result["blobs_scanned"], 1) + self.assertEqual(result["findings"], []) + self.assertEqual(self.names(scan_history(self.root)), {".env"}) + + def test_empty_commit_has_no_blobs(self): + self.git("commit", "--allow-empty", "-qm", "empty synthetic root") + result = scan_history(self.root) + self.assertEqual(result["blobs_scanned"], 0) + self.assertEqual(result["findings"], []) + + if __name__ == "__main__": unittest.main()