From b7e2886411e50d9babacb573bfa9e76da9d633f7 Mon Sep 17 00:00:00 2001 From: AutonomyProof Date: Tue, 18 Aug 2026 20:16:10 +0530 Subject: [PATCH] Assessment-lens grouping: Harness / Guardrail / Attack vector (0.22.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make AutonomyProof present as a structured security assessment, not a flat rule list. - rules/categories.py: central rule_id -> lens mapping (Harness gap / Guardrail gap / Attack vector) for all 40 rules; enforced complete + valid by tests. - Rule.category property; Finding gains a category field (set in make_finding / make_project_finding). - ScanResult.category_counts(); JSON report adds assessmentCounts + per-finding category; HTML report shows a per-lens breakdown; 'rules list' groups by lens and 'rules explain' prints the lens. Presentation only — no detection behavior changes (corpus unchanged). 642 tests, 100% branch coverage, ruff + ruff format + mypy all clean. Bumps 0.21.0 -> 0.22.0. Signed-off-by: AutonomyProof --- CHANGELOG.md | 10 +++ README.md | 9 ++- pyproject.toml | 2 +- src/autonomyproof/__init__.py | 2 +- src/autonomyproof/cli.py | 13 +++- src/autonomyproof/models.py | 8 +++ src/autonomyproof/reporters/html_reporter.py | 8 +++ src/autonomyproof/reporters/json_reporter.py | 1 + src/autonomyproof/rules/base.py | 8 +++ src/autonomyproof/rules/categories.py | 73 ++++++++++++++++++++ tests/test_categories.py | 62 +++++++++++++++++ tests/test_cli.py | 11 ++- 12 files changed, 200 insertions(+), 7 deletions(-) create mode 100644 src/autonomyproof/rules/categories.py create mode 100644 tests/test_categories.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d1c7bc3..59aac20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.22.0] + +### Added +- **Assessment-lens grouping.** Every rule and finding is now tagged with one of three + security-assessment lenses — **Harness gap**, **Guardrail gap**, or **Attack vector** — + so AutonomyProof reads as a structured assessment, not a flat rule list. `rules list` groups + by lens; `rules explain` shows the lens; JSON reports add `assessmentCounts` and a + `category` on each finding; the HTML report shows a per-lens breakdown. Central mapping in + `rules/categories.py`, enforced complete by tests. + ## [0.21.0] ### Added diff --git a/README.md b/README.md index a7bfbb2..c5a57da 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,11 @@ known-vulnerable framework dependencies (version-validated CVEs), and more. Run for details. Every finding carries **OWASP Agentic, NIST AI RMF, ISO 42001, MITRE ATLAS/ATT&CK, and CVE** mappings where a genuine one exists. +Rules and findings are grouped into three **assessment lenses** — **Harness gaps** (missing +runtime controls), **Guardrail gaps** (absent/disabled safety controls), and **Attack vectors** +(exploitable capability paths) — so a scan reads as a structured security assessment. +`autonomyproof rules list` shows the catalogue grouped this way. + ### How the analysis works (and its limits) AutonomyProof is AST-based static analysis. It resolves imports and follows source tracking — @@ -146,7 +151,7 @@ re-run `autonomyproof baseline .` and commit the updated file in the same PR. Use the action directly: ```yaml -- uses: autonomyproof/autonomyproof-cli@v0.21.0 +- uses: autonomyproof/autonomyproof-cli@v0.22.0 with: target: . fail-on: high @@ -173,7 +178,7 @@ Gate locally before a commit ever leaves your machine: # .pre-commit-config.yaml repos: - repo: https://github.com/autonomyproof/autonomyproof-cli - rev: v0.21.0 + rev: v0.22.0 hooks: - id: autonomyproof ``` diff --git a/pyproject.toml b/pyproject.toml index f9df065..37769b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "autonomyproof" -version = "0.21.0" +version = "0.22.0" description = "Open-source local scanner that finds unsafe capabilities and missing guardrails in Python AI-agent code." readme = "README.md" requires-python = ">=3.11" diff --git a/src/autonomyproof/__init__.py b/src/autonomyproof/__init__.py index cd2e516..d905d23 100644 --- a/src/autonomyproof/__init__.py +++ b/src/autonomyproof/__init__.py @@ -4,4 +4,4 @@ __all__ = ["__version__"] -__version__ = "0.21.0" +__version__ = "0.22.0" diff --git a/src/autonomyproof/cli.py b/src/autonomyproof/cli.py index 79ffb74..7f8e56f 100644 --- a/src/autonomyproof/cli.py +++ b/src/autonomyproof/cli.py @@ -32,6 +32,8 @@ ) from autonomyproof.models import Finding, ScanResult, Severity from autonomyproof.reporters import write_html, write_json, write_sarif +from autonomyproof.rules.base import Rule +from autonomyproof.rules.categories import CATEGORY_ORDER from autonomyproof.rules.registry import all_rules, get_rule from autonomyproof.scanner import Scanner from autonomyproof.scoring import SCORE_DISCLAIMER @@ -306,9 +308,15 @@ def rules() -> None: @rules.command("list") def rules_list() -> None: - """List every rule.""" + """List every rule, grouped by assessment lens.""" + rules_by_category: dict[str, list[Rule]] = {} for rule in all_rules(): - click.echo(f"{rule.id} {rule.default_severity.value:8} {rule.name}") + rules_by_category.setdefault(rule.category, []).append(rule) + for category in CATEGORY_ORDER: + group = rules_by_category.get(category, []) + click.echo(f"\n{category} ({len(group)})") + for rule in group: + click.echo(f" {rule.id} {rule.default_severity.value:8} {rule.name}") @rules.command("explain") @@ -320,6 +328,7 @@ def rules_explain(rule_id: str) -> None: except KeyError as exc: raise click.ClickException(f"Unknown rule: {rule_id}") from exc click.echo(f"{rule.id} — {rule.name}") + click.echo(f"Category: {rule.category}") click.echo(f"Severity: {rule.default_severity.value}") click.echo(f"Description: {rule.description}") click.echo(f"Risk: {rule.risk}") diff --git a/src/autonomyproof/models.py b/src/autonomyproof/models.py index fdd97ce..77597dc 100644 --- a/src/autonomyproof/models.py +++ b/src/autonomyproof/models.py @@ -64,6 +64,7 @@ class Finding: fingerprint: str framework: str | None = None toolName: str | None = None + category: str = "Attack vector" def to_dict(self) -> dict[str, object]: """Serialize to a JSON-ready dict with the severity as a string.""" @@ -127,3 +128,10 @@ def severity_counts(self) -> dict[str, int]: for finding in self.findings: counts[finding.severity.value] += 1 return counts + + def category_counts(self) -> dict[str, int]: + """Return a count of findings per assessment lens, in display order.""" + counts: dict[str, int] = {} + for finding in self.findings: + counts[finding.category] = counts.get(finding.category, 0) + 1 + return counts diff --git a/src/autonomyproof/reporters/html_reporter.py b/src/autonomyproof/reporters/html_reporter.py index a68d1c8..5391eab 100644 --- a/src/autonomyproof/reporters/html_reporter.py +++ b/src/autonomyproof/reporters/html_reporter.py @@ -7,6 +7,7 @@ from jinja2 import Environment, select_autoescape from autonomyproof.models import ScanResult +from autonomyproof.rules.categories import CATEGORY_ORDER from autonomyproof.scoring import SCORE_DISCLAIMER _TEMPLATE = """ @@ -55,6 +56,8 @@ {{ tools|length }} tool(s){% if project.branch %} · branch {{ project.branch }}{% endif %}

Critical: {{ counts.critical }} · High: {{ counts.high }} · Medium: {{ counts.medium }} · Low: {{ counts.low }}

+ {% if assessment %}

By assessment lens — + {% for category, count in assessment %}{{ category }}: {{ count }}{% if not loop.last %} · {% endif %}{% endfor %}

{% endif %} @@ -156,6 +159,11 @@ def render_html(result: ScanResult) -> str: capabilities=result.capabilities, findings=[f.to_dict() for f in result.findings], counts=result.severity_counts(), + assessment=[ + (category, count) + for category in CATEGORY_ORDER + if (count := result.category_counts().get(category, 0)) + ], errors=result.errors, disclaimer=SCORE_DISCLAIMER, ) diff --git a/src/autonomyproof/reporters/json_reporter.py b/src/autonomyproof/reporters/json_reporter.py index b96270f..68225ad 100644 --- a/src/autonomyproof/reporters/json_reporter.py +++ b/src/autonomyproof/reporters/json_reporter.py @@ -27,6 +27,7 @@ def build_report_dict(result: ScanResult) -> dict[str, object]: "score": result.score, "riskLevel": result.risk_level, "severityCounts": result.severity_counts(), + "assessmentCounts": result.category_counts(), "filesScanned": result.files_scanned, "rulesExecuted": result.rules_executed, "durationMs": result.duration_ms, diff --git a/src/autonomyproof/rules/base.py b/src/autonomyproof/rules/base.py index 6fe9b61..a975afa 100644 --- a/src/autonomyproof/rules/base.py +++ b/src/autonomyproof/rules/base.py @@ -12,6 +12,7 @@ from autonomyproof.frameworks import primary_framework from autonomyproof.models import Finding, Mappings, Severity from autonomyproof.redaction import redact +from autonomyproof.rules.categories import category_for def _code_tokens(analysis: FileAnalysis) -> str: @@ -108,6 +109,11 @@ class Rule: mappings: Mappings = Mappings() project_level: bool = False + @property + def category(self) -> str: + """The assessment lens (Harness gap / Guardrail gap / Attack vector) for this rule.""" + return category_for(self.id) + def check(self, ctx: RuleContext) -> Iterable[Finding]: """Yield findings for one file. Overridden by per-file rules.""" return [] @@ -144,6 +150,7 @@ def make_project_finding( fingerprint=compute_fingerprint(self.id, pctx.anchor_file, "", pattern), framework=pctx.framework, toolName=None, + category=self.category, ) def make_finding( @@ -182,4 +189,5 @@ def make_finding( ), framework=ctx.framework, toolName=resolved_tool, + category=self.category, ) diff --git a/src/autonomyproof/rules/categories.py b/src/autonomyproof/rules/categories.py new file mode 100644 index 0000000..499fe2b --- /dev/null +++ b/src/autonomyproof/rules/categories.py @@ -0,0 +1,73 @@ +"""Assessment lens each rule belongs to. + +AutonomyProof is a security *assessment* across three lenses, not a flat rule list: + +- **Harness gaps** — missing or weak runtime controls in the agent framework/config + (no limits, no timeout, no sandbox, no tracing). +- **Guardrail gaps** — safety controls that are absent, disabled, or bypassable + (no approval, disabled safety filter, agent can edit its own guardrails). +- **Attack vectors** — concrete exploitable capability or data-flow paths + (shell/RCE, SSRF, injection, destructive/escalation authority, supply-chain). +""" + +from __future__ import annotations + +HARNESS = "Harness gap" +GUARDRAIL = "Guardrail gap" +ATTACK_VECTOR = "Attack vector" + +# Fixed display order for reports and the CLI. +CATEGORY_ORDER = (HARNESS, GUARDRAIL, ATTACK_VECTOR) + +# Every registered rule id -> its assessment lens. Kept central so the whole assessment +# structure is reviewable in one place. `tests/` asserts this covers every rule exactly. +RULE_CATEGORY: dict[str, str] = { + # --- Harness gaps: missing/weak controls --- + "AG008": HARNESS, # missing execution limits + "AG009": HARNESS, # excessive/unbounded limits + "AG010": HARNESS, # missing action tracing + "AG016": HARNESS, # unrestricted sub-agent creation + "AG018": HARNESS, # external operation without a timeout + "AG020": HARNESS, # missing accountable-agent metadata + "AG027": HARNESS, # code-execution sandbox disabled + # --- Guardrail gaps: safety controls absent/disabled/bypassable --- + "AG007": GUARDRAIL, # dangerous operation without human approval + "AG011": GUARDRAIL, # persistent memory without tenant isolation + "AG013": GUARDRAIL, # MCP tool accepts unvalidated arguments + "AG015": GUARDRAIL, # agent can modify its own guardrails + "AG017": GUARDRAIL, # secret interpolated into model context + "AG032": GUARDRAIL, # model safety filter disabled + # --- Attack vectors: exploitable capability / data-flow paths --- + "AG001": ATTACK_VECTOR, # unrestricted shell execution + "AG002": ATTACK_VECTOR, # dynamic code execution + "AG003": ATTACK_VECTOR, # arbitrary filesystem access + "AG004": ATTACK_VECTOR, # sensitive credential-path access + "AG005": ATTACK_VECTOR, # unrestricted outbound HTTP + "AG006": ATTACK_VECTOR, # server-side request forgery + "AG012": ATTACK_VECTOR, # model-controlled SQL + "AG014": ATTACK_VECTOR, # bearer token forwarded without audience separation + "AG019": ATTACK_VECTOR, # destructive command exposure + "AG021": ATTACK_VECTOR, # insecure deserialization + "AG022": ATTACK_VECTOR, # disabled TLS verification + "AG023": ATTACK_VECTOR, # server-side template injection + "AG024": ATTACK_VECTOR, # dangerous framework capability flag + "AG025": ATTACK_VECTOR, # code/shell interpreter tool exposed + "AG026": ATTACK_VECTOR, # known-vulnerable dependency (supply chain) + "AG028": ATTACK_VECTOR, # code-executing agent or chain + "AG029": ATTACK_VECTOR, # unrestricted HTTP request tool + "AG030": ATTACK_VECTOR, # agent UI exposed via public tunnel + "AG031": ATTACK_VECTOR, # CORS wildcard origin with credentials + "AG033": ATTACK_VECTOR, # irreversible data destruction + "AG034": ATTACK_VECTOR, # cloud/infrastructure destruction + "AG035": ATTACK_VECTOR, # money movement without approval + "AG036": ATTACK_VECTOR, # persistence-sensitive file write + "AG037": ATTACK_VECTOR, # runtime package installation + "AG038": ATTACK_VECTOR, # IAM/privilege escalation + "AG039": ATTACK_VECTOR, # world-writable permission grant + "AG040": ATTACK_VECTOR, # insecure model-output handling +} + + +def category_for(rule_id: str) -> str: + """Return the assessment lens for ``rule_id`` (defaults to Attack vector).""" + return RULE_CATEGORY.get(rule_id, ATTACK_VECTOR) diff --git a/tests/test_categories.py b/tests/test_categories.py new file mode 100644 index 0000000..8b7c60a --- /dev/null +++ b/tests/test_categories.py @@ -0,0 +1,62 @@ +"""Tests for assessment-category grouping (Harness / Guardrail / Attack vector).""" + +from __future__ import annotations + +from pathlib import Path + +from autonomyproof.config import Config +from autonomyproof.reporters.html_reporter import render_html +from autonomyproof.reporters.json_reporter import build_report_dict +from autonomyproof.rules.categories import ( + ATTACK_VECTOR, + CATEGORY_ORDER, + RULE_CATEGORY, + category_for, +) +from autonomyproof.rules.registry import all_rules +from autonomyproof.scanner import Scanner + +_VULN = "import subprocess\ndef run(cmd):\n return subprocess.run(cmd, shell=True)\n" + + +def test_every_registered_rule_has_a_valid_category() -> None: + ids = {rule.id for rule in all_rules()} + # The mapping covers exactly the registered rules — no missing, no stale entries. + assert set(RULE_CATEGORY) == ids + assert all(value in CATEGORY_ORDER for value in RULE_CATEGORY.values()) + + +def test_rule_category_property_matches_mapping() -> None: + for rule in all_rules(): + assert rule.category == RULE_CATEGORY[rule.id] + + +def test_category_for_unknown_defaults_to_attack_vector() -> None: + assert category_for("AG999") == ATTACK_VECTOR + + +def test_all_three_lenses_are_represented() -> None: + present = {rule.category for rule in all_rules()} + assert present == set(CATEGORY_ORDER) + + +def test_finding_carries_category(tmp_path: Path) -> None: + (tmp_path / "agent.py").write_text(_VULN, encoding="utf-8") + result = Scanner(Config()).scan(tmp_path) + shell = next(f for f in result.findings if f.ruleId == "AG001") + assert shell.category == ATTACK_VECTOR + + +def test_category_counts_and_reports(tmp_path: Path) -> None: + (tmp_path / "agent.py").write_text(_VULN, encoding="utf-8") + result = Scanner(Config()).scan(tmp_path) + counts = result.category_counts() + assert counts.get(ATTACK_VECTOR, 0) >= 1 + # JSON report exposes the per-lens breakdown and per-finding category. + report = build_report_dict(result) + assert report["assessmentCounts"] == counts + assert all("category" in f for f in report["findings"]) # type: ignore[union-attr] + # HTML report shows the assessment breakdown. + html = render_html(result) + assert "By assessment lens" in html + assert ATTACK_VECTOR in html diff --git a/tests/test_cli.py b/tests/test_cli.py index fb5a7a0..a750a38 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -33,7 +33,7 @@ def _home(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: def test_version(runner: CliRunner) -> None: result = runner.invoke(cli.main, ["--version"]) assert result.exit_code == 0 - assert "0.21.0" in result.output + assert "0.22.0" in result.output def test_init_creates_and_is_idempotent(runner: CliRunner) -> None: @@ -49,6 +49,15 @@ def test_rules_list(runner: CliRunner) -> None: result = runner.invoke(cli.main, ["rules", "list"]) assert "AG001" in result.output assert result.output.count("AG0") >= 20 + # Grouped by assessment lens. + assert "Harness gap" in result.output + assert "Guardrail gap" in result.output + assert "Attack vector" in result.output + + +def test_rules_explain_shows_category(runner: CliRunner) -> None: + result = runner.invoke(cli.main, ["rules", "explain", "AG001"]) + assert "Category: Attack vector" in result.output def test_rules_explain(runner: CliRunner) -> None: