diff --git a/docs/authoring-evals.md b/docs/authoring-evals.md index a5c4743..20dfbf5 100644 --- a/docs/authoring-evals.md +++ b/docs/authoring-evals.md @@ -90,7 +90,11 @@ All arrays, all valid only on a `skill_should_trigger: true` evaluation: | `files_exist` | the filesystem | an artifact the run must produce | The bottom two are instant and free where a judged expectation costs a second -agent call, so reach for them when the thing you want is literal. Never assert +agent call, so reach for them when the thing you want is literal. They are also +the two that always produce an answer: a judged expectation has a third outcome, +**ungraded**, for when the judge could not be reached or did not reply with a +verdict. That is reported apart from a failure and excluded from the rate, since +a grader that never ran says nothing about your skill either way. Never assert your own skill's name in `logs_contain`: routing already grades that properly, and a substring match only proves the skill was staged. @@ -156,6 +160,8 @@ def setup_session(cache_dir): ... # once per skill; returns {name: value} fo def setup(workspace, case, ctx): ... # before each case; may return more placeholders def teardown(workspace, case, ctx): ... def check(run, case, ctx): ... # after each case; raise AssertionError to fail it + # (run.expects leaves it ungraded if the judge + # could not answer, rather than failing it) ``` `teardown` runs even when the agent itself blew up, and a `check` that raises diff --git a/docs/usage.md b/docs/usage.md index 17cd0f9..c3cd7d8 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -170,10 +170,10 @@ alone — those are illustrations of links rather than promises. ## Routing, and the bar it is held to `--min-accuracy` / `min_accuracy` defaults to `1`: every graded routing case has -to land on the right skill, the same way every behavioral expectation has to -hold. A routing miss is a defect and not a statistic — a description that fires -on its neighbour's prompt makes that neighbour worse — so it turns the run red -rather than sitting in a summary nobody reads. +to land on the right skill, the same way every behavioral expectation that was +graded has to hold. A routing miss is a defect and not a statistic — a +description that fires on its neighbour's prompt makes that neighbour worse — +so it turns the run red rather than sitting in a summary nobody reads. Loosening it is a one-line diff. `0` reports the score without gating on it, which is where a repo that has never measured its prompts should start; anything @@ -184,6 +184,16 @@ infrastructure checks — a run where nothing was graded, or where no skill activated anywhere, fails at any bar, since those numbers are an artifact rather than a result. +Behavioral is held to the same shape of bar, and carves out the same case. An +expectation its judge could not reach a verdict on — the API was unreachable, +the grader ran out of time, the reply came back without one — is reported as +**ungraded** rather than unmet, listed under its own heading, and left out of +the expectation rate. The run still fails, and the case carries an error saying +the grader could not answer. The distinction is what the report is for: an +`unexpected_behavior` expectation scored against no verdict would otherwise +assert the skill did the very thing it forbids, on the evidence of a dropped +connection. + Two failure modes are worth knowing before reading a report. A routing case that ends without the agent either activating a skill or answering is reported as an **error** rather than a missed trigger. And if the runner has its own skills diff --git a/skillscope/agent.py b/skillscope/agent.py index 53bce6d..ffc2630 100644 --- a/skillscope/agent.py +++ b/skillscope/agent.py @@ -263,9 +263,17 @@ def _find_file(files: list[str], expected: str) -> str | None: def _grade_with_llm( statement: str, run: "Run", judge_model: str | None, *, must_happen: bool -) -> tuple[bool, str]: +) -> tuple[bool | None, str]: """Ask a grader LLM whether the run satisfied a requirement. + The verdict is ``True`` (satisfied), ``False`` (not satisfied), or ``None`` + when the judge could not answer at all -- no CLI, no time left, no reply. + ``None`` is not a negative verdict and callers must not collapse it into + one: a grader that never ran is no evidence about the skill, and reporting + it as ``False`` publishes a network fault as a behavioral violation. The + ``must_happen=False`` polarity is where this bites hardest, because there + ``False`` asserts the prohibited thing *happened*. + ``must_happen`` selects the polarity: ``True`` means the agent was required to do ``statement``, ``False`` means it was required *not* to. The judge grades the requirement itself and returns ``True`` when it is satisfied, so @@ -278,7 +286,7 @@ def _grade_with_llm( """ claude_bin = shutil.which("claude") if not claude_bin: - return False, "llm_judge skipped: 'claude' CLI not on PATH" + return None, "llm_judge skipped: 'claude' CLI not on PATH" cmd_text = run.command_text if len(cmd_text) > 4000: @@ -326,7 +334,7 @@ def _grade_with_llm( if bound is not None: leftover = bound.remaining() if leftover <= 0: - return False, bound.message() + return None, bound.message() judge_timeout = bound.cap(judge_timeout) try: @@ -335,7 +343,7 @@ def _grade_with_llm( input=prompt_text, timeout=judge_timeout, env=claude_env(), ) except subprocess.TimeoutExpired: - return False, f"llm_judge timed out after {judge_timeout:g}s" + return None, f"llm_judge timed out after {judge_timeout:g}s" try: payload = json.loads((proc.stdout or "").strip()) @@ -359,21 +367,49 @@ def _grade_with_llm( if isinstance(parsed, dict) and "pass" in parsed: verdict = parsed if verdict is None: - return False, f"llm_judge gave no JSON verdict: {verdict_text[:200]!r}" + return None, f"llm_judge gave no JSON verdict: {verdict_text[:200]!r}" satisfied = bool(verdict.get("pass")) reason = str(verdict.get("reason", "")).strip() or "(no reason given)" return satisfied, f"llm_judge: {reason}" +class Ungradable(Exception): + """A check could not be decided, as distinct from deciding against it.""" + + @dataclass class Check: - """One graded expectation from a case.""" + """One expectation from a case, and what grading it concluded. + + ``graded`` is ``False`` when the check could not be decided at all -- an + LLM judge that never answered. Such a check is not a pass and not a + failure; it is an absence of evidence, and the distinction is what + separates reporting a broken grader from accusing a skill of the behavior + the expectation forbids. ``passed`` stays ``False`` there, so a reader that + only knows about pass and fail still refuses to call the case green -- but + anything attributing a result to the skill must read ``graded`` first. + """ kind: str expectation: str passed: bool detail: str = "" + graded: bool = True + + +def _judged(kind: str, expectation: str, verdict: bool | None, reason: str) -> Check: + """Turn a judge's verdict into a `Check`, keeping `None` as ungraded.""" + if verdict is None: + return Check(kind, expectation, False, reason, graded=False) + return Check(kind, expectation, verdict, reason) + + +def _verdict_label(check: Check) -> str: + """What the per-check line reports: met, unmet, or nothing concluded.""" + if not check.graded: + return "----" + return "PASS" if check.passed else "FAIL" class Run: @@ -434,22 +470,21 @@ def evaluate( for statement in expected_behavior: ok, reason = _grade_with_llm(statement, self, self.judge_model, must_happen=True) - checks.append(Check("expected_behavior", statement, ok, reason)) + checks.append(_judged("expected_behavior", statement, ok, reason)) for statement in unexpected_behavior: ok, reason = _grade_with_llm(statement, self, self.judge_model, must_happen=False) - checks.append(Check("unexpected_behavior", statement, ok, reason)) + checks.append(_judged("unexpected_behavior", statement, ok, reason)) for check in checks: suffix = f" -- {check.detail}" if check.detail else "" - _safe_print( - f" [{'PASS' if check.passed else 'FAIL'}] " - f"({check.kind}) {check.expectation}{suffix}" - ) + _safe_print(f" [{_verdict_label(check)}] ({check.kind}) {check.expectation}{suffix}") return checks # Asserting variants, for an `evals/hooks.py` that needs a check the - # dataset format cannot express. Each raises AssertionError on failure. + # dataset format cannot express. A check that failed raises + # AssertionError; one the judge could not decide raises `Ungradable`, + # which `run_case` records as an ungraded check rather than a failed one. def logs_contains(self, text: str) -> "Run": ok = text.lower() in self.logs.lower() @@ -474,7 +509,14 @@ def expects_not(self, statement: str) -> "Run": self._report(satisfied, "unexpected_behavior", f"{statement} -- {reason}") return self - def _report(self, passed: bool, kind: str, detail: str) -> None: + def _report(self, passed: bool | None, kind: str, detail: str) -> None: + if passed is None: + # Deliberately not an AssertionError. `run_case` catches that and + # records a failed check against the skill, which is the exact + # attribution this state exists to prevent; anything else reaches + # the outer handler and is reported as the run erroring. + _safe_print(f" [----] ({kind}) {detail}") + raise Ungradable(f"({kind}) {detail}") _safe_print(f" [{'PASS' if passed else 'FAIL'}] ({kind}) {detail}") assert passed, f"({kind}) {detail}" diff --git a/skillscope/behavior.py b/skillscope/behavior.py index 154f48c..3806b7c 100644 --- a/skillscope/behavior.py +++ b/skillscope/behavior.py @@ -25,7 +25,7 @@ from types import ModuleType from . import datasets, deadline -from .agent import Check, claude +from .agent import Check, Ungradable, claude from .datasets import Case @@ -61,6 +61,8 @@ def load_hooks(skill: str) -> ModuleType | None: setup(workspace, case, ctx) -> dict of extra template vars, per case teardown(workspace, case, ctx) check(run, case, ctx) -> raise AssertionError to fail the case + (`run.expects` raises `Ungradable` + when the judge could not answer) """ path = datasets.hooks_path(skill) if not path.is_file(): @@ -133,6 +135,19 @@ def run_case( checks.append(Check("hook", "evals/hooks.py check()", True)) except AssertionError as exc: checks.append(Check("hook", "evals/hooks.py check()", False, str(exc)[:400])) + except Ungradable as exc: + # `run.expects` could not get a verdict. Same standing as + # an ungraded dataset expectation: not a pass, but not + # evidence against the skill either. + checks.append( + Check( + "hook", + "evals/hooks.py check()", + False, + str(exc)[:400], + graded=False, + ) + ) finally: if hooks is not None and hasattr(hooks, "teardown"): hooks.teardown(workspace, case, case_ctx) @@ -142,8 +157,18 @@ def run_case( elapsed = round(time.perf_counter() - started, 2) passed = error is None and all(c.passed for c in checks) and bool(checks) + ungraded = [c for c in checks if not c.graded] if error is None and not checks: error = "case has no behavioral assertions to grade" + elif error is None and ungraded: + # Still not a pass -- nothing confirmed the skill behaved -- but the + # reason is that the grader could not answer, so it belongs in the + # errors column where it gets triaged as infrastructure rather than + # read as the skill having violated the expectation. + error = ( + f"{len(ungraded)} of {len(checks)} expectation(s) could not be " + f"graded: {ungraded[0].detail}" + ) print( f" [{'PASS' if passed else 'FAIL'}] {case.id}: " f"{sum(1 for c in checks if c.passed)}/{len(checks)} checks in {elapsed}s" @@ -186,6 +211,17 @@ def run(skills: list[str], cases: list[Case], model: str, effort: str) -> list[B return outcomes +def graded(check: dict) -> bool: + """Whether a check reached a verdict at all. + + A missing key means it did, so a check dict assembled by something + other than `asdict(Check)` still reads correctly. `summarize` is the + only place that has to tolerate the absence; everything downstream + reads what it emits, which always carries the counts. + """ + return check.get("graded", True) + + def summarize(outcomes: list[BehaviorOutcome], meta: dict) -> dict: per_skill: dict[str, dict] = {} for skill in sorted({o.skill for o in outcomes}): @@ -195,6 +231,8 @@ def summarize(outcomes: list[BehaviorOutcome], meta: dict) -> dict: "passed": sum(1 for o in subset if o.passed), "checks": sum(len(o.checks) for o in subset), "checks_passed": sum(1 for o in subset for c in o.checks if c["passed"]), + "checks_graded": sum(1 for o in subset for c in o.checks if graded(c)), + "checks_ungraded": sum(1 for o in subset for c in o.checks if not graded(c)), } return { "meta": meta, @@ -203,6 +241,8 @@ def summarize(outcomes: list[BehaviorOutcome], meta: dict) -> dict: "passed": sum(1 for o in outcomes if o.passed), "checks": sum(len(o.checks) for o in outcomes), "checks_passed": sum(1 for o in outcomes for c in o.checks if c["passed"]), + "checks_graded": sum(1 for o in outcomes for c in o.checks if graded(c)), + "checks_ungraded": sum(1 for o in outcomes for c in o.checks if not graded(c)), "errors": sum(1 for o in outcomes if o.error), }, "per_skill": per_skill, @@ -217,34 +257,64 @@ def render_markdown(summary: dict) -> str: "## Skill behavioral", "", f"**{totals['passed']}/{totals['cases']} cases passed** " - f"({totals['checks_passed']}/{totals['checks']} individual expectations) " + f"({totals['checks_passed']}/{totals['checks_graded']} graded expectations) " f"on `{meta['model']}` (effort `{meta['effort']}`).", "", - "| Skill | Cases | Passed | Expectations | Met |", - "| --- | --- | --- | --- | --- |", + "| Skill | Cases | Passed | Graded | Met | Ungraded |", + "| --- | --- | --- | --- | --- | --- |", ] for skill, stats in summary["per_skill"].items(): lines.append( f"| `{skill}` | {stats['cases']} | {stats['passed']} | " - f"{stats['checks']} | {stats['checks_passed']} |" + f"{stats['checks_graded']} | {stats['checks_passed']} | " + f"{stats['checks_ungraded']} |" ) + def row(case_id: str, kind: str, expectation: str, detail: str) -> str: + expectation = expectation.replace("|", "\\|") + detail = (detail or "").replace("|", "\\|").replace("\n", " ") + return f"| `{case_id}` | {kind} | {expectation[:120]} | {detail[:160]} |" + + header = ["| Case | Kind | Expectation | Detail |", "| --- | --- | --- | --- |"] + failures = [c for c in summary["cases"] if not c["passed"]] lines += ["", "### Unmet expectations", ""] - if not failures: - lines.append("None. Every behavioral case met every expectation.") + unmet = [] + for case in failures: + if case["error"]: + unmet.append(row(case["id"], "error", "(run failed)", case["error"])) + unmet += [ + row(case["id"], check["kind"], check["expectation"], check["detail"]) + for check in case["checks"] + if not check["passed"] and graded(check) + ] + if unmet: + lines += header + unmet + elif failures: + # `run_case` always records an error alongside an ungraded check, so + # this is unreachable from a real run. Guarded anyway: `render_markdown` + # is the one function that may be handed a summary built elsewhere, and + # "every expectation met" would be a false all-clear on a failed case. + lines.append("None unmet. See the ungraded expectations below.") else: - lines += ["| Case | Kind | Expectation | Detail |", "| --- | --- | --- | --- |"] - for case in failures: - if case["error"]: - detail = case["error"].replace("|", "\\|").replace("\n", " ") - lines.append(f"| `{case['id']}` | error | (run failed) | {detail[:160]} |") - for check in case["checks"]: - if check["passed"]: - continue - expectation = check["expectation"].replace("|", "\\|") - detail = (check["detail"] or "").replace("|", "\\|").replace("\n", " ") - lines.append( - f"| `{case['id']}` | {check['kind']} | {expectation[:120]} | {detail[:160]} |" - ) + lines.append("None. Every behavioral case met every expectation.") + + # Kept apart from the table above on purpose. These expectations were never + # decided, so listing them as unmet would report a grader that could not + # answer as a skill that misbehaved. + ungraded = [ + row(case["id"], check["kind"], check["expectation"], check["detail"]) + for case in summary["cases"] + for check in case["checks"] + if not graded(check) + ] + if ungraded: + lines += [ + "", + "### Ungraded expectations", + "", + "The judge could not reach a verdict on these, so they say nothing " + "about the skill either way and are excluded from the rate above.", + "", + ] + header + ungraded return "\n".join(lines) + "\n" diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 0bf0fb1..3c7928b 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -2356,6 +2356,15 @@ def fake_claude(model, *, skill, effort, seed=None): behavior.claude = original return outcome, made[0] + def test_an_ungraded_expectation_is_reported_as_an_error(self) -> None: + with mock.patch.object(agent.shutil, "which", return_value=None): + outcome, _ = self.run_case( + {"id": "a", "prompt": "run it", "expected_behavior": ["Do the thing"]} + ) + self.assertFalse(outcome.passed) + self.assertFalse(outcome.checks[0]["graded"]) + self.assertIn("could not be graded", outcome.error or "") + def test_a_passing_case(self) -> None: outcome, session = self.run_case( {"id": "a", "prompt": "run it", "logs_contain": ["detect.py"]}, @@ -2404,6 +2413,28 @@ def check(run, case, ctx): self.assertFalse(outcome.passed) self.assertTrue(any("scorer reported" in c["detail"] for c in outcome.checks)) + def test_a_hook_the_judge_could_not_answer_is_ungraded_not_failed(self) -> None: + calls: list[str] = [] + + class Hooks: + @staticmethod + def check(run, case, ctx): + run.expects_not("Claim measured results") + + @staticmethod + def teardown(workspace, case, ctx): + calls.append("teardown") + + with mock.patch.object(agent.shutil, "which", return_value=None): + outcome, _ = self.run_case( + {"id": "a", "prompt": "p", "logs_contain": []}, hooks=Hooks + ) + self.assertEqual(calls, ["teardown"]) + self.assertFalse(outcome.passed) + hook = next(c for c in outcome.checks if c["kind"] == "hook") + self.assertFalse(hook["graded"]) + self.assertIn("could not be graded", outcome.error or "") + def test_teardown_runs_even_when_the_agent_raises(self) -> None: calls: list[str] = [] @@ -2439,6 +2470,183 @@ def test_workspace_fixtures_are_staged(self) -> None: self.assertTrue(outcome.passed, outcome.checks) +class TestAJudgeThatCannotAnswer(unittest.TestCase): + """A grader that reached no verdict must not be read as having reached one. + + Every path out of `_grade_with_llm` that cannot produce an answer used to + return False, which is indistinguishable from the judge deciding against + the skill. For `unexpected_behavior` that inverts the default the function + documents: it asserts the prohibited behavior happened, on no evidence at + all, so a dropped connection was published as the skill misbehaving. + """ + + STATEMENT = "Claim measured results" + + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.workspace = Path(self.tmp.name) + self.addCleanup(self.tmp.cleanup) + + def make_run(self) -> agent.Run: + return agent.Run(workspace=self.workspace, events=stream(), judge_model=None) + + API_ERROR = "API Error: Connection dropped (ECONNRESET)" + + def no_cli(self): + # The judge's own binary is missing, so it cannot be called at all. + return mock.patch.object(agent.shutil, "which", return_value=None) + + def judge_reply(self, result: str): + # The CLI runs and returns `result` where a verdict should be. An API + # error arrives by exactly this route, which is what made it look like + # a verdict. + return self._patched( + run=mock.Mock( + return_value=mock.Mock( + stdout=json.dumps({"result": result}), stderr="", returncode=0 + ) + ) + ) + + def judge_times_out(self): + # The CLI is there and never comes back inside the cap. + return self._patched( + run=mock.Mock(side_effect=subprocess.TimeoutExpired(cmd="claude", timeout=1)) + ) + + def _patched(self, *, run): + return mock.patch.multiple( + agent, + shutil=mock.Mock(which=mock.Mock(return_value="/usr/bin/claude")), + subprocess=mock.Mock(run=run, TimeoutExpired=subprocess.TimeoutExpired), + ) + + def test_a_missing_cli_is_not_a_failed_expectation(self) -> None: + with self.no_cli(): + checks = self.make_run().evaluate(expected_behavior=[self.STATEMENT]) + self.assertFalse(checks[0].graded) + self.assertFalse(checks[0].passed) + self.assertIn("not on PATH", checks[0].detail) + + def test_an_api_error_is_not_a_failed_expectation(self) -> None: + with self.judge_reply(self.API_ERROR): + checks = self.make_run().evaluate(expected_behavior=[self.STATEMENT]) + self.assertFalse(checks[0].graded) + self.assertIn("ECONNRESET", checks[0].detail) + + def test_a_judge_that_times_out_is_not_a_failed_expectation(self) -> None: + with self.judge_times_out(): + checks = self.make_run().evaluate(expected_behavior=[self.STATEMENT]) + self.assertFalse(checks[0].graded) + self.assertIn("timed out", checks[0].detail) + + def test_a_spent_deadline_is_not_a_failed_expectation(self) -> None: + # The command ran out of time before the judge was ever called, which + # says as little about the skill as the judge refusing to answer. + previous = deadline.use(deadline.Deadline(0.0, command="behavioral")) + try: + with mock.patch.object(agent.shutil, "which", return_value="/usr/bin/claude"): + checks = self.make_run().evaluate(expected_behavior=[self.STATEMENT]) + finally: + deadline.use(previous) + self.assertFalse(checks[0].graded) + self.assertIn("--timeout", checks[0].detail) + + def test_a_prohibited_behavior_is_not_asserted_on_no_evidence(self) -> None: + # The polarity that made this worst: False here means the agent did it. + with self.judge_reply(self.API_ERROR): + checks = self.make_run().evaluate(unexpected_behavior=[self.STATEMENT]) + self.assertFalse(checks[0].graded) + + def test_an_ungraded_check_is_labelled_apart_from_a_failure(self) -> None: + buffer = io.StringIO() + with self.judge_reply(self.API_ERROR), contextlib.redirect_stdout(buffer): + self.make_run().evaluate(expected_behavior=[self.STATEMENT]) + self.assertIn("[----]", buffer.getvalue()) + self.assertNotIn("[FAIL]", buffer.getvalue()) + + def test_a_real_verdict_is_still_graded(self) -> None: + for verdict, expected in ((True, True), (False, False)): + with self.subTest(verdict=verdict): + reply = json.dumps({"pass": verdict, "reason": "because"}) + with self.judge_reply(reply): + checks = self.make_run().evaluate(expected_behavior=[self.STATEMENT]) + self.assertTrue(checks[0].graded) + self.assertIs(checks[0].passed, expected) + + def test_a_hook_whose_judge_never_answered_does_not_raise_assertion(self) -> None: + # `run_case` turns AssertionError into a check failed by the skill, so + # the ungradable signal must not arrive as one. + with self.no_cli(): + with self.assertRaises(agent.Ungradable): + self.make_run().expects_not(self.STATEMENT) + + def outcome(self, graded: bool) -> behavior.BehaviorOutcome: + check = { + "kind": "unexpected_behavior", + "expectation": self.STATEMENT, + "passed": False, + "detail": "llm_judge gave no JSON verdict", + "graded": graded, + } + return behavior.BehaviorOutcome( + id="a", + skill="s", + prompt="p", + passed=False, + elapsed_s=1.0, + checks=[check], + error=None if graded else "1 of 1 expectation(s) could not be graded", + ) + + def test_an_ungraded_check_is_excluded_from_the_rate(self) -> None: + summary = behavior.summarize( + [self.outcome(graded=False)], {"model": "opus", "effort": "high"} + ) + self.assertEqual(summary["totals"]["checks_graded"], 0) + self.assertEqual(summary["totals"]["checks_ungraded"], 1) + self.assertEqual(summary["totals"]["checks_passed"], 0) + self.assertEqual(summary["per_skill"]["s"]["checks_ungraded"], 1) + self.assertEqual(summary["per_skill"]["s"]["checks_graded"], 0) + + def test_the_rate_is_over_what_was_graded(self) -> None: + # The one claim both doc paragraphs make. Two checks, one ungraded, so + # a denominator of 2 would report the grader outage as a missed + # expectation. + outcome = self.outcome(graded=False) + outcome.checks.append( + { + "kind": "logs_contain", + "expectation": "x", + "passed": True, + "detail": "", + "graded": True, + } + ) + summary = behavior.summarize([outcome], {"model": "opus", "effort": "high"}) + report = behavior.render_markdown(summary) + self.assertIn("1/1 graded expectations", report) + + def test_a_report_without_the_graded_key_still_reads(self) -> None: + # A check dict built by anything other than asdict(Check) must not be + # read as ungraded just because it predates the key. + outcome = self.outcome(graded=True) + del outcome.checks[0]["graded"] + summary = behavior.summarize([outcome], {"model": "opus", "effort": "high"}) + self.assertEqual(summary["totals"]["checks_graded"], 1) + self.assertEqual(summary["totals"]["checks_ungraded"], 0) + + def test_the_report_keeps_ungraded_out_of_the_unmet_table(self) -> None: + summary = behavior.summarize( + [self.outcome(graded=False)], {"model": "opus", "effort": "high"} + ) + report = behavior.render_markdown(summary) + self.assertIn("### Ungraded expectations", report) + unmet, _, ungraded = report.partition("### Ungraded expectations") + self.assertNotIn(self.STATEMENT, unmet) + self.assertIn(self.STATEMENT, ungraded) + + class TestBehaviorReporting(unittest.TestCase): def test_summary_counts_cases_and_expectations(self) -> None: outcomes = [ @@ -2471,7 +2679,15 @@ def test_summary_counts_cases_and_expectations(self) -> None: summary = behavior.summarize(outcomes, {"model": "opus", "effort": "high"}) self.assertEqual( summary["totals"], - {"cases": 2, "passed": 1, "checks": 2, "checks_passed": 1, "errors": 0}, + { + "cases": 2, + "passed": 1, + "checks": 2, + "checks_passed": 1, + "checks_graded": 2, + "checks_ungraded": 0, + "errors": 0, + }, ) report = behavior.render_markdown(summary) self.assertIn("1/2 cases passed", report)