From faa44d6b3ce19a930e93d33884163e209cc7472d Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Wed, 7 Oct 2026 15:31:29 +0200 Subject: [PATCH] Report an expectation the judge could not grade as ungraded Every path out of _grade_with_llm that cannot reach a verdict returned False, the same value the judge returns when it decides against the skill. A missing CLI, a spent deadline, a judge timeout and a reply carrying no verdict were therefore all published as the skill having failed the expectation it was asked about. The polarity is what makes this worse than a mislabelled failure. For unexpected_behavior, False asserts the prohibited behaviour happened, on no evidence at all, which inverts the default the function documents: absence of evidence is supposed to count as the requirement being satisfied. One reported run recorded a skill as claiming measured results it never produced, on the strength of a dropped connection, with every other expectation in that case passing. Those four paths now return None, and a Check records whether it was graded. An ungraded check still refuses to call the case green, so the run stays red, but it is reported as an error against the run, listed apart from unmet expectations, and left out of the rate. An evals/hooks.py assertion the judge could not answer raises Ungradable rather than AssertionError so that it lands in the same place, instead of being recorded as a check the skill failed. Routing already draws this line, and usage.md already documents it for that half: a case whose own run errored counts towards neither side of the bar. Signed-off-by: John Lybeck --- docs/authoring-evals.md | 8 +- docs/usage.md | 18 +++- skillscope/agent.py | 70 ++++++++++--- skillscope/behavior.py | 110 ++++++++++++++++---- tests/test_skillscope.py | 218 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 384 insertions(+), 40 deletions(-) diff --git a/docs/authoring-evals.md b/docs/authoring-evals.md index a5c4743..20dfbf5 100644 --- a/docs/authoring-evals.md +++ b/docs/authoring-evals.md @@ -90,7 +90,11 @@ All arrays, all valid only on a `skill_should_trigger: true` evaluation: | `files_exist` | the filesystem | an artifact the run must produce | The bottom two are instant and free where a judged expectation costs a second -agent call, so reach for them when the thing you want is literal. Never assert +agent call, so reach for them when the thing you want is literal. They are also +the two that always produce an answer: a judged expectation has a third outcome, +**ungraded**, for when the judge could not be reached or did not reply with a +verdict. That is reported apart from a failure and excluded from the rate, since +a grader that never ran says nothing about your skill either way. Never assert your own skill's name in `logs_contain`: routing already grades that properly, and a substring match only proves the skill was staged. @@ -156,6 +160,8 @@ def setup_session(cache_dir): ... # once per skill; returns {name: value} fo def setup(workspace, case, ctx): ... # before each case; may return more placeholders def teardown(workspace, case, ctx): ... def check(run, case, ctx): ... # after each case; raise AssertionError to fail it + # (run.expects leaves it ungraded if the judge + # could not answer, rather than failing it) ``` `teardown` runs even when the agent itself blew up, and a `check` that raises diff --git a/docs/usage.md b/docs/usage.md index 17cd0f9..c3cd7d8 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -170,10 +170,10 @@ alone — those are illustrations of links rather than promises. ## Routing, and the bar it is held to `--min-accuracy` / `min_accuracy` defaults to `1`: every graded routing case has -to land on the right skill, the same way every behavioral expectation has to -hold. A routing miss is a defect and not a statistic — a description that fires -on its neighbour's prompt makes that neighbour worse — so it turns the run red -rather than sitting in a summary nobody reads. +to land on the right skill, the same way every behavioral expectation that was +graded has to hold. A routing miss is a defect and not a statistic — a +description that fires on its neighbour's prompt makes that neighbour worse — +so it turns the run red rather than sitting in a summary nobody reads. Loosening it is a one-line diff. `0` reports the score without gating on it, which is where a repo that has never measured its prompts should start; anything @@ -184,6 +184,16 @@ infrastructure checks — a run where nothing was graded, or where no skill activated anywhere, fails at any bar, since those numbers are an artifact rather than a result. +Behavioral is held to the same shape of bar, and carves out the same case. An +expectation its judge could not reach a verdict on — the API was unreachable, +the grader ran out of time, the reply came back without one — is reported as +**ungraded** rather than unmet, listed under its own heading, and left out of +the expectation rate. The run still fails, and the case carries an error saying +the grader could not answer. The distinction is what the report is for: an +`unexpected_behavior` expectation scored against no verdict would otherwise +assert the skill did the very thing it forbids, on the evidence of a dropped +connection. + Two failure modes are worth knowing before reading a report. A routing case that ends without the agent either activating a skill or answering is reported as an **error** rather than a missed trigger. And if the runner has its own skills diff --git a/skillscope/agent.py b/skillscope/agent.py index 53bce6d..ffc2630 100644 --- a/skillscope/agent.py +++ b/skillscope/agent.py @@ -263,9 +263,17 @@ def _find_file(files: list[str], expected: str) -> str | None: def _grade_with_llm( statement: str, run: "Run", judge_model: str | None, *, must_happen: bool -) -> tuple[bool, str]: +) -> tuple[bool | None, str]: """Ask a grader LLM whether the run satisfied a requirement. + The verdict is ``True`` (satisfied), ``False`` (not satisfied), or ``None`` + when the judge could not answer at all -- no CLI, no time left, no reply. + ``None`` is not a negative verdict and callers must not collapse it into + one: a grader that never ran is no evidence about the skill, and reporting + it as ``False`` publishes a network fault as a behavioral violation. The + ``must_happen=False`` polarity is where this bites hardest, because there + ``False`` asserts the prohibited thing *happened*. + ``must_happen`` selects the polarity: ``True`` means the agent was required to do ``statement``, ``False`` means it was required *not* to. The judge grades the requirement itself and returns ``True`` when it is satisfied, so @@ -278,7 +286,7 @@ def _grade_with_llm( """ claude_bin = shutil.which("claude") if not claude_bin: - return False, "llm_judge skipped: 'claude' CLI not on PATH" + return None, "llm_judge skipped: 'claude' CLI not on PATH" cmd_text = run.command_text if len(cmd_text) > 4000: @@ -326,7 +334,7 @@ def _grade_with_llm( if bound is not None: leftover = bound.remaining() if leftover <= 0: - return False, bound.message() + return None, bound.message() judge_timeout = bound.cap(judge_timeout) try: @@ -335,7 +343,7 @@ def _grade_with_llm( input=prompt_text, timeout=judge_timeout, env=claude_env(), ) except subprocess.TimeoutExpired: - return False, f"llm_judge timed out after {judge_timeout:g}s" + return None, f"llm_judge timed out after {judge_timeout:g}s" try: payload = json.loads((proc.stdout or "").strip()) @@ -359,21 +367,49 @@ def _grade_with_llm( if isinstance(parsed, dict) and "pass" in parsed: verdict = parsed if verdict is None: - return False, f"llm_judge gave no JSON verdict: {verdict_text[:200]!r}" + return None, f"llm_judge gave no JSON verdict: {verdict_text[:200]!r}" satisfied = bool(verdict.get("pass")) reason = str(verdict.get("reason", "")).strip() or "(no reason given)" return satisfied, f"llm_judge: {reason}" +class Ungradable(Exception): + """A check could not be decided, as distinct from deciding against it.""" + + @dataclass class Check: - """One graded expectation from a case.""" + """One expectation from a case, and what grading it concluded. + + ``graded`` is ``False`` when the check could not be decided at all -- an + LLM judge that never answered. Such a check is not a pass and not a + failure; it is an absence of evidence, and the distinction is what + separates reporting a broken grader from accusing a skill of the behavior + the expectation forbids. ``passed`` stays ``False`` there, so a reader that + only knows about pass and fail still refuses to call the case green -- but + anything attributing a result to the skill must read ``graded`` first. + """ kind: str expectation: str passed: bool detail: str = "" + graded: bool = True + + +def _judged(kind: str, expectation: str, verdict: bool | None, reason: str) -> Check: + """Turn a judge's verdict into a `Check`, keeping `None` as ungraded.""" + if verdict is None: + return Check(kind, expectation, False, reason, graded=False) + return Check(kind, expectation, verdict, reason) + + +def _verdict_label(check: Check) -> str: + """What the per-check line reports: met, unmet, or nothing concluded.""" + if not check.graded: + return "----" + return "PASS" if check.passed else "FAIL" class Run: @@ -434,22 +470,21 @@ def evaluate( for statement in expected_behavior: ok, reason = _grade_with_llm(statement, self, self.judge_model, must_happen=True) - checks.append(Check("expected_behavior", statement, ok, reason)) + checks.append(_judged("expected_behavior", statement, ok, reason)) for statement in unexpected_behavior: ok, reason = _grade_with_llm(statement, self, self.judge_model, must_happen=False) - checks.append(Check("unexpected_behavior", statement, ok, reason)) + checks.append(_judged("unexpected_behavior", statement, ok, reason)) for check in checks: suffix = f" -- {check.detail}" if check.detail else "" - _safe_print( - f" [{'PASS' if check.passed else 'FAIL'}] " - f"({check.kind}) {check.expectation}{suffix}" - ) + _safe_print(f" [{_verdict_label(check)}] ({check.kind}) {check.expectation}{suffix}") return checks # Asserting variants, for an `evals/hooks.py` that needs a check the - # dataset format cannot express. Each raises AssertionError on failure. + # dataset format cannot express. A check that failed raises + # AssertionError; one the judge could not decide raises `Ungradable`, + # which `run_case` records as an ungraded check rather than a failed one. def logs_contains(self, text: str) -> "Run": ok = text.lower() in self.logs.lower() @@ -474,7 +509,14 @@ def expects_not(self, statement: str) -> "Run": self._report(satisfied, "unexpected_behavior", f"{statement} -- {reason}") return self - def _report(self, passed: bool, kind: str, detail: str) -> None: + def _report(self, passed: bool | None, kind: str, detail: str) -> None: + if passed is None: + # Deliberately not an AssertionError. `run_case` catches that and + # records a failed check against the skill, which is the exact + # attribution this state exists to prevent; anything else reaches + # the outer handler and is reported as the run erroring. + _safe_print(f" [----] ({kind}) {detail}") + raise Ungradable(f"({kind}) {detail}") _safe_print(f" [{'PASS' if passed else 'FAIL'}] ({kind}) {detail}") assert passed, f"({kind}) {detail}" diff --git a/skillscope/behavior.py b/skillscope/behavior.py index 154f48c..3806b7c 100644 --- a/skillscope/behavior.py +++ b/skillscope/behavior.py @@ -25,7 +25,7 @@ from types import ModuleType from . import datasets, deadline -from .agent import Check, claude +from .agent import Check, Ungradable, claude from .datasets import Case @@ -61,6 +61,8 @@ def load_hooks(skill: str) -> ModuleType | None: setup(workspace, case, ctx) -> dict of extra template vars, per case teardown(workspace, case, ctx) check(run, case, ctx) -> raise AssertionError to fail the case + (`run.expects` raises `Ungradable` + when the judge could not answer) """ path = datasets.hooks_path(skill) if not path.is_file(): @@ -133,6 +135,19 @@ def run_case( checks.append(Check("hook", "evals/hooks.py check()", True)) except AssertionError as exc: checks.append(Check("hook", "evals/hooks.py check()", False, str(exc)[:400])) + except Ungradable as exc: + # `run.expects` could not get a verdict. Same standing as + # an ungraded dataset expectation: not a pass, but not + # evidence against the skill either. + checks.append( + Check( + "hook", + "evals/hooks.py check()", + False, + str(exc)[:400], + graded=False, + ) + ) finally: if hooks is not None and hasattr(hooks, "teardown"): hooks.teardown(workspace, case, case_ctx) @@ -142,8 +157,18 @@ def run_case( elapsed = round(time.perf_counter() - started, 2) passed = error is None and all(c.passed for c in checks) and bool(checks) + ungraded = [c for c in checks if not c.graded] if error is None and not checks: error = "case has no behavioral assertions to grade" + elif error is None and ungraded: + # Still not a pass -- nothing confirmed the skill behaved -- but the + # reason is that the grader could not answer, so it belongs in the + # errors column where it gets triaged as infrastructure rather than + # read as the skill having violated the expectation. + error = ( + f"{len(ungraded)} of {len(checks)} expectation(s) could not be " + f"graded: {ungraded[0].detail}" + ) print( f" [{'PASS' if passed else 'FAIL'}] {case.id}: " f"{sum(1 for c in checks if c.passed)}/{len(checks)} checks in {elapsed}s" @@ -186,6 +211,17 @@ def run(skills: list[str], cases: list[Case], model: str, effort: str) -> list[B return outcomes +def graded(check: dict) -> bool: + """Whether a check reached a verdict at all. + + A missing key means it did, so a check dict assembled by something + other than `asdict(Check)` still reads correctly. `summarize` is the + only place that has to tolerate the absence; everything downstream + reads what it emits, which always carries the counts. + """ + return check.get("graded", True) + + def summarize(outcomes: list[BehaviorOutcome], meta: dict) -> dict: per_skill: dict[str, dict] = {} for skill in sorted({o.skill for o in outcomes}): @@ -195,6 +231,8 @@ def summarize(outcomes: list[BehaviorOutcome], meta: dict) -> dict: "passed": sum(1 for o in subset if o.passed), "checks": sum(len(o.checks) for o in subset), "checks_passed": sum(1 for o in subset for c in o.checks if c["passed"]), + "checks_graded": sum(1 for o in subset for c in o.checks if graded(c)), + "checks_ungraded": sum(1 for o in subset for c in o.checks if not graded(c)), } return { "meta": meta, @@ -203,6 +241,8 @@ def summarize(outcomes: list[BehaviorOutcome], meta: dict) -> dict: "passed": sum(1 for o in outcomes if o.passed), "checks": sum(len(o.checks) for o in outcomes), "checks_passed": sum(1 for o in outcomes for c in o.checks if c["passed"]), + "checks_graded": sum(1 for o in outcomes for c in o.checks if graded(c)), + "checks_ungraded": sum(1 for o in outcomes for c in o.checks if not graded(c)), "errors": sum(1 for o in outcomes if o.error), }, "per_skill": per_skill, @@ -217,34 +257,64 @@ def render_markdown(summary: dict) -> str: "## Skill behavioral", "", f"**{totals['passed']}/{totals['cases']} cases passed** " - f"({totals['checks_passed']}/{totals['checks']} individual expectations) " + f"({totals['checks_passed']}/{totals['checks_graded']} graded expectations) " f"on `{meta['model']}` (effort `{meta['effort']}`).", "", - "| Skill | Cases | Passed | Expectations | Met |", - "| --- | --- | --- | --- | --- |", + "| Skill | Cases | Passed | Graded | Met | Ungraded |", + "| --- | --- | --- | --- | --- | --- |", ] for skill, stats in summary["per_skill"].items(): lines.append( f"| `{skill}` | {stats['cases']} | {stats['passed']} | " - f"{stats['checks']} | {stats['checks_passed']} |" + f"{stats['checks_graded']} | {stats['checks_passed']} | " + f"{stats['checks_ungraded']} |" ) + def row(case_id: str, kind: str, expectation: str, detail: str) -> str: + expectation = expectation.replace("|", "\\|") + detail = (detail or "").replace("|", "\\|").replace("\n", " ") + return f"| `{case_id}` | {kind} | {expectation[:120]} | {detail[:160]} |" + + header = ["| Case | Kind | Expectation | Detail |", "| --- | --- | --- | --- |"] + failures = [c for c in summary["cases"] if not c["passed"]] lines += ["", "### Unmet expectations", ""] - if not failures: - lines.append("None. Every behavioral case met every expectation.") + unmet = [] + for case in failures: + if case["error"]: + unmet.append(row(case["id"], "error", "(run failed)", case["error"])) + unmet += [ + row(case["id"], check["kind"], check["expectation"], check["detail"]) + for check in case["checks"] + if not check["passed"] and graded(check) + ] + if unmet: + lines += header + unmet + elif failures: + # `run_case` always records an error alongside an ungraded check, so + # this is unreachable from a real run. Guarded anyway: `render_markdown` + # is the one function that may be handed a summary built elsewhere, and + # "every expectation met" would be a false all-clear on a failed case. + lines.append("None unmet. See the ungraded expectations below.") else: - lines += ["| Case | Kind | Expectation | Detail |", "| --- | --- | --- | --- |"] - for case in failures: - if case["error"]: - detail = case["error"].replace("|", "\\|").replace("\n", " ") - lines.append(f"| `{case['id']}` | error | (run failed) | {detail[:160]} |") - for check in case["checks"]: - if check["passed"]: - continue - expectation = check["expectation"].replace("|", "\\|") - detail = (check["detail"] or "").replace("|", "\\|").replace("\n", " ") - lines.append( - f"| `{case['id']}` | {check['kind']} | {expectation[:120]} | {detail[:160]} |" - ) + lines.append("None. Every behavioral case met every expectation.") + + # Kept apart from the table above on purpose. These expectations were never + # decided, so listing them as unmet would report a grader that could not + # answer as a skill that misbehaved. + ungraded = [ + row(case["id"], check["kind"], check["expectation"], check["detail"]) + for case in summary["cases"] + for check in case["checks"] + if not graded(check) + ] + if ungraded: + lines += [ + "", + "### Ungraded expectations", + "", + "The judge could not reach a verdict on these, so they say nothing " + "about the skill either way and are excluded from the rate above.", + "", + ] + header + ungraded return "\n".join(lines) + "\n" diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 0bf0fb1..3c7928b 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -2356,6 +2356,15 @@ def fake_claude(model, *, skill, effort, seed=None): behavior.claude = original return outcome, made[0] + def test_an_ungraded_expectation_is_reported_as_an_error(self) -> None: + with mock.patch.object(agent.shutil, "which", return_value=None): + outcome, _ = self.run_case( + {"id": "a", "prompt": "run it", "expected_behavior": ["Do the thing"]} + ) + self.assertFalse(outcome.passed) + self.assertFalse(outcome.checks[0]["graded"]) + self.assertIn("could not be graded", outcome.error or "") + def test_a_passing_case(self) -> None: outcome, session = self.run_case( {"id": "a", "prompt": "run it", "logs_contain": ["detect.py"]}, @@ -2404,6 +2413,28 @@ def check(run, case, ctx): self.assertFalse(outcome.passed) self.assertTrue(any("scorer reported" in c["detail"] for c in outcome.checks)) + def test_a_hook_the_judge_could_not_answer_is_ungraded_not_failed(self) -> None: + calls: list[str] = [] + + class Hooks: + @staticmethod + def check(run, case, ctx): + run.expects_not("Claim measured results") + + @staticmethod + def teardown(workspace, case, ctx): + calls.append("teardown") + + with mock.patch.object(agent.shutil, "which", return_value=None): + outcome, _ = self.run_case( + {"id": "a", "prompt": "p", "logs_contain": []}, hooks=Hooks + ) + self.assertEqual(calls, ["teardown"]) + self.assertFalse(outcome.passed) + hook = next(c for c in outcome.checks if c["kind"] == "hook") + self.assertFalse(hook["graded"]) + self.assertIn("could not be graded", outcome.error or "") + def test_teardown_runs_even_when_the_agent_raises(self) -> None: calls: list[str] = [] @@ -2439,6 +2470,183 @@ def test_workspace_fixtures_are_staged(self) -> None: self.assertTrue(outcome.passed, outcome.checks) +class TestAJudgeThatCannotAnswer(unittest.TestCase): + """A grader that reached no verdict must not be read as having reached one. + + Every path out of `_grade_with_llm` that cannot produce an answer used to + return False, which is indistinguishable from the judge deciding against + the skill. For `unexpected_behavior` that inverts the default the function + documents: it asserts the prohibited behavior happened, on no evidence at + all, so a dropped connection was published as the skill misbehaving. + """ + + STATEMENT = "Claim measured results" + + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.workspace = Path(self.tmp.name) + self.addCleanup(self.tmp.cleanup) + + def make_run(self) -> agent.Run: + return agent.Run(workspace=self.workspace, events=stream(), judge_model=None) + + API_ERROR = "API Error: Connection dropped (ECONNRESET)" + + def no_cli(self): + # The judge's own binary is missing, so it cannot be called at all. + return mock.patch.object(agent.shutil, "which", return_value=None) + + def judge_reply(self, result: str): + # The CLI runs and returns `result` where a verdict should be. An API + # error arrives by exactly this route, which is what made it look like + # a verdict. + return self._patched( + run=mock.Mock( + return_value=mock.Mock( + stdout=json.dumps({"result": result}), stderr="", returncode=0 + ) + ) + ) + + def judge_times_out(self): + # The CLI is there and never comes back inside the cap. + return self._patched( + run=mock.Mock(side_effect=subprocess.TimeoutExpired(cmd="claude", timeout=1)) + ) + + def _patched(self, *, run): + return mock.patch.multiple( + agent, + shutil=mock.Mock(which=mock.Mock(return_value="/usr/bin/claude")), + subprocess=mock.Mock(run=run, TimeoutExpired=subprocess.TimeoutExpired), + ) + + def test_a_missing_cli_is_not_a_failed_expectation(self) -> None: + with self.no_cli(): + checks = self.make_run().evaluate(expected_behavior=[self.STATEMENT]) + self.assertFalse(checks[0].graded) + self.assertFalse(checks[0].passed) + self.assertIn("not on PATH", checks[0].detail) + + def test_an_api_error_is_not_a_failed_expectation(self) -> None: + with self.judge_reply(self.API_ERROR): + checks = self.make_run().evaluate(expected_behavior=[self.STATEMENT]) + self.assertFalse(checks[0].graded) + self.assertIn("ECONNRESET", checks[0].detail) + + def test_a_judge_that_times_out_is_not_a_failed_expectation(self) -> None: + with self.judge_times_out(): + checks = self.make_run().evaluate(expected_behavior=[self.STATEMENT]) + self.assertFalse(checks[0].graded) + self.assertIn("timed out", checks[0].detail) + + def test_a_spent_deadline_is_not_a_failed_expectation(self) -> None: + # The command ran out of time before the judge was ever called, which + # says as little about the skill as the judge refusing to answer. + previous = deadline.use(deadline.Deadline(0.0, command="behavioral")) + try: + with mock.patch.object(agent.shutil, "which", return_value="/usr/bin/claude"): + checks = self.make_run().evaluate(expected_behavior=[self.STATEMENT]) + finally: + deadline.use(previous) + self.assertFalse(checks[0].graded) + self.assertIn("--timeout", checks[0].detail) + + def test_a_prohibited_behavior_is_not_asserted_on_no_evidence(self) -> None: + # The polarity that made this worst: False here means the agent did it. + with self.judge_reply(self.API_ERROR): + checks = self.make_run().evaluate(unexpected_behavior=[self.STATEMENT]) + self.assertFalse(checks[0].graded) + + def test_an_ungraded_check_is_labelled_apart_from_a_failure(self) -> None: + buffer = io.StringIO() + with self.judge_reply(self.API_ERROR), contextlib.redirect_stdout(buffer): + self.make_run().evaluate(expected_behavior=[self.STATEMENT]) + self.assertIn("[----]", buffer.getvalue()) + self.assertNotIn("[FAIL]", buffer.getvalue()) + + def test_a_real_verdict_is_still_graded(self) -> None: + for verdict, expected in ((True, True), (False, False)): + with self.subTest(verdict=verdict): + reply = json.dumps({"pass": verdict, "reason": "because"}) + with self.judge_reply(reply): + checks = self.make_run().evaluate(expected_behavior=[self.STATEMENT]) + self.assertTrue(checks[0].graded) + self.assertIs(checks[0].passed, expected) + + def test_a_hook_whose_judge_never_answered_does_not_raise_assertion(self) -> None: + # `run_case` turns AssertionError into a check failed by the skill, so + # the ungradable signal must not arrive as one. + with self.no_cli(): + with self.assertRaises(agent.Ungradable): + self.make_run().expects_not(self.STATEMENT) + + def outcome(self, graded: bool) -> behavior.BehaviorOutcome: + check = { + "kind": "unexpected_behavior", + "expectation": self.STATEMENT, + "passed": False, + "detail": "llm_judge gave no JSON verdict", + "graded": graded, + } + return behavior.BehaviorOutcome( + id="a", + skill="s", + prompt="p", + passed=False, + elapsed_s=1.0, + checks=[check], + error=None if graded else "1 of 1 expectation(s) could not be graded", + ) + + def test_an_ungraded_check_is_excluded_from_the_rate(self) -> None: + summary = behavior.summarize( + [self.outcome(graded=False)], {"model": "opus", "effort": "high"} + ) + self.assertEqual(summary["totals"]["checks_graded"], 0) + self.assertEqual(summary["totals"]["checks_ungraded"], 1) + self.assertEqual(summary["totals"]["checks_passed"], 0) + self.assertEqual(summary["per_skill"]["s"]["checks_ungraded"], 1) + self.assertEqual(summary["per_skill"]["s"]["checks_graded"], 0) + + def test_the_rate_is_over_what_was_graded(self) -> None: + # The one claim both doc paragraphs make. Two checks, one ungraded, so + # a denominator of 2 would report the grader outage as a missed + # expectation. + outcome = self.outcome(graded=False) + outcome.checks.append( + { + "kind": "logs_contain", + "expectation": "x", + "passed": True, + "detail": "", + "graded": True, + } + ) + summary = behavior.summarize([outcome], {"model": "opus", "effort": "high"}) + report = behavior.render_markdown(summary) + self.assertIn("1/1 graded expectations", report) + + def test_a_report_without_the_graded_key_still_reads(self) -> None: + # A check dict built by anything other than asdict(Check) must not be + # read as ungraded just because it predates the key. + outcome = self.outcome(graded=True) + del outcome.checks[0]["graded"] + summary = behavior.summarize([outcome], {"model": "opus", "effort": "high"}) + self.assertEqual(summary["totals"]["checks_graded"], 1) + self.assertEqual(summary["totals"]["checks_ungraded"], 0) + + def test_the_report_keeps_ungraded_out_of_the_unmet_table(self) -> None: + summary = behavior.summarize( + [self.outcome(graded=False)], {"model": "opus", "effort": "high"} + ) + report = behavior.render_markdown(summary) + self.assertIn("### Ungraded expectations", report) + unmet, _, ungraded = report.partition("### Ungraded expectations") + self.assertNotIn(self.STATEMENT, unmet) + self.assertIn(self.STATEMENT, ungraded) + + class TestBehaviorReporting(unittest.TestCase): def test_summary_counts_cases_and_expectations(self) -> None: outcomes = [ @@ -2471,7 +2679,15 @@ def test_summary_counts_cases_and_expectations(self) -> None: summary = behavior.summarize(outcomes, {"model": "opus", "effort": "high"}) self.assertEqual( summary["totals"], - {"cases": 2, "passed": 1, "checks": 2, "checks_passed": 1, "errors": 0}, + { + "cases": 2, + "passed": 1, + "checks": 2, + "checks_passed": 1, + "checks_graded": 2, + "checks_ungraded": 0, + "errors": 0, + }, ) report = behavior.render_markdown(summary) self.assertIn("1/2 cases passed", report)