From abfa0039ef5adb168fd6500aa3688c369650def5 Mon Sep 17 00:00:00 2001 From: Andy Wang Date: Sat, 7 Nov 2026 11:38:30 +0800 Subject: [PATCH] Fail evals run on a missed per-group pass gate evals run --min-pass-rate gated a dashboard run on its whole pass rate alone. A suite can hold one group to its own bar (pass: { min_pass_rate: 1.0 }) and the dashboard now works each group's gate out for the run, but the CLI never read them: a run whose critical group slipped to 50% passed CI whenever the other groups lifted the whole-run rate over the threshold. gate() now reads the finished run's gates and gates_met. A run the dashboard says missed a gate fails with EXIT_GATE, with --wait alone or with --min-pass-rate, which stays the whole-run bar, so the run must clear both. Each missed gate prints as one line in Gate#to_s's words, its text passed through the table cell filter, and --json adds "gates" beside "pass_rate" and "verdict". A run without gates is gated and printed exactly as before, and without --wait the command still returns as soon as the run starts. Fixes #677 --- CHANGELOG.md | 8 ++ actionagent/test/cli_contract_test.rb | 34 ++++++ docs/framework/cli.md | 20 ++++ docs/framework/evaluations.md | 4 +- lib/active_agent/cli/eval_commands.rb | 51 ++++++++- test/cli/cli_test.rb | 154 +++++++++++++++++++++++++- 6 files changed, 261 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74ec8eddc..c6e7e0735 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`evals run` fails CI on a missed group gate** (`activeagent`). A run + whose critical group slipped to 50% passed `evals run --min-pass-rate` + whenever the other groups lifted the whole-run pass rate over the bar. + A finished run that the dashboard holds to per-group pass gates now fails + (exit 4) when any gated group missed its bar, with `--wait` alone or with + `--min-pass-rate`, which stays the whole-run bar; each missed gate prints + as one line, and `--json` adds `gates`. A run without gates is gated as + before (#677). - **The dashboard keeps a catalog's pass bars** (`actionagent`). A product's and a set's `pass` blocks (`min_pass_rate`, `threshold`, `gated`) parsed in `ActiveAgent::Evals::Catalog`, but neither diff --git a/actionagent/test/cli_contract_test.rb b/actionagent/test/cli_contract_test.rb index 9b22b0bac..3a12871af 100644 --- a/actionagent/test/cli_contract_test.rb +++ b/actionagent/test/cli_contract_test.rb @@ -10,6 +10,8 @@ # write what the MCP facade and the report collector actually serve, so a # change to either that breaks the CLI fails here. class CliContractTest < ActiveSupport::TestCase + include ActiveJob::TestHelper + URL = "http://localhost/activeagents" INGEST_KEY = "install-key-for-cli" @@ -139,6 +141,38 @@ def report(run_id) assert started["background"] end + test "evals run --wait fails on a catalog set's missed pass gate the dashboard worked out" do + catalog = ActionAgent::ScenarioCatalogImport.new(owner: nil, document: <<~YAML).call + catalog: shop + products: + - key: support + agent: support + pass: { min_pass_rate: 0.5 } + sets: + - key: critical + criteria: + - { key: response_present, type: response_present } + pass: { min_pass_rate: 1.0 } + scenarios: + - key: status_1 + prompt: Where is order 1? + - key: lookup_1 + prompt: Look order 1 up. + expect: { tools: [ find_order ] } + YAML + evaluation = catalog.sets.sole.materialize!(agent: @agent) + + code, out, err = perform_enqueued_jobs { cli("evals", "run", evaluation.name, "--wait", "--interval", "1") } + + assert_equal ActiveAgent::CLI::EXIT_GATE, code, err + assert_includes out, "complete: 1/2 (50.0%).", "the whole run clears the product's 50%" + assert_match(/Gates not met:\n critical · mock-model: 1\/2 \(50%\) passed, below 100%\n/, out) + assert_match(/FAIL$/, out) + + _, out, = cli("evals", "runs", evaluation.id.to_s, "--json") + assert_equal false, JSON.parse(out)["runs"].sole["gates_met"], "the run list carries the same verdict" + end + test "agents list and traces list read the key's agents and telemetry" do assert_equal [ "support" ], json("agents", "list")["agents"].map { |agent| agent["slug"] } assert_equal 60, json("traces", "list", "--since", "1h")["window_minutes"] diff --git a/docs/framework/cli.md b/docs/framework/cli.md index f5edb16d2..248eabe31 100644 --- a/docs/framework/cli.md +++ b/docs/framework/cli.md @@ -114,6 +114,26 @@ the run failed or passed less than the threshold. The rate is `0.8` or ACTIVEAGENT_API_KEY: ${{ secrets.ACTIVEAGENT_API_KEY }} ``` +A run held to per-group pass gates (the `pass` bars of the catalog set it +replays, see [Gating a suite](/framework/evaluations#gating-a-suite)) is +gated on them too: `--wait` alone exits 4 when the dashboard says any gated +group missed its bar, and `--min-pass-rate` stays the whole run's bar, so +the run must clear it and every gate. Each gate missed prints as one line: + +``` +Run 41 complete: 9/10 (90.0%), threshold 80.0%. +Gates not met: + critical · gpt-5-mini: 1/2 (50%) passed, below 100% +FAIL +``` + +Here the whole run clears 80% and still fails, because the critical group +is held to 100%. A group with `gated: false` is reported and never fails +the run. `--json` adds the run's `gates` beside `pass_rate` and `verdict`. +A run without gates is gated on its status and pass rate alone, as before, +and without `--wait` or `--min-pass-rate` the command returns once the run +starts, before any gate exists. + A scenario suite's replays run the agent on the dashboard, so the run needs execution enabled there and counts against its execution quota. diff --git a/docs/framework/evaluations.md b/docs/framework/evaluations.md index 58ef83be7..49628854d 100644 --- a/docs/framework/evaluations.md +++ b/docs/framework/evaluations.md @@ -884,8 +884,8 @@ group's pass count by its gate, naming the bar in its tooltip; and `to_h` gains `"pass_thresholds"`, `"groups"` (`summary_by_group`, each model's counts within each group) and `"gates"`. A report built without them has no gates table, no tone and none of the three keys. The command line's -`evals run --min-pass-rate` stays a gate on the whole run's pass rate -(see [Command Line](/framework/cli#gating-ci-on-a-pass-rate)); the +`evals run --wait` fails on a missed group gate as well as on the whole +run's pass rate (see [Command Line](/framework/cli#gating-ci-on-a-pass-rate)); the per-group gates travel with the report, in `to_h["gates"]`, for whatever reads the JSON. The mounted dashboard keeps a published report's `pass_thresholds` with the run and rebuilds the gates from its stored diff --git a/lib/active_agent/cli/eval_commands.rb b/lib/active_agent/cli/eval_commands.rb index 8777c96fa..2cde4d342 100644 --- a/lib/active_agent/cli/eval_commands.rb +++ b/lib/active_agent/cli/eval_commands.rb @@ -86,8 +86,8 @@ def evals_run(argv) p.on("--keys LIST", "Replay only these scenario keys, comma-separated") { |v| arguments[:keys] = list_option(v) } p.on("--group GROUP", "Replay only this scenario group") { |v| arguments[:group] = v } p.on("--sandbox ID", "Replay against this ready checkout sandbox") { |v| arguments[:sandbox_id] = v } - p.on("--wait", "Wait for the run to finish") { wait = true } - p.on("--min-pass-rate RATE", "Fail (exit #{EXIT_GATE}) below this pass rate: 0.8 or 80%; implies --wait") do |v| + p.on("--wait", "Wait for the run to finish; fail (exit #{EXIT_GATE}) when it failed or missed a pass gate") { wait = true } + p.on("--min-pass-rate RATE", "Also fail below this whole-run pass rate: 0.8 or 80%; implies --wait") do |v| min_pass_rate = pass_rate_option(v) wait = true end @@ -199,18 +199,28 @@ def wait_for_run(evaluation_id, run_id, timeout:, interval:) end end + # A finished run fails the gate when it failed, when its whole pass + # rate is under --min-pass-rate, or when it was held to per-group pass + # gates (the run's "gates", from the suite's or catalog set's `pass` + # bars) and the dashboard says it did not meet every one. A run + # without gates is gated on its status and rate alone, as it was. def gate(payload, min_pass_rate) run = payload.fetch("run", {}) rate = pass_rate(run) failed = run["status"] == "failed" below = min_pass_rate && (rate.nil? || rate < min_pass_rate) - verdict = failed || below ? "fail" : "pass" + gates = run_gates(run) + missed = !gates.nil? && run["gates_met"] != true + verdict = failed || below || missed ? "fail" : "pass" failures = payload.fetch("results", []) + summary = { "pass_rate" => rate, "min_pass_rate" => min_pass_rate } + summary["gates"] = gates if gates - emit(payload.merge("pass_rate" => rate, "min_pass_rate" => min_pass_rate, "verdict" => verdict)) do + emit(payload.merge(summary).merge("verdict" => verdict)) do @out.puts "Run #{run["id"]} #{run["status"]}: #{passed(run) || "no results"}" \ "#{rate ? format(" (%.1f%%)", rate * 100) : ""}#{min_pass_rate ? format(", threshold %.1f%%", min_pass_rate * 100) : ""}." @out.puts "Error: #{run["error_message"]}" if run["error_message"] + print_gates(gates, missed) if gates unless failures.empty? @out.puts "Failed results:" table(failures, { @@ -227,6 +237,39 @@ def gate(payload, min_pass_rate) verdict == "pass" ? EXIT_SUCCESS : EXIT_GATE end + # The run's per-group pass gates as the dashboard worked them out + # (EvaluationRun#gates), or nil for a run held to none. + def run_gates(run) + rows = run["gates"] + rows.select { |row| row.is_a?(Hash) } if rows.is_a?(Array) + end + + def print_gates(gates, missed) + unmet = gates.reject { |gate| gate["met"] == true } + unless missed + @out.puts "Gates: every gate met (#{gates.size})." + return + end + + @out.puts(unmet.empty? ? "Gates: no gate was reached." : "Gates not met:") + unmet.each { |gate| @out.puts " #{gate_sentence(gate)}" } + end + + # A gate the run missed, as ActiveAgent::Evals::Report::Gate#to_s + # reads it: "Orders · gpt-5-mini: 1/2 (50%) passed, below 100%". + def gate_sentence(gate) + passed = gate["passed"].to_i + total = gate["total"].to_i + passes = total.positive? ? "#{passed}/#{total} (#{whole_percent(passed.to_f / total)})" : "no results" + label = cell(gate["label"]) unless gate["label"].to_s.empty? + "#{cell(gate["name"] || gate["group"] || "All")}#{label ? " · #{label}" : ""}: #{passes} passed, below #{whole_percent(gate["min_pass_rate"])}" + end + + def whole_percent(fraction) + value = Float(fraction, exception: false) + value ? "#{(value * 100).round}%" : "?" + end + def pass_rate(run) evaluated = run["samples_evaluated"].to_i evaluated.positive? ? run["samples_passed"].to_f / evaluated : nil diff --git a/test/cli/cli_test.rb b/test/cli/cli_test.rb index 8925c6a04..54440b0c5 100644 --- a/test/cli/cli_test.rb +++ b/test/cli/cli_test.rb @@ -63,10 +63,28 @@ def stub_tool(name, *responses, arguments: nil, error: nil) stub end - def run_payload(status, passed: 0, evaluated: 0, results: []) - { evaluation: { id: 7, name: "Orders" }, - run: { id: 41, run_id: "dashboard-run-41-20261022100000", status: status, samples_evaluated: evaluated, samples_passed: passed }, - results: results } + def run_payload(status, passed: 0, evaluated: 0, results: [], gates: nil, gates_met: nil) + run = { id: 41, run_id: "dashboard-run-41-20261022100000", status: status, samples_evaluated: evaluated, samples_passed: passed } + run = run.merge(gates: gates, gates_met: gates_met) unless gates.nil? && gates_met.nil? + { evaluation: { id: 7, name: "Orders" }, run: run, results: results } + end + + # A run's gate as the dashboard serializes it (Report::Gate#to_h). + def gate_row(group, name, passed, total, min_pass_rate, gated: true, label: "mock/alpha") + rate = passed.to_f / total + { group: group, name: name, label: label, passed: passed, total: total, pass_rate: rate, + min_pass_rate: min_pass_rate, gated: gated, met: !gated || rate >= min_pass_rate } + end + + # Nine of ten passed; the critical group passed one of its two against + # 100%, the informational group none of its two but it is not gated. + def gated_run(critical: [ 1, 2 ], whole: [ 7, 8 ]) + gates = [ + gate_row("critical", "Critical", *critical, 1.0), + gate_row("informational", "Informational", 0, 2, 0.8, gated: false), + gate_row(nil, "All", *whole, 0.8) + ] + run_payload("complete", passed: 9, evaluated: 10, gates: gates, gates_met: gates.all? { |gate| gate[:met] }) end def envelope(run_id: "run-1") @@ -272,6 +290,134 @@ def test_evals_run_fails_the_gate_when_the_run_failed assert_match(/FAIL$/, out) end + def test_evals_run_wait_fails_an_unmet_group_gate_and_names_it + stub_tool("evaluations_run", run_payload("pending")) + stub_tool("evaluation_runs_get", gated_run) + + code, out, = cli("evals", "run", "7", "--wait") + + assert_equal CLI::EXIT_GATE, code + assert_includes out, "Run 41 complete: 9/10 (90.0%)." + assert_includes out, "Gates not met:\n Critical · mock/alpha: 1/2 (50%) passed, below 100%\n" + refute_includes out, "Informational", "an ungated group never fails the run" + assert_match(/FAIL$/, out) + end + + def test_evals_run_min_pass_rate_needs_the_rate_and_every_gate + stub_tool("evaluations_run", run_payload("pending")) + stub_tool("evaluation_runs_get", gated_run) + + code, out, = cli("evals", "run", "7", "--min-pass-rate", "0.8") + + assert_equal CLI::EXIT_GATE, code, "the rate clears 80% but the critical gate does not" + assert_includes out, "(90.0%), threshold 80.0%." + assert_includes out, "Critical · mock/alpha: 1/2 (50%) passed, below 100%" + end + + def test_evals_run_min_pass_rate_fails_below_the_rate_with_every_gate_met + stub_tool("evaluations_run", run_payload("pending")) + stub_tool("evaluation_runs_get", gated_run(critical: [ 2, 2 ])) + + code, out, = cli("evals", "run", "7", "--min-pass-rate", "0.95") + + assert_equal CLI::EXIT_GATE, code + assert_includes out, "(90.0%), threshold 95.0%." + assert_includes out, "Gates: every gate met (3)." + assert_match(/FAIL$/, out) + end + + def test_evals_run_passes_when_every_gated_group_is_met_whatever_an_ungated_group_scored + stub_tool("evaluations_run", run_payload("pending")) + stub_tool("evaluation_runs_get", gated_run(critical: [ 2, 2 ])) + + code, out, = cli("evals", "run", "7", "--wait") + + assert_equal 0, code, out + assert_includes out, "Gates: every gate met (3)." + assert_match(/PASS$/, out) + end + + def test_evals_run_trusts_the_dashboards_verdict_on_its_gates + stub_tool("evaluations_run", run_payload("pending")) + stub_tool("evaluation_runs_get", run_payload("complete", passed: 2, evaluated: 2, gates: [], gates_met: false)) + + code, out, = cli("evals", "run", "7", "--wait") + + assert_equal CLI::EXIT_GATE, code, "a run that reached no gate has not passed" + assert_includes out, "Gates: no gate was reached." + end + + def test_evals_run_json_adds_the_gates_and_keeps_the_payload + stub_tool("evaluations_run", run_payload("pending")) + stub_tool("evaluation_runs_get", gated_run) + + code, out, = cli("evals", "run", "7", "--wait", "--json") + + assert_equal CLI::EXIT_GATE, code + result = JSON.parse(out) + assert_equal "fail", result["verdict"] + assert_in_delta 0.9, result["pass_rate"] + assert_equal %w[Critical Informational All], result["gates"].map { |gate| gate["name"] } + assert_equal false, result.dig("run", "gates_met") + assert_equal "Orders", result.dig("evaluation", "name") + end + + def test_evals_run_without_gates_reads_as_before + stub_tool("evaluations_run", run_payload("pending")) + stub_tool("evaluation_runs_get", run_payload("complete", passed: 9, evaluated: 10)) + + code, out, = cli("evals", "run", "7", "--wait") + assert_equal 0, code + refute_includes out, "Gates" + assert_match(/PASS$/, out) + + code, out, = cli("evals", "run", "7", "--wait", "--json") + assert_equal 0, code + refute JSON.parse(out).key?("gates"), "a run without gates adds no key" + end + + def test_evals_run_failed_run_with_gates_still_fails + stub_tool("evaluations_run", run_payload("pending")) + stub_tool("evaluation_runs_get", run_payload("failed", gates: [ gate_row("critical", "Critical", 2, 2, 1.0) ], gates_met: true)) + + code, out, = cli("evals", "run", "7", "--wait") + + assert_equal CLI::EXIT_GATE, code + assert_match(/FAIL$/, out) + end + + def test_evals_run_gate_text_strips_control_characters_from_the_server + hostile = gate_row("critical", "Critical\e[2J\nNext", 1, 2, 1.0, label: "mock/\e]0;title\a") + stub_tool("evaluations_run", run_payload("pending")) + stub_tool("evaluation_runs_get", run_payload("complete", passed: 1, evaluated: 2, gates: [ hostile ], gates_met: false)) + + _, out, = cli("evals", "run", "7", "--wait") + + refute_includes out, "\e" + refute_includes out, "\a" + assert_includes out, "Critical [2J Next · mock/ ]0;title : 1/2 (50%) passed, below 100%" + end + + def test_evals_run_skips_a_gate_row_that_is_not_an_object + rows = [ "critical", 3, nil, gate_row("critical", "Critical", 1, 2, 1.0) ] + stub_tool("evaluations_run", run_payload("pending")) + stub_tool("evaluation_runs_get", run_payload("complete", passed: 1, evaluated: 2, gates: rows, gates_met: false)) + + code, out, = cli("evals", "run", "7", "--wait", "--json") + + assert_equal CLI::EXIT_GATE, code + assert_equal [ "Critical" ], JSON.parse(out)["gates"].map { |gate| gate["name"] } + end + + def test_evals_run_without_wait_ignores_gates + stub_tool("evaluations_run", gated_run) + + code, out, = cli("evals", "run", "7") + + assert_equal 0, code + refute_includes out, "Gates" + end + def test_evals_run_gives_up_after_the_timeout stub_tool("evaluations_run", run_payload("pending")) stub_tool("evaluation_runs_get", run_payload("running"))