diff --git a/pkg/evaluation/baseline.go b/pkg/evaluation/baseline.go index e9046989f..529e56e02 100644 --- a/pkg/evaluation/baseline.go +++ b/pkg/evaluation/baseline.go @@ -32,6 +32,14 @@ type Metrics struct { RelevanceRate float64 `json:"relevance_rate"` HasRelevance bool `json:"has_relevance"` + AssertionRate float64 `json:"assertion_rate"` + HasAssertions bool `json:"has_assertions"` + + PassK float64 `json:"pass_at_k,omitempty"` + HatK float64 `json:"pass_hat_k,omitempty"` + RepeatK int `json:"repeat_k,omitempty"` + HasRepeat bool `json:"has_repeat"` + TotalCost float64 `json:"total_cost"` } @@ -57,6 +65,16 @@ func metricsOfSummary(s Summary) Metrics { m.HasRelevance = true m.RelevanceRate = s.RelevancePassed / s.RelevanceTotal } + if s.AssertionsTotal > 0 { + m.HasAssertions = true + m.AssertionRate = float64(s.AssertionsPassed) / float64(s.AssertionsTotal) + } + if s.RepeatMetrics != nil { + m.HasRepeat = true + m.RepeatK = s.RepeatMetrics.K + m.PassK = s.RepeatMetrics.PassK + m.HatK = s.RepeatMetrics.HatK + } return m } @@ -216,6 +234,7 @@ func Compare(baseline *Baseline, current *EvalRun, tolerance float64) (Compariso {"size pass rate", c.Baseline.SizePassRate, c.Current.SizePassRate, c.Baseline.HasSizes, c.Current.HasSizes}, {"tool F1 mean", c.Baseline.ToolsF1Mean, c.Current.ToolsF1Mean, c.Baseline.HasTools, c.Current.HasTools}, {"relevance rate", c.Baseline.RelevanceRate, c.Current.RelevanceRate, c.Baseline.HasRelevance, c.Current.HasRelevance}, + {"assertion rate", c.Baseline.AssertionRate, c.Current.AssertionRate, c.Baseline.HasAssertions, c.Current.HasAssertions}, } { if !q.hasBase || !q.hasCur { continue @@ -254,6 +273,25 @@ func Compare(baseline *Baseline, current *EvalRun, tolerance float64) (Compariso Informational: true, }) + // pass@k and pass^k are informational: they measure consistency across + // repetitions but derive from the same per-eval pass/fail that the + // individual changes already gate on. + if c.Baseline.HasRepeat && c.Current.HasRepeat { + c.Deltas = append(c.Deltas, MetricDelta{ + Name: fmt.Sprintf("pass@%d", c.Current.RepeatK), + Baseline: c.Baseline.PassK, + Current: c.Current.PassK, + Delta: c.Current.PassK - c.Baseline.PassK, + Informational: true, + }, MetricDelta{ + Name: fmt.Sprintf("pass^%d", c.Current.RepeatK), + Baseline: c.Baseline.HatK, + Current: c.Current.HatK, + Delta: c.Current.HatK - c.Baseline.HatK, + Informational: true, + }) + } + for _, d := range c.Deltas { if d.Regressed && !d.Informational { c.Regressed = true diff --git a/pkg/evaluation/baseline_test.go b/pkg/evaluation/baseline_test.go index 316390927..779397fb9 100644 --- a/pkg/evaluation/baseline_test.go +++ b/pkg/evaluation/baseline_test.go @@ -387,3 +387,88 @@ func TestComparison_IsJSONSerializable(t *testing.T) { assert.True(t, round.Regressed) assert.InDelta(t, 0.05, round.Tolerance, 1e-9) } + +func assertionOnlyResult(title string, pass bool) Result { + r := Result{ + InputPath: title + ".json", + Title: title, + AssertionsTotal: 1, + AssertionResults: []AssertionResult{ + {Name: "check", Type: "contains", Passed: pass, Reason: "reason"}, + }, + Session: &session.Session{Title: title}, + } + if pass { + r.AssertionsPassed = 1 + } + return r +} + +func TestCompare_AssertionRateDropRegresses(t *testing.T) { + t.Parallel() + + baseline := saveAndLoad(t, newRun(assertionOnlyResult("a", true), assertionOnlyResult("b", true))) + got, err := Compare(baseline, newRun(assertionOnlyResult("a", true), assertionOnlyResult("b", false)), 0) + require.NoError(t, err) + assert.True(t, got.Regressed) + + var delta *MetricDelta + for i := range got.Deltas { + if got.Deltas[i].Name == "assertion rate" { + delta = &got.Deltas[i] + } + } + require.NotNil(t, delta) + assert.True(t, delta.Regressed) +} + +func TestCompare_AssertionRateNoDropIsClean(t *testing.T) { + t.Parallel() + + baseline := saveAndLoad(t, newRun(assertionOnlyResult("a", true))) + got, err := Compare(baseline, newRun(assertionOnlyResult("a", true)), 0) + require.NoError(t, err) + assert.False(t, got.Regressed) +} + +func TestMetricsOf_AssertionsFlag(t *testing.T) { + t.Parallel() + + got := MetricsOf(newRun(assertionOnlyResult("a", true))) + assert.True(t, got.HasAssertions) + assert.InDelta(t, 1.0, got.AssertionRate, 1e-9) + + empty := MetricsOf(newRun(sizeResult("a", true))) + assert.False(t, empty.HasAssertions) +} + +func TestMetricsOf_RepeatMetricsPopulated(t *testing.T) { + t.Parallel() + + run := newRun(sizeResult("a", true), sizeResult("a", true)) + run.Summary.RepeatMetrics = &RepeatMetrics{K: 2, PassK: 1.0, HatK: 1.0, Total: 1} + got := metricsOfSummary(run.Summary) + assert.True(t, got.HasRepeat) + assert.Equal(t, 2, got.RepeatK) + assert.InDelta(t, 1.0, got.PassK, 1e-9) +} + +func TestCompare_PassKIsInformational(t *testing.T) { + t.Parallel() + + baseRun := newRun(sizeResult("a", true), sizeResult("a", true)) + baseRun.Summary.RepeatMetrics = &RepeatMetrics{K: 2, PassK: 1.0, HatK: 1.0, Total: 1} + baseline := saveAndLoad(t, baseRun) + + curRun := newRun(sizeResult("a", true), sizeResult("a", false)) + curRun.Summary.RepeatMetrics = &RepeatMetrics{K: 2, PassK: 1.0, HatK: 0.0, Total: 1} + + got, err := Compare(baseline, curRun, 0) + require.NoError(t, err) + + for _, d := range got.Deltas { + if d.Name == "pass@2" || d.Name == "pass^2" { + assert.True(t, d.Informational, "%s must be informational", d.Name) + } + } +} diff --git a/pkg/evaluation/eval_test.go b/pkg/evaluation/eval_test.go index 827a5805b..db656a61c 100644 --- a/pkg/evaluation/eval_test.go +++ b/pkg/evaluation/eval_test.go @@ -1525,3 +1525,16 @@ func TestResultCheckResults_AssertionsDoNotFireWhenZero(t *testing.T) { assert.Empty(t, successes) assert.Empty(t, failures) } + +func TestComputeSummary_Assertions(t *testing.T) { + t.Parallel() + + results := []Result{ + {Title: "a", AssertionsTotal: 3, AssertionsPassed: 2}, + {Title: "b", AssertionsTotal: 2, AssertionsPassed: 2}, + {Title: "c", Error: "boom", AssertionsTotal: 1}, + } + s := computeSummary(results) + assert.Equal(t, 5, s.AssertionsTotal) + assert.Equal(t, 4, s.AssertionsPassed) +} diff --git a/pkg/evaluation/scoring.go b/pkg/evaluation/scoring.go index f5fd2952f..af68b84d5 100644 --- a/pkg/evaluation/scoring.go +++ b/pkg/evaluation/scoring.go @@ -88,6 +88,9 @@ func computeSummary(results []Result) Summary { summary.RelevanceTotal += r.RelevanceExpected summary.RelevancePassed += r.RelevancePassed + + summary.AssertionsTotal += r.AssertionsTotal + summary.AssertionsPassed += r.AssertionsPassed } return summary @@ -104,6 +107,7 @@ func printSummary(out io.Writer, summary Summary, duration time.Duration) { printMetric(out, "Sizes", summary.SizesPassed, summary.SizesTotal) printF1Score(out, "Tool Calls", summary.ToolsF1Sum, summary.ToolsCount) printMetric(out, "Relevance", int(summary.RelevancePassed), int(summary.RelevanceTotal)) + printMetric(out, "Assertions", summary.AssertionsPassed, summary.AssertionsTotal) if summary.RepeatMetrics != nil { rm := summary.RepeatMetrics diff --git a/pkg/evaluation/types.go b/pkg/evaluation/types.go index 95a2e02e0..8ef0d5e1b 100644 --- a/pkg/evaluation/types.go +++ b/pkg/evaluation/types.go @@ -119,15 +119,17 @@ func (r *Result) checkResults() (successes, failures []string) { // Summary contains aggregate statistics across all evaluations. type Summary struct { - TotalEvals int `json:"total_evals"` - FailedEvals int `json:"failed_evals"` - TotalCost float64 `json:"total_cost"` - SizesPassed int `json:"sizes_passed"` - SizesTotal int `json:"sizes_total"` - ToolsF1Sum float64 `json:"tools_f1_sum"` - ToolsCount int `json:"tools_count"` - RelevancePassed float64 `json:"relevance_passed"` - RelevanceTotal float64 `json:"relevance_total"` + TotalEvals int `json:"total_evals"` + FailedEvals int `json:"failed_evals"` + TotalCost float64 `json:"total_cost"` + SizesPassed int `json:"sizes_passed"` + SizesTotal int `json:"sizes_total"` + ToolsF1Sum float64 `json:"tools_f1_sum"` + ToolsCount int `json:"tools_count"` + RelevancePassed float64 `json:"relevance_passed"` + RelevanceTotal float64 `json:"relevance_total"` + AssertionsPassed int `json:"assertions_passed"` + AssertionsTotal int `json:"assertions_total"` // RepeatMetrics is populated only when --repeat > 1. RepeatMetrics *RepeatMetrics `json:"repeat_metrics,omitempty"` diff --git a/pkg/evaluation/verify.go b/pkg/evaluation/verify.go deleted file mode 100644 index 3bab27be7..000000000 --- a/pkg/evaluation/verify.go +++ /dev/null @@ -1,79 +0,0 @@ -package evaluation - -import ( - "bytes" - "context" - "errors" - "fmt" - "log/slog" - "os/exec" - "strings" - "time" -) - -// verifyResult holds the outcome of running a verify script inside the -// evaluation container. -type verifyResult struct { - Passed bool - ExitCode int - Output string -} - -// runVerifyScript executes a shell verify script and returns its outcome. -// The script is run with sh -c; a zero exit code means pass. Output is -// capped at maxVerifyOutputBytes to avoid unbounded memory from chatty -// scripts. -func runVerifyScript(ctx context.Context, script, containerRuntime, containerName string) (verifyResult, error) { - if script == "" { - return verifyResult{Passed: true}, nil - } - - ctx, cancel := context.WithTimeout(ctx, 60*time.Second) - defer cancel() - - args := []string{"exec", containerName, "sh", "-c", script} - cmd := exec.CommandContext(ctx, containerRuntime, args...) - - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - err := cmd.Run() - - output := strings.TrimSpace(stdout.String()) - if errOutput := strings.TrimSpace(stderr.String()); errOutput != "" { - if output != "" { - output += "\n" - } - output += errOutput - } - - // Cap output to avoid unbounded memory. - if len(output) > maxVerifyOutputBytes { - output = output[:maxVerifyOutputBytes] + "...(truncated)" - } - - exitCode := 0 - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - exitCode = exitErr.ExitCode() - } else { - return verifyResult{ExitCode: -1, Output: output}, fmt.Errorf("running verify script: %w", err) - } - } - - slog.DebugContext(ctx, "Verify script completed", - "exit_code", exitCode, - "output_length", len(output), - ) - - return verifyResult{ - Passed: exitCode == 0, - ExitCode: exitCode, - Output: output, - }, nil -} - -// maxVerifyOutputBytes caps the output captured from a verify script. -const maxVerifyOutputBytes = 8192 diff --git a/pkg/evaluation/verify_test.go b/pkg/evaluation/verify_test.go deleted file mode 100644 index 0c5b6c355..000000000 --- a/pkg/evaluation/verify_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package evaluation - -import ( - "os" - "path/filepath" - "runtime" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestRunVerifyScript_EmptyScriptPasses(t *testing.T) { - t.Parallel() - result, err := runVerifyScript(t.Context(), "", "docker", "c") - require.NoError(t, err) - assert.True(t, result.Passed) -} - -func TestRunVerifyScript_ZeroExitCodePasses(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("POSIX shell required") - } - t.Parallel() - - // Use a fake "container runtime" script that execs the verify command - // directly (no real container needed). - fake := writeFakeExecRuntime(t, "echo 'all good'; exit 0") - - result, err := runVerifyScript(t.Context(), "verify", fake, "container-1") - require.NoError(t, err) - assert.True(t, result.Passed) - assert.Equal(t, 0, result.ExitCode) - assert.Contains(t, result.Output, "all good") -} - -func TestRunVerifyScript_NonZeroExitCodeFails(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("POSIX shell required") - } - t.Parallel() - - fake := writeFakeExecRuntime(t, "echo 'file missing'; exit 1") - - result, err := runVerifyScript(t.Context(), "verify", fake, "container-1") - require.NoError(t, err) - assert.False(t, result.Passed) - assert.Equal(t, 1, result.ExitCode) - assert.Contains(t, result.Output, "file missing") -} - -func TestRunVerifyScript_OutputCapped(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("POSIX shell required") - } - t.Parallel() - - // Generate output larger than the cap. - fake := writeFakeExecRuntime(t, "dd if=/dev/zero bs=1 count=20000 2>/dev/null | tr '\\0' 'x'; exit 0") - - result, err := runVerifyScript(t.Context(), "verify", fake, "container-1") - require.NoError(t, err) - assert.True(t, result.Passed) - assert.LessOrEqual(t, len(result.Output), maxVerifyOutputBytes+20) // +20 for truncation marker -} - -// writeFakeExecRuntime creates a shell script that ignores docker exec args -// and runs the given shell command instead, returning the script path. -func writeFakeExecRuntime(t *testing.T, shCmd string) string { - t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, "fake-runtime") - // The script ignores its arguments (exec container sh -c ...) and runs - // shCmd directly, simulating what the container would do. - script := "#!/bin/sh\n" + shCmd + "\n" - require.NoError(t, os.WriteFile(path, []byte(script), 0o755)) - return path -} diff --git a/pkg/session/branch.go b/pkg/session/branch.go index 3037e6d62..16faaade2 100644 --- a/pkg/session/branch.go +++ b/pkg/session/branch.go @@ -367,10 +367,6 @@ func cloneEvalResultChecks(src EvalResultChecks) EvalResultChecks { assertions.Results = slices.Clone(src.Assertions.Results) cp.Assertions = &assertions } - if src.Verify != nil { - verify := *src.Verify - cp.Verify = &verify - } return cp } diff --git a/pkg/session/session.go b/pkg/session/session.go index 953b836d2..6ca408973 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -642,7 +642,6 @@ type EvalResultChecks struct { ToolCalls *ToolCallsCheck `json:"tool_calls,omitempty"` Relevance *RelevanceCheck `json:"relevance,omitempty"` Assertions *AssertionsCheck `json:"assertions,omitempty"` - Verify *VerifyCheck `json:"verify,omitempty"` } // SizeCheck contains the result of the response size check. @@ -689,18 +688,10 @@ type AssertionResult struct { Reason string `json:"reason,omitempty"` } -// VerifyCheck contains the result of the post-agent verify script. -type VerifyCheck struct { - Passed bool `json:"passed"` - ExitCode int `json:"exit_code"` - Output string `json:"output,omitempty"` -} - // EvalCriteria contains the evaluation criteria for a session. type EvalCriteria struct { Relevance []string `json:"relevance"` // Statements that should be true about the response Assertions []Assertion `json:"assertions,omitempty"` // Code-based assertions evaluated against the agent output - Verify string `json:"verify,omitempty"` // Shell script for post-agent outcome verification WorkingDir string `json:"working_dir,omitempty"` // Subdirectory under evals/working_dirs/ Size string `json:"size,omitempty"` // Expected response size: S, M, L, XL Setup string `json:"setup,omitempty"` // Optional sh script to run in the container before docker agent run --exec diff --git a/pkg/session/session_test.go b/pkg/session/session_test.go index e45045ef6..a1a8b5514 100644 --- a/pkg/session/session_test.go +++ b/pkg/session/session_test.go @@ -520,14 +520,6 @@ func TestEvalCriteriaUnmarshalJSON(t *testing.T) { Assertions: []Assertion{{Name: "has greeting", Type: "contains", Value: "hello"}}, }, }, - { - name: "valid with verify", - input: `{"relevance":[],"verify":"test -f output.txt"}`, - want: EvalCriteria{ - Relevance: []string{}, - Verify: "test -f output.txt", - }, - }, { name: "empty object", input: `{}`,