feat: include assertions and pass@k in baseline regression comparison

pkg/evaluation/baseline.go

@@ -32,6 +32,14 @@ type Metrics struct {
 	RelevanceRate float64 `json:"relevance_rate"`
 	HasRelevance  bool    `json:"has_relevance"`
 
+	AssertionRate float64 `json:"assertion_rate"`
+	HasAssertions bool    `json:"has_assertions"`
+
+	PassK     float64 `json:"pass_at_k,omitempty"`
+	HatK      float64 `json:"pass_hat_k,omitempty"`
+	RepeatK   int     `json:"repeat_k,omitempty"`
+	HasRepeat bool    `json:"has_repeat"`
+
 	TotalCost float64 `json:"total_cost"`
 }
 
@@ -57,6 +65,16 @@ func metricsOfSummary(s Summary) Metrics {
 		m.HasRelevance = true
 		m.RelevanceRate = s.RelevancePassed / s.RelevanceTotal
 	}
+	if s.AssertionsTotal > 0 {
+		m.HasAssertions = true
+		m.AssertionRate = float64(s.AssertionsPassed) / float64(s.AssertionsTotal)
+	}
+	if s.RepeatMetrics != nil {
+		m.HasRepeat = true
+		m.RepeatK = s.RepeatMetrics.K
+		m.PassK = s.RepeatMetrics.PassK
+		m.HatK = s.RepeatMetrics.HatK
+	}
 	return m
 }
 
@@ -216,6 +234,7 @@ func Compare(baseline *Baseline, current *EvalRun, tolerance float64) (Compariso
 		{"size pass rate", c.Baseline.SizePassRate, c.Current.SizePassRate, c.Baseline.HasSizes, c.Current.HasSizes},
 		{"tool F1 mean", c.Baseline.ToolsF1Mean, c.Current.ToolsF1Mean, c.Baseline.HasTools, c.Current.HasTools},
 		{"relevance rate", c.Baseline.RelevanceRate, c.Current.RelevanceRate, c.Baseline.HasRelevance, c.Current.HasRelevance},
+		{"assertion rate", c.Baseline.AssertionRate, c.Current.AssertionRate, c.Baseline.HasAssertions, c.Current.HasAssertions},
 	} {
 		if !q.hasBase || !q.hasCur {
 			continue
@@ -254,6 +273,25 @@ func Compare(baseline *Baseline, current *EvalRun, tolerance float64) (Compariso
 		Informational: true,
 	})
 
+	// pass@k and pass^k are informational: they measure consistency across
+	// repetitions but derive from the same per-eval pass/fail that the
+	// individual changes already gate on.
+	if c.Baseline.HasRepeat && c.Current.HasRepeat {
+		c.Deltas = append(c.Deltas, MetricDelta{
+			Name:          fmt.Sprintf("pass@%d", c.Current.RepeatK),
+			Baseline:      c.Baseline.PassK,
+			Current:       c.Current.PassK,
+			Delta:         c.Current.PassK - c.Baseline.PassK,
+			Informational: true,
+		}, MetricDelta{
+			Name:          fmt.Sprintf("pass^%d", c.Current.RepeatK),
+			Baseline:      c.Baseline.HatK,
+			Current:       c.Current.HatK,
+			Delta:         c.Current.HatK - c.Baseline.HatK,
+			Informational: true,
+		})
+	}
+
 	for _, d := range c.Deltas {
 		if d.Regressed && !d.Informational {
 			c.Regressed = true

pkg/evaluation/baseline_test.go

@@ -387,3 +387,88 @@ func TestComparison_IsJSONSerializable(t *testing.T) {
 	assert.True(t, round.Regressed)
 	assert.InDelta(t, 0.05, round.Tolerance, 1e-9)
 }
+
+func assertionOnlyResult(title string, pass bool) Result {
+	r := Result{
+		InputPath:       title + ".json",
+		Title:           title,
+		AssertionsTotal: 1,
+		AssertionResults: []AssertionResult{
+			{Name: "check", Type: "contains", Passed: pass, Reason: "reason"},
+		},
+		Session: &session.Session{Title: title},
+	}
+	if pass {
+		r.AssertionsPassed = 1
+	}
+	return r
+}
+
+func TestCompare_AssertionRateDropRegresses(t *testing.T) {
+	t.Parallel()
+
+	baseline := saveAndLoad(t, newRun(assertionOnlyResult("a", true), assertionOnlyResult("b", true)))
+	got, err := Compare(baseline, newRun(assertionOnlyResult("a", true), assertionOnlyResult("b", false)), 0)
+	require.NoError(t, err)
+	assert.True(t, got.Regressed)
+
+	var delta *MetricDelta
+	for i := range got.Deltas {
+		if got.Deltas[i].Name == "assertion rate" {
+			delta = &got.Deltas[i]
+		}
+	}
+	require.NotNil(t, delta)
+	assert.True(t, delta.Regressed)
+}
+
+func TestCompare_AssertionRateNoDropIsClean(t *testing.T) {
+	t.Parallel()
+
+	baseline := saveAndLoad(t, newRun(assertionOnlyResult("a", true)))
+	got, err := Compare(baseline, newRun(assertionOnlyResult("a", true)), 0)
+	require.NoError(t, err)
+	assert.False(t, got.Regressed)
+}
+
+func TestMetricsOf_AssertionsFlag(t *testing.T) {
+	t.Parallel()
+
+	got := MetricsOf(newRun(assertionOnlyResult("a", true)))
+	assert.True(t, got.HasAssertions)
+	assert.InDelta(t, 1.0, got.AssertionRate, 1e-9)
+
+	empty := MetricsOf(newRun(sizeResult("a", true)))
+	assert.False(t, empty.HasAssertions)
+}
+
+func TestMetricsOf_RepeatMetricsPopulated(t *testing.T) {
+	t.Parallel()
+
+	run := newRun(sizeResult("a", true), sizeResult("a", true))
+	run.Summary.RepeatMetrics = &RepeatMetrics{K: 2, PassK: 1.0, HatK: 1.0, Total: 1}
+	got := metricsOfSummary(run.Summary)
+	assert.True(t, got.HasRepeat)
+	assert.Equal(t, 2, got.RepeatK)
+	assert.InDelta(t, 1.0, got.PassK, 1e-9)
+}
+
+func TestCompare_PassKIsInformational(t *testing.T) {
+	t.Parallel()
+
+	baseRun := newRun(sizeResult("a", true), sizeResult("a", true))
+	baseRun.Summary.RepeatMetrics = &RepeatMetrics{K: 2, PassK: 1.0, HatK: 1.0, Total: 1}
+	baseline := saveAndLoad(t, baseRun)
+
+	curRun := newRun(sizeResult("a", true), sizeResult("a", false))
+	curRun.Summary.RepeatMetrics = &RepeatMetrics{K: 2, PassK: 1.0, HatK: 0.0, Total: 1}
+
+	got, err := Compare(baseline, curRun, 0)
+	require.NoError(t, err)
+
+	for _, d := range got.Deltas {
+		if d.Name == "pass@2" || d.Name == "pass^2" {
+			assert.True(t, d.Informational, "%s must be informational", d.Name)
+		}
+	}
+}

pkg/evaluation/eval_test.go

@@ -1525,3 +1525,16 @@ func TestResultCheckResults_AssertionsDoNotFireWhenZero(t *testing.T) {
 	assert.Empty(t, successes)
 	assert.Empty(t, failures)
 }
+
+func TestComputeSummary_Assertions(t *testing.T) {
+	t.Parallel()
+
+	results := []Result{
+		{Title: "a", AssertionsTotal: 3, AssertionsPassed: 2},
+		{Title: "b", AssertionsTotal: 2, AssertionsPassed: 2},
+		{Title: "c", Error: "boom", AssertionsTotal: 1},
+	}
+	s := computeSummary(results)
+	assert.Equal(t, 5, s.AssertionsTotal)
+	assert.Equal(t, 4, s.AssertionsPassed)
+}

pkg/evaluation/scoring.go

@@ -88,6 +88,9 @@ func computeSummary(results []Result) Summary {
 
 		summary.RelevanceTotal += r.RelevanceExpected
 		summary.RelevancePassed += r.RelevancePassed
+
+		summary.AssertionsTotal += r.AssertionsTotal
+		summary.AssertionsPassed += r.AssertionsPassed
 	}
 
 	return summary
@@ -104,6 +107,7 @@ func printSummary(out io.Writer, summary Summary, duration time.Duration) {
 	printMetric(out, "Sizes", summary.SizesPassed, summary.SizesTotal)
 	printF1Score(out, "Tool Calls", summary.ToolsF1Sum, summary.ToolsCount)
 	printMetric(out, "Relevance", int(summary.RelevancePassed), int(summary.RelevanceTotal))
+	printMetric(out, "Assertions", summary.AssertionsPassed, summary.AssertionsTotal)
 
 	if summary.RepeatMetrics != nil {
 		rm := summary.RepeatMetrics

pkg/evaluation/types.go

@@ -119,15 +119,17 @@ func (r *Result) checkResults() (successes, failures []string) {
 
 // Summary contains aggregate statistics across all evaluations.
 type Summary struct {
-	TotalEvals      int     `json:"total_evals"`
-	FailedEvals     int     `json:"failed_evals"`
-	TotalCost       float64 `json:"total_cost"`
-	SizesPassed     int     `json:"sizes_passed"`
-	SizesTotal      int     `json:"sizes_total"`
-	ToolsF1Sum      float64 `json:"tools_f1_sum"`
-	ToolsCount      int     `json:"tools_count"`
-	RelevancePassed float64 `json:"relevance_passed"`
-	RelevanceTotal  float64 `json:"relevance_total"`
+	TotalEvals       int     `json:"total_evals"`
+	FailedEvals      int     `json:"failed_evals"`
+	TotalCost        float64 `json:"total_cost"`
+	SizesPassed      int     `json:"sizes_passed"`
+	SizesTotal       int     `json:"sizes_total"`
+	ToolsF1Sum       float64 `json:"tools_f1_sum"`
+	ToolsCount       int     `json:"tools_count"`
+	RelevancePassed  float64 `json:"relevance_passed"`
+	RelevanceTotal   float64 `json:"relevance_total"`
+	AssertionsPassed int     `json:"assertions_passed"`
+	AssertionsTotal  int     `json:"assertions_total"`
 
 	// RepeatMetrics is populated only when --repeat > 1.
 	RepeatMetrics *RepeatMetrics `json:"repeat_metrics,omitempty"`

pkg/session/branch.go

@@ -367,10 +367,6 @@ func cloneEvalResultChecks(src EvalResultChecks) EvalResultChecks {
 		assertions.Results = slices.Clone(src.Assertions.Results)
 		cp.Assertions = &assertions
 	}
-	if src.Verify != nil {
-		verify := *src.Verify
-		cp.Verify = &verify
-	}
 	return cp
 }