From 8853cf4f71b5378ba6a392bdd26810486c7afa65 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 15:52:35 +0200 Subject: [PATCH 1/3] fix(evaluation): remove public llm-grader assertion --- .agents/conventions.md | 7 +- .../docs/docs/next/graders/llm-graders.mdx | 4 +- .../docs/v4.42.4/evaluation/eval-cases.mdx | 16 +-- .../docs/docs/v4.42.4/evaluation/examples.mdx | 10 +- .../v4.42.4/graders/execution-metrics.mdx | 2 +- .../docs/docs/v4.42.4/graders/llm-graders.mdx | 28 ++--- .../v4.42.4/guides/benchmark-provenance.mdx | 2 +- .../docs/v4.42.4/guides/evaluation-types.mdx | 2 +- .../docs/docs/v4.42.4/guides/human-review.mdx | 4 +- .../guides/skill-improvement-workflow.mdx | 4 +- .../integrations/agent-skills-evals.mdx | 10 +- .../docs/v4.42.4/integrations/phoenix.mdx | 2 +- .../docs/docs/v4.42.4/tools/convert.mdx | 2 +- .../docs/docs/v4.42.4/tools/import.mdx | 2 +- packages/core/src/evaluation/evaluate.ts | 4 +- .../loaders/eval-yaml-transpiler.ts | 18 +-- .../src/evaluation/loaders/grader-parser.ts | 15 ++- .../core/src/evaluation/template-variables.ts | 4 +- .../evaluation/validation/eval-file.schema.ts | 15 +-- .../evaluation/validation/eval-validator.ts | 6 +- packages/core/src/evaluation/yaml-parser.ts | 6 +- .../loaders/eval-yaml-transpiler.test.ts | 22 ++-- .../evaluation/loaders/grader-parser.test.ts | 116 +++++++++++------- .../evaluation/loaders/jsonl-parser.test.ts | 4 +- .../evaluation/preprocessors-yaml.test.ts | 2 +- .../evaluation/source-traceability.test.ts | 8 +- .../validation/eval-file-schema.test.ts | 29 +++++ .../validation/eval-validator.test.ts | 28 +++++ .../evaluation/yaml-parser-metadata.test.ts | 8 +- packages/sdk/src/assertion.ts | 1 - packages/sdk/src/graders.ts | 36 ------ packages/sdk/src/index.ts | 3 - .../references/breaking-changes.md | 8 +- 33 files changed, 239 insertions(+), 189 deletions(-) diff --git a/.agents/conventions.md b/.agents/conventions.md index dab30d4d9..341306576 100644 --- a/.agents/conventions.md +++ b/.agents/conventions.md @@ -141,6 +141,7 @@ Grader types use kebab-case everywhere. - Internal TypeScript may keep shared LLM grader implementation names, but new authored evals should not use `llm-grader` as a public type. - Output `scores[].type`: use the authored grader type when available, such as `"llm-rubric"`, `"g-eval"`, or `"is-json"`. - Registry keys: `registry.register('llm-rubric', ...)` for the authored rubric surface, with `llm-grader` retained as internal/shared implementation plumbing. +- Public SDK assertion helpers and authored YAML schemas must not expose `llm-grader`; use `llmRubricGrader()` / `type: llm-rubric` or `type: agent-rubric`. Source of truth: `GRADER_KIND_VALUES` in `packages/core/src/evaluation/types.ts`. @@ -149,10 +150,10 @@ Backward compatibility: - Snake_case is accepted in YAML by `normalizeGraderType()` in `grader-parser.ts`, for example `llm_rubric` -> `llm-rubric`. - Single-word types such as `contains`, `equals`, `regex`, `latency`, and `cost` are unchanged. -Two type definitions exist and must stay in sync: +Public and internal type definitions intentionally differ for internal-only grader plumbing: -- `EvaluatorKind` in `packages/core/src/evaluation/types.ts` -- `AssertionType` in `packages/sdk/src/assertion.ts` +- `GraderKind` in `packages/core/src/evaluation/types.ts` includes internal/runtime keys such as `llm-grader`. +- `AssertionType` in `packages/sdk/src/assertion.ts` includes public authored assertion types only. ## Python Scripts diff --git a/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx b/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx index 273050425..9bc10298f 100644 --- a/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx +++ b/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx @@ -50,8 +50,8 @@ Use `provider:` when you want different `llm-rubric` entries in the same eval to Use `type: agent-rubric` when a Promptfoo-style rubric needs an agentic grader that can inspect the workspace instead of only judging the final answer text. -AgentV routes `agent-rubric` through the same `llm-rubric` / `llm-grader` -scoring path, so results still appear as normal `EvaluationScore` entries. +AgentV routes `agent-rubric` through the same shared LLM rubric scoring path, +so results still appear as normal `EvaluationScore` entries. ```yaml assert: diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-cases.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-cases.mdx index 5d29059cd..a510ffb16 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-cases.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-cases.mdx @@ -86,7 +86,7 @@ tests: target: gpt4_target assertions: - name: depth_check - type: llm-grader + type: llm-rubric prompt: ./graders/depth.md ``` @@ -111,7 +111,7 @@ tests: skip_defaults: true assertions: - name: custom_eval - type: llm-grader + type: llm-rubric # Does NOT get latency_check ``` @@ -330,14 +330,14 @@ The `criteria` field is a **data field** that describes what the response should ### No `assertions` — implicit LLM grader -When a test has no `assertions` field, a default `llm-grader` grader runs automatically and uses `criteria` as the evaluation prompt: +When a test has no `assertions` field, a default `llm-rubric` grader runs automatically and uses `criteria` as the evaluation prompt: ```yaml tests: - id: simple-eval criteria: Assistant correctly explains the bug and proposes a fix input: "Debug this function..." - # No assertions → default llm-grader evaluates against criteria + # No assertions → default llm-rubric evaluates against criteria ``` Suite-level `preprocessors` also apply to this implicit grader. That matters when the agent output is a `ContentFile` block rather than plain text: @@ -355,13 +355,13 @@ tests: ### `assertions` present — explicit graders only -When `assertions` is defined, only the declared graders run. No implicit grader is added. Graders that are declared (such as `llm-grader`, `code-grader`, or `rubrics`) receive `criteria` as input automatically. +When `assertions` is defined, only the declared graders run. No implicit grader is added. Graders that are declared (such as `llm-rubric`, `script`, or `rubrics`) receive `criteria` as input automatically. If `assertions` contains only deterministic graders (like `contains` or `regex`), the `criteria` field is not evaluated and a warning is emitted: ``` Warning: Test 'my-test': criteria is defined but no grader in assertions -will evaluate it. Add 'type: llm-grader' to assertions, or remove criteria +will evaluate it. Add 'type: llm-rubric' to assertions, or remove criteria if it is documentation-only. ``` @@ -373,7 +373,7 @@ tests: criteria: Response is helpful and mentions the fix input: "Debug this function..." assertions: - - type: llm-grader # explicit — receives criteria automatically + - type: llm-rubric # explicit — receives criteria automatically - type: contains value: "fix" ``` @@ -390,7 +390,7 @@ tests: criteria: Response is helpful and mentions the fix input: "Debug this function..." assertions: - - type: llm-grader + - type: llm-rubric preprocessors: - type: xlsx command: ["bun", "run", "scripts/preprocessors/xlsx-to-json.ts"] diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/examples.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/examples.mdx index c8ab57ce2..8dd45bfb6 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/examples.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/examples.mdx @@ -91,7 +91,7 @@ tests: command: [uv, run, validate_json.py] cwd: ./graders - name: content_evaluator - type: llm-grader + type: llm-rubric prompt: ./graders/semantic_correctness.md input: |- @@ -108,7 +108,7 @@ tests: ## File Output Preprocessing -Convert a binary file output into text before the `llm-grader` sees it: +Convert a binary file output into text before the `llm-rubric` sees it: ```yaml description: Grade spreadsheet output via a preprocessor @@ -183,15 +183,15 @@ assertions: threshold: 0.6 assertions: - name: grader-gpt-5-mini - type: llm-grader + type: llm-rubric target: grader_gpt_5_mini prompt: ../prompts/grader-pass-fail-v1.md - name: grader-claude-haiku - type: llm-grader + type: llm-rubric target: grader_claude_haiku prompt: ../prompts/grader-pass-fail-v1.md - name: grader-gemini-flash - type: llm-grader + type: llm-rubric target: grader_gemini_flash prompt: ../prompts/grader-pass-fail-v1.md ``` diff --git a/apps/web/src/content/docs/docs/v4.42.4/graders/execution-metrics.mdx b/apps/web/src/content/docs/docs/v4.42.4/graders/execution-metrics.mdx index 59591f8e7..5f3210e3a 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/graders/execution-metrics.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/graders/execution-metrics.mdx @@ -129,7 +129,7 @@ tests: assertions: # Semantic quality - name: quality - type: llm-grader + type: llm-rubric prompt: ./prompts/code-quality.md # Efficiency constraints diff --git a/apps/web/src/content/docs/docs/v4.42.4/graders/llm-graders.mdx b/apps/web/src/content/docs/docs/v4.42.4/graders/llm-graders.mdx index b05ef7d25..bd75259bd 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/graders/llm-graders.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/graders/llm-graders.mdx @@ -12,14 +12,14 @@ LLM graders use a language model to evaluate agent responses against custom crit ## Default Grader -When a test defines `criteria` but has **no `assertions` field**, a default `llm-grader` runs automatically. The built-in prompt evaluates the response against your `criteria` and `expected_output`: +When a test defines `criteria` but has **no `assertions` field**, a default `llm-rubric` runs automatically. The built-in prompt evaluates the response against your `criteria` and `expected_output`: ```yaml tests: - id: simple-eval criteria: Correctly explains the bug and proposes a fix input: "Debug this function..." - # No assertions needed — default llm-grader evaluates against criteria + # No assertions needed — default llm-rubric evaluates against criteria ``` When `assertions` **is** present, no default grader is added. To use an LLM grader alongside other graders, declare it explicitly. See [How criteria and assertions interact](/docs/v4.42.4/evaluation/eval-cases/#how-criteria-and-assertions-interact). @@ -31,12 +31,12 @@ Reference an LLM grader in your eval file: ```yaml assertions: - name: semantic_check - type: llm-grader + type: llm-rubric prompt: file://graders/correctness.md target: grader_gpt_5_mini # optional: route this grader to a named LLM target ``` -Use `target:` when you want different `llm-grader` entries in the same eval to run on different grader models. This is useful for grader panels, majority-vote ensembles, and grader A/B benchmarks. +Use `target:` when you want different `llm-rubric` entries in the same eval to run on different grader models. This is useful for grader panels, majority-vote ensembles, and grader A/B benchmarks. ## Prompt Files @@ -100,26 +100,26 @@ tests: row: 1 assertions: - name: dexter_semantic - type: llm-grader + type: llm-rubric prompt: file://prompts/dexter-grader.md - rubrics: + value: - operator: correctness - criteria: Uses the provided ticker and company. + outcome: Uses the provided ticker and company. ``` ## Per-Grader Target -By default, an `llm-grader` uses `defaults.grader` from the resolved config graph. +By default, an `llm-rubric` uses `defaults.grader` from the resolved config graph. Override it per grader when you need multiple grader models in one run: ```yaml assertions: - name: grader-gpt - type: llm-grader + type: llm-rubric target: grader_gpt_5_mini prompt: ./prompts/pass-fail.md - name: grader-haiku - type: llm-grader + type: llm-rubric target: grader_claude_haiku prompt: ./prompts/pass-fail.md ``` @@ -178,7 +178,7 @@ When using TypeScript templates, configure them in YAML with optional `config` d ```yaml assertions: - name: custom-eval - type: llm-grader + type: llm-rubric prompt: command: [bun, run, ../prompts/custom-grader.ts] config: @@ -190,7 +190,7 @@ The `config` object is available as `ctx.config` inside the template function. ## Preprocessing File Outputs -If an agent returns a `ContentFile` block instead of plain text, you can preprocess that file into text before `llm-grader` builds the candidate prompt. +If an agent returns a `ContentFile` block instead of plain text, you can preprocess that file into text before `llm-rubric` builds the candidate prompt. AgentV always tries a default UTF-8 text read first. That is enough for text-based formats such as CSV, JSON, SQL, Markdown, YAML, HTML, XML, and plain text. For binary formats such as `.xlsx`, `.pdf`, or `.docx`, add a preprocessor command: @@ -205,7 +205,7 @@ tests: input: Generate the spreadsheet report assertions: - name: spreadsheet-check - type: llm-grader + type: llm-rubric prompt: | Check whether the transformed spreadsheet text contains the revenue rows: @@ -220,7 +220,7 @@ Resolution order: - if no preprocessor matches, AgentV falls back to a UTF-8 text read - if the fallback read looks binary or invalid, the grader receives a warning note instead of failing the test run -The implicit default `llm-grader` also inherits suite-level `preprocessors`, so you can omit `assertions` and still preprocess file outputs before grading. +The implicit default `llm-rubric` also inherits suite-level `preprocessors`, so you can omit `assertions` and still preprocess file outputs before grading. See [`examples/features/preprocessors/`](../../../../../examples/features/preprocessors/) for a runnable example with a file-producing target and a custom preprocessor script. diff --git a/apps/web/src/content/docs/docs/v4.42.4/guides/benchmark-provenance.mdx b/apps/web/src/content/docs/docs/v4.42.4/guides/benchmark-provenance.mdx index fa4ddddc9..2186f8cde 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/guides/benchmark-provenance.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/guides/benchmark-provenance.mdx @@ -212,7 +212,7 @@ description: Generated finance research cases with row-level provenance. assertions: - name: answer-quality - type: llm-grader + type: llm-rubric prompt: ./graders/finance-answer.md required: true diff --git a/apps/web/src/content/docs/docs/v4.42.4/guides/evaluation-types.mdx b/apps/web/src/content/docs/docs/v4.42.4/guides/evaluation-types.mdx index 4a107a917..256125d9d 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/guides/evaluation-types.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/guides/evaluation-types.mdx @@ -67,7 +67,7 @@ AgentV's eval tooling is designed for **execution quality**: - **`EVAL.yaml`** — define test cases with inputs, expected outputs, and assertions - **`evals.json`** — lightweight skill evaluation format (prompt/expected-output pairs) - **`agentv eval`** — execute evaluations and collect results -- **Graders** — `llm-grader`, `code-grader`, `tool-trajectory`, `rubrics`, `contains`, `regex`, and others all measure execution behavior +- **Graders** — `llm-rubric`, `script`, `tool-trajectory`, `rubrics`, `contains`, `regex`, and others all measure execution behavior These tools assume the skill is already loaded and invoked. They measure what happens *after* routing, not the routing decision itself. diff --git a/apps/web/src/content/docs/docs/v4.42.4/guides/human-review.mdx b/apps/web/src/content/docs/docs/v4.42.4/guides/human-review.mdx index 7ee5b25a1..0090a4759 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/guides/human-review.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/guides/human-review.mdx @@ -96,7 +96,7 @@ The `feedback.json` file is a structured annotation of a single eval run. It rec "notes": "Missing coverage of multi-document queries.", "evaluator_overrides": { "code-grader:format-check": "Too strict — penalized valid output with trailing newline", - "llm-grader:quality": "Score 0.6 seems fair, answer was incomplete" + "llm-rubric:quality": "Score 0.6 seems fair, answer was incomplete" }, "workspace_notes": "Workspace had stale cached files from previous run — may have affected retrieval results." }, @@ -148,7 +148,7 @@ For workspace evaluations with multiple graders (code graders, LLM graders, tool "verdict": "needs_improvement", "evaluator_overrides": { "code-grader:test-pass": "Tests pass but the refactored code has a subtle race condition the tests don't cover", - "llm-grader:quality": "Score 0.9 is too high — the agent left dead code behind", + "llm-rubric:quality": "Score 0.9 is too high — the agent left dead code behind", "tool-trajectory:efficiency": "Used 12 tool calls where 5 would suffice, but the result is correct" }, "workspace_notes": "Agent cloned the repo correctly but didn't clean up temp files." diff --git a/apps/web/src/content/docs/docs/v4.42.4/guides/skill-improvement-workflow.mdx b/apps/web/src/content/docs/docs/v4.42.4/guides/skill-improvement-workflow.mdx index d69f5dfd6..44b6945ac 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/guides/skill-improvement-workflow.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/guides/skill-improvement-workflow.mdx @@ -234,7 +234,7 @@ tests: content: "Review this Python function for bugs:..." assertions: - name: assertion-1 - type: llm-grader + type: llm-rubric prompt: "Identifies the division by zero risk" # Replace with type: contains for deterministic checks: # - type: contains @@ -242,7 +242,7 @@ tests: ``` After converting, you can: -- Replace `llm-grader` assertions with faster deterministic graders (`contains`, `regex`, `equals`) +- Replace `llm-rubric` assertions with faster deterministic graders (`contains`, `regex`, `equals`) - Add `workspace` configuration for file-system isolation - Use `code-grader` for custom scoring logic - Define `tool-trajectory` assertions to check tool usage patterns diff --git a/apps/web/src/content/docs/docs/v4.42.4/integrations/agent-skills-evals.mdx b/apps/web/src/content/docs/docs/v4.42.4/integrations/agent-skills-evals.mdx index bee673238..ef4e56038 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/integrations/agent-skills-evals.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/integrations/agent-skills-evals.mdx @@ -59,7 +59,7 @@ When AgentV loads `evals.json`, it promotes fields to its internal representatio |---|---|---| | `prompt` | `input` | Wrapped as `[{role: "user", content: prompt}]` | | `expected_output` | `expected_output` + `criteria` | Used as reference answer and evaluation criteria | -| `assertions[]` | `assertions[]` | Each string becomes `{type: llm-grader, prompt: text}` | +| `assertions[]` | `assertions[]` | Each string becomes `{type: llm-rubric, value: text}` | | `files[]` | `file_paths` | Resolved relative to evals.json, copied into workspace | | `skill_name` | `metadata.skill_name` | Carried as metadata | | `id` (number) | `id` (string) | Converted via `String(id)` | @@ -171,7 +171,7 @@ tests: # Replace with type: is_json, contains, or regex for deterministic checks assertions: - name: assertion-1 - type: llm-grader + type: llm-rubric prompt: "Output identifies November as the highest revenue month" ``` @@ -236,17 +236,17 @@ tests: An empathetic response that offers to track the order and provides next steps. assertions: - name: acknowledges-frustration - type: llm-grader + type: llm-rubric prompt: Response acknowledges the customer's frustration - name: looks-up-order type: contains value: "12345" - name: has-next-steps - type: llm-grader + type: llm-rubric prompt: Response provides clear next steps ``` -Notice how the EVAL.yaml version can mix `llm-grader` (for subjective checks) with `contains` (for deterministic checks) — the order number check is now instant and free. +Notice how the EVAL.yaml version can mix `llm-rubric` (for subjective checks) with `contains` (for deterministic checks) — the order number check is now instant and free. ## References diff --git a/apps/web/src/content/docs/docs/v4.42.4/integrations/phoenix.mdx b/apps/web/src/content/docs/docs/v4.42.4/integrations/phoenix.mdx index 3cef1a53d..121e18893 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/integrations/phoenix.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/integrations/phoenix.mdx @@ -52,7 +52,7 @@ bun packages/phoenix-adapter/src/cli.ts run \ | `regex` | Converts to deterministic Phoenix evaluator logic | | `equals` | Converts to deterministic Phoenix evaluator logic | | `is-json` | Converts to deterministic Phoenix evaluator logic | -| `llm-grader`, rubrics, `code-grader`, `tool-trajectory`, assert-set, metrics, and custom families | Reported as unsupported in the adapter report | +| `llm-rubric`, rubrics, `code-grader`, `tool-trajectory`, assert-set, metrics, and custom families | Reported as unsupported in the adapter report | Unsupported families do not fail conversion by default. Add `--fail-on-unsupported` when a parity report should fail CI if any suite needs a diff --git a/apps/web/src/content/docs/docs/v4.42.4/tools/convert.mdx b/apps/web/src/content/docs/docs/v4.42.4/tools/convert.mdx index 8ff6aba85..2f34ce4f7 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/tools/convert.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/tools/convert.mdx @@ -38,7 +38,7 @@ Converts an [Agent Skills `evals.json`](/docs/v4.42.4/integrations/agent-skills- - Maps `prompt` → `input` message array - Maps `expected_output` → `expected_output` -- Maps `assertions` → `assertions` graders (llm-grader) +- Maps `assertions` → `assertions` graders (`llm-rubric`) - Resolves `files[]` paths relative to the evals.json directory - Adds TODO comments for AgentV-specific features (workspace setup, code graders, rubrics) diff --git a/apps/web/src/content/docs/docs/v4.42.4/tools/import.mdx b/apps/web/src/content/docs/docs/v4.42.4/tools/import.mdx index 63df559b9..debd57109 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/tools/import.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/tools/import.mdx @@ -51,7 +51,7 @@ Default output: `EVAL.yaml` beside the promptfoo config file. - per-test `vars`, `description`, `threshold`, `metadata`, prompt filters, and provider filters - simple prompt templates are preserved as AgentV `{{var}}` input templates instead of being eagerly flattened - deterministic assertions that map directly to AgentV: `equals`, `contains`, `icontains`, `regex`, `starts-with`, `ends-with`, `contains-any`, `contains-all`, `icontains-any`, `icontains-all`, `is-json`, `latency`, `cost` -- rubric-style assertions mapped to `llm-grader`: `llm-rubric`, `g-eval`, `factuality`, `context-faithfulness`, `context-recall` +- rubric-style assertions mapped to `llm-rubric`: `llm-rubric`, `g-eval`, `factuality`, `context-faithfulness`, `context-recall` ### What still needs manual migration diff --git a/packages/core/src/evaluation/evaluate.ts b/packages/core/src/evaluation/evaluate.ts index 87984840b..b3f3366e5 100644 --- a/packages/core/src/evaluation/evaluate.ts +++ b/packages/core/src/evaluation/evaluate.ts @@ -136,7 +136,7 @@ export interface ConversationTurnInput { * Matches the YAML `assert` block structure. */ export interface EvalAssertionInput { - /** Assertion type (e.g., 'contains', 'llm-grader', 'script') */ + /** Assertion type (e.g., 'contains', 'llm-rubric', 'script') */ readonly type: string; /** Score/check metric name */ readonly metric?: string; @@ -148,7 +148,7 @@ export interface EvalAssertionInput { readonly required?: boolean; /** Minimum score (0-1) for this evaluator to pass. Independent of `required` gate. */ readonly min_score?: number; - /** Prompt file for llm-grader */ + /** Prompt file for LLM rubric assertions */ readonly prompt?: string; /** Command for script grader */ readonly command?: string | readonly string[]; diff --git a/packages/core/src/evaluation/loaders/eval-yaml-transpiler.ts b/packages/core/src/evaluation/loaders/eval-yaml-transpiler.ts index a404d4799..2d35fae17 100644 --- a/packages/core/src/evaluation/loaders/eval-yaml-transpiler.ts +++ b/packages/core/src/evaluation/loaders/eval-yaml-transpiler.ts @@ -160,12 +160,9 @@ function assertionToNaturalLanguage(entry: RawAssertEntry): string | null { return `Output ends with '${entry.value}'`; case 'llm-grader': - // Expand each rubric item to its own assertion string - // Return the first one — callers handle arrays via assertionToNaturalLanguageList - if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) { - return null; // handled by list expansion below - } - return typeof entry.prompt === 'string' ? entry.prompt : null; + throw new Error( + "Authored assertion type 'llm-grader' has been removed. Use 'llm-rubric' for free-form rubric checks or 'agent-rubric' for agentic rubric checks.", + ); case 'tool-trajectory': { const expectedArr = Array.isArray(entry.expected) ? entry.expected : []; @@ -227,7 +224,7 @@ function assertionToNaturalLanguage(entry: RawAssertEntry): string | null { /** * Expand a single assertion entry into zero or more NL strings. - * Most assertions produce exactly one string; llm-grader with rubrics expands to many. + * Most assertions produce exactly one string; llm-rubric with structured values expands to many. */ function assertionToNaturalLanguageList(entry: RawAssertEntry): string[] { if (entry.type === 'llm-rubric') { @@ -242,13 +239,6 @@ function assertionToNaturalLanguageList(entry: RawAssertEntry): string[] { .filter((value): value is string => typeof value === 'string'); } } - if (entry.type === 'llm-grader') { - if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) { - return (entry.rubrics as Array<{ outcome?: string; criteria?: string; id?: string }>) - .map((r) => r.outcome ?? r.criteria ?? r.id) - .filter((s): s is string => typeof s === 'string'); - } - } const nl = assertionToNaturalLanguage(entry); return nl !== null ? [nl] : []; } diff --git a/packages/core/src/evaluation/loaders/grader-parser.ts b/packages/core/src/evaluation/loaders/grader-parser.ts index 116286405..1ecbeaf32 100644 --- a/packages/core/src/evaluation/loaders/grader-parser.ts +++ b/packages/core/src/evaluation/loaders/grader-parser.ts @@ -49,8 +49,7 @@ function removedGraderReplacement(type: string): string | undefined { code_grader: 'script', code_judge: 'script', composite: 'assert-set', - llm_judge: 'llm-grader', - llm_grader: 'llm-grader', + llm_judge: 'llm-rubric', tool_trajectory: 'tool-trajectory', field_accuracy: 'field-accuracy', token_usage: 'token-usage', @@ -66,6 +65,14 @@ function removedGraderReplacement(type: string): string | undefined { return replacements[type]; } +function removedPublicLlmGraderMessage(type: string): string | undefined { + const normalizedType = type.replace(/_/g, '-'); + if (normalizedType !== 'llm-grader' && normalizedType !== 'llm-judge') { + return undefined; + } + return "Use 'llm-rubric' for free-form rubric checks or 'agent-rubric' for agentic rubric checks."; +} + const UNSUPPORTED_PROMPTFOO_ASSERTION_TYPES = new Set([ 'answer-relevance', 'bleu', @@ -654,6 +661,10 @@ async function parseGraderList( `${rawName ? ` for evaluator '${rawName}'` : ''}. ${staleMessage}`, ); } + const llmGraderMessage = removedPublicLlmGraderMessage(normalizedType); + if (llmGraderMessage) { + throw new Error(`Unsupported grader '${rawType}' in '${evalId}'. ${llmGraderMessage}`); + } const replacement = removedGraderReplacement(normalizedType); if (replacement) { throw new Error( diff --git a/packages/core/src/evaluation/template-variables.ts b/packages/core/src/evaluation/template-variables.ts index b4019038c..492a5e9d9 100644 --- a/packages/core/src/evaluation/template-variables.ts +++ b/packages/core/src/evaluation/template-variables.ts @@ -10,8 +10,8 @@ * - {{ rubric }} — rubric data, as structured JSON when available or criteria text otherwise * - {{ metadata }} — per-test metadata as formatted JSON * - {{ metadata_json }} — per-test metadata as compact JSON - * - {{ rubrics }} — llm-grader rubrics as formatted JSON - * - {{ rubrics_json }} — llm-grader rubrics as compact JSON + * - {{ rubrics }} — rubric rows as formatted JSON + * - {{ rubrics_json }} — rubric rows as compact JSON * - {{ file_changes }} — file diff (if available) * - {{ tool_calls }} — formatted summary of tool calls from agent execution * diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 12b72f0f5..3edd616b5 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -95,6 +95,9 @@ function staleToolTrajectoryMessage(value: Record): string { function staleAuthoredAssertionMessage(value: Record): string | undefined { if (typeof value.type !== 'string') return undefined; const type = value.type.replace(/_/g, '-'); + if (type === 'llm-grader' || type === 'llm-judge') { + return "Authored assertion type 'llm-grader' has been removed. Use 'llm-rubric' for free-form rubric checks or 'agent-rubric' for agentic rubric checks."; + } if (type === 'skill-trigger') return staleSkillTriggerMessage(value); if (type === 'tool-trajectory') return staleToolTrajectoryMessage(value); return undefined; @@ -192,17 +195,6 @@ const ScriptGraderSchema = EvaluatorCommonSchema.extend({ config: z.record(z.unknown()).optional(), }); -const LlmGraderSchema = EvaluatorCommonSchema.extend({ - type: z.literal('llm-grader'), - prompt: PromptSchema.optional(), - rubrics: z.array(RubricItemSchema).optional(), - model: z.string().optional(), - provider: z.string().optional(), - config: z.record(z.unknown()).optional(), - max_steps: z.number().int().min(1).max(50).optional(), - temperature: z.number().min(0).max(2).optional(), -}); - const IncludeSchema = z .object({ include: z.string().min(1), @@ -340,7 +332,6 @@ const PromptfooAssertionSchema = EvaluatorCommonSchema.extend({ /** Union of all grader types */ const EvaluatorSchema = z.union([ ScriptGraderSchema, - LlmGraderSchema, PromptfooAssertionSchema, IncludeSchema, ToolTrajectorySchema, diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 949250fe5..e62a9b46d 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -67,7 +67,6 @@ const REMOVED_ASSERTION_TYPE_REPLACEMENTS = new Map([ ['rubric', 'llm-rubric with value'], ['script-grader', 'script'], ['code-judge', 'script'], - ['llm-judge', 'llm-grader'], ]); const UNSUPPORTED_PROMPTFOO_ASSERTION_TYPES = new Set([ @@ -186,6 +185,9 @@ function staleToolTrajectoryMessage(value: JsonObject): string { function staleAuthoredAssertionMessage(value: JsonObject): string | undefined { if (typeof value.type !== 'string') return undefined; const type = value.type.replace(/_/g, '-'); + if (type === 'llm-grader' || type === 'llm-judge') { + return "Authored assertion type 'llm-grader' has been removed. Use 'llm-rubric' for free-form rubric checks or 'agent-rubric' for agentic rubric checks."; + } if (type === 'skill-trigger') return staleSkillTriggerMessage(value); if (type === 'tool-trajectory') return staleToolTrajectoryMessage(value); return undefined; @@ -2627,7 +2629,7 @@ function validateAssertArray( } // String items in the assert array are valid shorthand — the parser collects them - // into a single rubrics/llm-grader evaluator. Filter them out before object validation. + // into a single llm-rubric evaluator. Filter them out before object validation. const objectItems: { item: JsonObject; index: number }[] = []; for (let i = 0; i < assertField.length; i++) { const item = assertField[i]; diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 5e7b3d303..373cb0a8a 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -2909,7 +2909,11 @@ function collectSingleGraderSourceReferences( } } - if (evaluator.type === 'llm-grader') { + if ( + evaluator.type === 'llm-grader' || + evaluator.type === 'llm-rubric' || + evaluator.type === 'agent-rubric' + ) { const resolvedPromptPath = evaluator.resolvedPromptPath ?? evaluator.promptPath; if (resolvedPromptPath) { references.push({ diff --git a/packages/core/test/evaluation/loaders/eval-yaml-transpiler.test.ts b/packages/core/test/evaluation/loaders/eval-yaml-transpiler.test.ts index 595089b30..21b2a8ac2 100644 --- a/packages/core/test/evaluation/loaders/eval-yaml-transpiler.test.ts +++ b/packages/core/test/evaluation/loaders/eval-yaml-transpiler.test.ts @@ -283,7 +283,7 @@ describe('transpileEvalYaml — NL assertions', () => { expect(evals[0].assertions).toContain('Output is valid JSON'); }); - it('converts llm-grader prompt to NL', () => { + it('converts llm-rubric text to NL', () => { const suite = { tests: [ { @@ -291,7 +291,7 @@ describe('transpileEvalYaml — NL assertions', () => { input: 'test', assert: [ { type: 'skill-used', value: 's' }, - { type: 'llm-grader', prompt: 'The answer is clear and concise' }, + { type: 'llm-rubric', value: 'The answer is clear and concise' }, ], }, ], @@ -301,7 +301,7 @@ describe('transpileEvalYaml — NL assertions', () => { expect(evals[0].assertions).toContain('The answer is clear and concise'); }); - it('converts llm-grader with rubrics to multiple assertions (rubrics variant)', () => { + it('converts llm-rubric with structured values to multiple assertions', () => { const suite = { tests: [ { @@ -310,8 +310,8 @@ describe('transpileEvalYaml — NL assertions', () => { assert: [ { type: 'skill-used', value: 's' }, { - type: 'llm-grader', - rubrics: [ + type: 'llm-rubric', + value: [ { id: 'r1', outcome: 'Correct result returned' }, { id: 'r2', outcome: 'No unnecessary steps' }, ], @@ -326,7 +326,7 @@ describe('transpileEvalYaml — NL assertions', () => { expect(evals[0].assertions).toContain('No unnecessary steps'); }); - it('converts llm-grader with rubrics to multiple assertions', () => { + it('rejects public llm-grader assertions with migration guidance', () => { const suite = { tests: [ { @@ -336,7 +336,8 @@ describe('transpileEvalYaml — NL assertions', () => { { type: 'skill-used', value: 's' }, { type: 'llm-grader', - rubrics: [ + prompt: 'The answer is clear and concise', + value: [ { id: 'r1', outcome: 'Response is accurate' }, { id: 'r2', outcome: 'Formatting is correct' }, ], @@ -345,10 +346,9 @@ describe('transpileEvalYaml — NL assertions', () => { }, ], }; - const { files } = transpileEvalYaml(suite); - const evals = files.get('s')?.evals; - expect(evals[0].assertions).toContain('Response is accurate'); - expect(evals[0].assertions).toContain('Formatting is correct'); + expect(() => transpileEvalYaml(suite)).toThrow( + "Authored assertion type 'llm-grader' has been removed", + ); }); it('converts tool-trajectory to NL', () => { diff --git a/packages/core/test/evaluation/loaders/grader-parser.test.ts b/packages/core/test/evaluation/loaders/grader-parser.test.ts index bf2f92967..b6d5184db 100644 --- a/packages/core/test/evaluation/loaders/grader-parser.test.ts +++ b/packages/core/test/evaluation/loaders/grader-parser.test.ts @@ -412,7 +412,7 @@ describe('parseGraders - deterministic assertion types', () => { { assert: [ { metric: 'rubric', type: 'llm-rubric', value: 'Judge whether it is helpful' }, - { metric: 'grader', type: 'llm-grader' }, + { metric: 'contains-check', type: 'contains', value: 'ok' }, ], }, undefined, @@ -427,7 +427,11 @@ describe('parseGraders - deterministic assertion types', () => { type: 'llm-rubric', prompt: 'Grade {{ output }} against {{ rubric }}', }); - expect((evaluators?.[1] as LlmGraderConfig).prompt).toBeUndefined(); + expect(evaluators?.[1]).toMatchObject({ + name: 'contains-check', + type: 'contains', + value: 'ok', + }); }); it('rejects explicit g-eval until AgentV implements promptfoo two-call semantics', async () => { @@ -775,7 +779,7 @@ describe('parseGraders - kebab-case type normalization', () => { assert: [ { metric: 'kebab-llm', - type: 'llm-grader', + type: 'llm-rubric', prompt: 'test prompt', provider: 'grader-low-cost-a', }, @@ -785,7 +789,7 @@ describe('parseGraders - kebab-case type normalization', () => { const evaluators = await parseGraders(rawEvalCase, undefined, [tempDir], 'test-case'); expect(evaluators).toHaveLength(1); - expect(evaluators?.[0].type).toBe('llm-grader'); + expect(evaluators?.[0].type).toBe('llm-rubric'); expect((evaluators?.[0] as LlmGraderConfig).target).toBe('grader-low-cost-a'); }); @@ -843,7 +847,23 @@ describe('parseGraders - kebab-case type normalization', () => { expect(evaluators?.[0].type).toBe('is-json'); }); - it('rejects removed snake_case grader aliases', async () => { + it('rejects public llm-grader assertion type with migration guidance', async () => { + const rawEvalCase = { + assert: [ + { + metric: 'public-llm', + type: 'llm-grader', + prompt: 'test prompt', + }, + ], + }; + + await expect(parseGraders(rawEvalCase, undefined, [tempDir], 'test-case')).rejects.toThrow( + "Unsupported grader 'llm-grader' in 'test-case'. Use 'llm-rubric' for free-form rubric checks or 'agent-rubric' for agentic rubric checks.", + ); + }); + + it('rejects removed snake_case llm-grader alias with migration guidance', async () => { const rawEvalCase = { assert: [ { @@ -855,7 +875,7 @@ describe('parseGraders - kebab-case type normalization', () => { }; await expect(parseGraders(rawEvalCase, undefined, [tempDir], 'test-case')).rejects.toThrow( - "Unsupported grader 'llm_grader' in 'test-case'. Use 'llm-grader' instead.", + "Unsupported grader 'llm_grader' in 'test-case'. Use 'llm-rubric' for free-form rubric checks or 'agent-rubric' for agentic rubric checks.", ); }); @@ -883,8 +903,7 @@ describe('parseGraders - score_ranges rubrics', () => { assert: [ { metric: 'correctness', - type: 'llm-grader', - rubrics: [ + type: 'llm-rubric',value: [ { id: 'accuracy', weight: 2.0, @@ -905,8 +924,8 @@ describe('parseGraders - score_ranges rubrics', () => { expect(evaluators).toHaveLength(1); const config = evaluators?.[0]; - expect(config?.type).toBe('llm-grader'); - if (config?.type === 'llm-grader') { + expect(config?.type).toBe('llm-rubric'); + if (config?.type === 'llm-rubric') { expect(config.rubrics).toHaveLength(1); const rubric = config.rubrics?.[0]; expect(rubric?.id).toBe('accuracy'); @@ -921,8 +940,7 @@ describe('parseGraders - score_ranges rubrics', () => { assert: [ { metric: 'correctness', - type: 'llm-grader', - rubrics: [ + type: 'llm-rubric',value: [ { id: 'accuracy', required_min_score: 7, @@ -948,8 +966,7 @@ describe('parseGraders - score_ranges rubrics', () => { assert: [ { metric: 'overlapping', - type: 'llm-grader', - rubrics: [ + type: 'llm-rubric',value: [ { id: 'test', score_ranges: [ @@ -972,8 +989,7 @@ describe('parseGraders - score_ranges rubrics', () => { assert: [ { metric: 'incomplete', - type: 'llm-grader', - rubrics: [ + type: 'llm-rubric',value: [ { id: 'test', score_ranges: [ @@ -991,14 +1007,12 @@ describe('parseGraders - score_ranges rubrics', () => { ).rejects.toThrow(/coverage/i); }); - it('skips rubric items that use legacy description field without outcome', async () => { - const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); + it('keeps legacy description objects as promptfoo-compatible llm-rubric value', async () => { const rawEvalCase = { assert: [ { metric: 'legacy', - type: 'llm-grader', - rubrics: [ + type: 'llm-rubric',value: [ { id: 'r1', description: 'Must be polite', // Legacy field name — no longer supported @@ -1014,12 +1028,18 @@ describe('parseGraders - score_ranges rubrics', () => { expect(evaluators).toHaveLength(1); const config = evaluators?.[0]; - if (config?.type === 'llm-grader') { - // Rubric should be skipped since it has no 'outcome' field - expect(config.rubrics ?? []).toHaveLength(0); + if (config?.type !== 'llm-rubric') { + throw new Error('expected llm-rubric config'); } - expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('missing outcome')); - warnSpy.mockRestore(); + expect(config.rubrics).toBeUndefined(); + expect(config.value).toEqual([ + { + id: 'r1', + description: 'Must be polite', + weight: 1.0, + required: true, + }, + ]); }); }); @@ -1029,8 +1049,7 @@ describe('parseGraders - score_ranges shorthand map', () => { assert: [ { metric: 'shorthand-test', - type: 'llm-grader', - rubrics: [ + type: 'llm-rubric',value: [ { id: 'accuracy', weight: 2.0, @@ -1051,8 +1070,8 @@ describe('parseGraders - score_ranges shorthand map', () => { expect(evaluators).toHaveLength(1); const config = evaluators?.[0]; - expect(config?.type).toBe('llm-grader'); - if (config?.type === 'llm-grader') { + expect(config?.type).toBe('llm-rubric'); + if (config?.type === 'llm-rubric') { expect(config.rubrics).toHaveLength(1); const rubric = config.rubrics?.[0]; expect(rubric?.id).toBe('accuracy'); @@ -1082,8 +1101,7 @@ describe('parseGraders - score_ranges shorthand map', () => { assert: [ { metric: 'bad-start', - type: 'llm-grader', - rubrics: [ + type: 'llm-rubric',value: [ { id: 'test', score_ranges: { @@ -1107,8 +1125,7 @@ describe('parseGraders - score_ranges shorthand map', () => { assert: [ { metric: 'array-format', - type: 'llm-grader', - rubrics: [ + type: 'llm-rubric',value: [ { id: 'accuracy', score_ranges: [ @@ -1127,7 +1144,7 @@ describe('parseGraders - score_ranges shorthand map', () => { expect(evaluators).toHaveLength(1); const config = evaluators?.[0]; - if (config?.type === 'llm-grader') { + if (config?.type === 'llm-rubric') { expect(config.rubrics?.[0]?.score_ranges).toHaveLength(4); } }); @@ -2129,10 +2146,18 @@ describe('parseGraders - required field', () => { expect(config.required).toBe(true); }); - it('parses required with min_score on llm-grader evaluator', async () => { + it('parses required with min_score on llm-rubric evaluator', async () => { const evaluators = await parseGraders( { - assert: [{ metric: 'grader', type: 'llm-grader', required: true, min_score: 0.7 }], + assert: [ + { + metric: 'grader', + type: 'llm-rubric', + value: 'Judge whether it is helpful', + required: true, + min_score: 0.7, + }, + ], }, undefined, [tempDir], @@ -2201,7 +2226,7 @@ describe('parseGraders - assert-set grouping', () => { beforeAll(async () => { tempDir = path.join(os.tmpdir(), `agentv-test-assert-set-${Date.now()}`); await mkdir(tempDir, { recursive: true }); - // Create dummy prompt files for llm-grader members (must include required template fields) + // Create dummy prompt files for llm-rubric members (must include required template fields) await writeFile(path.join(tempDir, 'safety.md'), 'Evaluate safety of {{ output }}'); await writeFile(path.join(tempDir, 'quality.md'), 'Evaluate quality of {{ output }}'); }); @@ -2218,8 +2243,8 @@ describe('parseGraders - assert-set grouping', () => { metric: 'combined', type: 'assert-set', assert: [ - { metric: 'safety', type: 'llm-grader', prompt: './safety.md' }, - { metric: 'quality', type: 'llm-grader', prompt: './quality.md' }, + { metric: 'safety', type: 'llm-rubric', prompt: './safety.md' }, + { metric: 'quality', type: 'llm-rubric', prompt: './quality.md' }, ], threshold: 0.7, }, @@ -2482,7 +2507,14 @@ describe('parseGraders - file:// prefix prompt resolution', () => { it('file:// prefix resolves existing file', async () => { const evaluators = await parseGraders( { - assert: [{ metric: 'quality', type: 'llm-grader', prompt: 'file://grader.md' }], + assert: [ + { + metric: 'quality', + type: 'llm-rubric', + value: 'Judge the answer quality', + prompt: 'file://grader.md', + }, + ], }, undefined, [tempDir], @@ -2498,7 +2530,7 @@ describe('parseGraders - file:// prefix prompt resolution', () => { await expect( parseGraders( { - assert: [{ metric: 'missing', type: 'llm-grader', prompt: 'file://nonexistent.md' }], + assert: [{ metric: 'missing', type: 'llm-rubric', prompt: 'file://nonexistent.md' }], }, undefined, [tempDir], @@ -2510,7 +2542,7 @@ describe('parseGraders - file:// prefix prompt resolution', () => { it('bare path is always treated as inline text even if file exists', async () => { const evaluators = await parseGraders( { - assert: [{ metric: 'quality', type: 'llm-grader', prompt: 'grader.md' }], + assert: [{ metric: 'quality', type: 'llm-rubric', prompt: 'grader.md' }], }, undefined, [tempDir], diff --git a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts index 7839ecef1..f7ab50ae2 100644 --- a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts +++ b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts @@ -196,7 +196,7 @@ describe('loadTestsFromJsonl', () => { ); await expect(loadTestsFromJsonl(jsonlPath, tempDir)).rejects.toThrow( - "Unsupported grader 'llm_judge' in sidecar. Use 'llm-grader' instead.", + "Unsupported grader 'llm_judge' in sidecar. Use 'llm-rubric' instead.", ); }); @@ -217,7 +217,7 @@ describe('loadTestsFromJsonl', () => { const jsonlPath = path.join(tempDir, 'with-assert.jsonl'); await writeFile( jsonlPath, - '{"id": "test-1", "criteria": "Goal", "input": [{"role": "user", "content": "Query"}], "assert": [{"metric": "rubric-check", "type": "llm-grader", "rubrics": [{"id": "r1", "description": "Must be polite", "weight": 1.0, "required": true}]}]}\n', + '{"id": "test-1", "criteria": "Goal", "input": [{"role": "user", "content": "Query"}], "assert": [{"metric": "rubric-check", "type": "llm-rubric", "value": [{"id": "r1", "outcome": "Must be polite", "weight": 1.0, "required": true}]}]}\n', ); const cases = await loadTestsFromJsonl(jsonlPath, tempDir); diff --git a/packages/core/test/evaluation/preprocessors-yaml.test.ts b/packages/core/test/evaluation/preprocessors-yaml.test.ts index 81d7f9030..9af2773ee 100644 --- a/packages/core/test/evaluation/preprocessors-yaml.test.ts +++ b/packages/core/test/evaluation/preprocessors-yaml.test.ts @@ -53,7 +53,7 @@ tests: criteria: works assert: - name: grade - type: llm-grader + type: llm-rubric prompt: Evaluate {{ output }} preprocessors: - type: xlsx diff --git a/packages/core/test/evaluation/source-traceability.test.ts b/packages/core/test/evaluation/source-traceability.test.ts index e53358923..4b2bd44a9 100644 --- a/packages/core/test/evaluation/source-traceability.test.ts +++ b/packages/core/test/evaluation/source-traceability.test.ts @@ -46,10 +46,12 @@ tests: criteria: ok assert: - metric: prompt-file - type: llm-grader + type: llm-rubric + value: Check the answer with the prompt file. prompt: file://graders/prompt.md - metric: prompt-script - type: llm-grader + type: agent-rubric + value: Check the answer with the prompt script. prompt: command: - bun @@ -110,7 +112,7 @@ tests: ); expect(promptScriptDefinition?.definition).toMatchObject({ name: 'prompt-script', - type: 'llm-grader', + type: 'agent-rubric', config: { secret_token: '[redacted]', apiKey: '[redacted]', diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index b1e4f0755..3a8e31983 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -592,6 +592,35 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(true); }); + it('rejects public llm-grader assertions with migration guidance', () => { + const result = EvalFileSchema.safeParse({ + tests: [ + { + ...baseTest, + assert: [ + { + type: 'llm-grader', + prompt: 'Judge whether the answer is helpful', + }, + ], + }, + ], + }); + + expect(result.success).toBe(false); + if (!result.success) { + const messages = collectIssueMessages(result.error.issues); + expect( + messages.some( + (message) => + message.includes("Authored assertion type 'llm-grader' has been removed") && + message.includes("'llm-rubric'") && + message.includes("'agent-rubric'"), + ), + ).toBe(true); + } + }); + it('rejects stale skill-trigger assertions with migration guidance', () => { const positive = EvalFileSchema.safeParse({ tests: [ diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 1400545f0..76906bb23 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -2068,6 +2068,34 @@ tests: expect(result.errors.filter((e) => e.severity === 'error')).toHaveLength(0); }); + it('rejects public llm-grader assertions with migration guidance', async () => { + const filePath = path.join(tempDir, 'assert-llm-grader-removed.yaml'); + await writeFile( + filePath, + `prompts: + - "Review the answer" +tests: + - id: test-1 + assert: + - type: llm-grader + prompt: "Judge whether the answer is helpful" +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'tests[0].assert[0].type', + message: expect.stringContaining("Authored assertion type 'llm-grader' has been removed"), + }), + ); + expect(result.errors[0].message).toContain('llm-rubric'); + expect(result.errors[0].message).toContain('agent-rubric'); + }); + it('accepts promptfoo-compatible skill-used assertions', async () => { const filePath = path.join(tempDir, 'assert-skill-used.yaml'); await writeFile( diff --git a/packages/core/test/evaluation/yaml-parser-metadata.test.ts b/packages/core/test/evaluation/yaml-parser-metadata.test.ts index 8cde13fd1..e24c0b4ad 100644 --- a/packages/core/test/evaluation/yaml-parser-metadata.test.ts +++ b/packages/core/test/evaluation/yaml-parser-metadata.test.ts @@ -253,8 +253,8 @@ tests: ticker: AAPL assert: - name: dexter_rubric - type: llm-grader - rubrics: + type: llm-rubric + value: - id: factual operator: correctness criteria: "Uses the supplied company and ticker" @@ -264,8 +264,8 @@ tests: expect(suite.tests[0].input[0].content).toBe('Use Apple (AAPL)'); expect(suite.tests[0].question).toBe('Use Apple (AAPL)'); const grader = suite.tests[0].assertions?.[0]; - expect(grader?.type).toBe('llm-grader'); - if (grader?.type === 'llm-grader') { + expect(grader?.type).toBe('llm-rubric'); + if (grader?.type === 'llm-rubric') { expect(grader.rubrics?.[0]).toMatchObject({ id: 'factual', operator: 'correctness', diff --git a/packages/sdk/src/assertion.ts b/packages/sdk/src/assertion.ts index d5075d0c6..0a832ab03 100644 --- a/packages/sdk/src/assertion.ts +++ b/packages/sdk/src/assertion.ts @@ -37,7 +37,6 @@ export type AssertionContext = ScriptGraderInput; */ export type AssertionType = // kebab-case (canonical internal form) - | 'llm-grader' | 'llm-rubric' | 'agent-rubric' | 'script' diff --git a/packages/sdk/src/graders.ts b/packages/sdk/src/graders.ts index 70f16cd56..905bc8151 100644 --- a/packages/sdk/src/graders.ts +++ b/packages/sdk/src/graders.ts @@ -79,25 +79,6 @@ export interface GraderPromptScriptConfig { readonly config?: Readonly>; } -export interface LlmGraderOptions extends GraderHelperOptions { - readonly prompt?: string | GraderPromptScriptConfig; - readonly rubrics?: readonly GraderRubric[]; - readonly provider?: string; - readonly config?: Readonly>; - readonly maxSteps?: number; - readonly temperature?: number; -} - -export interface LlmGraderConfig extends EvalAssertionConfig, GraderCommonConfig { - readonly type: 'llm-grader'; - readonly prompt?: string | GraderPromptScriptConfig; - readonly rubrics?: readonly GraderRubric[]; - readonly provider?: string; - readonly config?: Readonly>; - readonly maxSteps?: number; - readonly temperature?: number; -} - export interface ScriptGraderProviderOptions { readonly maxCalls?: number; } @@ -134,7 +115,6 @@ export type GraderHelperConfig = | RegexGraderConfig | IsJsonGraderConfig | LlmRubricGraderConfig - | LlmGraderConfig | ScriptGraderConfig; function withCommon( @@ -213,21 +193,6 @@ export function llmRubricGrader( ); } -export function llmGrader(options: LlmGraderOptions = {}): LlmGraderConfig { - return withCommon( - { - type: 'llm-grader', - ...(options.prompt !== undefined ? { prompt: options.prompt } : {}), - ...(options.rubrics !== undefined ? { rubrics: options.rubrics } : {}), - ...(options.provider !== undefined ? { provider: options.provider } : {}), - ...(options.config !== undefined ? { config: options.config } : {}), - ...(options.maxSteps !== undefined ? { maxSteps: options.maxSteps } : {}), - ...(options.temperature !== undefined ? { temperature: options.temperature } : {}), - }, - options, - ); -} - /** @deprecated Use scriptGrader. */ export function codeGrader( command: GraderCommand, @@ -260,7 +225,6 @@ export const graders = Object.freeze({ isJson: isJsonGrader, json: jsonGrader, llmRubric: llmRubricGrader, - llmGrader, codeGrader, script: scriptGrader, scriptGrader, diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 691b53bcf..8c6e44933 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -177,7 +177,6 @@ export { graders, isJsonGrader, jsonGrader, - llmGrader, llmRubricGrader, regexGrader, scriptGrader, @@ -201,8 +200,6 @@ export { type GraderRubricOperator, type GraderScoreRange, type IsJsonGraderConfig, - type LlmGraderConfig, - type LlmGraderOptions, type LlmRubricGraderConfig, type RegexGraderConfig, type RegexGraderOptions, diff --git a/skills-data/agentv-eval-migrations/references/breaking-changes.md b/skills-data/agentv-eval-migrations/references/breaking-changes.md index aadd4f176..7ac6b3871 100644 --- a/skills-data/agentv-eval-migrations/references/breaking-changes.md +++ b/skills-data/agentv-eval-migrations/references/breaking-changes.md @@ -74,7 +74,7 @@ v4.42.4 docs and schema used `assertions` for suite-level and per-test graders: ```yaml assertions: - name: correctness - type: llm-grader + type: llm-rubric prompt: ./graders/correctness.md tests: @@ -961,7 +961,7 @@ assert: - `type: g-eval` -> `type: llm-rubric`. - `type: code-grader`, `code-judge`, `code_grader`, or `code_judge` -> `type: script`. -- `type: llm_judge` or `llm_grader` -> `type: llm-grader`. +- `type: llm_judge` or `llm_grader` -> `type: llm-rubric`. - Convert multi-word snake_case deterministic types to kebab-case: `is_json` -> `is-json`, `contains_all` -> `contains-all`, `starts_with` -> `starts-with`, and so on. @@ -990,9 +990,9 @@ v4.42.4 LLM grader docs allowed both: ```yaml assertions: - - type: llm-grader + - type: llm-rubric prompt: ./graders/correctness.md - - type: llm-grader + - type: llm-rubric prompt: file://graders/correctness.md ``` From f51572f0a33ccfb24ac54315e40fb11240d5781d Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 15:57:56 +0200 Subject: [PATCH 2/3] chore(test): format grader parser rubric cases --- .../evaluation/loaders/grader-parser.test.ts | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/core/test/evaluation/loaders/grader-parser.test.ts b/packages/core/test/evaluation/loaders/grader-parser.test.ts index b6d5184db..d5615b134 100644 --- a/packages/core/test/evaluation/loaders/grader-parser.test.ts +++ b/packages/core/test/evaluation/loaders/grader-parser.test.ts @@ -903,7 +903,8 @@ describe('parseGraders - score_ranges rubrics', () => { assert: [ { metric: 'correctness', - type: 'llm-rubric',value: [ + type: 'llm-rubric', + value: [ { id: 'accuracy', weight: 2.0, @@ -940,7 +941,8 @@ describe('parseGraders - score_ranges rubrics', () => { assert: [ { metric: 'correctness', - type: 'llm-rubric',value: [ + type: 'llm-rubric', + value: [ { id: 'accuracy', required_min_score: 7, @@ -966,7 +968,8 @@ describe('parseGraders - score_ranges rubrics', () => { assert: [ { metric: 'overlapping', - type: 'llm-rubric',value: [ + type: 'llm-rubric', + value: [ { id: 'test', score_ranges: [ @@ -989,7 +992,8 @@ describe('parseGraders - score_ranges rubrics', () => { assert: [ { metric: 'incomplete', - type: 'llm-rubric',value: [ + type: 'llm-rubric', + value: [ { id: 'test', score_ranges: [ @@ -1012,7 +1016,8 @@ describe('parseGraders - score_ranges rubrics', () => { assert: [ { metric: 'legacy', - type: 'llm-rubric',value: [ + type: 'llm-rubric', + value: [ { id: 'r1', description: 'Must be polite', // Legacy field name — no longer supported @@ -1049,7 +1054,8 @@ describe('parseGraders - score_ranges shorthand map', () => { assert: [ { metric: 'shorthand-test', - type: 'llm-rubric',value: [ + type: 'llm-rubric', + value: [ { id: 'accuracy', weight: 2.0, @@ -1101,7 +1107,8 @@ describe('parseGraders - score_ranges shorthand map', () => { assert: [ { metric: 'bad-start', - type: 'llm-rubric',value: [ + type: 'llm-rubric', + value: [ { id: 'test', score_ranges: { @@ -1125,7 +1132,8 @@ describe('parseGraders - score_ranges shorthand map', () => { assert: [ { metric: 'array-format', - type: 'llm-rubric',value: [ + type: 'llm-rubric', + value: [ { id: 'accuracy', score_ranges: [ From 154dbf2178c0ba8d0ffb953edf74712fec6270c8 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 16:02:05 +0200 Subject: [PATCH 3/3] test(cli): update pipeline fixture rubric type --- .../test/commands/eval/pipeline/fixtures/input-test.eval.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/cli/test/commands/eval/pipeline/fixtures/input-test.eval.yaml b/apps/cli/test/commands/eval/pipeline/fixtures/input-test.eval.yaml index 40c08b24e..b96121313 100644 --- a/apps/cli/test/commands/eval/pipeline/fixtures/input-test.eval.yaml +++ b/apps/cli/test/commands/eval/pipeline/fixtures/input-test.eval.yaml @@ -12,8 +12,8 @@ tests: command: node grader-score-1.js weight: 1 - metric: relevance - type: llm-grader - prompt: Did the response echo the input? + type: llm-rubric + value: Did the response echo the input? weight: 2 vars: input: hello world