diff --git a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx index 235918396..d8c0b12fa 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx @@ -66,10 +66,10 @@ agentv eval evals/my-eval.yaml --experiment without_skills ``` The experiment label chooses the result bucket and is propagated to each entry -in `index.jsonl`. CLI `--experiment` wins over `experiment.name` in the eval -file. If neither is set, AgentV writes to the `default` bucket. The eval file -stays the same across experiments; what changes is the runtime condition. -Dashboards can filter and compare results by experiment. +in `index.jsonl`. CLI `--experiment` wins over `tags.experiment` authored in +the eval file. If neither is set, AgentV writes to the `default` bucket. The +eval file can stay the same across experiments; what changes is the runtime +condition. Dashboards can filter and compare results by experiment. ### Run Specific Test diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx index 8a9fa9838..601507e79 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx @@ -67,10 +67,10 @@ agentv eval evals/my-eval.yaml --experiment without_skills ``` The experiment label chooses the result bucket and is propagated to each entry -in `index.jsonl`. CLI `--experiment` wins over `experiment.name` in the eval -file. If neither is set, AgentV writes to the `default` bucket. The eval file -stays the same across experiments; what changes is the runtime condition. -Dashboards can filter and compare results by experiment. +in `index.jsonl`. CLI `--experiment` wins over `tags.experiment` authored in +the eval file. If neither is set, AgentV writes to the `default` bucket. The +eval file can stay the same across experiments; what changes is the runtime +condition. Dashboards can filter and compare results by experiment. ### Run Specific Test diff --git a/docs/adr/0013-stabilize-eval-authoring-contract.md b/docs/adr/0013-stabilize-eval-authoring-contract.md index 7aecc2d7f..4e01da7d3 100644 --- a/docs/adr/0013-stabilize-eval-authoring-contract.md +++ b/docs/adr/0013-stabilize-eval-authoring-contract.md @@ -55,7 +55,8 @@ The preferred eval authoring contract is: ```yaml name: code-generation-quality -experiment: with-skills +tags: + experiment: with-skills target: copilot-sdk evaluate_options: repeat: 3 @@ -81,12 +82,13 @@ artifact routing, cache keys that should track executable behavior, or result comparison semantics. Source identity belongs to `eval_path` and run metadata, not to the display name. -`experiment` remains the optional top-level string run/result grouping label. -It names the condition being measured, such as `baseline`, `candidate`, +The reserved `tags.experiment` key is the eval-authored run/result grouping +label. It names the condition being measured, such as `baseline`, `candidate`, `with-skills`, or `without-skills`. It is not a runtime-policy object, not a separate artifact type, and not a storage path namespace. It should not repeat the suite name, category, target, provider, or model; those are separate -dimensions in run metadata. +dimensions in run metadata. Authored eval YAML does not expose a top-level +`experiment` field. Top-level `description` is not part of the preferred eval authoring contract. Existing files that contain it may be read as legacy display metadata, but it diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index f03ce5bcf..450abe7a2 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -430,8 +430,10 @@ function stripLocalOnlyExecutionDefaults( } function rejectAuthoredRuntimeContainers(suite: JsonObject): void { - if (suite.experiment !== undefined && typeof suite.experiment !== 'string') { - throw new Error("Invalid top-level 'experiment': use a string run/result grouping label."); + if (suite.experiment !== undefined) { + throw new Error( + "Top-level 'experiment' has been removed from authored eval YAML. Use tags.experiment in the eval file or CLI --experiment at run time.", + ); } if (suite.policy !== undefined) { throw new Error( diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 12b72f0f5..57907e3e6 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -923,6 +923,9 @@ const TagsSchema = z.union([ const TOP_LEVEL_IMPORTS_MESSAGE = "Top-level 'imports' is not supported. Run eval files directly with CLI multi-file selection and tags for grouping. For raw case files, use tests: file://... or string entries under tests. For reusable scenarios, use scenarios: [file://...]. For reusable config, use prompts: file://..., default_test: file://..., and environment: file://... for coding-agent testbeds."; +const TOP_LEVEL_EXPERIMENT_MESSAGE = + "Top-level 'experiment' has been removed from authored eval YAML. Use tags.experiment in the eval file or CLI --experiment at run time."; + // --------------------------------------------------------------------------- // Top-level eval file // --------------------------------------------------------------------------- @@ -995,8 +998,8 @@ export const EvalFileSchemaInput: z.ZodType = z.object({ .optional(), providers: EvalProvidersSchema.optional(), model: z.never().optional(), - // Run/result grouping label and flat run controls - experiment: z.string().min(1).optional(), + // Flat run controls. Group authored YAML with tags.experiment or CLI --experiment. + experiment: z.never({ invalid_type_error: TOP_LEVEL_EXPERIMENT_MESSAGE }).optional(), repeat: z.never().optional(), runs: z.never().optional(), early_exit: z.never().optional(), diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 949250fe5..f618305f0 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -290,6 +290,10 @@ const KNOWN_TEST_EXECUTION_FIELDS = new Set([ /** Removed top-level fields with migration hints. */ const REMOVED_TOP_LEVEL_FIELDS = new Map([ + [ + 'experiment', + "Top-level 'experiment' has been removed from authored eval YAML. Use tags.experiment in the eval file or CLI --experiment at run time.", + ], [ 'imports', "Top-level 'imports' is not supported. Run eval files directly with CLI multi-file selection and tags for grouping. For raw case files, use tests: file://... or string entries under tests. For reusable scenarios, use scenarios: [file://...]. For reusable config, use prompts: file://..., default_test: file://..., and environment: file://... for coding-agent testbeds.", @@ -547,14 +551,6 @@ export async function validateEvalFile(filePath: string): Promise { evalPath, [ 'name: runtime-suite', - 'experiment: release-gate', + 'tags:', + ' experiment: release-gate', 'providers:', ' - id: agentv:codex-cli', ' label: codex', @@ -49,12 +50,12 @@ describe('eval.yaml flat runtime controls and tests imports', () => { expect(suite.experimentConfig).toMatchObject({ target: 'codex', - name: 'release-gate', threshold: 0.7, repeat: { count: 2, strategy: 'pass_any' }, timeoutSeconds: 30, budgetUsd: 1.5, }); + expect(suite.tags).toEqual({ experiment: 'release-gate' }); expect(suite.targetSpec).toBeUndefined(); expect(suite.targets).toEqual(['codex']); }); @@ -667,7 +668,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { await expect(loadTestSuite(evalPath, tempDir)).rejects.toThrow(/evaluate_options\.repeat/); }); - it('rejects top-level execution blocks and non-string experiment values', async () => { + it('rejects top-level execution blocks and removed experiment values', async () => { const legacyPath = path.join(tempDir, 'legacy.eval.yaml'); await writeFile( legacyPath, @@ -702,8 +703,25 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ].join('\n'), ); - await expect(loadTestSuite(removedPath, tempDir)).rejects.toThrow( - /top-level 'experiment' must be a string/, + await expect(loadTestSuite(removedPath, tempDir)).rejects.toThrow(/tags\.experiment/); + + const stringExperimentPath = path.join(tempDir, 'string-experiment.eval.yaml'); + await writeFile( + stringExperimentPath, + [ + 'experiment: release-gate', + 'prompts:', + ' - "{{ input }}"', + 'tests:', + ' - id: one', + ' criteria: ok', + ' vars:', + ' input: hello', + ].join('\n'), + ); + + await expect(loadTestSuite(stringExperimentPath, tempDir)).rejects.toThrow( + /tags\.experiment.*CLI --experiment/, ); }); @@ -1256,7 +1274,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ); await expect(loadTestSuite(parentPath, tempDir)).rejects.toThrow( - /top-level 'experiment' must be a string/, + /tags\.experiment.*CLI --experiment/, ); }); diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index b1e4f0755..a1fe09b33 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -293,7 +293,7 @@ describe('EvalFileSchema input shorthand', () => { const result = EvalFileSchema.safeParse({ name: 'wrapper', description: 'Wrapper eval', - experiment: 'release-gate', + tags: { experiment: 'release-gate' }, providers: [{ id: 'agentv:codex-cli', label: 'codex' }], threshold: 0.8, timeout_seconds: 300, @@ -337,6 +337,18 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(true); }); + it('rejects top-level experiment strings with a migration hint', () => { + const result = EvalFileSchema.safeParse({ + experiment: 'release-gate', + tests: [baseTest], + }); + + expect(result.success).toBe(false); + expect(result.error?.issues.some((issue) => issue.path.join('.') === 'experiment')).toBe(true); + expect(JSON.stringify(result.error?.issues)).toContain('tags.experiment'); + expect(JSON.stringify(result.error?.issues)).toContain('CLI --experiment'); + }); + it('accepts Promptfoo-style colon provider specs', () => { const result = EvalFileSchema.safeParse({ name: 'colon-providers', @@ -790,6 +802,7 @@ describe('EvalFileSchema input shorthand', () => { }); expect(result.success).toBe(false); + expect(JSON.stringify(result.error?.issues)).toContain('tags.experiment'); }); it('rejects lifecycle commands under authored policy blocks', () => { diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 1400545f0..af52e7098 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -1066,7 +1066,9 @@ tests: (error) => error.severity === 'error' && error.location === 'experiment' && - error.message.includes("Top-level 'experiment' must be a string"), + error.message.includes("Top-level 'experiment' has been removed") && + error.message.includes('tags.experiment') && + error.message.includes('CLI --experiment'), ), ).toBe(true); }); @@ -1130,7 +1132,9 @@ tests: (error) => error.severity === 'error' && error.location === 'experiment' && - error.message.includes("Top-level 'experiment' must be a string"), + error.message.includes("Top-level 'experiment' has been removed") && + error.message.includes('tags.experiment') && + error.message.includes('CLI --experiment'), ), ).toBe(true); }); @@ -1159,7 +1163,9 @@ tests: (error) => error.severity === 'error' && error.location === 'experiment' && - error.message.includes("Top-level 'experiment' must be a string"), + error.message.includes("Top-level 'experiment' has been removed") && + error.message.includes('tags.experiment') && + error.message.includes('CLI --experiment'), ), ).toBe(true); }); @@ -1187,7 +1193,9 @@ tests: (error) => error.severity === 'error' && error.location === 'experiment' && - error.message.includes("Top-level 'experiment' must be a string"), + error.message.includes("Top-level 'experiment' has been removed") && + error.message.includes('tags.experiment') && + error.message.includes('CLI --experiment'), ), ).toBe(true); }); @@ -1364,7 +1372,7 @@ tests: [] expect(importsError?.message).toContain('CLI multi-file selection'); }); - it('rejects removed execution blocks when experiment label is present', async () => { + it('rejects removed execution blocks when top-level experiment is present', async () => { const filePath = path.join(tempDir, 'runtime-conflict.yaml'); await writeFile( filePath, @@ -1381,6 +1389,14 @@ tests: const result = await validateEvalFile(filePath); expect(result.valid).toBe(false); + expect( + result.errors.some( + (error) => + error.location === 'experiment' && + error.message.includes('tags.experiment') && + error.message.includes('CLI --experiment'), + ), + ).toBe(true); expect( result.errors.some( (error) => @@ -2475,7 +2491,9 @@ tests: "./cases-shorthand-workspace.yaml" (error) => error.severity === 'error' && error.location === 'experiment' && - error.message.includes("Top-level 'experiment' must be a string"), + error.message.includes("Top-level 'experiment' has been removed") && + error.message.includes('tags.experiment') && + error.message.includes('CLI --experiment'), ), ).toBe(true); }); diff --git a/packages/sdk/src/eval.ts b/packages/sdk/src/eval.ts index 8878599b8..47a5a3889 100644 --- a/packages/sdk/src/eval.ts +++ b/packages/sdk/src/eval.ts @@ -259,11 +259,8 @@ export interface EvalConfig { readonly defaults?: EvalDefaultsConfig; readonly defaultTest?: EvalDefaultTest | string; readonly tests: readonly EvalTest[] | string; - /** - * @deprecated A top-level `experiment` label no longer sets the run's - * experiment namespace. Use `tags: { experiment: '' }` instead. - */ - readonly experiment?: string; + /** @deprecated Rejected. Use `tags: { experiment: '' }` or CLI --experiment. */ + readonly experiment?: never; readonly repeat?: EvalRepeat; readonly timeoutSeconds?: number; readonly threshold?: number; @@ -375,11 +372,10 @@ function validateTopLevelRuntimeFields(definition: EvalConfig): void { "defineEval() no longer accepts top-level 'graders'. Put grader providers in 'providers' and select them with defaults.grader, defaultTest.options.provider, tests[].options.provider, or assertion provider.", ); } - if ( - Object.prototype.hasOwnProperty.call(rawDefinition, 'experiment') && - typeof rawDefinition.experiment !== 'string' - ) { - throw new Error("defineEval() expects top-level 'experiment' to be a string label."); + if (Object.prototype.hasOwnProperty.call(rawDefinition, 'experiment')) { + throw new Error( + "defineEval() no longer accepts top-level 'experiment'. Use tags: { experiment: '' } or CLI --experiment.", + ); } for (const field of ['model', 'policy', 'execution', 'runs', 'earlyExit']) { if (Object.prototype.hasOwnProperty.call(rawDefinition, field)) { diff --git a/packages/sdk/test/eval-authoring.test.ts b/packages/sdk/test/eval-authoring.test.ts index d983724fb..ee203b74b 100644 --- a/packages/sdk/test/eval-authoring.test.ts +++ b/packages/sdk/test/eval-authoring.test.ts @@ -7,7 +7,7 @@ describe('YAML-aligned eval authoring helpers', () => { const suite = defineEval({ name: 'sdk-yaml-suite', inputFiles: ['fixtures/shared-system.md'], - experiment: 'sdk-yaml-run', + tags: { experiment: 'sdk-yaml-run' }, providers: [ { label: 'sdk-codex', @@ -97,7 +97,7 @@ describe('YAML-aligned eval authoring helpers', () => { expect(lowered).toEqual({ name: 'sdk-yaml-suite', input_files: ['fixtures/shared-system.md'], - experiment: 'sdk-yaml-run', + tags: { experiment: 'sdk-yaml-run' }, providers: [ { label: 'sdk-codex', @@ -363,11 +363,11 @@ describe('YAML-aligned eval authoring helpers', () => { ).toThrow(/top-level 'preprocessors'.*defaultTest\.options\.transform/); }); - it('rejects removed experiment authoring blocks', () => { + it('rejects removed top-level experiment authoring', () => { expect(() => defineEval({ name: 'removed-experiment', - experiment: { target: 'mock' }, + experiment: 'sdk-baseline', prompts: ['{{ input }}'], tests: [ { @@ -377,7 +377,7 @@ describe('YAML-aligned eval authoring helpers', () => { }, ], } as never), - ).toThrow(/top-level 'experiment'/); + ).toThrow(/top-level 'experiment'.*tags.*experiment.*CLI --experiment/); }); it('rejects removed top-level repeat aliases', () => { diff --git a/skills-data/agentv-eval-migrations/references/breaking-changes.md b/skills-data/agentv-eval-migrations/references/breaking-changes.md index aadd4f176..982ea8582 100644 --- a/skills-data/agentv-eval-migrations/references/breaking-changes.md +++ b/skills-data/agentv-eval-migrations/references/breaking-changes.md @@ -539,11 +539,13 @@ experiment: ### Current Shape -Current schema accepts top-level `experiment` only as a non-empty string run -grouping label. Current docs also support promptfoo-shaped `tags.experiment`. +Current schema rejects top-level `experiment` in authored eval YAML. Use the +promptfoo-shaped `tags.experiment` key when the label belongs in the eval file, +or pass CLI `--experiment` when the label is run-time context. ```yaml -experiment: with-skills +tags: + experiment: with-skills target: codex timeout_seconds: 600 evaluate_options: @@ -552,14 +554,6 @@ evaluate_options: strategy: pass_any ``` -or: - -```yaml -tags: - experiment: with-skills -target: codex -``` - ### Migration Steps - If `experiment` is an object, move runtime fields out: @@ -568,7 +562,8 @@ target: codex - budget -> `evaluate_options.budget_usd` - timeout -> top-level `timeout_seconds` - threshold -> top-level `threshold` -- Keep only a string label in `experiment`, or use `tags.experiment`. +- Move string experiment labels to `tags.experiment`, or supply them at run time + with CLI `--experiment`. ### Verification @@ -577,6 +572,8 @@ bun apps/cli/src/cli.ts validate path/to/eval.eval.yaml rg -n "^experiment:" path/to/evals ``` +Any match under authored eval YAML should be removed or migrated. + ### Compatibility Notes Do not describe a v4.42.4 eval as if `experiment:` was already the main runtime diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 21a7f4c28..8724355b7 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -142,7 +142,7 @@ tests: ## Eval File Structure **Required:** `tests` (array or string raw-case path) or `scenarios` -**Optional:** `name`, `description`, `experiment`, `version`, `author`, `tags`, `license`, `requires`, `target`, `targets`, `prompts`, `default_test`, `timeout_seconds`, `evaluate_options`, `threshold`, `suite`, `environment`, `env`, `extensions`, `assert` +**Optional:** `name`, `description`, `version`, `author`, `tags`, `license`, `requires`, `target`, `targets`, `prompts`, `default_test`, `timeout_seconds`, `evaluate_options`, `threshold`, `suite`, `environment`, `env`, `extensions`, `assert` **Test fields:** diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index 526d83ea2..5567dc1ec 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -3100,8 +3100,7 @@ "not": {} }, "experiment": { - "type": "string", - "minLength": 1 + "not": {} }, "repeat": { "not": {}