fix(eval): reject top-level experiment authoring

apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx

@@ -66,10 +66,10 @@ agentv eval evals/my-eval.yaml --experiment without_skills
 ```
 
 The experiment label chooses the result bucket and is propagated to each entry
-in `index.jsonl`. CLI `--experiment` wins over `experiment.name` in the eval
-file. If neither is set, AgentV writes to the `default` bucket. The eval file
-stays the same across experiments; what changes is the runtime condition.
-Dashboards can filter and compare results by experiment.
+in `index.jsonl`. CLI `--experiment` wins over `tags.experiment` authored in
+the eval file. If neither is set, AgentV writes to the `default` bucket. The
+eval file can stay the same across experiments; what changes is the runtime
+condition. Dashboards can filter and compare results by experiment.
 
 ### Run Specific Test
 

apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx

@@ -67,10 +67,10 @@ agentv eval evals/my-eval.yaml --experiment without_skills
 ```
 
 The experiment label chooses the result bucket and is propagated to each entry
-in `index.jsonl`. CLI `--experiment` wins over `experiment.name` in the eval
-file. If neither is set, AgentV writes to the `default` bucket. The eval file
-stays the same across experiments; what changes is the runtime condition.
-Dashboards can filter and compare results by experiment.
+in `index.jsonl`. CLI `--experiment` wins over `tags.experiment` authored in
+the eval file. If neither is set, AgentV writes to the `default` bucket. The
+eval file can stay the same across experiments; what changes is the runtime
+condition. Dashboards can filter and compare results by experiment.
 
 ### Run Specific Test
 

docs/adr/0013-stabilize-eval-authoring-contract.md

@@ -55,7 +55,8 @@ The preferred eval authoring contract is:
 
 ```yaml
 name: code-generation-quality
-experiment: with-skills
+tags:
+  experiment: with-skills
 target: copilot-sdk
 evaluate_options:
   repeat: 3
@@ -81,12 +82,13 @@ artifact routing, cache keys that should track executable behavior, or result
 comparison semantics. Source identity belongs to `eval_path` and run metadata,
 not to the display name.
 
-`experiment` remains the optional top-level string run/result grouping label.
-It names the condition being measured, such as `baseline`, `candidate`,
+The reserved `tags.experiment` key is the eval-authored run/result grouping
+label. It names the condition being measured, such as `baseline`, `candidate`,
 `with-skills`, or `without-skills`. It is not a runtime-policy object, not a
 separate artifact type, and not a storage path namespace. It should not repeat
 the suite name, category, target, provider, or model; those are separate
-dimensions in run metadata.
+dimensions in run metadata. Authored eval YAML does not expose a top-level
+`experiment` field.
 
 Top-level `description` is not part of the preferred eval authoring contract.
 Existing files that contain it may be read as legacy display metadata, but it

packages/core/src/evaluation/loaders/config-loader.ts

@@ -430,8 +430,10 @@ function stripLocalOnlyExecutionDefaults(
 }
 
 function rejectAuthoredRuntimeContainers(suite: JsonObject): void {
-  if (suite.experiment !== undefined && typeof suite.experiment !== 'string') {
-    throw new Error("Invalid top-level 'experiment': use a string run/result grouping label.");
+  if (suite.experiment !== undefined) {
+    throw new Error(
+      "Top-level 'experiment' has been removed from authored eval YAML. Use tags.experiment in the eval file or CLI --experiment at run time.",
+    );
   }
   if (suite.policy !== undefined) {
     throw new Error(

packages/core/src/evaluation/validation/eval-file.schema.ts

@@ -923,6 +923,9 @@ const TagsSchema = z.union([
 const TOP_LEVEL_IMPORTS_MESSAGE =
   "Top-level 'imports' is not supported. Run eval files directly with CLI multi-file selection and tags for grouping. For raw case files, use tests: file://... or string entries under tests. For reusable scenarios, use scenarios: [file://...]. For reusable config, use prompts: file://..., default_test: file://..., and environment: file://... for coding-agent testbeds.";
 
+const TOP_LEVEL_EXPERIMENT_MESSAGE =
+  "Top-level 'experiment' has been removed from authored eval YAML. Use tags.experiment in the eval file or CLI --experiment at run time.";
+
 // ---------------------------------------------------------------------------
 // Top-level eval file
 // ---------------------------------------------------------------------------
@@ -995,8 +998,8 @@ export const EvalFileSchemaInput: z.ZodType = z.object({
     .optional(),
   providers: EvalProvidersSchema.optional(),
   model: z.never().optional(),
-  // Run/result grouping label and flat run controls
-  experiment: z.string().min(1).optional(),
+  // Flat run controls. Group authored YAML with tags.experiment or CLI --experiment.
+  experiment: z.never({ invalid_type_error: TOP_LEVEL_EXPERIMENT_MESSAGE }).optional(),
   repeat: z.never().optional(),
   runs: z.never().optional(),
   early_exit: z.never().optional(),

packages/core/src/evaluation/validation/eval-validator.ts

@@ -290,6 +290,10 @@ const KNOWN_TEST_EXECUTION_FIELDS = new Set([
 
 /** Removed top-level fields with migration hints. */
 const REMOVED_TOP_LEVEL_FIELDS = new Map<string, string>([
+  [
+    'experiment',
+    "Top-level 'experiment' has been removed from authored eval YAML. Use tags.experiment in the eval file or CLI --experiment at run time.",
+  ],
   [
     'imports',
     "Top-level 'imports' is not supported. Run eval files directly with CLI multi-file selection and tags for grouping. For raw case files, use tests: file://... or string entries under tests. For reusable scenarios, use scenarios: [file://...]. For reusable config, use prompts: file://..., default_test: file://..., and environment: file://... for coding-agent testbeds.",
@@ -547,14 +551,6 @@ export async function validateEvalFile(filePath: string): Promise<ValidationResu
 
   // Validate metadata fields
   validateMetadata(parsed, absolutePath, errors);
-  if (parsed.experiment !== undefined && typeof parsed.experiment !== 'string') {
-    errors.push({
-      severity: 'error',
-      filePath: absolutePath,
-      location: 'experiment',
-      message: "Top-level 'experiment' must be a string run/result grouping label.",
-    });
-  }
 
   // Match promptfoo-style validation: unknown authored suite fields are hard
   // errors, not advisory warnings, because they otherwise silently do nothing.

packages/core/src/evaluation/yaml-parser.ts

@@ -2621,9 +2621,9 @@ function parentEnvironmentLocation(suite: RawTestSuite): string | undefined {
 }
 
 function readSuiteRuntimeBlock(suite: RawTestSuite, evalFilePath: string): JsonObject | undefined {
-  if (suite.experiment !== undefined && typeof suite.experiment !== 'string') {
+  if (suite.experiment !== undefined) {
     throw new Error(
-      `Invalid eval runtime config in ${evalFilePath}: top-level 'experiment' must be a string run/result grouping label.`,
+      `Invalid eval runtime config in ${evalFilePath}: top-level 'experiment' has been removed from authored eval YAML. Use tags.experiment in the eval file or CLI --experiment at run time.`,
     );
   }
   if (suite.policy !== undefined) {
@@ -2689,11 +2689,9 @@ function normalizeSuiteExperimentConfig(parsed: JsonObject): ExperimentConfig |
   readSuiteRuntimeBlock(suite, 'eval file');
   const suiteTargets = extractTargetsFromSuite(parsed);
   const singleSuiteTarget = suiteTargets?.length === 1 ? suiteTargets[0] : undefined;
-  const experimentName = asString(suite.experiment);
   const budgetUsd = extractBudgetUsd(parsed);
   const evaluateOptions = isJsonObject(suite.evaluate_options) ? suite.evaluate_options : undefined;
   const runtimeConfig: JsonObject = {
-    ...(experimentName !== undefined ? { name: experimentName } : {}),
     ...(singleSuiteTarget !== undefined ? { target: singleSuiteTarget } : {}),
     ...(evaluateOptions?.repeat !== undefined ? { repeat: evaluateOptions.repeat } : {}),
     ...(suite.timeout_seconds !== undefined ? { timeout_seconds: suite.timeout_seconds } : {}),

packages/core/test/evaluation/eval-inline-experiment.test.ts

@@ -23,7 +23,8 @@ describe('eval.yaml flat runtime controls and tests imports', () => {
       evalPath,
       [
         'name: runtime-suite',
-        'experiment: release-gate',
+        'tags:',
+        '  experiment: release-gate',
         'providers:',
         '  - id: agentv:codex-cli',
         '    label: codex',
@@ -49,12 +50,12 @@ describe('eval.yaml flat runtime controls and tests imports', () => {
 
     expect(suite.experimentConfig).toMatchObject({
       target: 'codex',
-      name: 'release-gate',
       threshold: 0.7,
       repeat: { count: 2, strategy: 'pass_any' },
       timeoutSeconds: 30,
       budgetUsd: 1.5,
     });
+    expect(suite.tags).toEqual({ experiment: 'release-gate' });
     expect(suite.targetSpec).toBeUndefined();
     expect(suite.targets).toEqual(['codex']);
   });
@@ -667,7 +668,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => {
     await expect(loadTestSuite(evalPath, tempDir)).rejects.toThrow(/evaluate_options\.repeat/);
   });
 
-  it('rejects top-level execution blocks and non-string experiment values', async () => {
+  it('rejects top-level execution blocks and removed experiment values', async () => {
     const legacyPath = path.join(tempDir, 'legacy.eval.yaml');
     await writeFile(
       legacyPath,
@@ -702,8 +703,25 @@ describe('eval.yaml flat runtime controls and tests imports', () => {
       ].join('\n'),
     );
 
-    await expect(loadTestSuite(removedPath, tempDir)).rejects.toThrow(
-      /top-level 'experiment' must be a string/,
+    await expect(loadTestSuite(removedPath, tempDir)).rejects.toThrow(/tags\.experiment/);
+
+    const stringExperimentPath = path.join(tempDir, 'string-experiment.eval.yaml');
+    await writeFile(
+      stringExperimentPath,
+      [
+        'experiment: release-gate',
+        'prompts:',
+        '  - "{{ input }}"',
+        'tests:',
+        '  - id: one',
+        '    criteria: ok',
+        '    vars:',
+        '      input: hello',
+      ].join('\n'),
+    );
+
+    await expect(loadTestSuite(stringExperimentPath, tempDir)).rejects.toThrow(
+      /tags\.experiment.*CLI --experiment/,
     );
   });
 
@@ -1256,7 +1274,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => {
     );
 
     await expect(loadTestSuite(parentPath, tempDir)).rejects.toThrow(
-      /top-level 'experiment' must be a string/,
+      /tags\.experiment.*CLI --experiment/,
     );
   });
 

packages/core/test/evaluation/validation/eval-file-schema.test.ts

@@ -293,7 +293,7 @@ describe('EvalFileSchema input shorthand', () => {
     const result = EvalFileSchema.safeParse({
       name: 'wrapper',
       description: 'Wrapper eval',
-      experiment: 'release-gate',
+      tags: { experiment: 'release-gate' },
       providers: [{ id: 'agentv:codex-cli', label: 'codex' }],
       threshold: 0.8,
       timeout_seconds: 300,
@@ -337,6 +337,18 @@ describe('EvalFileSchema input shorthand', () => {
     expect(result.success).toBe(true);
   });
 
+  it('rejects top-level experiment strings with a migration hint', () => {
+    const result = EvalFileSchema.safeParse({
+      experiment: 'release-gate',
+      tests: [baseTest],
+    });
+
+    expect(result.success).toBe(false);
+    expect(result.error?.issues.some((issue) => issue.path.join('.') === 'experiment')).toBe(true);
+    expect(JSON.stringify(result.error?.issues)).toContain('tags.experiment');
+    expect(JSON.stringify(result.error?.issues)).toContain('CLI --experiment');
+  });
+
   it('accepts Promptfoo-style colon provider specs', () => {
     const result = EvalFileSchema.safeParse({
       name: 'colon-providers',
@@ -790,6 +802,7 @@ describe('EvalFileSchema input shorthand', () => {
     });
 
     expect(result.success).toBe(false);
+    expect(JSON.stringify(result.error?.issues)).toContain('tags.experiment');
   });
 
   it('rejects lifecycle commands under authored policy blocks', () => {

packages/core/test/evaluation/validation/eval-validator.test.ts

@@ -1066,7 +1066,9 @@ tests:
         (error) =>
           error.severity === 'error' &&
           error.location === 'experiment' &&
-          error.message.includes("Top-level 'experiment' must be a string"),
+          error.message.includes("Top-level 'experiment' has been removed") &&
+          error.message.includes('tags.experiment') &&
+          error.message.includes('CLI --experiment'),
       ),
     ).toBe(true);
   });
@@ -1130,7 +1132,9 @@ tests:
         (error) =>
           error.severity === 'error' &&
           error.location === 'experiment' &&
-          error.message.includes("Top-level 'experiment' must be a string"),
+          error.message.includes("Top-level 'experiment' has been removed") &&
+          error.message.includes('tags.experiment') &&
+          error.message.includes('CLI --experiment'),
       ),
     ).toBe(true);
   });
@@ -1159,7 +1163,9 @@ tests:
         (error) =>
           error.severity === 'error' &&
           error.location === 'experiment' &&
-          error.message.includes("Top-level 'experiment' must be a string"),
+          error.message.includes("Top-level 'experiment' has been removed") &&
+          error.message.includes('tags.experiment') &&
+          error.message.includes('CLI --experiment'),
       ),
     ).toBe(true);
   });
@@ -1187,7 +1193,9 @@ tests:
         (error) =>
           error.severity === 'error' &&
           error.location === 'experiment' &&
-          error.message.includes("Top-level 'experiment' must be a string"),
+          error.message.includes("Top-level 'experiment' has been removed") &&
+          error.message.includes('tags.experiment') &&
+          error.message.includes('CLI --experiment'),
       ),
     ).toBe(true);
   });
@@ -1364,7 +1372,7 @@ tests: []
     expect(importsError?.message).toContain('CLI multi-file selection');
   });
 
-  it('rejects removed execution blocks when experiment label is present', async () => {
+  it('rejects removed execution blocks when top-level experiment is present', async () => {
     const filePath = path.join(tempDir, 'runtime-conflict.yaml');
     await writeFile(
       filePath,
@@ -1381,6 +1389,14 @@ tests:
     const result = await validateEvalFile(filePath);
 
     expect(result.valid).toBe(false);
+    expect(
+      result.errors.some(
+        (error) =>
+          error.location === 'experiment' &&
+          error.message.includes('tags.experiment') &&
+          error.message.includes('CLI --experiment'),
+      ),
+    ).toBe(true);
     expect(
       result.errors.some(
         (error) =>
@@ -2475,7 +2491,9 @@ tests: "./cases-shorthand-workspace.yaml"
           (error) =>
             error.severity === 'error' &&
             error.location === 'experiment' &&
-            error.message.includes("Top-level 'experiment' must be a string"),
+            error.message.includes("Top-level 'experiment' has been removed") &&
+            error.message.includes('tags.experiment') &&
+            error.message.includes('CLI --experiment'),
         ),
       ).toBe(true);
     });

packages/sdk/src/eval.ts

@@ -259,11 +259,8 @@ export interface EvalConfig {
   readonly defaults?: EvalDefaultsConfig;
   readonly defaultTest?: EvalDefaultTest | string;
   readonly tests: readonly EvalTest[] | string;
-  /**
-   * @deprecated A top-level `experiment` label no longer sets the run's
-   * experiment namespace. Use `tags: { experiment: '<name>' }` instead.
-   */
-  readonly experiment?: string;
+  /** @deprecated Rejected. Use `tags: { experiment: '<name>' }` or CLI --experiment. */
+  readonly experiment?: never;
   readonly repeat?: EvalRepeat;
   readonly timeoutSeconds?: number;
   readonly threshold?: number;
@@ -375,11 +372,10 @@ function validateTopLevelRuntimeFields(definition: EvalConfig): void {
       "defineEval() no longer accepts top-level 'graders'. Put grader providers in 'providers' and select them with defaults.grader, defaultTest.options.provider, tests[].options.provider, or assertion provider.",
     );
   }
-  if (
-    Object.prototype.hasOwnProperty.call(rawDefinition, 'experiment') &&
-    typeof rawDefinition.experiment !== 'string'
-  ) {
-    throw new Error("defineEval() expects top-level 'experiment' to be a string label.");
+  if (Object.prototype.hasOwnProperty.call(rawDefinition, 'experiment')) {
+    throw new Error(
+      "defineEval() no longer accepts top-level 'experiment'. Use tags: { experiment: '<name>' } or CLI --experiment.",
+    );
   }
   for (const field of ['model', 'policy', 'execution', 'runs', 'earlyExit']) {
     if (Object.prototype.hasOwnProperty.call(rawDefinition, field)) {

packages/sdk/test/eval-authoring.test.ts

@@ -7,7 +7,7 @@ describe('YAML-aligned eval authoring helpers', () => {
     const suite = defineEval({
       name: 'sdk-yaml-suite',
       inputFiles: ['fixtures/shared-system.md'],
-      experiment: 'sdk-yaml-run',
+      tags: { experiment: 'sdk-yaml-run' },
       providers: [
         {
           label: 'sdk-codex',
@@ -97,7 +97,7 @@ describe('YAML-aligned eval authoring helpers', () => {
     expect(lowered).toEqual({
       name: 'sdk-yaml-suite',
       input_files: ['fixtures/shared-system.md'],
-      experiment: 'sdk-yaml-run',
+      tags: { experiment: 'sdk-yaml-run' },
       providers: [
         {
           label: 'sdk-codex',
@@ -363,11 +363,11 @@ describe('YAML-aligned eval authoring helpers', () => {
     ).toThrow(/top-level 'preprocessors'.*defaultTest\.options\.transform/);
   });
 
-  it('rejects removed experiment authoring blocks', () => {
+  it('rejects removed top-level experiment authoring', () => {
     expect(() =>
       defineEval({
         name: 'removed-experiment',
-        experiment: { target: 'mock' },
+        experiment: 'sdk-baseline',
         prompts: ['{{ input }}'],
         tests: [
           {
@@ -377,7 +377,7 @@ describe('YAML-aligned eval authoring helpers', () => {
           },
         ],
       } as never),
-    ).toThrow(/top-level 'experiment'/);
+    ).toThrow(/top-level 'experiment'.*tags.*experiment.*CLI --experiment/);
   });
 
   it('rejects removed top-level repeat aliases', () => {