diff --git a/apps/web/src/content/docs/docs/next/targets/configuration.mdx b/apps/web/src/content/docs/docs/next/targets/configuration.mdx index 18b5b34c3..6e9042230 100644 --- a/apps/web/src/content/docs/docs/next/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/next/targets/configuration.mdx @@ -295,20 +295,50 @@ tests: - id: test-2 ``` -The string is a configured provider `label` or `id`. Use object form when an -eval needs a local provider variant: +The string is usually a configured provider `label` or `id`. Promptfoo-style +provider spec strings such as `openai:gpt-4.1-mini` are also accepted and lower +to an inline provider definition. Use object form when an eval needs a local +provider variant: ```yaml providers: - id: codex-app-server label: codex-high-reasoning - runtime: host - config: - command: ["codex", "app-server"] - model: gpt-5-codex - reasoning_effort: high + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex + reasoning_effort: high ``` +AgentV accepts the Promptfoo provider declaration layer where the semantics +match: strings, object form, provider maps, and `inputs`. In object form, `id` +is the backend/spec and `label` is the stable AgentV result and selection +identity. + +```yaml +providers: + - openai:gpt-4.1-mini + - openai:gpt-4: + label: gpt4-low-temp + config: + temperature: 0 + inputs: + prompt: User prompt text + - id: openai:responses:gpt-5.4 + label: gpt5-responses + inputs: + prompt: + type: text + description: User prompt text +``` + +`runtime`, provider-local `environment`, and provider `hooks` are AgentV +extensions. Direct Promptfoo runs do not execute those fields. A Promptfoo +export path must either lower a supported AgentV extension into Promptfoo +providers/extensions or reject the config with a clear diagnostic instead of +silently dropping runtime or testbed behavior. + Use `defaults.grader` for the project default grader. `default_test.options.provider` and `tests[].options.provider` can choose a grader provider for LLM-backed assertions before an assertion-level `provider` override takes final precedence. @@ -443,7 +473,6 @@ Provider hooks can be scoped to an eval-local provider object: providers: - id: codex-app-server label: codex-with-skills - extends: default hooks: before_each: command: ["setup-plugins.sh", "skills"] diff --git a/packages/core/src/evaluation/loaders/config-graph.ts b/packages/core/src/evaluation/loaders/config-graph.ts index 5bc206208..453277d2b 100644 --- a/packages/core/src/evaluation/loaders/config-graph.ts +++ b/packages/core/src/evaluation/loaders/config-graph.ts @@ -2,7 +2,7 @@ import { readFile } from 'node:fs/promises'; import path from 'node:path'; import { isPlainConfigObject } from '../../config-overlays.js'; -import { normalizeProviderDefinition } from '../providers/targets.js'; +import { expandProviderDefinitionEntries } from '../providers/targets.js'; import { parseYamlValue } from '../yaml-loader.js'; const FILE_PROTOCOL = 'file://'; @@ -214,12 +214,17 @@ function parseArray(value: unknown, location: string): readonly unknown[] { } function parseProviders(value: unknown, location: string): readonly NormalizedTargetConfig[] { - return parseArray(value, location).map((entry, index) => - parseProvider(entry, `${location}[${index}]`), - ); + return expandProviderDefinitionEntries(parseArray(value, location), { + location, + stringMode: 'all', + }).map((entry) => parseProvider(entry.rawDefinition, entry.definition, `${location}`)); } -function parseProvider(value: unknown, location: string): NormalizedTargetConfig { +function parseProvider( + value: unknown, + definition: ReturnType[number]['definition'], + location: string, +): NormalizedTargetConfig { if (!isPlainConfigObject(value)) { throw new Error(`Invalid ${location}: provider must be an object.`); } @@ -229,7 +234,6 @@ function parseProvider(value: unknown, location: string): NormalizedTargetConfig } } - const definition = normalizeProviderDefinition(value, { location }); const id = definition.name; const provider = readRequiredString(definition.provider, `${location}.id`); if (AMBIGUOUS_PROVIDER_ALIASES.has(provider)) { diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 450abe7a2..d755ebcb8 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -12,7 +12,8 @@ import { import { getAgentvConfigDir } from '../../paths.js'; import { createEvalConfigEnv, interpolateEnv } from '../interpolation.js'; import { - normalizeProviderDefinition, + expandProviderDefinitionEntries, + isProviderSpecString, resolveProviderDefinitionEnvironments, } from '../providers/targets.js'; import type { ProviderDefinition } from '../providers/types.js'; @@ -326,14 +327,44 @@ function parseProviderDefinitions( if (!Array.isArray(rawProviders)) { return Promise.resolve(undefined); } - const definitions = rawProviders.map((entry, index) => - normalizeProviderDefinition(entry, { location: `${configPath}:providers[${index}]` }), - ); + const definitions = expandProviderDefinitionEntries(rawProviders, { + location: `${configPath}:providers`, + stringMode: 'all', + }).map((entry) => entry.definition); return resolveProviderDefinitionEnvironments(definitions, baseDir, { location: `${configPath}:providers`, }); } +function parseInlineProviderRefs(rawProviders: readonly unknown[]): readonly EvalTargetRef[] { + const refs: EvalTargetRef[] = []; + rawProviders.forEach((entry, index) => { + const location = `providers[${index}]`; + if (typeof entry === 'string' && !isProviderSpecString(entry)) { + const name = entry.trim(); + if (name.length === 0) { + throw new Error(`Invalid ${location}: provider reference must be non-empty.`); + } + refs.push({ name }); + return; + } + + refs.push(...parseEvalProviderRefs(entry, location)); + }); + return refs; +} + +function parseEvalProviderRefs(raw: unknown, location: string): readonly EvalTargetRef[] { + const entries = expandProviderDefinitionEntries([raw], { + location: location.replace(/\[\d+\]$/, ''), + stringMode: 'spec-only', + }); + + return entries.map((entry) => + providerDefinitionToRef(entry.rawDefinition, entry.rawId, entry.definition), + ); +} + function mergeExecutionConfig( defaults: ExecutionDefaults | undefined, graph: ComposableConfigGraph['execution'], @@ -543,7 +574,7 @@ export function extractTargetRefsFromSuite( } const entries = Array.isArray(rawProviders) ? rawProviders : [rawProviders]; - const refs = entries.map((entry, index) => parseEvalProviderRef(entry, `providers[${index}]`)); + const refs = parseInlineProviderRefs(entries); assertUniqueProviderRefs(refs); return refs.length > 0 ? refs : undefined; } @@ -558,29 +589,18 @@ export function extractTargetsFromSuite(suite: JsonObject): readonly string[] | return names.length > 0 ? names : undefined; } -function parseEvalProviderRef(raw: unknown, location: string): EvalTargetRef { - if (typeof raw === 'string') { - const name = raw.trim(); - if (name.length === 0) { - throw new Error(`Invalid ${location}: provider reference must be non-empty.`); - } - return { name }; - } - - if (!isJsonObject(raw)) { - throw new Error(`Invalid ${location}: use a provider reference string or provider object.`); - } - +function providerDefinitionToRef( + raw: Record, + rawId: string, + definition: ProviderDefinition, +): EvalTargetRef { const hooks = parseTargetHooks(raw.hooks); - const definition = normalizeProviderDefinition( - Object.fromEntries(Object.entries(raw).filter(([key]) => key !== 'hooks')), - { location }, - ) as ProviderDefinition; - + const rawLabel = + typeof raw.label === 'string' && raw.label.trim().length > 0 ? raw.label.trim() : undefined; return { name: definition.name, - id: typeof raw.id === 'string' ? raw.id.trim() : definition.provider, - ...(definition.label !== undefined ? { label: definition.label } : {}), + id: rawId, + ...(rawLabel !== undefined ? { label: rawLabel } : {}), definition, ...(hooks !== undefined ? { hooks } : {}), }; diff --git a/packages/core/src/evaluation/providers/targets-file.ts b/packages/core/src/evaluation/providers/targets-file.ts index 0910c67df..bdbe20a32 100644 --- a/packages/core/src/evaluation/providers/targets-file.ts +++ b/packages/core/src/evaluation/providers/targets-file.ts @@ -3,7 +3,10 @@ import { access, readFile } from 'node:fs/promises'; import path from 'node:path'; import { parseYamlValue } from '../yaml-loader.js'; -import { normalizeProviderDefinition, resolveProviderDefinitionEnvironments } from './targets.js'; +import { + expandProviderDefinitionEntries, + resolveProviderDefinitionEnvironments, +} from './targets.js'; import { TARGETS_SCHEMA_V2 } from './types.js'; import type { ProviderDefinition } from './types.js'; @@ -32,24 +35,6 @@ function extractProvidersArray(parsed: unknown, absolutePath: string): unknown[] return providers; } -function assertProviderDefinition( - value: unknown, - index: number, - filePath: string, -): ProviderDefinition { - if (!isRecord(value)) { - throw new Error(`providers entry at index ${index} in ${filePath} must be an object`); - } - - const id = value.id; - - if (typeof id !== 'string' || id.trim().length === 0) { - throw new Error(`providers entry at index ${index} in ${filePath} is missing a valid 'id'`); - } - - return normalizeProviderDefinition(value, { location: `providers[${index}]` }); -} - async function fileExists(filePath: string): Promise { try { await access(filePath, constants.F_OK); @@ -71,9 +56,10 @@ export async function readProviderDefinitions( const parsed = parseYamlValue(raw); const providers = extractProvidersArray(parsed, absolutePath); - const definitions = providers.map((entry, index) => - assertProviderDefinition(entry, index, absolutePath), - ); + const definitions = expandProviderDefinitionEntries(providers, { + location: 'providers', + stringMode: 'all', + }).map((entry) => entry.definition); return resolveProviderDefinitionEnvironments(definitions, path.dirname(absolutePath), { location: 'providers', }); diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index ed9c6ba1e..94ccd178f 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -699,6 +699,99 @@ export interface NormalizeProviderDefinitionOptions { readonly location?: string; } +export type ExpandedProviderDefinitionEntry = { + readonly rawId: string; + readonly rawDefinition: Record; + readonly definition: ProviderDefinition; +}; + +export function isProviderSpecString(value: string): boolean { + const trimmed = value.trim(); + return trimmed.startsWith('package:') || trimmed.startsWith('file://') || trimmed.includes(':'); +} + +export function expandProviderDefinitionEntries( + entries: readonly unknown[], + options: { + readonly location?: string; + readonly stringMode?: 'all' | 'spec-only'; + } = {}, +): readonly ExpandedProviderDefinitionEntry[] { + const location = options.location ?? 'providers'; + const stringMode = options.stringMode ?? 'all'; + const expanded: ExpandedProviderDefinitionEntry[] = []; + + entries.forEach((entry, index) => { + const entryLocation = `${location}[${index}]`; + if (typeof entry === 'string') { + const id = entry.trim(); + if (id.length === 0) { + throw new Error(`Invalid ${entryLocation}: provider string must be non-empty.`); + } + if (stringMode === 'spec-only' && !isProviderSpecString(id)) { + return; + } + const rawDefinition = { id }; + expanded.push({ + rawId: id, + rawDefinition, + definition: normalizeProviderDefinition(rawDefinition, { location: entryLocation }), + }); + return; + } + + if (!isRecord(entry)) { + throw new Error(`Invalid ${entryLocation}: provider must be a string or object.`); + } + + if (typeof entry.id === 'string' && entry.id.trim().length > 0) { + expanded.push({ + rawId: entry.id.trim(), + rawDefinition: entry, + definition: normalizeProviderDefinition(entry, { location: entryLocation }), + }); + return; + } + + if ( + entry.provider !== undefined || + entry.name !== undefined || + entry.container !== undefined || + entry.install !== undefined + ) { + normalizeProviderDefinition(entry, { location: entryLocation }); + return; + } + + const mapEntries = Object.entries(entry); + if (mapEntries.length === 0) { + throw new Error(`Invalid ${entryLocation}: provider map must not be empty.`); + } + + for (const [providerId, providerOptions] of mapEntries) { + if (providerId.trim().length === 0) { + throw new Error(`Invalid ${entryLocation}: provider map key must be non-empty.`); + } + if (!isRecord(providerOptions)) { + throw new Error( + `Invalid ${entryLocation}.${providerId}: provider map value must be an object.`, + ); + } + const { id: _ignoredId, ...optionsWithoutId } = providerOptions; + const rawDefinition = { ...optionsWithoutId, id: providerId }; + expanded.push({ + rawId: providerId, + rawDefinition, + definition: normalizeProviderDefinition(rawDefinition, { + location: `${entryLocation}.${providerId}`, + }), + }); + } + }); + + return expanded; +} + function normalizePublicProviderId(providerId: string): { readonly provider: string; readonly config: Record; diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index dd0de8504..4ce170068 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -466,6 +466,7 @@ export interface ProviderDefinition { readonly prompts?: unknown | undefined; readonly transform?: unknown | undefined; readonly delay?: number | unknown | undefined; + readonly inputs?: unknown | undefined; readonly provider?: ProviderKind | string; /** Original public providers[].id backend/spec string after public provider normalization. */ readonly provider_spec?: string | undefined; diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 57907e3e6..3ad3ab87c 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -661,43 +661,68 @@ const EvalLocalTargetSchema = z const EvalTargetSchema = z.union([z.string().min(1), EvalLocalTargetSchema]); const EvalTargetsSchema = z.union([EvalTargetSchema, z.array(EvalTargetSchema).min(1)]); -const EvalProviderObjectSchema = z +const EvalProviderExtensionFieldsSchema = z.object({ + runtime: z + .union([ + z.enum(['host', 'profile', 'sandbox']), + z + .object({ + mode: z.enum(['host', 'profile', 'sandbox']), + }) + .passthrough(), + ]) + .optional(), + environment: EnvironmentSchema.optional(), + hooks: TargetHooksSchema.optional(), +}); +const EvalProviderOptionsSchema = z .object({ - id: z.string().min(1), label: z.string().min(1).optional(), config: JsonRecordSchema.optional(), - runtime: z - .union([ - z.enum(['host', 'profile', 'sandbox']), - z - .object({ - mode: z.enum(['host', 'profile', 'sandbox']), - }) - .passthrough(), - ]) - .optional(), prompts: PromptsSchema.optional(), transform: z.union([z.string(), JsonObjectSchema]).optional(), delay: z.number().min(0).optional(), env: z.record(z.string()).optional(), - environment: EnvironmentSchema.optional(), - hooks: TargetHooksSchema.optional(), - provider: z - .never({ - invalid_type_error: - 'providers[].provider has been removed. Use providers[].id for the backend and providers[].label for the stable identity.', - }) - .optional(), - name: z - .never({ - invalid_type_error: 'providers[].name has been removed. Use providers[].label.', - }) - .optional(), - container: z.never().optional(), - install: z.never().optional(), + inputs: JsonRecordSchema.optional(), }) + .merge(EvalProviderExtensionFieldsSchema) .strict(); -const EvalProviderSchema = z.union([z.string().min(1), EvalProviderObjectSchema]); +const EvalProviderObjectSchema = EvalProviderOptionsSchema.extend({ + id: z.string().min(1), + provider: z + .never({ + invalid_type_error: + 'providers[].provider has been removed. Use providers[].id for the backend and providers[].label for the stable identity.', + }) + .optional(), + name: z + .never({ + invalid_type_error: 'providers[].name has been removed. Use providers[].label.', + }) + .optional(), + container: z.never().optional(), + install: z.never().optional(), +}).strict(); +const EvalProviderMapOptionsSchema = EvalProviderOptionsSchema.extend({ + id: z.string().min(1).optional(), +}).strict(); +const EvalProviderMapSchema = z + .record(z.string().min(1), EvalProviderMapOptionsSchema) + .refine( + (value) => + !Object.prototype.hasOwnProperty.call(value, 'id') && + !Object.prototype.hasOwnProperty.call(value, 'label') && + !Object.prototype.hasOwnProperty.call(value, 'config'), + { + message: + 'Provider maps must be keyed by provider id, e.g. { "openai:gpt-4": { label, config } }.', + }, + ); +const EvalProviderSchema = z.union([ + z.string().min(1), + EvalProviderObjectSchema, + EvalProviderMapSchema, +]); const EvalProvidersSchema = z.union([EvalProviderSchema, z.array(EvalProviderSchema).min(1)]); // --------------------------------------------------------------------------- diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index f4705dfc9..9523f1e18 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -1064,13 +1064,21 @@ describe('extractTargetsFromSuite and extractTargetRefsFromSuite', () => { ]); }); - it('preserves colon provider specs as selection identity and lowers backend config', () => { + it('lowers Promptfoo provider string specs and maps to inline provider refs', () => { const suite: JsonObject = { providers: [ 'openai:gpt-4.1-mini', + { + 'openai:gpt-4': { + label: 'gpt4-map', + config: { temperature: 0 }, + inputs: { prompt: 'User prompt text' }, + }, + }, { id: 'openai:responses:gpt-5.4', label: 'gpt5-responses', + inputs: { prompt: 'User prompt text' }, }, { id: 'anthropic:messages:claude-sonnet-4-6', @@ -1097,6 +1105,7 @@ describe('extractTargetsFromSuite and extractTargetRefsFromSuite', () => { expect(extractTargetsFromSuite(suite)).toEqual([ 'openai:gpt-4.1-mini', + 'gpt4-map', 'gpt5-responses', 'anthropic:messages:claude-sonnet-4-6', 'exec:node ./provider.js', @@ -1106,7 +1115,31 @@ describe('extractTargetsFromSuite and extractTargetRefsFromSuite', () => { 'openai:codex-desktop', ]); expect(extractTargetRefsFromSuite(suite)).toEqual([ - { name: 'openai:gpt-4.1-mini' }, + { + name: 'openai:gpt-4.1-mini', + id: 'openai:gpt-4.1-mini', + definition: expect.objectContaining({ + id: 'openai:gpt-4.1-mini', + name: 'openai:gpt-4.1-mini', + label: 'openai:gpt-4.1-mini', + provider: 'openai', + model: 'gpt-4.1-mini', + }), + }, + { + name: 'gpt4-map', + id: 'openai:gpt-4', + label: 'gpt4-map', + definition: expect.objectContaining({ + id: 'gpt4-map', + name: 'gpt4-map', + label: 'gpt4-map', + provider: 'openai', + model: 'gpt-4', + temperature: 0, + inputs: { prompt: 'User prompt text' }, + }), + }, { name: 'gpt5-responses', id: 'openai:responses:gpt-5.4', @@ -1118,12 +1151,12 @@ describe('extractTargetsFromSuite and extractTargetRefsFromSuite', () => { provider: 'openai', model: 'gpt-5.4', api_format: 'responses', + inputs: { prompt: 'User prompt text' }, }), }, { name: 'anthropic:messages:claude-sonnet-4-6', id: 'anthropic:messages:claude-sonnet-4-6', - label: 'anthropic:messages:claude-sonnet-4-6', definition: expect.objectContaining({ id: 'anthropic:messages:claude-sonnet-4-6', name: 'anthropic:messages:claude-sonnet-4-6', @@ -1134,7 +1167,6 @@ describe('extractTargetsFromSuite and extractTargetRefsFromSuite', () => { { name: 'exec:node ./provider.js', id: 'exec:node ./provider.js', - label: 'exec:node ./provider.js', definition: expect.objectContaining({ id: 'exec:node ./provider.js', name: 'exec:node ./provider.js', @@ -1145,7 +1177,6 @@ describe('extractTargetsFromSuite and extractTargetRefsFromSuite', () => { { name: 'gateway:openai:responses:gpt-5.4', id: 'gateway:openai:responses:gpt-5.4', - label: 'gateway:openai:responses:gpt-5.4', definition: expect.objectContaining({ id: 'gateway:openai:responses:gpt-5.4', name: 'gateway:openai:responses:gpt-5.4', @@ -1178,7 +1209,6 @@ describe('extractTargetsFromSuite and extractTargetRefsFromSuite', () => { { name: 'openai:codex-desktop', id: 'openai:codex-desktop', - label: 'openai:codex-desktop', definition: expect.objectContaining({ id: 'openai:codex-desktop', name: 'openai:codex-desktop', diff --git a/packages/core/test/evaluation/providers/targets-file.test.ts b/packages/core/test/evaluation/providers/targets-file.test.ts index b954f87a9..5c31928b3 100644 --- a/packages/core/test/evaluation/providers/targets-file.test.ts +++ b/packages/core/test/evaluation/providers/targets-file.test.ts @@ -66,6 +66,39 @@ describe('readProviderDefinitions', () => { ]); }); + it('accepts Promptfoo provider strings and maps in provider catalogs', async () => { + const filePath = await writeProvidersYaml(`providers: + - openai:gpt-4.1-mini + - openai:gpt-4: + label: gpt4-map + config: + temperature: 0 + inputs: + prompt: User prompt text +`); + + const definitions = await readProviderDefinitions(filePath); + + expect(definitions).toEqual([ + expect.objectContaining({ + id: 'openai:gpt-4.1-mini', + name: 'openai:gpt-4.1-mini', + label: 'openai:gpt-4.1-mini', + provider: 'openai', + model: 'gpt-4.1-mini', + }), + expect.objectContaining({ + id: 'gpt4-map', + name: 'gpt4-map', + label: 'gpt4-map', + provider: 'openai', + model: 'gpt-4', + temperature: 0, + inputs: { prompt: 'User prompt text' }, + }), + ]); + }); + it('accepts colon provider specs and preserves unlabeled specs as stable identity', async () => { const filePath = await writeProvidersYaml(`providers: - id: openai:gpt-4.1-mini diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index a1fe09b33..9c30a04d2 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -366,6 +366,30 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(true); }); + it('accepts Promptfoo provider maps and provider inputs', () => { + const result = EvalFileSchema.safeParse({ + name: 'promptfoo-provider-maps', + prompts: ['{{ prompt }}'], + providers: [ + { + 'openai:gpt-4': { + label: 'gpt4', + config: { temperature: 0 }, + inputs: { prompt: 'User prompt text' }, + }, + }, + { + id: 'openai:gpt-4.1-mini', + label: 'mini', + inputs: { prompt: { type: 'text', description: 'User prompt text' } }, + }, + ], + tests: [baseTest], + }); + + expect(result.success).toBe(true); + }); + it('rejects invalid evaluate_options.max_concurrency', () => { const result = EvalFileSchema.safeParse({ providers: ['openai:codex'], diff --git a/scripts/export-promptfoo-config.test.ts b/scripts/export-promptfoo-config.test.ts index 73d591dcf..3063fca26 100644 --- a/scripts/export-promptfoo-config.test.ts +++ b/scripts/export-promptfoo-config.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { copyFileSync, existsSync, mkdtempSync, readFileSync } from 'node:fs'; +import { copyFileSync, existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import YAML from 'yaml'; @@ -14,6 +14,14 @@ function outputPath(name: string): string { return path.join(mkdtempSync(path.join(tmpdir(), 'agentv-promptfoo-export-')), name); } +function writeAgentvConfig(content: string): { readonly input: string; readonly output: string } { + const dir = mkdtempSync(path.join(tmpdir(), 'agentv-promptfoo-export-')); + const input = path.join(dir, 'input.agentv.yaml'); + const output = path.join(dir, 'promptfooconfig.yaml'); + writeFileSync(input, content, 'utf8'); + return { input, output }; +} + function parseYamlFile(filePath: string): Record { return YAML.parse(readFileSync(filePath, 'utf8')) as Record; } @@ -98,6 +106,121 @@ describe('exportPromptfooConfig', () => { }); }); + it('preserves Promptfoo-compatible provider maps and inputs', () => { + const { input, output } = writeAgentvConfig(`providers: + - openai:gpt-4: + label: gpt4-map + config: + temperature: 0 + inputs: + prompt: User prompt text + - id: openai:responses:gpt-5.4 + label: responses-object + inputs: + prompt: + type: text + description: User prompt text +prompts: + - "{{ prompt }}" +tests: + - vars: + prompt: hello +`); + + exportPromptfooConfig({ inputPath: input, outputPath: output }); + + const exported = parseYamlFile(output); + expect(exported.providers).toEqual([ + { + 'openai:gpt-4': { + label: 'gpt4-map', + config: { temperature: 0 }, + inputs: { prompt: 'User prompt text' }, + }, + }, + { + id: 'openai:responses:gpt-5.4', + label: 'responses-object', + inputs: { + prompt: { + type: 'text', + description: 'User prompt text', + }, + }, + }, + ]); + }); + + it('rejects AgentV-only provider runtime settings that cannot be lowered', () => { + const { input, output } = writeAgentvConfig(`providers: + - id: openai:gpt-4 + label: sandboxed + runtime: sandbox +tests: + - vars: {} +`); + + expect(() => exportPromptfooConfig({ inputPath: input, outputPath: output })).toThrow( + PromptfooExportDiagnostic, + ); + try { + exportPromptfooConfig({ inputPath: input, outputPath: output }); + } catch (error) { + expect(error).toBeInstanceOf(PromptfooExportDiagnostic); + expect((error as PromptfooExportDiagnostic).code).toBe('unsupported_provider_runtime'); + expect((error as Error).message).toContain('providers[].runtime is AgentV-only'); + } + }); + + it('rejects AgentV-only provider-local environment and hooks', () => { + const providerEnvironment = writeAgentvConfig(`providers: + - id: openai:gpt-4 + label: with-env + environment: + type: host + workdir: ./workspace +tests: + - vars: {} +`); + const providerHooks = writeAgentvConfig(`providers: + - openai:gpt-4: + label: with-hooks + hooks: + before_each: + command: ["echo", "setup"] +tests: + - vars: {} +`); + + expect(() => + exportPromptfooConfig({ + inputPath: providerEnvironment.input, + outputPath: providerEnvironment.output, + }), + ).toThrow(PromptfooExportDiagnostic); + try { + exportPromptfooConfig({ + inputPath: providerEnvironment.input, + outputPath: providerEnvironment.output, + }); + } catch (error) { + expect(error).toBeInstanceOf(PromptfooExportDiagnostic); + expect((error as PromptfooExportDiagnostic).code).toBe('unsupported_provider_environment'); + expect((error as Error).message).toContain('provider-local testbed setup'); + } + + expect(() => + exportPromptfooConfig({ inputPath: providerHooks.input, outputPath: providerHooks.output }), + ).toThrow(PromptfooExportDiagnostic); + try { + exportPromptfooConfig({ inputPath: providerHooks.input, outputPath: providerHooks.output }); + } catch (error) { + expect(error).toBeInstanceOf(PromptfooExportDiagnostic); + expect((error as PromptfooExportDiagnostic).code).toBe('unsupported_provider_hooks'); + expect((error as Error).message).toContain('AgentV-only provider hooks'); + } + }); + it('lowers agentv:codex-cli to a generated Promptfoo file provider and preserves label', () => { const output = outputPath('promptfooconfig.yaml'); exportPromptfooConfig({ diff --git a/scripts/export-promptfoo-config.ts b/scripts/export-promptfoo-config.ts index 7e26481e1..7731ad4ee 100644 --- a/scripts/export-promptfoo-config.ts +++ b/scripts/export-promptfoo-config.ts @@ -264,7 +264,41 @@ function lowerProviderId(id: string): string { return lowered; } +function assertPromptfooExportableProviderExtensions(provider: JsonMap, label: string): void { + if (provider.environment !== undefined) { + throw new PromptfooExportDiagnostic( + 'unsupported_provider_environment', + `${label}.environment is AgentV-only provider-local testbed setup. Move shared setup to top-level environment for supported host export, or remove the provider-local environment before exporting to Promptfoo.`, + ); + } + if (provider.hooks !== undefined) { + throw new PromptfooExportDiagnostic( + 'unsupported_provider_hooks', + `${label}.hooks are AgentV-only provider hooks and cannot be represented in Promptfoo provider declarations. Move compatible lifecycle work to top-level extensions or remove hooks before export.`, + ); + } + + const runtime = provider.runtime; + if (runtime === undefined || runtime === 'host') { + return; + } + if ( + runtime && + typeof runtime === 'object' && + !Array.isArray(runtime) && + (runtime as JsonMap).mode === 'host' && + Object.keys(runtime as JsonMap).length === 1 + ) { + return; + } + throw new PromptfooExportDiagnostic( + 'unsupported_provider_runtime', + `${label}.runtime is AgentV-only. Promptfoo export can omit runtime: host, but profile/sandbox or runtime-specific settings must be removed or represented by a custom Promptfoo provider wrapper.`, + ); +} + function lowerProviderObject(provider: JsonMap): JsonMap { + assertPromptfooExportableProviderExtensions(provider, 'providers[]'); const lowered: JsonMap = {}; for (const [key, value] of Object.entries(provider)) { if (AGENTV_ONLY_PROVIDER_KEYS.has(key)) { @@ -292,7 +326,12 @@ function lowerProviderEntry(provider: unknown, environment?: HostEnvironmentExpo const providerObject = provider as JsonMap; if (looksLikeProviderOptionsMap(providerObject)) { const [[id, options]] = Object.entries(providerObject); - return { [lowerProviderId(id)]: options }; + const optionObject = + options && typeof options === 'object' && !Array.isArray(options) + ? (options as JsonMap) + : {}; + assertPromptfooExportableProviderExtensions(optionObject, `providers[].${id}`); + return { [lowerProviderId(id)]: lowerProviderObject(optionObject) }; } return lowerProviderObject(providerObject); } diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 8724355b7..1bc7f1551 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -20,14 +20,14 @@ Promptfoo parity matrix: https://agentv.dev/docs/reference/promptfoo-parity/ Treat YAML as the canonical portable model. Prefer authoring `.eval.yaml` / `EVAL.yaml` first, then use TypeScript helpers, Python scripts, or executable graders only when they lower to the same fields or when the evaluation logic must actually run code. Eval files define what is tested and how it runs: prompts, datasets, assertions, -task fixtures, top-level `target`, and suite run controls. Use field-local file +task fixtures, top-level `providers`, and suite run controls. Use field-local file refs such as `tests: file://...`, `prompts: file://...`, `default_test: file://...`, and `environment: file://...`. String-valued `tests` and string entries inside `tests[]` are raw-case refs for direct paths, directories, and globs. Run several full eval suites directly with CLI multi-file selection and tags. Use scoped `run:` on individual tests only for `threshold`, `repeat`, -`timeout_seconds`, and legacy `budget_usd`; keep target selection at top-level -`target` or CLI `--target`, put suite budget caps under +`timeout_seconds`, and legacy `budget_usd`; keep provider selection at top-level +`providers` or CLI `--provider`, put suite budget caps under `evaluate_options.budget_usd`, authored concurrency under `evaluate_options.max_concurrency`, suite repeat policy under `evaluate_options.repeat`, coding-agent testbed setup under `environment`, @@ -122,7 +122,8 @@ tests: ```yaml description: Example eval -target: default +providers: + - default prompts: - "{{ prompt }}" @@ -142,7 +143,7 @@ tests: ## Eval File Structure **Required:** `tests` (array or string raw-case path) or `scenarios` -**Optional:** `name`, `description`, `version`, `author`, `tags`, `license`, `requires`, `target`, `targets`, `prompts`, `default_test`, `timeout_seconds`, `evaluate_options`, `threshold`, `suite`, `environment`, `env`, `extensions`, `assert` +**Optional:** `name`, `description`, `version`, `author`, `tags`, `license`, `requires`, `providers`, `prompts`, `default_test`, `timeout_seconds`, `evaluate_options`, `threshold`, `suite`, `environment`, `env`, `extensions`, `assert` **Test fields:** @@ -152,11 +153,21 @@ tests: | `vars` | yes when the prompt needs row data | Prompt-template variables for this row | | `vars.expected_output` | no | Conventional reference-answer var consumed by explicit graders | | `assert` | yes | Graders: deterministic checks, `llm-rubric` / `agent-rubric` checks, script graders, or plain string rubric criteria | -| `execution` | no | Per-case grader/default overrides such as `skip_defaults`; target selection belongs in top-level `target` or CLI `--target` | +| `execution` | no | Per-case grader/default overrides such as `skip_defaults`; provider selection belongs in top-level `providers` or CLI `--provider` | | `environment` | no | Per-case coding-agent testbed config (overrides suite-level) | | `metadata` | no | Arbitrary key-value pairs passed to setup/teardown scripts | | `conversation_id` | no | Thread grouping | +**Provider declarations:** AgentV accepts Promptfoo-shaped provider entries +where they map cleanly: strings such as `openai:gpt-4.1-mini`, object form with +`id`, `label`, `config`, `env`, `prompts`, `transform`, `delay`, and `inputs`, +and provider maps such as `{ "openai:gpt-4": { label, config } }`. In object +form, `id` is the backend/spec and `label` is the stable AgentV selection and +result identity. AgentV-only `runtime`, provider-local `environment`, and +provider `hooks` must be explicit extensions; direct Promptfoo runs do not +execute them, and export must lower supported cases or reject unsupported ones +clearly. + ## Prompt Templates and Vars Use top-level `prompts` plus `tests[].vars` for the Promptfoo-compatible canonical @@ -167,7 +178,8 @@ repeat samples. ```yaml description: Prompt matrix example -target: default +providers: + - default prompts: - id: support-chat diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index 5567dc1ec..4a344262e 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -2033,10 +2033,6 @@ { "type": "object", "properties": { - "id": { - "type": "string", - "minLength": 1 - }, "label": { "type": "string", "minLength": 1 @@ -2045,25 +2041,6 @@ "type": "object", "additionalProperties": {} }, - "runtime": { - "anyOf": [ - { - "type": "string", - "enum": ["host", "profile", "sandbox"] - }, - { - "type": "object", - "properties": { - "mode": { - "type": "string", - "enum": ["host", "profile", "sandbox"] - } - }, - "required": ["mode"], - "additionalProperties": true - } - ] - }, "prompts": { "anyOf": [ { @@ -2224,6 +2201,29 @@ "type": "string" } }, + "inputs": { + "type": "object", + "additionalProperties": {} + }, + "runtime": { + "anyOf": [ + { + "type": "string", + "enum": ["host", "profile", "sandbox"] + }, + { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["host", "profile", "sandbox"] + } + }, + "required": ["mode"], + "additionalProperties": true + } + ] + }, "environment": { "anyOf": [ { @@ -2539,6 +2539,10 @@ }, "additionalProperties": false }, + "id": { + "type": "string", + "minLength": 1 + }, "provider": { "not": {} }, @@ -2554,24 +2558,12 @@ }, "required": ["id"], "additionalProperties": false - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { + }, + { + "type": "object", + "additionalProperties": { "type": "object", "properties": { - "id": { - "type": "string", - "minLength": 1 - }, "label": { "type": "string", "minLength": 1 @@ -2580,25 +2572,6 @@ "type": "object", "additionalProperties": {} }, - "runtime": { - "anyOf": [ - { - "type": "string", - "enum": ["host", "profile", "sandbox"] - }, - { - "type": "object", - "properties": { - "mode": { - "type": "string", - "enum": ["host", "profile", "sandbox"] - } - }, - "required": ["mode"], - "additionalProperties": true - } - ] - }, "prompts": { "anyOf": [ { @@ -2759,6 +2732,29 @@ "type": "string" } }, + "inputs": { + "type": "object", + "additionalProperties": {} + }, + "runtime": { + "anyOf": [ + { + "type": "string", + "enum": ["host", "profile", "sandbox"] + }, + { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["host", "profile", "sandbox"] + } + }, + "required": ["mode"], + "additionalProperties": true + } + ] + }, "environment": { "anyOf": [ { @@ -3074,21 +3070,1077 @@ }, "additionalProperties": false }, - "provider": { - "not": {} - }, - "name": { - "not": {} - }, - "container": { - "not": {} - }, - "install": { - "not": {} + "id": { + "type": "string", + "minLength": 1 } }, - "required": ["id"], "additionalProperties": false + }, + "propertyNames": { + "minLength": 1 + } + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": { + "label": { + "type": "string", + "minLength": 1 + }, + "config": { + "type": "object", + "additionalProperties": {} + }, + "prompts": { + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] + }, + "minItems": 1 + } + ] + }, + "transform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + }, + "delay": { + "type": "number", + "minimum": 0 + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "inputs": { + "type": "object", + "additionalProperties": {} + }, + "runtime": { + "anyOf": [ + { + "type": "string", + "enum": ["host", "profile", "sandbox"] + }, + { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["host", "profile", "sandbox"] + } + }, + "required": ["mode"], + "additionalProperties": true + } + ] + }, + "environment": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\s*file:\\/\\/" + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "allOf": [ + {}, + { + "type": "object", + "properties": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + } + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "host" + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "allOf": [ + {}, + { + "type": "object", + "properties": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + } + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "docker" + }, + "context": { + "type": "string", + "minLength": 1 + }, + "dockerfile": { + "type": "string", + "minLength": 1 + }, + "image": { + "type": "string", + "minLength": 1 + }, + "resources": { + "type": "object", + "properties": { + "cpus": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + }, + "memory": { + "type": "string", + "minLength": 1 + }, + "disk": { + "type": "string", + "minLength": 1 + }, + "gpu": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + } + }, + "additionalProperties": false + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "minLength": 1 + }, + "target": { + "type": "string", + "minLength": 1 + }, + "access": { + "type": "string", + "enum": ["ro", "rw"] + }, + "read_only": { + "type": "boolean" + } + }, + "required": ["source", "target"], + "additionalProperties": false + } + }, + "secrets": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + } + ] + }, + "hooks": { + "type": "object", + "properties": { + "before_all": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + }, + "before_each": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + }, + "after_each": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + }, + "after_all": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "id": { + "type": "string", + "minLength": 1 + }, + "provider": { + "not": {} + }, + "name": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + } + }, + "required": ["id"], + "additionalProperties": false + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "label": { + "type": "string", + "minLength": 1 + }, + "config": { + "type": "object", + "additionalProperties": {} + }, + "prompts": { + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] + }, + "minItems": 1 + } + ] + }, + "transform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + }, + "delay": { + "type": "number", + "minimum": 0 + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "inputs": { + "type": "object", + "additionalProperties": {} + }, + "runtime": { + "anyOf": [ + { + "type": "string", + "enum": ["host", "profile", "sandbox"] + }, + { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["host", "profile", "sandbox"] + } + }, + "required": ["mode"], + "additionalProperties": true + } + ] + }, + "environment": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\s*file:\\/\\/" + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "allOf": [ + {}, + { + "type": "object", + "properties": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + } + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "host" + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "allOf": [ + {}, + { + "type": "object", + "properties": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + } + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "docker" + }, + "context": { + "type": "string", + "minLength": 1 + }, + "dockerfile": { + "type": "string", + "minLength": 1 + }, + "image": { + "type": "string", + "minLength": 1 + }, + "resources": { + "type": "object", + "properties": { + "cpus": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + }, + "memory": { + "type": "string", + "minLength": 1 + }, + "disk": { + "type": "string", + "minLength": 1 + }, + "gpu": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + } + }, + "additionalProperties": false + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "minLength": 1 + }, + "target": { + "type": "string", + "minLength": 1 + }, + "access": { + "type": "string", + "enum": ["ro", "rw"] + }, + "read_only": { + "type": "boolean" + } + }, + "required": ["source", "target"], + "additionalProperties": false + } + }, + "secrets": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + } + ] + }, + "hooks": { + "type": "object", + "properties": { + "before_all": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + }, + "before_each": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + }, + "after_each": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + }, + "after_all": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "id": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "propertyNames": { + "minLength": 1 + } } ] },