From 84d3d3fe4407ad228b84091664c77e3865dc8da8 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 15:07:19 +0200 Subject: [PATCH 1/6] feat(package): expose agentv sdk facade --- apps/cli/package.json | 31 +- apps/cli/src/cli-app.ts | 200 ++++++++ apps/cli/src/cli.ts | 2 +- apps/cli/src/commands/create/commands.ts | 4 +- apps/cli/src/commands/eval/commands/vitest.ts | 2 +- apps/cli/src/config.ts | 4 + apps/cli/src/contracts.ts | 14 + apps/cli/src/index.ts | 204 +------- apps/cli/src/provider.ts | 20 + apps/cli/src/sdk.ts | 9 + .../test/commands/create/assertion.test.ts | 2 +- apps/cli/test/package-exports.test.ts | 91 ++++ apps/cli/test/setup-core-build.ts | 7 +- apps/cli/tsup.config.ts | 16 +- packages/core/package.json | 7 +- packages/core/src/evaluation/config.ts | 4 +- packages/core/src/evaluation/evaluate.ts | 4 +- .../src/evaluation/loaders/ts-eval-loader.ts | 4 +- .../evaluation/registry/grader-registry.ts | 2 +- .../src/evaluation/script-grader-runtime.ts | 122 +++++ .../src/evaluation/script-grader-schemas.ts | 391 +++++++++++++++ .../src/evaluation/vitest-workspace-grader.ts | 433 +++++++++++++++++ packages/core/src/script-grader.ts | 3 + .../loaders/fixtures/sdk-define-eval.eval.ts | 6 +- packages/core/tsup.config.ts | 1 + packages/sdk/package.json | 14 +- packages/sdk/src/deprecation.ts | 2 +- packages/sdk/src/eval.ts | 4 +- packages/sdk/src/index.ts | 18 +- packages/sdk/src/prompt-template.ts | 2 +- packages/sdk/src/runtime.ts | 132 +---- packages/sdk/src/schemas.ts | 453 +++--------------- packages/sdk/src/target-client.ts | 2 +- packages/sdk/src/vitest.ts | 440 +---------------- packages/sdk/test/evaluate-export.test.ts | 2 +- packages/sdk/test/package-graph.test.ts | 5 +- scripts/publish.ts | 2 +- scripts/release.ts | 13 +- 38 files changed, 1463 insertions(+), 1209 deletions(-) create mode 100644 apps/cli/src/cli-app.ts create mode 100644 apps/cli/src/config.ts create mode 100644 apps/cli/src/contracts.ts create mode 100644 apps/cli/src/provider.ts create mode 100644 apps/cli/src/sdk.ts create mode 100644 apps/cli/test/package-exports.test.ts create mode 100644 packages/core/src/evaluation/script-grader-runtime.ts create mode 100644 packages/core/src/evaluation/script-grader-schemas.ts create mode 100644 packages/core/src/evaluation/vitest-workspace-grader.ts create mode 100644 packages/core/src/script-grader.ts diff --git a/apps/cli/package.json b/apps/cli/package.json index 11014db1a..b46b795f6 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -14,17 +14,42 @@ "bin": { "agentv": "./dist/cli.js" }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "./sdk": { + "types": "./dist/sdk.d.ts", + "import": "./dist/sdk.js" + }, + "./provider": { + "types": "./dist/provider.d.ts", + "import": "./dist/provider.js" + }, + "./contracts": { + "types": "./dist/contracts.d.ts", + "import": "./dist/contracts.js" + }, + "./config": { + "types": "./dist/config.d.ts", + "import": "./dist/config.js" + }, + "./package.json": "./package.json" + }, "files": ["dist", "README.md"], "scripts": { "dev": "bun src/cli.ts", - "build": "(cd ../../packages/sdk && bun run build) && tsup && bun run copy-readme", + "build": "tsup && bun run copy-readme", "copy-readme": "bun -e \"import { cpSync } from 'fs'; cpSync('../../README.md', 'README.md')\"", "prepublishOnly": "node -e \"if(process.env.ALLOW_PUBLISH!=='1'){console.error('ERROR: Use bun run publish:next, then bun run promote:latest');process.exit(1)}\"", - "typecheck": "(cd ../../packages/sdk && bun run build) && tsc --noEmit", + "typecheck": "tsc --noEmit", "lint": "biome check .", "format": "biome format --write .", "fix": "biome check --write .", - "test": "(cd ../../packages/sdk && bun run build) && bun test", + "test": "bun test", "test:watch": "bun test --watch" }, "dependencies": { diff --git a/apps/cli/src/cli-app.ts b/apps/cli/src/cli-app.ts new file mode 100644 index 000000000..8bc82505d --- /dev/null +++ b/apps/cli/src/cli-app.ts @@ -0,0 +1,200 @@ +import path from 'node:path'; +import { + loadConfig, + loadEnvPathFiles, + runBeforeSessionHook, + runEnvFromEntries, +} from '@agentv/core'; +import { binary, run, subcommands } from 'cmd-ts'; +import { findRepoRoot } from './commands/eval/shared.js'; + +import packageJson from '../package.json' with { type: 'json' }; +import { compareCommand } from './commands/compare/index.js'; +import { convertCommand } from './commands/convert/index.js'; +import { createCommand } from './commands/create/index.js'; +import { doctorCommand } from './commands/doctor/index.js'; +import { evalCommand } from './commands/eval/index.js'; +import { gradeCommand } from './commands/grade/index.js'; +import { importCommand } from './commands/import/index.js'; +import { initCmdTsCommand } from './commands/init/index.js'; +import { inspectCommand } from './commands/inspect/index.js'; +import { pipelineCommand } from './commands/pipeline/index.js'; +import { prepareCommand } from './commands/prepare/index.js'; +import { resultsCommand } from './commands/results/index.js'; +import { resultsServeCommand } from './commands/results/serve.js'; +import { runsCommand } from './commands/runs/index.js'; +import { selfCommand } from './commands/self/index.js'; +import { skillsCommand } from './commands/skills/index.js'; +import { transpileCommand } from './commands/transpile/index.js'; +import { trendCommand } from './commands/trend/index.js'; +import { trimCommand } from './commands/trim/index.js'; +import { validateCommand } from './commands/validate/index.js'; +import { getUpdateNotice } from './update-check.js'; + +export const app = subcommands({ + name: 'agentv', + description: 'AgentV CLI', + version: packageJson.version, + cmds: { + dashboard: resultsServeCommand, + eval: evalCommand, + grade: gradeCommand, + import: importCommand, + compare: compareCommand, + convert: convertCommand, + create: createCommand, + doctor: doctorCommand, + init: initCmdTsCommand, + pipeline: pipelineCommand, + prepare: prepareCommand, + results: resultsCommand, + runs: runsCommand, + self: selfCommand, + skills: skillsCommand, + serve: resultsServeCommand, + inspect: inspectCommand, + trend: trendCommand, + transpile: transpileCommand, + trim: trimCommand, + validate: validateCommand, + }, +}); + +/** + * Known eval subcommand names — used to decide whether to inject the + * implicit `run` subcommand for backward-compatible `agentv eval `. + */ +const EVAL_SUBCOMMANDS = new Set(['run', 'assert', 'aggregate', 'bundle', 'vitest']); +const VITEST_VERIFIER_RE = /(?:^|[/\\])(?:EVAL|[^/\\]+[.-](?:test|spec))\.[cm]?[jt]sx?$/i; + +/** + * Top-level CLI command names (excluding `eval` itself). + * Used to ensure `eval` is the top-level subcommand, not nested. + */ +const TOP_LEVEL_COMMANDS = new Set([ + 'import', + 'inspect', + 'compare', + 'convert', + 'create', + 'dashboard', + 'doctor', + 'grade', + 'init', + 'pipeline', + 'prepare', + 'results', + 'runs', + 'self', + 'skills', + 'serve', + 'studio', + 'trend', + 'transpile', + 'trim', + 'validate', +]); + +export function usesDeprecatedStudioAlias(argv: string[]): boolean { + return argv[2] === 'studio'; +} + +export function shouldRunBeforeSessionHook(argv: string[]): boolean { + return !(argv[2] === 'eval' && argv[3] === 'vitest'); +} + +export function inferEvalSubcommand(arg: string | undefined): 'run' | 'vitest' { + return arg && VITEST_VERIFIER_RE.test(arg) ? 'vitest' : 'run'; +} + +/** + * Preprocess argv for convenience aliases: + * - `--eval-id` → `--test-id` + * - `agentv eval ` → `agentv eval run ` + * (backward compat: `eval` used to be a direct command, now it's a group) + */ +export function preprocessArgv(argv: string[]): string[] { + const result = [...argv]; + + if (result[2] === 'studio') { + result[2] = 'dashboard'; + } + + // Rewrite --eval-id → --test-id (convenience alias) + for (let i = 0; i < result.length; i++) { + if (result[i] === '--eval-id') { + result[i] = '--test-id'; + } else if (result[i].startsWith('--eval-id=')) { + result[i] = `--test-id=${result[i].slice('--eval-id='.length)}`; + } + } + + // Implicit eval subcommand: `agentv eval []` injects the inferred command + // when the first arg after `eval` is absent or is not a known eval subcommand. + // Backward-compat: `eval` used to be a direct command; now it is a subcommands group. + // Bare `agentv eval` falls through to the run handler so its TTY check can launch + // the interactive wizard. Vitest-looking verifier files use the protocol adapter + // directly so deterministic workspace graders can stay short in eval YAML. + // Only applies when `eval` is the top-level subcommand. + // Exception: `--help` / `-h` should show the eval group help, not run's help. + const evalIdx = result.indexOf('eval'); + if (evalIdx !== -1) { + // Ensure no top-level command appears before `eval` in the argv — + // if one does, `eval` is a nested subcommand. + const isTopLevel = !result.slice(0, evalIdx).some((arg) => TOP_LEVEL_COMMANDS.has(arg)); + if (isTopLevel) { + const nextArg = result[evalIdx + 1]; + const isHelp = nextArg === '--help' || nextArg === '-h'; + const isKnownSubcommand = nextArg !== undefined && EVAL_SUBCOMMANDS.has(nextArg); + if (!isHelp && !isKnownSubcommand) { + result.splice(evalIdx + 1, 0, inferEvalSubcommand(nextArg)); + } + } + } + + return result; +} + +export async function runCli(argv: string[] = process.argv): Promise { + // Kick off update check: reads from local cache (fast), spawns a detached + // child to refresh if stale. The notice is printed on process exit so it + // appears after command output, even if the command calls process.exit(). + let updateNotice: string | null = null; + process.on('exit', () => { + if (updateNotice) process.stderr.write(`\n${updateNotice}\n`); + }); + getUpdateNotice(packageJson.version).then((n) => { + updateNotice = n; + }); + + const processedArgv = preprocessArgv(argv); + if (usesDeprecatedStudioAlias(argv)) { + process.stderr.write( + 'Warning: `agentv studio` is deprecated and will be removed in a future release. Use `agentv dashboard` instead.\n', + ); + } + + if (shouldRunBeforeSessionHook(processedArgv)) { + // Load env_path/env_from and run the before_session hook once at startup, + // before any command executes. Uses cwd as the search root for + // .agentv/config.yaml so validate/eval commands see the injected vars. + const cwd = process.cwd(); + const repoRoot = await findRepoRoot(cwd); + const sessionConfig = await loadConfig(path.join(cwd, '_'), repoRoot); + const configDir = sessionConfig?.configDir ?? repoRoot; + + if (sessionConfig?.env_path) { + await loadEnvPathFiles(sessionConfig.env_path, configDir); + } + if (sessionConfig?.env_from) { + await runEnvFromEntries(sessionConfig.env_from, { cwd: configDir }); + } + + const beforeSessionCommand = sessionConfig?.hooks?.before_session; + if (beforeSessionCommand) { + runBeforeSessionHook(beforeSessionCommand); + } + } + + await run(binary(app), processedArgv); +} diff --git a/apps/cli/src/cli.ts b/apps/cli/src/cli.ts index 9c7946ed1..62668b071 100644 --- a/apps/cli/src/cli.ts +++ b/apps/cli/src/cli.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node import { killAllTrackedChildren } from '@agentv/core'; -import { runCli } from './index.js'; +import { runCli } from './cli-app.js'; // Forward SIGINT/SIGTERM to spawned provider subprocesses before exiting. // Without this, Dashboard's `child.kill('SIGTERM')` against the CLI orphans diff --git a/apps/cli/src/commands/create/commands.ts b/apps/cli/src/commands/create/commands.ts index 8924e200d..6f1a047b9 100644 --- a/apps/cli/src/commands/create/commands.ts +++ b/apps/cli/src/commands/create/commands.ts @@ -4,7 +4,7 @@ import { command, option, optional, positional, string } from 'cmd-ts'; const ASSERTION_TEMPLATES: Record = { default: `#!/usr/bin/env bun -import { defineAssertion } from '@agentv/sdk'; +import { defineAssertion } from 'agentv'; export default defineAssertion(({ output }) => { // TODO: Implement your assertion logic @@ -18,7 +18,7 @@ export default defineAssertion(({ output }) => { }); `, score: `#!/usr/bin/env bun -import { defineAssertion } from '@agentv/sdk'; +import { defineAssertion } from 'agentv'; export default defineAssertion(({ output }) => { // TODO: Implement your scoring logic (0.0 to 1.0) diff --git a/apps/cli/src/commands/eval/commands/vitest.ts b/apps/cli/src/commands/eval/commands/vitest.ts index d8b6a20fc..0a3e1d8a7 100644 --- a/apps/cli/src/commands/eval/commands/vitest.ts +++ b/apps/cli/src/commands/eval/commands/vitest.ts @@ -1,6 +1,6 @@ import { command, flag, number, option, optional, restPositionals, string } from 'cmd-ts'; -import { runScriptGrader, runVitestWorkspaceGrader } from '@agentv/sdk'; +import { runScriptGrader, runVitestWorkspaceGrader } from '@agentv/core/script-grader'; function parseCommand(value: string | undefined): readonly string[] | undefined { const trimmed = value?.trim(); diff --git a/apps/cli/src/config.ts b/apps/cli/src/config.ts new file mode 100644 index 000000000..a8d4a0c33 --- /dev/null +++ b/apps/cli/src/config.ts @@ -0,0 +1,4 @@ +export { + defineConfig, + type AgentVTsConfig as AgentVConfig, +} from '@agentv/core'; diff --git a/apps/cli/src/contracts.ts b/apps/cli/src/contracts.ts new file mode 100644 index 000000000..812eb8bfa --- /dev/null +++ b/apps/cli/src/contracts.ts @@ -0,0 +1,14 @@ +export * from './provider.js'; +export * from './config.js'; +export type { + AssertEntry, + ConversationTurnInput, + EvalAssertionInput, + EvalRunArtifacts, + EvalRunResult, + EvalSummary, + EvalTestInput, + EvaluationResultWire, + TraceSummaryWire, + TraceWire, +} from '@agentv/core'; diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index 8bc82505d..320fe230d 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -1,200 +1,8 @@ -import path from 'node:path'; -import { - loadConfig, - loadEnvPathFiles, - runBeforeSessionHook, - runEnvFromEntries, -} from '@agentv/core'; -import { binary, run, subcommands } from 'cmd-ts'; -import { findRepoRoot } from './commands/eval/shared.js'; - -import packageJson from '../package.json' with { type: 'json' }; -import { compareCommand } from './commands/compare/index.js'; -import { convertCommand } from './commands/convert/index.js'; -import { createCommand } from './commands/create/index.js'; -import { doctorCommand } from './commands/doctor/index.js'; -import { evalCommand } from './commands/eval/index.js'; -import { gradeCommand } from './commands/grade/index.js'; -import { importCommand } from './commands/import/index.js'; -import { initCmdTsCommand } from './commands/init/index.js'; -import { inspectCommand } from './commands/inspect/index.js'; -import { pipelineCommand } from './commands/pipeline/index.js'; -import { prepareCommand } from './commands/prepare/index.js'; -import { resultsCommand } from './commands/results/index.js'; -import { resultsServeCommand } from './commands/results/serve.js'; -import { runsCommand } from './commands/runs/index.js'; -import { selfCommand } from './commands/self/index.js'; -import { skillsCommand } from './commands/skills/index.js'; -import { transpileCommand } from './commands/transpile/index.js'; -import { trendCommand } from './commands/trend/index.js'; -import { trimCommand } from './commands/trim/index.js'; -import { validateCommand } from './commands/validate/index.js'; -import { getUpdateNotice } from './update-check.js'; - -export const app = subcommands({ - name: 'agentv', - description: 'AgentV CLI', - version: packageJson.version, - cmds: { - dashboard: resultsServeCommand, - eval: evalCommand, - grade: gradeCommand, - import: importCommand, - compare: compareCommand, - convert: convertCommand, - create: createCommand, - doctor: doctorCommand, - init: initCmdTsCommand, - pipeline: pipelineCommand, - prepare: prepareCommand, - results: resultsCommand, - runs: runsCommand, - self: selfCommand, - skills: skillsCommand, - serve: resultsServeCommand, - inspect: inspectCommand, - trend: trendCommand, - transpile: transpileCommand, - trim: trimCommand, - validate: validateCommand, - }, -}); - /** - * Known eval subcommand names — used to decide whether to inject the - * implicit `run` subcommand for backward-compatible `agentv eval `. + * Public TypeScript facade for the `agentv` package. + * + * CLI implementation code lives in `cli-app.ts` and must not import through + * this facade. Keep this file focused on user-facing programmatic APIs. */ -const EVAL_SUBCOMMANDS = new Set(['run', 'assert', 'aggregate', 'bundle', 'vitest']); -const VITEST_VERIFIER_RE = /(?:^|[/\\])(?:EVAL|[^/\\]+[.-](?:test|spec))\.[cm]?[jt]sx?$/i; - -/** - * Top-level CLI command names (excluding `eval` itself). - * Used to ensure `eval` is the top-level subcommand, not nested. - */ -const TOP_LEVEL_COMMANDS = new Set([ - 'import', - 'inspect', - 'compare', - 'convert', - 'create', - 'dashboard', - 'doctor', - 'grade', - 'init', - 'pipeline', - 'prepare', - 'results', - 'runs', - 'self', - 'skills', - 'serve', - 'studio', - 'trend', - 'transpile', - 'trim', - 'validate', -]); - -export function usesDeprecatedStudioAlias(argv: string[]): boolean { - return argv[2] === 'studio'; -} - -export function shouldRunBeforeSessionHook(argv: string[]): boolean { - return !(argv[2] === 'eval' && argv[3] === 'vitest'); -} - -export function inferEvalSubcommand(arg: string | undefined): 'run' | 'vitest' { - return arg && VITEST_VERIFIER_RE.test(arg) ? 'vitest' : 'run'; -} - -/** - * Preprocess argv for convenience aliases: - * - `--eval-id` → `--test-id` - * - `agentv eval ` → `agentv eval run ` - * (backward compat: `eval` used to be a direct command, now it's a group) - */ -export function preprocessArgv(argv: string[]): string[] { - const result = [...argv]; - - if (result[2] === 'studio') { - result[2] = 'dashboard'; - } - - // Rewrite --eval-id → --test-id (convenience alias) - for (let i = 0; i < result.length; i++) { - if (result[i] === '--eval-id') { - result[i] = '--test-id'; - } else if (result[i].startsWith('--eval-id=')) { - result[i] = `--test-id=${result[i].slice('--eval-id='.length)}`; - } - } - - // Implicit eval subcommand: `agentv eval []` injects the inferred command - // when the first arg after `eval` is absent or is not a known eval subcommand. - // Backward-compat: `eval` used to be a direct command; now it is a subcommands group. - // Bare `agentv eval` falls through to the run handler so its TTY check can launch - // the interactive wizard. Vitest-looking verifier files use the protocol adapter - // directly so deterministic workspace graders can stay short in eval YAML. - // Only applies when `eval` is the top-level subcommand. - // Exception: `--help` / `-h` should show the eval group help, not run's help. - const evalIdx = result.indexOf('eval'); - if (evalIdx !== -1) { - // Ensure no top-level command appears before `eval` in the argv — - // if one does, `eval` is a nested subcommand. - const isTopLevel = !result.slice(0, evalIdx).some((arg) => TOP_LEVEL_COMMANDS.has(arg)); - if (isTopLevel) { - const nextArg = result[evalIdx + 1]; - const isHelp = nextArg === '--help' || nextArg === '-h'; - const isKnownSubcommand = nextArg !== undefined && EVAL_SUBCOMMANDS.has(nextArg); - if (!isHelp && !isKnownSubcommand) { - result.splice(evalIdx + 1, 0, inferEvalSubcommand(nextArg)); - } - } - } - - return result; -} - -export async function runCli(argv: string[] = process.argv): Promise { - // Kick off update check: reads from local cache (fast), spawns a detached - // child to refresh if stale. The notice is printed on process exit so it - // appears after command output, even if the command calls process.exit(). - let updateNotice: string | null = null; - process.on('exit', () => { - if (updateNotice) process.stderr.write(`\n${updateNotice}\n`); - }); - getUpdateNotice(packageJson.version).then((n) => { - updateNotice = n; - }); - - const processedArgv = preprocessArgv(argv); - if (usesDeprecatedStudioAlias(argv)) { - process.stderr.write( - 'Warning: `agentv studio` is deprecated and will be removed in a future release. Use `agentv dashboard` instead.\n', - ); - } - - if (shouldRunBeforeSessionHook(processedArgv)) { - // Load env_path/env_from and run the before_session hook once at startup, - // before any command executes. Uses cwd as the search root for - // .agentv/config.yaml so validate/eval commands see the injected vars. - const cwd = process.cwd(); - const repoRoot = await findRepoRoot(cwd); - const sessionConfig = await loadConfig(path.join(cwd, '_'), repoRoot); - const configDir = sessionConfig?.configDir ?? repoRoot; - - if (sessionConfig?.env_path) { - await loadEnvPathFiles(sessionConfig.env_path, configDir); - } - if (sessionConfig?.env_from) { - await runEnvFromEntries(sessionConfig.env_from, { cwd: configDir }); - } - - const beforeSessionCommand = sessionConfig?.hooks?.before_session; - if (beforeSessionCommand) { - runBeforeSessionHook(beforeSessionCommand); - } - } - - await run(binary(app), processedArgv); -} +export * from './sdk.js'; +export * from './config.js'; diff --git a/apps/cli/src/provider.ts b/apps/cli/src/provider.ts new file mode 100644 index 000000000..421ca5de5 --- /dev/null +++ b/apps/cli/src/provider.ts @@ -0,0 +1,20 @@ +export { + createBuiltinProviderRegistry, + createProvider, + resolveAndCreateProvider, + resolveDelegatedProviderDefinition, + resolveProviderDefinition, + resolveProviderDefinitionEnvironments, + type EnvLookup, + type Message, + type Provider, + type ProviderDefinition, + type ProviderFactoryFn, + type ProviderKind, + type ProviderRequest, + type ProviderResponse, + type ProviderStreamCallbacks, + type ProviderTokenUsage, + type ResolvedProviderBackend, + type ToolCall, +} from '@agentv/core'; diff --git a/apps/cli/src/sdk.ts b/apps/cli/src/sdk.ts new file mode 100644 index 000000000..1da20f351 --- /dev/null +++ b/apps/cli/src/sdk.ts @@ -0,0 +1,9 @@ +/** + * User-facing SDK helpers for `agentv/sdk`. + * + * This leaf entrypoint intentionally avoids importing CLI command modules so + * script graders and TypeScript eval files can import helpers without loading + * the command tree. + */ +export * from '@agentv/sdk'; +export * from './config.js'; diff --git a/apps/cli/test/commands/create/assertion.test.ts b/apps/cli/test/commands/create/assertion.test.ts index e7f5e8d38..14d4971d5 100644 --- a/apps/cli/test/commands/create/assertion.test.ts +++ b/apps/cli/test/commands/create/assertion.test.ts @@ -33,7 +33,7 @@ describe('agentv create assertion', () => { path.join(cwd, '.agentv', 'assertions', 'word-count.ts'), 'utf8', ); - expect(content).toContain("import { defineAssertion } from '@agentv/sdk';"); + expect(content).toContain("import { defineAssertion } from 'agentv';"); expect(content).toContain("const text = output ?? '';"); expect(content).toContain('pass,'); expect(content).toContain('score: pass ? 1 : 0,'); diff --git a/apps/cli/test/package-exports.test.ts b/apps/cli/test/package-exports.test.ts new file mode 100644 index 000000000..911b7b6b3 --- /dev/null +++ b/apps/cli/test/package-exports.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'bun:test'; +import { constants, accessSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const repoRoot = path.resolve(import.meta.dir, '../../..'); +const cliPackageDir = path.join(repoRoot, 'apps/cli'); +const cliDistDir = path.join(cliPackageDir, 'dist'); + +function readJson(filePath: string) { + return JSON.parse(readFileSync(path.join(repoRoot, filePath), 'utf8')) as { + readonly private?: boolean; + readonly exports?: Record; + readonly dependencies?: Record; + readonly devDependencies?: Record; + }; +} + +function assertBuilt(fileName: string): string { + const distPath = path.join(cliDistDir, fileName); + try { + accessSync(distPath, constants.R_OK); + } catch { + throw new Error('agentv package artifact is not built. Run `bun run build` first.'); + } + return distPath; +} + +async function importDist(fileName: string): Promise> { + return (await import(pathToFileURL(assertBuilt(fileName)).href)) as Record; +} + +describe('agentv package exports', () => { + it('publishes agentv as the only public package facade', () => { + const agentvPackage = readJson('apps/cli/package.json'); + const corePackage = readJson('packages/core/package.json'); + const sdkPackage = readJson('packages/sdk/package.json'); + + expect(corePackage.private).toBe(true); + expect(sdkPackage.private).toBe(true); + expect(agentvPackage.exports).toMatchObject({ + '.': { + types: './dist/index.d.ts', + import: './dist/index.js', + }, + './sdk': { + types: './dist/sdk.d.ts', + import: './dist/sdk.js', + }, + './provider': { + types: './dist/provider.d.ts', + import: './dist/provider.js', + }, + './config': { + types: './dist/config.d.ts', + import: './dist/config.js', + }, + }); + }); + + it('keeps the public facade separate from CLI command assembly', () => { + const publicIndex = readFileSync(path.join(cliPackageDir, 'src/index.ts'), 'utf8'); + const publicSdk = readFileSync(path.join(cliPackageDir, 'src/sdk.ts'), 'utf8'); + const cliEntry = readFileSync(path.join(cliPackageDir, 'src/cli.ts'), 'utf8'); + + expect(publicIndex).not.toContain('./cli-app'); + expect(publicSdk).not.toContain('./cli-app'); + expect(cliEntry).toContain('./cli-app.js'); + }); + + it('loads SDK, provider, and config helpers from built agentv artifacts', async () => { + const rootFacade = await importDist('index.js'); + const sdkFacade = await importDist('sdk.js'); + const providerFacade = await importDist('provider.js'); + const configFacade = await importDist('config.js'); + + expect(typeof rootFacade.evaluate).toBe('function'); + expect(typeof rootFacade.defineScriptGrader).toBe('function'); + expect(typeof rootFacade.defineAssertion).toBe('function'); + expect(typeof rootFacade.definePromptTemplate).toBe('function'); + expect(typeof rootFacade.defineEval).toBe('function'); + expect(typeof rootFacade.graders).toBe('object'); + expect(typeof rootFacade.defineConfig).toBe('function'); + + expect(sdkFacade.evaluate).toBe(rootFacade.evaluate); + expect(sdkFacade.defineScriptGrader).toBe(rootFacade.defineScriptGrader); + expect(typeof providerFacade.createBuiltinProviderRegistry).toBe('function'); + expect(typeof providerFacade.createProvider).toBe('function'); + expect(configFacade.defineConfig).toBe(rootFacade.defineConfig); + }); +}); diff --git a/apps/cli/test/setup-core-build.ts b/apps/cli/test/setup-core-build.ts index 3c0a87b53..0aba22c85 100644 --- a/apps/cli/test/setup-core-build.ts +++ b/apps/cli/test/setup-core-build.ts @@ -1,7 +1,7 @@ /** * Pre-flight check for CLI integration tests. * - * CLI integration tests depend on @agentv/core and @agentv/sdk being built + * CLI integration tests depend on @agentv/core being built * (they import from the dist output). Rather than building packages inside * the test — which is slow and hides staleness issues — we simply verify dist exists and * fail fast with a clear message if it doesn't. @@ -9,7 +9,7 @@ * CI runs `bun run build` before `bun run test`, so dist is available in * the normal merge gate. For ad-hoc local runs, build first: * - * bun --filter @agentv/core build && bun --filter @agentv/sdk build && bun --filter agentv test + * bun --filter @agentv/core build && bun --filter agentv test */ import { constants, accessSync } from 'node:fs'; @@ -21,7 +21,6 @@ const __dirname = path.dirname(__filename); const projectRoot = path.resolve(__dirname, '../../..'); const distEntries = [ ['@agentv/core', path.join(projectRoot, 'packages/core/dist/index.js')], - ['@agentv/sdk', path.join(projectRoot, 'packages/sdk/dist/index.js')], ] as const; export function assertCoreBuild(): void { @@ -30,7 +29,7 @@ export function assertCoreBuild(): void { accessSync(distEntry, constants.R_OK); } catch { throw new Error( - `${packageName} is not built. Run \`bun --filter @agentv/core build && bun --filter @agentv/sdk build\` first.`, + `${packageName} is not built. Run \`bun --filter @agentv/core build\` first.`, ); } } diff --git a/apps/cli/tsup.config.ts b/apps/cli/tsup.config.ts index cf84e118f..b674fbdbc 100644 --- a/apps/cli/tsup.config.ts +++ b/apps/cli/tsup.config.ts @@ -3,11 +3,23 @@ import path from 'node:path'; import { defineConfig } from 'tsup'; export default defineConfig({ - entry: ['src/index.ts', 'src/cli.ts'], + entry: [ + 'src/index.ts', + 'src/sdk.ts', + 'src/provider.ts', + 'src/contracts.ts', + 'src/config.ts', + 'src/cli.ts', + ], format: ['esm'], sourcemap: true, clean: true, - dts: false, + dts: { + resolve: true, + compilerOptions: { + composite: false, + }, + }, target: 'node20', platform: 'node', tsconfig: './tsconfig.build.json', diff --git a/packages/core/package.json b/packages/core/package.json index bb28021b6..7c4bea5e7 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,7 @@ { "name": "@agentv/core", "version": "5.3.1-next.1", + "private": true, "description": "Primitive runtime components for AgentV", "type": "module", "repository": { @@ -23,6 +24,11 @@ "types": "./dist/evaluation/validation/index.d.ts", "import": "./dist/evaluation/validation/index.js", "require": "./dist/evaluation/validation/index.cjs" + }, + "./script-grader": { + "types": "./dist/script-grader.d.ts", + "import": "./dist/script-grader.js", + "require": "./dist/script-grader.cjs" } }, "scripts": { @@ -38,7 +44,6 @@ "diagnostics:azure": "bun src/diagnostics/azure-deployment-diag.ts", "generate:schema": "bun scripts/generate-eval-schema.ts" }, - "files": ["dist", "README.md"], "dependencies": { "@agentclientprotocol/sdk": "^0.14.1", "@earendil-works/pi-ai": "^0.74.0", diff --git a/packages/core/src/evaluation/config.ts b/packages/core/src/evaluation/config.ts index 136606247..e6b3c3387 100644 --- a/packages/core/src/evaluation/config.ts +++ b/packages/core/src/evaluation/config.ts @@ -7,7 +7,7 @@ * @example * ```typescript * // agentv.config.ts - * import { defineConfig } from '@agentv/core'; + * import { defineConfig } from 'agentv'; * * export default defineConfig({ * execution: { @@ -147,7 +147,7 @@ export type AgentVConfig = z.infer; * * @example * ```typescript - * import { defineConfig } from '@agentv/core'; + * import { defineConfig } from 'agentv'; * * export default defineConfig({ * execution: { maxConcurrency: 5 }, diff --git a/packages/core/src/evaluation/evaluate.ts b/packages/core/src/evaluation/evaluate.ts index 87984840b..159027e03 100644 --- a/packages/core/src/evaluation/evaluate.ts +++ b/packages/core/src/evaluation/evaluate.ts @@ -7,7 +7,7 @@ * * @example Inline tests with config objects * ```typescript - * import { evaluate } from '@agentv/core'; + * import { evaluate } from 'agentv'; * * const results = await evaluate({ * prompts: ['{{ question }}'], @@ -27,7 +27,7 @@ * * @example Inline tests with task function and custom assertion * ```typescript - * import { evaluate } from '@agentv/core'; + * import { evaluate } from 'agentv'; * * const { summary } = await evaluate({ * prompts: ['{{ text }}'], diff --git a/packages/core/src/evaluation/loaders/ts-eval-loader.ts b/packages/core/src/evaluation/loaders/ts-eval-loader.ts index c6d6e862d..9d6e00255 100644 --- a/packages/core/src/evaluation/loaders/ts-eval-loader.ts +++ b/packages/core/src/evaluation/loaders/ts-eval-loader.ts @@ -15,8 +15,8 @@ import type { ProviderFactoryFn } from '../providers/provider-registry.js'; import type { ProviderDefinition } from '../providers/types.js'; import { type EvalSuiteResult, loadTestSuiteFromYamlObject } from '../yaml-parser.js'; -const SDK_EVAL_SUITE_SYMBOL = Symbol.for('@agentv/sdk/eval-suite'); -const SDK_TO_EVAL_YAML_OBJECT_SYMBOL = Symbol.for('@agentv/sdk/to-eval-yaml-object'); +const SDK_EVAL_SUITE_SYMBOL = Symbol.for('agentv/eval-suite'); +const SDK_TO_EVAL_YAML_OBJECT_SYMBOL = Symbol.for('agentv/to-eval-yaml-object'); const TS_EVAL_CONFIG_NAME_RE = /^.+\.eval\.(?:m)?ts$/i; const TS_EVAL_CONFIG_GLOB = '*.eval.ts,*.eval.mts' as const; diff --git a/packages/core/src/evaluation/registry/grader-registry.ts b/packages/core/src/evaluation/registry/grader-registry.ts index 7f23f7c08..b8a9465b2 100644 --- a/packages/core/src/evaluation/registry/grader-registry.ts +++ b/packages/core/src/evaluation/registry/grader-registry.ts @@ -4,7 +4,7 @@ * Replaces the hardcoded switch/case dispatch in the orchestrator with * a registry of named factory functions. Built-in evaluators are registered * at startup; users can add custom evaluators via `defineAssertion()` in - * `@agentv/sdk` or by dropping files in `.agentv/assertions/`. + * `agentv` or by dropping files in `.agentv/assertions/`. */ import type { EvaluationContext, EvaluationScore, Grader } from '../graders/types.js'; diff --git a/packages/core/src/evaluation/script-grader-runtime.ts b/packages/core/src/evaluation/script-grader-runtime.ts new file mode 100644 index 000000000..c683a7b4d --- /dev/null +++ b/packages/core/src/evaluation/script-grader-runtime.ts @@ -0,0 +1,122 @@ +/** + * Runtime for script grader evaluators. + * Handles stdin parsing, validation, error handling, and output formatting. + */ +import { readFileSync } from 'node:fs'; +import { toCamelCaseDeep } from './case-conversion.js'; + +import { + type ScriptGraderInput, + ScriptGraderInputSchema, + type ScriptGraderResult, + ScriptGraderResultSchema, +} from './script-grader-schemas.js'; + +/** + * Handler function type for script graders. + */ +export type ScriptGraderHandler = ( + input: ScriptGraderInput, +) => ScriptGraderResult | Promise; + +/** + * Read stdin synchronously (works in both Node.js and Bun). + */ +function readStdin(): string { + return readFileSync(0, 'utf8'); +} + +/** + * Clamp a value to the range [0, 1]. + */ +function clampScore(value: number): number { + if (Number.isNaN(value) || !Number.isFinite(value)) { + return 0; + } + return Math.max(0, Math.min(1, value)); +} + +/** + * Format an error for output. + */ +function formatError(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + return String(error); +} + +/** + * Run a script grader handler with full stdin/stdout handling. + * This is the internal implementation called by defineScriptGrader. + */ +export async function runScriptGrader(handler: ScriptGraderHandler): Promise { + try { + // 1. Read stdin + const stdin = readStdin(); + + // 2. Parse JSON + const rawInput = JSON.parse(stdin) as Record; + + // 3. Convert snake_case to camelCase + const camelInput = toCamelCaseDeep(rawInput); + + // 4. Validate input with Zod + const input = ScriptGraderInputSchema.parse(camelInput); + + // 5. Set up lazy file-backed output loading if applicable + if (input.outputPath && (input.output === null || input.output === undefined)) { + let cachedOutput: ScriptGraderInput['output'] | undefined; + const filePath = input.outputPath; + Object.defineProperty(input, 'output', { + get() { + if (cachedOutput === undefined) { + cachedOutput = JSON.parse(readFileSync(filePath, 'utf8')); + } + return cachedOutput; + }, + configurable: true, + enumerable: true, + }); + } + + // 6. Run handler + const rawResult = await handler(input); + + // 7. Validate and normalize output + const result = ScriptGraderResultSchema.parse({ + ...rawResult, + score: clampScore(rawResult.score), + checks: rawResult.checks?.map((check) => ({ + ...check, + ...(check.score !== undefined ? { score: clampScore(check.score) } : {}), + })), + }); + + // 8. Output JSON + console.log(JSON.stringify(result, null, 2)); + } catch (error) { + // Output failure result + const errorMessage = formatError(error); + const errorResult: ScriptGraderResult = { + pass: false, + score: 0, + reason: `Evaluation failed: ${errorMessage}`, + checks: [ + { + text: 'Script grader execution', + pass: false, + reason: errorMessage, + }, + ], + }; + console.log(JSON.stringify(errorResult, null, 2)); + process.exit(1); + } +} + +/** @deprecated Use ScriptGraderHandler. */ +export type CodeGraderHandler = ScriptGraderHandler; + +/** @deprecated Use runScriptGrader. */ +export const runCodeGrader = runScriptGrader; diff --git a/packages/core/src/evaluation/script-grader-schemas.ts b/packages/core/src/evaluation/script-grader-schemas.ts new file mode 100644 index 000000000..f4ca17f99 --- /dev/null +++ b/packages/core/src/evaluation/script-grader-schemas.ts @@ -0,0 +1,391 @@ +/** + * Zod schemas for script grader input/output validation. + * Provides both compile-time types and runtime validation. + * + * ## Content model + * + * `Message.content` accepts `string | object[] | object`: + * - `string` — backward-compatible plain text (most common case) + * - `object[]` — typed content blocks for multimodal messages, plus AgentV + * eval input blocks such as `{ type: "file", value, path, text }` + * - `object` — structured YAML/JSON content such as expected outputs + * + * Content variants: + * - `ContentText` — `{ type: 'text', text: string }` + * - `ContentImage` — `{ type: 'image', media_type: string, path: string }` (file path, not base64) + * - `ContentFile` — `{ type: 'file', media_type: string, path: string }` + * + * To add a new content variant: + * 1. Define a new Zod schema with a unique `type` literal + * 2. Add it to `ContentSchema` discriminated union + * 3. Re-export from `index.ts` + */ +import { z } from 'zod'; + +/** + * Token usage metrics schema. + */ +export const TokenUsageSchema = z.object({ + input: z.number(), + output: z.number(), + cached: z.number().optional(), + reasoning: z.number().optional(), +}); + +/** + * Derived trace summary schema (camelCase for TypeScript ergonomics). + * + * This is a compact read model for metric-style graders. Full transcript/tool + * evidence lives in the canonical `Trace` under `messages` and `events`. + */ +export const TraceSummarySchema = z.object({ + eventCount: z.number(), + toolCalls: z.record(z.string(), z.number()), + errorCount: z.number(), + toolDurations: z.record(z.string(), z.array(z.number())).optional(), + llmCallCount: z.number().optional(), +}); + +export const TRACE_SOURCE_KINDS = [ + 'agentv_run', + 'otlp', + 'phoenix', + 'langfuse', + 'pi_session', + 'imported_transcript', + 'compact_transcript', +] as const; + +export const TRACE_EVENT_TYPES = [ + 'message', + 'model_turn', + 'tool_call', + 'tool_result', + 'final_response', + 'error', +] as const; + +export const TRACE_TOOL_STATUSES = ['ok', 'error', 'timeout', 'cancelled', 'unknown'] as const; + +export const TRACE_REDACTION_LEVELS = ['none', 'partial', 'full'] as const; + +const MetadataSchema = z.record(z.string(), z.unknown()); + +export const TraceRedactionStateSchema = z.object({ + level: z.enum(TRACE_REDACTION_LEVELS), + fields: z.array(z.string()).optional(), + reason: z.string().optional(), +}); + +export const TraceErrorSchema = z.object({ + message: z.string(), + name: z.string().optional(), + code: z.string().optional(), + stack: z.string().optional(), + metadata: MetadataSchema.optional(), +}); + +export const TraceSourceSchema = z.object({ + kind: z.enum(TRACE_SOURCE_KINDS), + path: z.string().optional(), + url: z.string().optional(), + provider: z.string().optional(), + format: z.string().optional(), + version: z.string().optional(), + metadata: MetadataSchema.optional(), +}); + +export const TraceSessionSchema = z.object({ + sessionId: z.string().optional(), + conversationId: z.string().optional(), + cwd: z.string().optional(), + startedAt: z.string().optional(), + endedAt: z.string().optional(), + metadata: MetadataSchema.optional(), +}); + +export const TraceBranchSchema = z.object({ + selectedLeafId: z.string().optional(), + selectedPathIds: z.array(z.string()).optional(), + includedEventIds: z.array(z.string()).optional(), + omittedEventIds: z.array(z.string()).optional(), + selectionReason: z.string().optional(), +}); + +export const TraceSourceRefSchema = z.object({ + eventId: z.string().optional(), + messageId: z.string().optional(), + spanId: z.string().optional(), + traceId: z.string().optional(), + rawKind: z.string().optional(), + path: z.string().optional(), + line: z.number().int().nonnegative().optional(), + metadata: MetadataSchema.optional(), +}); + +export const TraceRawEvidenceSchema = z.object({ + kind: z.string(), + ref: z.string().optional(), + mediaType: z.string().optional(), + content: z.unknown().optional(), + redacted: z.boolean().optional(), + metadata: MetadataSchema.optional(), +}); + +export const TraceMessageSchema = z.object({ + role: z.string(), + name: z.string().optional(), + content: z.unknown().optional(), + redaction: TraceRedactionStateSchema.optional(), + tokenUsage: TokenUsageSchema.optional(), + metadata: MetadataSchema.optional(), +}); + +export const TraceModelSchema = z.object({ + provider: z.string().optional(), + name: z.string().optional(), + invocationId: z.string().optional(), + tokenUsage: TokenUsageSchema.optional(), + metadata: MetadataSchema.optional(), +}); + +export const TraceToolSchema = z.object({ + name: z.string(), + callId: z.string().optional(), + input: z.unknown().optional(), + output: z.unknown().optional(), + status: z.enum(TRACE_TOOL_STATUSES).optional(), + error: TraceErrorSchema.optional(), + redaction: TraceRedactionStateSchema.optional(), + metadata: MetadataSchema.optional(), +}); + +export const TraceEventSchema = z.object({ + eventId: z.string(), + parentEventId: z.string().optional(), + ordinal: z.number().int().nonnegative(), + type: z.enum(TRACE_EVENT_TYPES), + timestamp: z.string().optional(), + durationMs: z.number().nonnegative().optional(), + durationInferred: z.boolean().optional(), + turnIndex: z.number().int().nonnegative().optional(), + message: TraceMessageSchema.optional(), + model: TraceModelSchema.optional(), + tool: TraceToolSchema.optional(), + error: TraceErrorSchema.optional(), + sourceRef: TraceSourceRefSchema.optional(), + rawEvidence: z.array(TraceRawEvidenceSchema).optional(), + redaction: TraceRedactionStateSchema.optional(), + metadata: MetadataSchema.optional(), +}); + +/** + * Derived trace artifact shape used by import/replay helpers. + * + * This is not a persisted public trace contract. Grader authors normally + * receive the result-local `Trace` shape below. + */ +export const TraceArtifactSchema = z.object({ + source: TraceSourceSchema, + session: TraceSessionSchema, + branch: TraceBranchSchema.optional(), + events: z.array(TraceEventSchema), + tokenUsage: TokenUsageSchema.optional(), + costUsd: z.number().optional(), + durationMs: z.number().optional(), + startedAt: z.string().optional(), + endedAt: z.string().optional(), + metadata: MetadataSchema.optional(), +}); + +/** + * Tool call schema. + */ +export const ToolCallSchema = z.object({ + tool: z.string(), + input: z.unknown().optional(), + output: z.unknown().optional(), + id: z.string().optional(), + startTime: z.string().optional(), + endTime: z.string().optional(), + durationMs: z.number().optional(), +}); + +// --------------------------------------------------------------------------- +// Content block schemas (discriminated union on `type`) +// --------------------------------------------------------------------------- + +/** Text content block. */ +export const ContentTextSchema = z.object({ + type: z.literal('text'), + text: z.string(), +}); + +/** + * Image content block. + * `path` is a filesystem path — never inline base64. + */ +export const ContentImageSchema = z.object({ + type: z.literal('image'), + media_type: z.string(), + path: z.string(), +}); + +/** File content block. */ +export const ContentFileSchema = z.object({ + type: z.literal('file'), + media_type: z.string(), + path: z.string(), +}); + +/** Discriminated union of all content block types. */ +export const ContentSchema = z.discriminatedUnion('type', [ + ContentTextSchema, + ContentImageSchema, + ContentFileSchema, +]); + +const MessageContentBlockSchema = z.union([ContentSchema, z.record(z.unknown())]); + +/** + * Unified message schema for input, expected, and output messages. + * + * `content` is a plain string, an array of structured blocks, or a + * structured object from YAML/JSON eval files. Use `getTextContent()` from + * `agentv/contracts` to extract plain text when the content is textual. + */ +export const MessageSchema = z.object({ + role: z.enum(['assistant', 'user', 'system', 'tool']), + content: z + .union([z.string(), z.array(MessageContentBlockSchema), z.record(z.unknown())]) + .optional(), + toolCalls: z.array(ToolCallSchema).optional(), + name: z.string().optional(), + startTime: z.string().optional(), + endTime: z.string().optional(), + durationMs: z.number().optional(), + metadata: z.record(z.unknown()).optional(), +}); + +/** + * Derived evaluation trace read model exposed to custom assertions and script graders. + * + * Top-level summary fields (`eventCount`, `toolCalls`, `errorCount`) remain + * available for existing metric graders; full transcript/tool evidence is under + * `messages` and structured execution events under `events`. + */ +export const TraceSchema = TraceSummarySchema.extend({ + messages: z.array(MessageSchema), + events: z.array(TraceEventSchema), + tokenUsage: TokenUsageSchema.optional(), + costUsd: z.number().optional(), + durationMs: z.number().optional(), + startTime: z.string().optional(), + endTime: z.string().optional(), + metadata: MetadataSchema.optional(), +}); + +/** + * Script grader input schema (camelCase, converted from snake_case wire format). + * + * `output` is the final answer/scored result only. Transcript-aware graders + * should inspect `messages`, `trace.messages`, or `trace.events`. + */ +export const ScriptGraderInputSchema = z.object({ + criteria: z.string(), + expectedOutput: z.array(MessageSchema), + output: z.string().nullable().optional(), + messages: z.array(MessageSchema).optional().default([]), + /** Path to a temp file containing the output JSON (used for large payloads). */ + outputPath: z.string().optional(), + inputFiles: z.array(z.string()), + input: z.array(MessageSchema), + metadata: z.record(z.unknown()).nullable().optional(), + trace: TraceSchema.nullable().optional(), + traceSummary: TraceSummarySchema.nullable().optional(), + tokenUsage: TokenUsageSchema.nullable().optional(), + costUsd: z.number().nullable().optional(), + durationMs: z.number().nullable().optional(), + startTime: z.string().nullable().optional(), + endTime: z.string().nullable().optional(), + fileChanges: z.string().nullable().optional(), + workspacePath: z.string().nullable().optional(), + config: z.record(z.unknown()).nullable().optional(), +}); + +export const ScriptGraderCheckSchema = z.object({ + id: z.string().optional(), + text: z.string(), + pass: z.boolean(), + score: z.number().min(0).max(1).optional(), + reason: z.string(), + evidence: z.string().optional(), +}); + +/** + * Script grader result schema (validated before output). + */ +export const ScriptGraderResultSchema = z.object({ + pass: z.boolean(), + score: z.number().min(0).max(1), + reason: z.string(), + checks: z.array(ScriptGraderCheckSchema).optional().default([]), + /** Optional structured details for domain-specific metrics (e.g., TP/TN/FP/FN counts, alignments). */ + details: z.record(z.unknown()).optional(), +}); + +/** + * Inferred types from schemas. + */ +export type ScriptGraderInput = z.infer; +export type ScriptGraderCheck = z.infer; +export type ScriptGraderResult = z.infer; + +export type TraceSummary = z.infer; +export type Trace = z.infer; +export type TraceArtifact = z.infer; +export type TraceSource = z.infer; +export type TraceSession = z.infer; +export type TraceBranch = z.infer; +export type TraceEvent = z.infer; +export type TraceMessage = z.infer; +export type TraceModel = z.infer; +export type TraceTool = z.infer; +export type TraceError = z.infer; +export type TraceSourceRef = z.infer; +export type TraceRawEvidence = z.infer; +export type TraceRedactionState = z.infer; +export type Message = z.infer; +export type ToolCall = z.infer; +export type TokenUsage = z.infer; + +export type ContentText = z.infer; +export type ContentImage = z.infer; +export type ContentFile = z.infer; +export type Content = z.infer; + +/** + * Prompt template input schema (camelCase, converted from snake_case wire format). + * Uses the same schema as ScriptGraderInput since the orchestrator sends identical payloads. + */ +export const PromptTemplateInputSchema = ScriptGraderInputSchema; + +export type PromptTemplateInput = ScriptGraderInput; + +// ── Backward-compat aliases (deprecated) ──────────────────────────────────────── +/** @deprecated Use ScriptGraderInputSchema */ +export const CodeGraderInputSchema = ScriptGraderInputSchema; +/** @deprecated Use ScriptGraderResultSchema */ +export const CodeGraderResultSchema = ScriptGraderResultSchema; +/** @deprecated Use ScriptGraderInput */ +export type CodeGraderInput = ScriptGraderInput; +/** @deprecated Use ScriptGraderResult */ +export type CodeGraderResult = ScriptGraderResult; +/** @deprecated Use ScriptGraderInputSchema */ +export const CodeJudgeInputSchema = ScriptGraderInputSchema; +/** @deprecated Use ScriptGraderResultSchema */ +export const CodeJudgeResultSchema = ScriptGraderResultSchema; +/** @deprecated Use ScriptGraderInput */ +export type CodeJudgeInput = ScriptGraderInput; +/** @deprecated Use ScriptGraderResult */ +export type CodeJudgeResult = ScriptGraderResult; diff --git a/packages/core/src/evaluation/vitest-workspace-grader.ts b/packages/core/src/evaluation/vitest-workspace-grader.ts new file mode 100644 index 000000000..5b0c061e5 --- /dev/null +++ b/packages/core/src/evaluation/vitest-workspace-grader.ts @@ -0,0 +1,433 @@ +/** + * Vitest workspace verifier adapter. + * + * This module keeps deterministic workspace verification in familiar Vitest + * tests while translating the JSON reporter output into AgentV's script-grader + * result contract. + */ +import { spawn } from 'node:child_process'; +import { copyFile, mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import nodePath from 'node:path'; + +import { runScriptGrader } from './script-grader-runtime.js'; +import { + type ScriptGraderInput, + type ScriptGraderResult, + ScriptGraderResultSchema, +} from './script-grader-schemas.js'; + +export interface VitestWorkspaceGraderOptions { + /** + * Vitest verifier file(s). By default these are relative to the prepared + * workspace. When `copyTestFilesToWorkspace` is true, relative paths resolve + * from `testFileRoot` instead. + * + * When provided without `command`, the adapter runs + * `bunx vitest run --reporter=json --outputFile `. + */ + readonly testFile?: string | readonly string[]; + /** + * Copy `testFile` entries into a temporary directory inside the workspace + * before running Vitest. Use this for hidden verifier files that live beside + * the eval instead of inside the prepared workspace. + */ + readonly copyTestFilesToWorkspace?: boolean; + /** + * Base directory for copied `testFile` entries. Defaults to `process.cwd()`. + */ + readonly testFileRoot?: string; + /** + * Full command to run. Use this for package scripts such as + * `["bun", "run", "verify:workspace"]`. + */ + readonly command?: readonly string[]; + /** + * Base Vitest command used with `testFile`. Defaults to + * `["bunx", "vitest", "run"]`. + */ + readonly vitestCommand?: readonly string[]; + /** Workspace-relative directory to run the command in. Defaults to workspace root. */ + readonly cwd?: string; + /** Append `--reporter=json --outputFile ` to the command. Defaults to true for testFile mode. */ + readonly appendReporterArgs?: boolean; + /** Read Vitest JSON from this path instead of stdout. Relative paths resolve under `cwd`. */ + readonly outputFile?: string; + readonly timeoutMs?: number; + readonly env?: Readonly>; + readonly passWithNoTests?: boolean; +} + +interface CommandResult { + readonly exitCode: number | null; + readonly stdout: string; + readonly stderr: string; +} + +interface VitestAssertionResult { + readonly ancestorTitles?: readonly string[]; + readonly fullName?: string; + readonly title?: string; + readonly status?: string; + readonly failureMessages?: readonly string[]; + readonly duration?: number; +} + +interface VitestFileResult { + readonly name?: string; + readonly assertionResults?: readonly VitestAssertionResult[]; +} + +interface VitestJsonReport { + readonly success?: boolean; + readonly numTotalTests?: number; + readonly numPassedTests?: number; + readonly numFailedTests?: number; + readonly numPendingTests?: number; + readonly numTodoTests?: number; + readonly testResults?: readonly VitestFileResult[]; +} + +function workspacePathFrom(input: ScriptGraderInput): string | undefined { + const workspacePath = input.workspacePath ?? process.env.AGENTV_WORKSPACE_PATH; + return workspacePath?.trim() ? workspacePath : undefined; +} + +function truncate(value: string, maxLength = 2000): string { + if (value.length <= maxLength) { + return value; + } + return `${value.slice(0, maxLength)}\n...(truncated)`; +} + +function resolveInsideWorkspace( + workspacePath: string, + relativePath: string, + label: string, + options: { readonly allowRoot?: boolean } = {}, +): string { + if (!relativePath.trim()) { + throw new Error(`${label} must not be empty.`); + } + + if (nodePath.isAbsolute(relativePath)) { + throw new Error(`${label} must be relative to the workspace: ${relativePath}`); + } + + const root = nodePath.resolve(workspacePath); + const resolvedPath = nodePath.resolve(root, relativePath); + const relativeToRoot = nodePath.relative(root, resolvedPath); + if (relativeToRoot === '' && options.allowRoot === true) { + return resolvedPath; + } + + if ( + relativeToRoot === '' || + relativeToRoot.startsWith('..') || + nodePath.isAbsolute(relativeToRoot) + ) { + throw new Error(`${label} must stay inside the workspace: ${relativePath}`); + } + + return resolvedPath; +} + +function normalizeTestFiles(testFile: string | readonly string[] | undefined): readonly string[] { + if (testFile === undefined) { + return []; + } + return typeof testFile === 'string' ? [testFile] : [...testFile]; +} + +function buildCommand( + options: VitestWorkspaceGraderOptions, + testFiles: readonly string[] = normalizeTestFiles(options.testFile), +) { + if (options.command && options.command.length > 0) { + return [...options.command]; + } + + const command = [...(options.vitestCommand ?? ['bunx', 'vitest', 'run'])]; + command.push(...testFiles); + return command; +} + +function parseJsonObjectFromText(text: string): unknown { + const trimmed = text.trim(); + if (!trimmed) { + return undefined; + } + + try { + return JSON.parse(trimmed); + } catch { + const start = trimmed.indexOf('{'); + const end = trimmed.lastIndexOf('}'); + if (start >= 0 && end > start) { + return JSON.parse(trimmed.slice(start, end + 1)); + } + throw new Error('Vitest output did not contain a JSON object.'); + } +} + +function isVitestJsonReport(value: unknown): value is VitestJsonReport { + return ( + typeof value === 'object' && + value !== null && + Array.isArray((value as VitestJsonReport).testResults) + ); +} + +function assertionText(file: VitestFileResult, assertion: VitestAssertionResult): string { + const title = + assertion.fullName ?? + [...(assertion.ancestorTitles ?? []), assertion.title].filter(Boolean).join(' '); + return title || file.name || 'Vitest assertion'; +} + +export function vitestReportToScriptGraderResult( + report: VitestJsonReport, + options: Pick = {}, +): ScriptGraderResult { + const checks = (report.testResults ?? []).flatMap((file) => + (file.assertionResults ?? []).map((item) => { + const pass = item.status === 'passed'; + const evidence = + item.failureMessages && item.failureMessages.length > 0 + ? truncate(item.failureMessages.join('\n\n')) + : undefined; + return { + text: assertionText(file, item), + pass, + reason: pass ? 'Vitest test passed.' : 'Vitest test failed.', + ...(evidence !== undefined ? { evidence } : {}), + }; + }), + ); + + if (checks.length === 0) { + const pass = options.passWithNoTests === true; + return ScriptGraderResultSchema.parse({ + pass, + score: pass ? 1 : 0, + reason: pass ? 'Vitest reported no tests; configured to pass.' : 'Vitest reported no tests.', + checks: [ + { + text: 'Vitest reported no tests', + pass, + reason: pass ? 'passWithNoTests is enabled.' : 'No Vitest tests were discovered.', + }, + ], + details: { + vitest_success: report.success ?? false, + num_total_tests: report.numTotalTests ?? 0, + num_passed_tests: report.numPassedTests ?? 0, + num_failed_tests: report.numFailedTests ?? 0, + num_pending_tests: report.numPendingTests ?? 0, + num_todo_tests: report.numTodoTests ?? 0, + }, + }); + } + + const passedCount = checks.filter((item) => item.pass).length; + const pass = passedCount === checks.length; + return ScriptGraderResultSchema.parse({ + pass, + score: passedCount / checks.length, + reason: `${passedCount}/${checks.length} Vitest tests passed.`, + checks, + details: { + vitest_success: report.success ?? pass, + num_total_tests: report.numTotalTests ?? checks.length, + num_passed_tests: report.numPassedTests ?? passedCount, + num_failed_tests: report.numFailedTests ?? checks.length - passedCount, + num_pending_tests: report.numPendingTests ?? 0, + num_todo_tests: report.numTodoTests ?? 0, + }, + }); +} + +/** @deprecated Use vitestReportToScriptGraderResult. */ +export const vitestReportToCodeGraderResult = vitestReportToScriptGraderResult; + +function runCommand( + command: readonly string[], + options: { + readonly cwd: string; + readonly timeoutMs?: number; + readonly env?: Readonly>; + }, +): Promise { + if (command.length === 0) { + return Promise.reject(new Error('Vitest command must not be empty.')); + } + + return new Promise((resolve, reject) => { + const child = spawn(command[0], command.slice(1), { + cwd: options.cwd, + env: { ...process.env, ...options.env }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + let timedOut = false; + const timeout = + options.timeoutMs !== undefined + ? setTimeout(() => { + timedOut = true; + child.kill('SIGTERM'); + }, options.timeoutMs) + : undefined; + + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { + stdout += chunk; + }); + child.stderr.on('data', (chunk) => { + stderr += chunk; + }); + child.on('error', reject); + child.on('close', (exitCode) => { + if (timeout) { + clearTimeout(timeout); + } + if (timedOut) { + reject(new Error(`Vitest command timed out after ${options.timeoutMs}ms.`)); + return; + } + resolve({ exitCode, stdout, stderr }); + }); + }); +} + +function resolveSourceTestFile(testFileRoot: string, testFile: string): string { + if (!testFile.trim()) { + throw new Error('testFile entries must not be empty.'); + } + + return nodePath.isAbsolute(testFile) + ? nodePath.resolve(testFile) + : nodePath.resolve(testFileRoot, testFile); +} + +async function copyTestFilesIntoWorkspace( + testFiles: readonly string[], + options: Pick, + cwd: string, +): Promise<{ readonly testFiles: readonly string[]; readonly tempDir?: string }> { + if (testFiles.length === 0) { + return { testFiles }; + } + + const tempDir = await mkdtemp(nodePath.join(cwd, '.agentv-vitest-')); + const testFileRoot = nodePath.resolve(options.testFileRoot ?? process.cwd()); + const copiedFiles = await Promise.all( + testFiles.map(async (testFile, index) => { + const sourcePath = resolveSourceTestFile(testFileRoot, testFile); + const destinationPath = nodePath.join(tempDir, `${index}-${nodePath.basename(testFile)}`); + await copyFile(sourcePath, destinationPath); + return nodePath.relative(cwd, destinationPath); + }), + ); + return { testFiles: copiedFiles, tempDir }; +} + +async function readVitestReport( + commandResult: CommandResult, + outputFile: string | undefined, +): Promise { + const rawReport = + outputFile !== undefined ? await readFile(outputFile, 'utf8') : commandResult.stdout; + const parsed = parseJsonObjectFromText(rawReport); + if (!isVitestJsonReport(parsed)) { + throw new Error('Vitest JSON report did not include testResults[].'); + } + return parsed; +} + +export async function runVitestWorkspaceGrader( + options: VitestWorkspaceGraderOptions, + input: ScriptGraderInput, +): Promise { + const workspacePath = workspacePathFrom(input); + if (!workspacePath) { + return { + pass: false, + score: 0, + reason: 'Vitest workspace verifier requires workspace_path.', + checks: [ + { + text: 'Vitest workspace verifier requires workspace_path', + pass: false, + reason: + 'Configure an environment recipe in the eval YAML so AgentV can pass workspace_path.', + }, + ], + }; + } + + const tempDirs: string[] = []; + try { + const cwd = options.cwd + ? resolveInsideWorkspace(workspacePath, options.cwd, 'cwd', { allowRoot: true }) + : workspacePath; + const testFiles = normalizeTestFiles(options.testFile); + const preparedTestFiles = + options.copyTestFilesToWorkspace === true + ? await copyTestFilesIntoWorkspace(testFiles, options, cwd) + : { testFiles }; + if (preparedTestFiles.tempDir) { + tempDirs.push(preparedTestFiles.tempDir); + } + + const command = buildCommand(options, preparedTestFiles.testFiles); + const appendReporterArgs = options.appendReporterArgs ?? options.command === undefined; + let outputFile = options.outputFile + ? resolveInsideWorkspace(cwd, options.outputFile, 'outputFile') + : undefined; + + if (appendReporterArgs) { + const tempDir = await mkdtemp(nodePath.join(tmpdir(), 'agentv-vitest-')); + tempDirs.push(tempDir); + outputFile = nodePath.join(tempDir, 'results.json'); + command.push('--reporter=json', `--outputFile=${outputFile}`); + } + + const result = await runCommand(command, { + cwd, + timeoutMs: options.timeoutMs, + env: { + AGENTV_WORKSPACE_PATH: workspacePath, + ...(outputFile !== undefined ? { AGENTV_VITEST_JSON_PATH: outputFile } : {}), + ...options.env, + }, + }); + + const report = await readVitestReport(result, outputFile); + return vitestReportToScriptGraderResult(report, options); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + return { + pass: false, + score: 0, + reason: 'Vitest workspace verifier failed to run.', + checks: [ + { + text: 'Vitest workspace verifier failed to run', + pass: false, + reason, + }, + ], + }; + } finally { + for (const tempDir of tempDirs.reverse()) { + await rm(tempDir, { recursive: true, force: true }).catch(() => {}); + } + } +} + +export function defineVitestWorkspaceGrader(options: VitestWorkspaceGraderOptions): void { + runScriptGrader((input) => runVitestWorkspaceGrader(options, input)); +} diff --git a/packages/core/src/script-grader.ts b/packages/core/src/script-grader.ts new file mode 100644 index 000000000..8c23eb328 --- /dev/null +++ b/packages/core/src/script-grader.ts @@ -0,0 +1,3 @@ +export * from './evaluation/script-grader-runtime.js'; +export * from './evaluation/script-grader-schemas.js'; +export * from './evaluation/vitest-workspace-grader.js'; diff --git a/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts b/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts index fc156b6a3..1ee6bf87e 100644 --- a/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts @@ -1,9 +1,9 @@ -const EVAL_SUITE_SYMBOL = Symbol.for('@agentv/sdk/eval-suite'); -const TO_EVAL_YAML_OBJECT_SYMBOL = Symbol.for('@agentv/sdk/to-eval-yaml-object'); +const EVAL_SUITE_SYMBOL = Symbol.for('agentv/eval-suite'); +const TO_EVAL_YAML_OBJECT_SYMBOL = Symbol.for('agentv/to-eval-yaml-object'); const suite = { name: 'sdk-define-eval-suite', - description: 'YAML-aligned TypeScript suite authored with @agentv/sdk', + description: 'YAML-aligned TypeScript suite authored with agentv', tags: ['sdk', 'typescript', 'yaml'], providers: [ { diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index 4fd7ceabd..b8a153329 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'tsup'; export default defineConfig({ entry: [ 'src/index.ts', + 'src/script-grader.ts', 'src/evaluation/validation/index.ts', 'src/evaluation/providers/sdk-child-runner.ts', ], diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 853823df0..5d1a2346a 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,7 +1,8 @@ { "name": "@agentv/sdk", "version": "5.3.1-next.1", - "description": "Evaluation SDK for AgentV - build custom assertions and script graders", + "private": true, + "description": "Internal implementation package for AgentV TypeScript SDK helpers", "type": "module", "repository": { "type": "git", @@ -21,19 +22,16 @@ } }, "scripts": { - "prepublishOnly": "node -e \"if(process.env.ALLOW_PUBLISH!=='1'){console.error('ERROR: Use bun run publish:next, then bun run promote:latest');process.exit(1)}\"", - "build:deps": "bun --cwd ../.. --filter @agentv/core build", - "build": "bun run build:deps && tsup", + "build": "tsup", "dev": "tsup --watch", - "typecheck": "bun run build:deps && tsc --noEmit", + "typecheck": "tsc --noEmit", "lint": "biome check .", "format": "biome format --write .", "fix": "biome check --write .", - "test": "bun run build:deps && bun test" + "test": "bun test" }, - "files": ["dist", "README.md"], "dependencies": { - "@agentv/core": "5.3.1-next.1", + "@agentv/core": "workspace:*", "yaml": "^2.8.3", "zod": "^3.23.8" } diff --git a/packages/sdk/src/deprecation.ts b/packages/sdk/src/deprecation.ts index 8bcbea344..2d083acc9 100644 --- a/packages/sdk/src/deprecation.ts +++ b/packages/sdk/src/deprecation.ts @@ -13,7 +13,7 @@ import type { ScriptGraderInput } from './schemas.js'; * * Previously populated text convenience accessors; now a no-op pass-through since * those fields were removed. script graders should extract text from `Message.content` - * using `getTextContent()` from `@agentv/core` instead. + * using `getTextContent()` from `agentv/contracts` instead. */ export function enrichInput(input: ScriptGraderInput): ScriptGraderInput { return input; diff --git a/packages/sdk/src/eval.ts b/packages/sdk/src/eval.ts index 8878599b8..fa70a134e 100644 --- a/packages/sdk/src/eval.ts +++ b/packages/sdk/src/eval.ts @@ -1,7 +1,7 @@ import { stringify as stringifyYaml } from 'yaml'; -const EVAL_SUITE_SYMBOL = Symbol.for('@agentv/sdk/eval-suite'); -const TO_EVAL_YAML_OBJECT_SYMBOL = Symbol.for('@agentv/sdk/to-eval-yaml-object'); +const EVAL_SUITE_SYMBOL = Symbol.for('agentv/eval-suite'); +const TO_EVAL_YAML_OBJECT_SYMBOL = Symbol.for('agentv/to-eval-yaml-object'); const KNOWN_SNAKE_CASE_KEYS = { afterAll: 'after_all', diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 691b53bcf..177ec48f9 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -6,7 +6,7 @@ * @example Custom assertion (simplest way to add evaluation logic) * ```typescript * #!/usr/bin/env bun - * import { defineAssertion } from '@agentv/sdk'; + * import { defineAssertion } from 'agentv'; * * export default defineAssertion(({ output, criteria }) => { * const answer = output ?? ''; @@ -21,7 +21,7 @@ * @example script grader (full control) * ```typescript * #!/usr/bin/env bun - * import { defineScriptGrader } from '@agentv/sdk'; + * import { defineScriptGrader } from 'agentv'; * * export default defineScriptGrader(({ output, traceSummary }) => { * return { @@ -39,7 +39,7 @@ * @example Vitest workspace verifier adapter (custom wrapper form) * ```typescript * #!/usr/bin/env bun - * import { defineVitestWorkspaceGrader } from '@agentv/sdk'; + * import { defineVitestWorkspaceGrader } from 'agentv'; * * export default defineVitestWorkspaceGrader({ * testFile: 'graders/welcome-banner.test.ts', @@ -50,7 +50,7 @@ * @example Workspace grader (small file checks) * ```typescript * #!/usr/bin/env bun - * import { defineWorkspaceGrader } from '@agentv/sdk'; + * import { defineWorkspaceGrader } from 'agentv'; * * export default defineWorkspaceGrader(async ({ workspace }) => [ * await workspace.file('app/page.tsx').contains('Status: All systems ready'), @@ -283,7 +283,7 @@ export type { PromptTemplateHandler }; * * @example * ```typescript - * import { defineScriptGrader } from '@agentv/sdk'; + * import { defineScriptGrader } from 'agentv'; * * export default defineScriptGrader(({ trace }) => { * if (!trace) { @@ -302,7 +302,7 @@ export type { PromptTemplateHandler }; * * @example With typed config * ```typescript - * import { defineScriptGrader, z } from '@agentv/sdk'; + * import { defineScriptGrader, z } from 'agentv'; * * const ConfigSchema = z.object({ * maxToolCalls: z.number().default(10), @@ -338,7 +338,7 @@ export function defineCodeGrader(handler: ScriptGraderHandler): void { * * @example * ```typescript - * import { definePromptTemplate } from '@agentv/sdk'; + * import { definePromptTemplate } from 'agentv'; * * export default definePromptTemplate((ctx) => { * const question = ctx.input @@ -376,7 +376,7 @@ export function definePromptTemplate(handler: PromptTemplateHandler): void { * * @example Simple pass/fail * ```typescript - * import { defineAssertion } from '@agentv/sdk'; + * import { defineAssertion } from 'agentv'; * * export default defineAssertion(({ output }) => { * const text = output ?? ''; @@ -389,7 +389,7 @@ export function definePromptTemplate(handler: PromptTemplateHandler): void { * * @example Granular scoring * ```typescript - * import { defineAssertion } from '@agentv/sdk'; + * import { defineAssertion } from 'agentv'; * * export default defineAssertion(({ output, traceSummary }) => { * const text = output ?? ''; diff --git a/packages/sdk/src/prompt-template.ts b/packages/sdk/src/prompt-template.ts index 45b8f6f35..5c0ef16cf 100644 --- a/packages/sdk/src/prompt-template.ts +++ b/packages/sdk/src/prompt-template.ts @@ -68,7 +68,7 @@ export async function runPromptTemplate(handler: PromptTemplateHandler): Promise * * @example * ```typescript - * import { definePromptTemplate, type ScriptGraderInput } from '@agentv/sdk'; + * import { definePromptTemplate, type ScriptGraderInput } from 'agentv'; * * export default definePromptTemplate((ctx: ScriptGraderInput) => { * const question = ctx.input diff --git a/packages/sdk/src/runtime.ts b/packages/sdk/src/runtime.ts index cf0259d72..0b5054848 100644 --- a/packages/sdk/src/runtime.ts +++ b/packages/sdk/src/runtime.ts @@ -1,126 +1,6 @@ -/** - * Runtime for script grader evaluators. - * Handles stdin parsing, validation, error handling, and output formatting. - */ -import { readFileSync } from 'node:fs'; -import { toCamelCaseDeep } from '@agentv/core'; - -import { enrichInput } from './deprecation.js'; -import { - type ScriptGraderInput, - ScriptGraderInputSchema, - type ScriptGraderResult, - ScriptGraderResultSchema, -} from './schemas.js'; - -/** - * Handler function type for script graders. - */ -export type ScriptGraderHandler = ( - input: ScriptGraderInput, -) => ScriptGraderResult | Promise; - -/** - * Read stdin synchronously (works in both Node.js and Bun). - */ -function readStdin(): string { - return readFileSync(0, 'utf8'); -} - -/** - * Clamp a value to the range [0, 1]. - */ -function clampScore(value: number): number { - if (Number.isNaN(value) || !Number.isFinite(value)) { - return 0; - } - return Math.max(0, Math.min(1, value)); -} - -/** - * Format an error for output. - */ -function formatError(error: unknown): string { - if (error instanceof Error) { - return error.message; - } - return String(error); -} - -/** - * Run a script grader handler with full stdin/stdout handling. - * This is the internal implementation called by defineScriptGrader. - */ -export async function runScriptGrader(handler: ScriptGraderHandler): Promise { - try { - // 1. Read stdin - const stdin = readStdin(); - - // 2. Parse JSON - const rawInput = JSON.parse(stdin) as Record; - - // 3. Convert snake_case to camelCase - const camelInput = toCamelCaseDeep(rawInput); - - // 4. Validate input with Zod - const input = ScriptGraderInputSchema.parse(camelInput); - - // 5. Set up lazy file-backed output loading if applicable - if (input.outputPath && (input.output === null || input.output === undefined)) { - let cachedOutput: ScriptGraderInput['output'] | undefined; - const filePath = input.outputPath; - Object.defineProperty(input, 'output', { - get() { - if (cachedOutput === undefined) { - cachedOutput = JSON.parse(readFileSync(filePath, 'utf8')); - } - return cachedOutput; - }, - configurable: true, - enumerable: true, - }); - } - - // 6. Enrich input — no-op pass-through - enrichInput(input); - - // 7. Run handler - const rawResult = await handler(input); - - // 8. Validate and normalize output - const result = ScriptGraderResultSchema.parse({ - ...rawResult, - score: clampScore(rawResult.score), - checks: rawResult.checks?.map((check) => ({ - ...check, - ...(check.score !== undefined ? { score: clampScore(check.score) } : {}), - })), - }); - - // 9. Output JSON - console.log(JSON.stringify(result, null, 2)); - } catch (error) { - // Output failure result - const errorMessage = formatError(error); - const errorResult: ScriptGraderResult = { - pass: false, - score: 0, - reason: `Evaluation failed: ${errorMessage}`, - checks: [ - { - text: 'Script grader execution', - pass: false, - reason: errorMessage, - }, - ], - }; - console.log(JSON.stringify(errorResult, null, 2)); - process.exit(1); - } -} - -/** @deprecated Use ScriptGraderHandler. */ -export type CodeGraderHandler = ScriptGraderHandler; - -/** @deprecated Use runScriptGrader. */ -export const runCodeGrader = runScriptGrader; +export { + runCodeGrader, + runScriptGrader, + type CodeGraderHandler, + type ScriptGraderHandler, +} from '@agentv/core/script-grader'; diff --git a/packages/sdk/src/schemas.ts b/packages/sdk/src/schemas.ts index f2ca874e2..686df63c4 100644 --- a/packages/sdk/src/schemas.ts +++ b/packages/sdk/src/schemas.ts @@ -1,391 +1,64 @@ -/** - * Zod schemas for script grader input/output validation. - * Provides both compile-time types and runtime validation. - * - * ## Content model - * - * `Message.content` accepts `string | object[] | object`: - * - `string` — backward-compatible plain text (most common case) - * - `object[]` — typed content blocks for multimodal messages, plus AgentV - * eval input blocks such as `{ type: "file", value, path, text }` - * - `object` — structured YAML/JSON content such as expected outputs - * - * Content variants: - * - `ContentText` — `{ type: 'text', text: string }` - * - `ContentImage` — `{ type: 'image', media_type: string, path: string }` (file path, not base64) - * - `ContentFile` — `{ type: 'file', media_type: string, path: string }` - * - * To add a new content variant: - * 1. Define a new Zod schema with a unique `type` literal - * 2. Add it to `ContentSchema` discriminated union - * 3. Re-export from `index.ts` - */ -import { z } from 'zod'; - -/** - * Token usage metrics schema. - */ -export const TokenUsageSchema = z.object({ - input: z.number(), - output: z.number(), - cached: z.number().optional(), - reasoning: z.number().optional(), -}); - -/** - * Derived trace summary schema (camelCase for TypeScript ergonomics). - * - * This is a compact read model for metric-style graders. Full transcript/tool - * evidence lives in the canonical `Trace` under `messages` and `events`. - */ -export const TraceSummarySchema = z.object({ - eventCount: z.number(), - toolCalls: z.record(z.string(), z.number()), - errorCount: z.number(), - toolDurations: z.record(z.string(), z.array(z.number())).optional(), - llmCallCount: z.number().optional(), -}); - -export const TRACE_SOURCE_KINDS = [ - 'agentv_run', - 'otlp', - 'phoenix', - 'langfuse', - 'pi_session', - 'imported_transcript', - 'compact_transcript', -] as const; - -export const TRACE_EVENT_TYPES = [ - 'message', - 'model_turn', - 'tool_call', - 'tool_result', - 'final_response', - 'error', -] as const; - -export const TRACE_TOOL_STATUSES = ['ok', 'error', 'timeout', 'cancelled', 'unknown'] as const; - -export const TRACE_REDACTION_LEVELS = ['none', 'partial', 'full'] as const; - -const MetadataSchema = z.record(z.string(), z.unknown()); - -export const TraceRedactionStateSchema = z.object({ - level: z.enum(TRACE_REDACTION_LEVELS), - fields: z.array(z.string()).optional(), - reason: z.string().optional(), -}); - -export const TraceErrorSchema = z.object({ - message: z.string(), - name: z.string().optional(), - code: z.string().optional(), - stack: z.string().optional(), - metadata: MetadataSchema.optional(), -}); - -export const TraceSourceSchema = z.object({ - kind: z.enum(TRACE_SOURCE_KINDS), - path: z.string().optional(), - url: z.string().optional(), - provider: z.string().optional(), - format: z.string().optional(), - version: z.string().optional(), - metadata: MetadataSchema.optional(), -}); - -export const TraceSessionSchema = z.object({ - sessionId: z.string().optional(), - conversationId: z.string().optional(), - cwd: z.string().optional(), - startedAt: z.string().optional(), - endedAt: z.string().optional(), - metadata: MetadataSchema.optional(), -}); - -export const TraceBranchSchema = z.object({ - selectedLeafId: z.string().optional(), - selectedPathIds: z.array(z.string()).optional(), - includedEventIds: z.array(z.string()).optional(), - omittedEventIds: z.array(z.string()).optional(), - selectionReason: z.string().optional(), -}); - -export const TraceSourceRefSchema = z.object({ - eventId: z.string().optional(), - messageId: z.string().optional(), - spanId: z.string().optional(), - traceId: z.string().optional(), - rawKind: z.string().optional(), - path: z.string().optional(), - line: z.number().int().nonnegative().optional(), - metadata: MetadataSchema.optional(), -}); - -export const TraceRawEvidenceSchema = z.object({ - kind: z.string(), - ref: z.string().optional(), - mediaType: z.string().optional(), - content: z.unknown().optional(), - redacted: z.boolean().optional(), - metadata: MetadataSchema.optional(), -}); - -export const TraceMessageSchema = z.object({ - role: z.string(), - name: z.string().optional(), - content: z.unknown().optional(), - redaction: TraceRedactionStateSchema.optional(), - tokenUsage: TokenUsageSchema.optional(), - metadata: MetadataSchema.optional(), -}); - -export const TraceModelSchema = z.object({ - provider: z.string().optional(), - name: z.string().optional(), - invocationId: z.string().optional(), - tokenUsage: TokenUsageSchema.optional(), - metadata: MetadataSchema.optional(), -}); - -export const TraceToolSchema = z.object({ - name: z.string(), - callId: z.string().optional(), - input: z.unknown().optional(), - output: z.unknown().optional(), - status: z.enum(TRACE_TOOL_STATUSES).optional(), - error: TraceErrorSchema.optional(), - redaction: TraceRedactionStateSchema.optional(), - metadata: MetadataSchema.optional(), -}); - -export const TraceEventSchema = z.object({ - eventId: z.string(), - parentEventId: z.string().optional(), - ordinal: z.number().int().nonnegative(), - type: z.enum(TRACE_EVENT_TYPES), - timestamp: z.string().optional(), - durationMs: z.number().nonnegative().optional(), - durationInferred: z.boolean().optional(), - turnIndex: z.number().int().nonnegative().optional(), - message: TraceMessageSchema.optional(), - model: TraceModelSchema.optional(), - tool: TraceToolSchema.optional(), - error: TraceErrorSchema.optional(), - sourceRef: TraceSourceRefSchema.optional(), - rawEvidence: z.array(TraceRawEvidenceSchema).optional(), - redaction: TraceRedactionStateSchema.optional(), - metadata: MetadataSchema.optional(), -}); - -/** - * Derived trace artifact shape used by import/replay helpers. - * - * This is not a persisted public trace contract. Grader authors normally - * receive the result-local `Trace` shape below. - */ -export const TraceArtifactSchema = z.object({ - source: TraceSourceSchema, - session: TraceSessionSchema, - branch: TraceBranchSchema.optional(), - events: z.array(TraceEventSchema), - tokenUsage: TokenUsageSchema.optional(), - costUsd: z.number().optional(), - durationMs: z.number().optional(), - startedAt: z.string().optional(), - endedAt: z.string().optional(), - metadata: MetadataSchema.optional(), -}); - -/** - * Tool call schema. - */ -export const ToolCallSchema = z.object({ - tool: z.string(), - input: z.unknown().optional(), - output: z.unknown().optional(), - id: z.string().optional(), - startTime: z.string().optional(), - endTime: z.string().optional(), - durationMs: z.number().optional(), -}); - -// --------------------------------------------------------------------------- -// Content block schemas (discriminated union on `type`) -// --------------------------------------------------------------------------- - -/** Text content block. */ -export const ContentTextSchema = z.object({ - type: z.literal('text'), - text: z.string(), -}); - -/** - * Image content block. - * `path` is a filesystem path — never inline base64. - */ -export const ContentImageSchema = z.object({ - type: z.literal('image'), - media_type: z.string(), - path: z.string(), -}); - -/** File content block. */ -export const ContentFileSchema = z.object({ - type: z.literal('file'), - media_type: z.string(), - path: z.string(), -}); - -/** Discriminated union of all content block types. */ -export const ContentSchema = z.discriminatedUnion('type', [ - ContentTextSchema, - ContentImageSchema, +export { + CodeGraderInputSchema, + CodeGraderResultSchema, + CodeJudgeInputSchema, + CodeJudgeResultSchema, ContentFileSchema, -]); - -const MessageContentBlockSchema = z.union([ContentSchema, z.record(z.unknown())]); - -/** - * Unified message schema for input, expected, and output messages. - * - * `content` is a plain string, an array of structured blocks, or a - * structured object from YAML/JSON eval files. Use `getTextContent()` from - * `@agentv/core` to extract plain text when the content is textual. - */ -export const MessageSchema = z.object({ - role: z.enum(['assistant', 'user', 'system', 'tool']), - content: z - .union([z.string(), z.array(MessageContentBlockSchema), z.record(z.unknown())]) - .optional(), - toolCalls: z.array(ToolCallSchema).optional(), - name: z.string().optional(), - startTime: z.string().optional(), - endTime: z.string().optional(), - durationMs: z.number().optional(), - metadata: z.record(z.unknown()).optional(), -}); - -/** - * Derived evaluation trace read model exposed to custom assertions and script graders. - * - * Top-level summary fields (`eventCount`, `toolCalls`, `errorCount`) remain - * available for existing metric graders; full transcript/tool evidence is under - * `messages` and structured execution events under `events`. - */ -export const TraceSchema = TraceSummarySchema.extend({ - messages: z.array(MessageSchema), - events: z.array(TraceEventSchema), - tokenUsage: TokenUsageSchema.optional(), - costUsd: z.number().optional(), - durationMs: z.number().optional(), - startTime: z.string().optional(), - endTime: z.string().optional(), - metadata: MetadataSchema.optional(), -}); - -/** - * Script grader input schema (camelCase, converted from snake_case wire format). - * - * `output` is the final answer/scored result only. Transcript-aware graders - * should inspect `messages`, `trace.messages`, or `trace.events`. - */ -export const ScriptGraderInputSchema = z.object({ - criteria: z.string(), - expectedOutput: z.array(MessageSchema), - output: z.string().nullable().optional(), - messages: z.array(MessageSchema).optional().default([]), - /** Path to a temp file containing the output JSON (used for large payloads). */ - outputPath: z.string().optional(), - inputFiles: z.array(z.string()), - input: z.array(MessageSchema), - metadata: z.record(z.unknown()).nullable().optional(), - trace: TraceSchema.nullable().optional(), - traceSummary: TraceSummarySchema.nullable().optional(), - tokenUsage: TokenUsageSchema.nullable().optional(), - costUsd: z.number().nullable().optional(), - durationMs: z.number().nullable().optional(), - startTime: z.string().nullable().optional(), - endTime: z.string().nullable().optional(), - fileChanges: z.string().nullable().optional(), - workspacePath: z.string().nullable().optional(), - config: z.record(z.unknown()).nullable().optional(), -}); - -export const ScriptGraderCheckSchema = z.object({ - id: z.string().optional(), - text: z.string(), - pass: z.boolean(), - score: z.number().min(0).max(1).optional(), - reason: z.string(), - evidence: z.string().optional(), -}); - -/** - * Script grader result schema (validated before output). - */ -export const ScriptGraderResultSchema = z.object({ - pass: z.boolean(), - score: z.number().min(0).max(1), - reason: z.string(), - checks: z.array(ScriptGraderCheckSchema).optional().default([]), - /** Optional structured details for domain-specific metrics (e.g., TP/TN/FP/FN counts, alignments). */ - details: z.record(z.unknown()).optional(), -}); - -/** - * Inferred types from schemas. - */ -export type ScriptGraderInput = z.infer; -export type ScriptGraderCheck = z.infer; -export type ScriptGraderResult = z.infer; - -export type TraceSummary = z.infer; -export type Trace = z.infer; -export type TraceArtifact = z.infer; -export type TraceSource = z.infer; -export type TraceSession = z.infer; -export type TraceBranch = z.infer; -export type TraceEvent = z.infer; -export type TraceMessage = z.infer; -export type TraceModel = z.infer; -export type TraceTool = z.infer; -export type TraceError = z.infer; -export type TraceSourceRef = z.infer; -export type TraceRawEvidence = z.infer; -export type TraceRedactionState = z.infer; -export type Message = z.infer; -export type ToolCall = z.infer; -export type TokenUsage = z.infer; - -export type ContentText = z.infer; -export type ContentImage = z.infer; -export type ContentFile = z.infer; -export type Content = z.infer; - -/** - * Prompt template input schema (camelCase, converted from snake_case wire format). - * Uses the same schema as ScriptGraderInput since the orchestrator sends identical payloads. - */ -export const PromptTemplateInputSchema = ScriptGraderInputSchema; - -export type PromptTemplateInput = ScriptGraderInput; - -// ── Backward-compat aliases (deprecated) ──────────────────────────────────────── -/** @deprecated Use ScriptGraderInputSchema */ -export const CodeGraderInputSchema = ScriptGraderInputSchema; -/** @deprecated Use ScriptGraderResultSchema */ -export const CodeGraderResultSchema = ScriptGraderResultSchema; -/** @deprecated Use ScriptGraderInput */ -export type CodeGraderInput = ScriptGraderInput; -/** @deprecated Use ScriptGraderResult */ -export type CodeGraderResult = ScriptGraderResult; -/** @deprecated Use ScriptGraderInputSchema */ -export const CodeJudgeInputSchema = ScriptGraderInputSchema; -/** @deprecated Use ScriptGraderResultSchema */ -export const CodeJudgeResultSchema = ScriptGraderResultSchema; -/** @deprecated Use ScriptGraderInput */ -export type CodeJudgeInput = ScriptGraderInput; -/** @deprecated Use ScriptGraderResult */ -export type CodeJudgeResult = ScriptGraderResult; + ContentImageSchema, + ContentSchema, + ContentTextSchema, + MessageSchema, + PromptTemplateInputSchema, + ScriptGraderCheckSchema, + ScriptGraderInputSchema, + ScriptGraderResultSchema, + TokenUsageSchema, + TRACE_EVENT_TYPES, + TRACE_REDACTION_LEVELS, + TRACE_SOURCE_KINDS, + TRACE_TOOL_STATUSES, + ToolCallSchema, + TraceArtifactSchema, + TraceBranchSchema, + TraceErrorSchema, + TraceEventSchema, + TraceMessageSchema, + TraceModelSchema, + TraceRawEvidenceSchema, + TraceRedactionStateSchema, + TraceSchema, + TraceSessionSchema, + TraceSourceRefSchema, + TraceSourceSchema, + TraceSummarySchema, + TraceToolSchema, + type CodeGraderInput, + type CodeGraderResult, + type CodeJudgeInput, + type CodeJudgeResult, + type Content, + type ContentFile, + type ContentImage, + type ContentText, + type Message, + type PromptTemplateInput, + type ScriptGraderCheck, + type ScriptGraderInput, + type ScriptGraderResult, + type TokenUsage, + type ToolCall, + type Trace, + type TraceArtifact, + type TraceBranch, + type TraceError, + type TraceEvent, + type TraceMessage, + type TraceModel, + type TraceRawEvidence, + type TraceRedactionState, + type TraceSession, + type TraceSource, + type TraceSourceRef, + type TraceSummary, + type TraceTool, +} from '@agentv/core/script-grader'; diff --git a/packages/sdk/src/target-client.ts b/packages/sdk/src/target-client.ts index f07c4d9a9..1556f433c 100644 --- a/packages/sdk/src/target-client.ts +++ b/packages/sdk/src/target-client.ts @@ -110,7 +110,7 @@ export class TargetInvocationError extends Error { * * @example * ```typescript - * import { createTargetClient, defineScriptGrader } from '@agentv/sdk'; + * import { createTargetClient, defineScriptGrader } from 'agentv'; * * export default defineScriptGrader(async ({ input, criteria, output }) => { * const target = createTargetClient(); diff --git a/packages/sdk/src/vitest.ts b/packages/sdk/src/vitest.ts index d55d71a23..734bc206b 100644 --- a/packages/sdk/src/vitest.ts +++ b/packages/sdk/src/vitest.ts @@ -1,433 +1,7 @@ -/** - * Vitest workspace verifier adapter. - * - * This module keeps deterministic workspace verification in familiar Vitest - * tests while translating the JSON reporter output into AgentV's script-grader - * result contract. - */ -import { spawn } from 'node:child_process'; -import { copyFile, mkdtemp, readFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import nodePath from 'node:path'; - -import { runScriptGrader } from './runtime.js'; -import { - type ScriptGraderInput, - type ScriptGraderResult, - ScriptGraderResultSchema, -} from './schemas.js'; - -export interface VitestWorkspaceGraderOptions { - /** - * Vitest verifier file(s). By default these are relative to the prepared - * workspace. When `copyTestFilesToWorkspace` is true, relative paths resolve - * from `testFileRoot` instead. - * - * When provided without `command`, the adapter runs - * `bunx vitest run --reporter=json --outputFile `. - */ - readonly testFile?: string | readonly string[]; - /** - * Copy `testFile` entries into a temporary directory inside the workspace - * before running Vitest. Use this for hidden verifier files that live beside - * the eval instead of inside the prepared workspace. - */ - readonly copyTestFilesToWorkspace?: boolean; - /** - * Base directory for copied `testFile` entries. Defaults to `process.cwd()`. - */ - readonly testFileRoot?: string; - /** - * Full command to run. Use this for package scripts such as - * `["bun", "run", "verify:workspace"]`. - */ - readonly command?: readonly string[]; - /** - * Base Vitest command used with `testFile`. Defaults to - * `["bunx", "vitest", "run"]`. - */ - readonly vitestCommand?: readonly string[]; - /** Workspace-relative directory to run the command in. Defaults to workspace root. */ - readonly cwd?: string; - /** Append `--reporter=json --outputFile ` to the command. Defaults to true for testFile mode. */ - readonly appendReporterArgs?: boolean; - /** Read Vitest JSON from this path instead of stdout. Relative paths resolve under `cwd`. */ - readonly outputFile?: string; - readonly timeoutMs?: number; - readonly env?: Readonly>; - readonly passWithNoTests?: boolean; -} - -interface CommandResult { - readonly exitCode: number | null; - readonly stdout: string; - readonly stderr: string; -} - -interface VitestAssertionResult { - readonly ancestorTitles?: readonly string[]; - readonly fullName?: string; - readonly title?: string; - readonly status?: string; - readonly failureMessages?: readonly string[]; - readonly duration?: number; -} - -interface VitestFileResult { - readonly name?: string; - readonly assertionResults?: readonly VitestAssertionResult[]; -} - -interface VitestJsonReport { - readonly success?: boolean; - readonly numTotalTests?: number; - readonly numPassedTests?: number; - readonly numFailedTests?: number; - readonly numPendingTests?: number; - readonly numTodoTests?: number; - readonly testResults?: readonly VitestFileResult[]; -} - -function workspacePathFrom(input: ScriptGraderInput): string | undefined { - const workspacePath = input.workspacePath ?? process.env.AGENTV_WORKSPACE_PATH; - return workspacePath?.trim() ? workspacePath : undefined; -} - -function truncate(value: string, maxLength = 2000): string { - if (value.length <= maxLength) { - return value; - } - return `${value.slice(0, maxLength)}\n...(truncated)`; -} - -function resolveInsideWorkspace( - workspacePath: string, - relativePath: string, - label: string, - options: { readonly allowRoot?: boolean } = {}, -): string { - if (!relativePath.trim()) { - throw new Error(`${label} must not be empty.`); - } - - if (nodePath.isAbsolute(relativePath)) { - throw new Error(`${label} must be relative to the workspace: ${relativePath}`); - } - - const root = nodePath.resolve(workspacePath); - const resolvedPath = nodePath.resolve(root, relativePath); - const relativeToRoot = nodePath.relative(root, resolvedPath); - if (relativeToRoot === '' && options.allowRoot === true) { - return resolvedPath; - } - - if ( - relativeToRoot === '' || - relativeToRoot.startsWith('..') || - nodePath.isAbsolute(relativeToRoot) - ) { - throw new Error(`${label} must stay inside the workspace: ${relativePath}`); - } - - return resolvedPath; -} - -function normalizeTestFiles(testFile: string | readonly string[] | undefined): readonly string[] { - if (testFile === undefined) { - return []; - } - return typeof testFile === 'string' ? [testFile] : [...testFile]; -} - -function buildCommand( - options: VitestWorkspaceGraderOptions, - testFiles: readonly string[] = normalizeTestFiles(options.testFile), -) { - if (options.command && options.command.length > 0) { - return [...options.command]; - } - - const command = [...(options.vitestCommand ?? ['bunx', 'vitest', 'run'])]; - command.push(...testFiles); - return command; -} - -function parseJsonObjectFromText(text: string): unknown { - const trimmed = text.trim(); - if (!trimmed) { - return undefined; - } - - try { - return JSON.parse(trimmed); - } catch { - const start = trimmed.indexOf('{'); - const end = trimmed.lastIndexOf('}'); - if (start >= 0 && end > start) { - return JSON.parse(trimmed.slice(start, end + 1)); - } - throw new Error('Vitest output did not contain a JSON object.'); - } -} - -function isVitestJsonReport(value: unknown): value is VitestJsonReport { - return ( - typeof value === 'object' && - value !== null && - Array.isArray((value as VitestJsonReport).testResults) - ); -} - -function assertionText(file: VitestFileResult, assertion: VitestAssertionResult): string { - const title = - assertion.fullName ?? - [...(assertion.ancestorTitles ?? []), assertion.title].filter(Boolean).join(' '); - return title || file.name || 'Vitest assertion'; -} - -export function vitestReportToScriptGraderResult( - report: VitestJsonReport, - options: Pick = {}, -): ScriptGraderResult { - const checks = (report.testResults ?? []).flatMap((file) => - (file.assertionResults ?? []).map((item) => { - const pass = item.status === 'passed'; - const evidence = - item.failureMessages && item.failureMessages.length > 0 - ? truncate(item.failureMessages.join('\n\n')) - : undefined; - return { - text: assertionText(file, item), - pass, - reason: pass ? 'Vitest test passed.' : 'Vitest test failed.', - ...(evidence !== undefined ? { evidence } : {}), - }; - }), - ); - - if (checks.length === 0) { - const pass = options.passWithNoTests === true; - return ScriptGraderResultSchema.parse({ - pass, - score: pass ? 1 : 0, - reason: pass ? 'Vitest reported no tests; configured to pass.' : 'Vitest reported no tests.', - checks: [ - { - text: 'Vitest reported no tests', - pass, - reason: pass ? 'passWithNoTests is enabled.' : 'No Vitest tests were discovered.', - }, - ], - details: { - vitest_success: report.success ?? false, - num_total_tests: report.numTotalTests ?? 0, - num_passed_tests: report.numPassedTests ?? 0, - num_failed_tests: report.numFailedTests ?? 0, - num_pending_tests: report.numPendingTests ?? 0, - num_todo_tests: report.numTodoTests ?? 0, - }, - }); - } - - const passedCount = checks.filter((item) => item.pass).length; - const pass = passedCount === checks.length; - return ScriptGraderResultSchema.parse({ - pass, - score: passedCount / checks.length, - reason: `${passedCount}/${checks.length} Vitest tests passed.`, - checks, - details: { - vitest_success: report.success ?? pass, - num_total_tests: report.numTotalTests ?? checks.length, - num_passed_tests: report.numPassedTests ?? passedCount, - num_failed_tests: report.numFailedTests ?? checks.length - passedCount, - num_pending_tests: report.numPendingTests ?? 0, - num_todo_tests: report.numTodoTests ?? 0, - }, - }); -} - -/** @deprecated Use vitestReportToScriptGraderResult. */ -export const vitestReportToCodeGraderResult = vitestReportToScriptGraderResult; - -function runCommand( - command: readonly string[], - options: { - readonly cwd: string; - readonly timeoutMs?: number; - readonly env?: Readonly>; - }, -): Promise { - if (command.length === 0) { - return Promise.reject(new Error('Vitest command must not be empty.')); - } - - return new Promise((resolve, reject) => { - const child = spawn(command[0], command.slice(1), { - cwd: options.cwd, - env: { ...process.env, ...options.env }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - - let stdout = ''; - let stderr = ''; - let timedOut = false; - const timeout = - options.timeoutMs !== undefined - ? setTimeout(() => { - timedOut = true; - child.kill('SIGTERM'); - }, options.timeoutMs) - : undefined; - - child.stdout.setEncoding('utf8'); - child.stderr.setEncoding('utf8'); - child.stdout.on('data', (chunk) => { - stdout += chunk; - }); - child.stderr.on('data', (chunk) => { - stderr += chunk; - }); - child.on('error', reject); - child.on('close', (exitCode) => { - if (timeout) { - clearTimeout(timeout); - } - if (timedOut) { - reject(new Error(`Vitest command timed out after ${options.timeoutMs}ms.`)); - return; - } - resolve({ exitCode, stdout, stderr }); - }); - }); -} - -function resolveSourceTestFile(testFileRoot: string, testFile: string): string { - if (!testFile.trim()) { - throw new Error('testFile entries must not be empty.'); - } - - return nodePath.isAbsolute(testFile) - ? nodePath.resolve(testFile) - : nodePath.resolve(testFileRoot, testFile); -} - -async function copyTestFilesIntoWorkspace( - testFiles: readonly string[], - options: Pick, - cwd: string, -): Promise<{ readonly testFiles: readonly string[]; readonly tempDir?: string }> { - if (testFiles.length === 0) { - return { testFiles }; - } - - const tempDir = await mkdtemp(nodePath.join(cwd, '.agentv-vitest-')); - const testFileRoot = nodePath.resolve(options.testFileRoot ?? process.cwd()); - const copiedFiles = await Promise.all( - testFiles.map(async (testFile, index) => { - const sourcePath = resolveSourceTestFile(testFileRoot, testFile); - const destinationPath = nodePath.join(tempDir, `${index}-${nodePath.basename(testFile)}`); - await copyFile(sourcePath, destinationPath); - return nodePath.relative(cwd, destinationPath); - }), - ); - return { testFiles: copiedFiles, tempDir }; -} - -async function readVitestReport( - commandResult: CommandResult, - outputFile: string | undefined, -): Promise { - const rawReport = - outputFile !== undefined ? await readFile(outputFile, 'utf8') : commandResult.stdout; - const parsed = parseJsonObjectFromText(rawReport); - if (!isVitestJsonReport(parsed)) { - throw new Error('Vitest JSON report did not include testResults[].'); - } - return parsed; -} - -export async function runVitestWorkspaceGrader( - options: VitestWorkspaceGraderOptions, - input: ScriptGraderInput, -): Promise { - const workspacePath = workspacePathFrom(input); - if (!workspacePath) { - return { - pass: false, - score: 0, - reason: 'Vitest workspace verifier requires workspace_path.', - checks: [ - { - text: 'Vitest workspace verifier requires workspace_path', - pass: false, - reason: - 'Configure an environment recipe in the eval YAML so AgentV can pass workspace_path.', - }, - ], - }; - } - - const tempDirs: string[] = []; - try { - const cwd = options.cwd - ? resolveInsideWorkspace(workspacePath, options.cwd, 'cwd', { allowRoot: true }) - : workspacePath; - const testFiles = normalizeTestFiles(options.testFile); - const preparedTestFiles = - options.copyTestFilesToWorkspace === true - ? await copyTestFilesIntoWorkspace(testFiles, options, cwd) - : { testFiles }; - if (preparedTestFiles.tempDir) { - tempDirs.push(preparedTestFiles.tempDir); - } - - const command = buildCommand(options, preparedTestFiles.testFiles); - const appendReporterArgs = options.appendReporterArgs ?? options.command === undefined; - let outputFile = options.outputFile - ? resolveInsideWorkspace(cwd, options.outputFile, 'outputFile') - : undefined; - - if (appendReporterArgs) { - const tempDir = await mkdtemp(nodePath.join(tmpdir(), 'agentv-vitest-')); - tempDirs.push(tempDir); - outputFile = nodePath.join(tempDir, 'results.json'); - command.push('--reporter=json', `--outputFile=${outputFile}`); - } - - const result = await runCommand(command, { - cwd, - timeoutMs: options.timeoutMs, - env: { - AGENTV_WORKSPACE_PATH: workspacePath, - ...(outputFile !== undefined ? { AGENTV_VITEST_JSON_PATH: outputFile } : {}), - ...options.env, - }, - }); - - const report = await readVitestReport(result, outputFile); - return vitestReportToScriptGraderResult(report, options); - } catch (error) { - const reason = error instanceof Error ? error.message : String(error); - return { - pass: false, - score: 0, - reason: 'Vitest workspace verifier failed to run.', - checks: [ - { - text: 'Vitest workspace verifier failed to run', - pass: false, - reason, - }, - ], - }; - } finally { - for (const tempDir of tempDirs.reverse()) { - await rm(tempDir, { recursive: true, force: true }).catch(() => {}); - } - } -} - -export function defineVitestWorkspaceGrader(options: VitestWorkspaceGraderOptions): void { - runScriptGrader((input) => runVitestWorkspaceGrader(options, input)); -} +export { + defineVitestWorkspaceGrader, + runVitestWorkspaceGrader, + vitestReportToCodeGraderResult, + vitestReportToScriptGraderResult, + type VitestWorkspaceGraderOptions, +} from '@agentv/core/script-grader'; diff --git a/packages/sdk/test/evaluate-export.test.ts b/packages/sdk/test/evaluate-export.test.ts index 8b3a2e54e..94e436c39 100644 --- a/packages/sdk/test/evaluate-export.test.ts +++ b/packages/sdk/test/evaluate-export.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from 'bun:test'; import { evaluate } from '../src/index.js'; describe('evaluate export', () => { - it('runs the core programmatic evaluate API through @agentv/sdk', async () => { + it('runs the core programmatic evaluate API through the internal SDK implementation', async () => { const { results, summary } = await evaluate({ prompts: ['{{ input }}'], tests: [ diff --git a/packages/sdk/test/package-graph.test.ts b/packages/sdk/test/package-graph.test.ts index e43f96f24..8562d9599 100644 --- a/packages/sdk/test/package-graph.test.ts +++ b/packages/sdk/test/package-graph.test.ts @@ -6,6 +6,7 @@ const repoRoot = path.resolve(import.meta.dir, '../../..'); function readJson(filePath: string) { return JSON.parse(readFileSync(path.join(repoRoot, filePath), 'utf8')) as { + readonly private?: boolean; readonly dependencies?: Record; readonly devDependencies?: Record; }; @@ -31,7 +32,9 @@ describe('core/sdk package graph', () => { const sdkPackage = readJson('packages/sdk/package.json'); const corePackage = readJson('packages/core/package.json'); - expect(sdkPackage.dependencies?.['@agentv/core']).toBe(corePackage.version); + expect(sdkPackage.private).toBe(true); + expect(corePackage.private).toBe(true); + expect(sdkPackage.dependencies?.['@agentv/core']).toBe('workspace:*'); expect(corePackage.dependencies?.['@agentv/sdk']).toBeUndefined(); expect(corePackage.devDependencies?.['@agentv/sdk']).toBeUndefined(); }); diff --git a/scripts/publish.ts b/scripts/publish.ts index c06e968ca..c49b3086d 100644 --- a/scripts/publish.ts +++ b/scripts/publish.ts @@ -25,7 +25,7 @@ if (requestedTag !== undefined && requestedTag !== 'next') { const npmTag = requestedTag ?? 'latest'; const publishArgs = ['--tag', npmTag, '--access', 'public']; -const PACKAGES = ['packages/core', 'packages/sdk', 'apps/cli']; +const PACKAGES = ['apps/cli']; interface PackageJson { name: string; diff --git a/scripts/release.ts b/scripts/release.ts index df7df8106..d588064e4 100644 --- a/scripts/release.ts +++ b/scripts/release.ts @@ -32,11 +32,7 @@ const VALID_BUMP_TYPES: BumpType[] = ['patch', 'minor', 'major']; const NEXT_PRERELEASE_TAG = 'next'; // Packages to update (relative to repo root) -const PACKAGE_PATHS = [ - 'packages/core/package.json', - 'packages/sdk/package.json', - 'apps/cli/package.json', -]; +const PACKAGE_PATHS = ['apps/cli/package.json']; // The primary package that determines the version const PRIMARY_PACKAGE = 'apps/cli/package.json'; @@ -70,12 +66,6 @@ function writePackageJson(path: string, pkg: PackageJson): void { writeFileSync(path, `${JSON.stringify(pkg, null, 2)}\n`, 'utf-8'); } -function syncInternalRuntimeDependencies(pkg: PackageJson, version: string): void { - if (pkg.name === '@agentv/sdk' && pkg.dependencies?.['@agentv/core']) { - pkg.dependencies['@agentv/core'] = version; - } -} - function bumpVersion(currentVersion: string, bumpType: BumpType): string { const stablePart = currentVersion.split('-')[0]; const parts = stablePart.split('.').map(Number); @@ -408,7 +398,6 @@ async function main() { const pkg = readPackageJson(fullPath); const oldVersion = pkg.version; pkg.version = newVersion; - syncInternalRuntimeDependencies(pkg, newVersion); writePackageJson(fullPath, pkg); console.log(` ✓ ${pkg.name}: ${oldVersion} → ${newVersion}`); } From fd92254922ed6e2ba4a12af8f11fd438912a74f7 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 15:07:40 +0200 Subject: [PATCH 2/6] docs: move examples to agentv imports --- README.md | 21 +- .../docs/docs/next/evaluation/eval-files.mdx | 2 +- .../docs/next/evaluation/running-evals.mdx | 4 +- .../content/docs/docs/next/evaluation/sdk.mdx | 56 ++-- .../docs/next/graders/custom-assertions.mdx | 10 +- .../docs/docs/next/graders/llm-graders.mdx | 4 +- .../docs/docs/next/graders/script-graders.mdx | 10 +- .../docs/next/reference/promptfoo-parity.mdx | 2 +- .../docs/next/targets/custom-providers.mdx | 6 +- examples/README.md | 8 +- examples/features/README.md | 2 +- examples/features/batch-cli/bun.lock | 17 - .../graders/check-batch-cli-output.ts | 2 +- examples/features/batch-cli/package.json | 2 +- .../copilot-transcript-replay/README.md | 2 +- .../graders/transcript-quality.ts | 2 +- .../features/deterministic-graders/README.md | 2 +- .../graders/assertions.ts | 2 +- .../deterministic-graders/package.json | 2 +- examples/features/eval-assert-demo/README.md | 2 +- examples/features/execution-metrics/bun.lock | 17 - .../features/execution-metrics/package.json | 2 +- .../scripts/check-efficiency.ts | 2 +- .../scripts/check-metrics-present.ts | 2 +- examples/features/import-claude/README.md | 2 +- .../graders/transcript-quality.ts | 2 +- examples/features/nlp-metrics/README.md | 2 +- examples/features/nlp-metrics/graders/bleu.ts | 2 +- .../nlp-metrics/graders/levenshtein.ts | 2 +- .../features/nlp-metrics/graders/rouge.ts | 2 +- .../nlp-metrics/graders/similarity.ts | 2 +- .../features/prompt-template-sdk/README.md | 4 +- .../features/prompt-template-sdk/bun.lock | 17 - .../features/prompt-template-sdk/package.json | 2 +- .../prompts/custom-evaluator.ts | 2 +- examples/features/script-grader-sdk/README.md | 6 +- examples/features/script-grader-sdk/bun.lock | 17 - .../features/script-grader-sdk/package.json | 2 +- .../scripts/verify-attachments.ts | 2 +- .../script-grader-with-llm-calls/README.md | 2 +- .../script-grader-with-llm-calls/bun.lock | 17 - .../script-grader-with-llm-calls/package.json | 2 +- .../scripts/contextual-precision.ts | 2 +- .../scripts/contextual-recall.ts | 2 +- .../scripts/utils.ts | 2 +- examples/features/sdk-config-file/README.md | 2 +- .../features/sdk-config-file/agentv.config.ts | 2 +- .../features/sdk-config-file/package.json | 3 +- .../features/sdk-custom-assertion/README.md | 2 +- .../features/sdk-custom-assertion/bun.lock | 17 - .../sdk-custom-assertion/package.json | 2 +- .../features/sdk-eval-authoring/README.md | 2 +- .../sdk-eval-authoring/evals/greeting.eval.ts | 4 +- .../features/sdk-eval-authoring/package.json | 2 +- .../sdk-programmatic-api-advanced/evaluate.ts | 2 +- .../package.json | 3 +- .../features/sdk-programmatic-api/README.md | 4 +- .../features/sdk-programmatic-api/evaluate.ts | 4 +- .../sdk-programmatic-api/package.json | 3 +- .../graders/tool-args-f1.ts | 2 +- .../graders/tool-call-f1.ts | 2 +- .../trace-evaluation/graders/error-spans.ts | 2 +- .../trace-evaluation/graders/span-count.ts | 2 +- .../trace-evaluation/graders/span-duration.ts | 2 +- .../features/trace-evaluation/package.json | 2 +- .../graders/trial-consistency.ts | 2 +- .../features/vitest-workspace-grader/bun.lock | 163 ---------- examples/showcase/cross-repo-sync/bun.lock | 80 ----- .../showcase/cross-repo-sync/package.json | 2 +- .../cross-repo-sync/scripts/validate-sync.ts | 2 +- examples/showcase/export-screening/bun.lock | 17 - .../evals/validate_risk_output.ts | 2 +- .../showcase/export-screening/package.json | 2 +- examples/showcase/grader-conformance/bun.lock | 20 -- .../graders/keyword-grader.ts | 2 +- .../showcase/grader-conformance/package.json | 2 +- .../showcase/tool-evaluation-plugins/bun.lock | 17 - .../tool-evaluation-plugins/package.json | 2 +- .../scripts/efficiency-scorer.ts | 2 +- .../scripts/pairwise-tool-compare.ts | 2 +- .../scripts/tool-selection-grader.ts | 2 +- .../graders/recovery-check.ts | 2 +- .../trace-evaluation/graders/replay-proof.ts | 2 +- .../showcase/trace-evaluation/package.json | 2 +- packages/core/README.md | 81 +---- packages/sdk/README.md | 306 +----------------- 86 files changed, 151 insertions(+), 899 deletions(-) delete mode 100644 examples/features/batch-cli/bun.lock delete mode 100644 examples/features/execution-metrics/bun.lock delete mode 100644 examples/features/prompt-template-sdk/bun.lock delete mode 100644 examples/features/script-grader-sdk/bun.lock delete mode 100644 examples/features/script-grader-with-llm-calls/bun.lock delete mode 100644 examples/features/sdk-custom-assertion/bun.lock delete mode 100644 examples/features/vitest-workspace-grader/bun.lock delete mode 100644 examples/showcase/cross-repo-sync/bun.lock delete mode 100644 examples/showcase/export-screening/bun.lock delete mode 100644 examples/showcase/grader-conformance/bun.lock delete mode 100644 examples/showcase/tool-evaluation-plugins/bun.lock diff --git a/README.md b/README.md index 9646b3efb..2bd1bd875 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,7 @@ Run bundle layout: Use `evaluate()` when your application owns the run: ```typescript -import { evaluate } from '@agentv/sdk'; +import { evaluate } from 'agentv'; const { results, summary } = await evaluate({ experiment: 'with-skills', @@ -242,14 +242,25 @@ console.log(`${summary.passed}/${summary.total} passed`); Use `*.eval.ts` when you want AgentV to run a TypeScript eval config: ```typescript -import type { EvalConfig } from '@agentv/sdk'; +import type { EvalConfig } from 'agentv'; const config: EvalConfig = { description: 'Code generation quality', tags: { experiment: 'with-skills' }, - target: { - extends: 'copilot-sdk', - model: 'claude-sonnet-4.6', + providers: [ + { + id: 'copilot-sdk', + label: 'copilot', + config: { model: 'claude-sonnet-4.6' }, + }, + { + id: 'openai:gpt-5-mini', + label: 'grader', + }, + ], + defaults: { + provider: 'copilot', + grader: 'grader', }, repeat: 3, threshold: 0.8, diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx index d478335f1..45e966da6 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx @@ -43,7 +43,7 @@ For TypeScript config authoring, use an explicit `*.eval.ts` or `*.eval.mts` file with a typed default export: ```typescript -import type { EvalConfig } from '@agentv/sdk'; +import type { EvalConfig } from 'agentv'; const config: EvalConfig = { providers: ['local-mini'], diff --git a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx index 235918396..0816c2ca1 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx @@ -578,7 +578,7 @@ eval_patterns: ### TypeScript config (`agentv.config.ts`) ```typescript -import { defineConfig } from '@agentv/core'; +import { defineConfig } from 'agentv/config'; export default defineConfig({ execution: { @@ -602,7 +602,7 @@ agentv eval evals/suite.yaml --cache-path .agentv/response-cache Project TypeScript config can set the project default: ```typescript -import { defineConfig } from '@agentv/core'; +import { defineConfig } from 'agentv/config'; export default defineConfig({ cache: { diff --git a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx index 19b6863c2..26e1f440d 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx @@ -6,37 +6,29 @@ sidebar: slug: docs/evaluation/sdk --- -YAML remains AgentV's canonical, portable eval format. The SDK surfaces below are for cases where you want to generate YAML-shaped definitions in code, embed eval runs inside another application, or write executable graders and prompt templates. For authoring helpers, `@agentv/sdk` is AgentV's public lightweight SDK package. - -AgentV currently provides two npm packages for programmatic use: - -- **`@agentv/sdk`** — user-facing SDK for `evaluate()`, YAML-aligned eval authoring, custom assertions, and script graders -- **`@agentv/core`** — core implementation package and typed configuration +YAML remains AgentV's canonical, portable eval format. The SDK surfaces below are for cases where you want to generate YAML-shaped definitions in code, embed eval runs inside another application, or write executable graders and prompt templates. The public TypeScript SDK and programmatic API live in the `agentv` package. ## Installation ```bash # User-facing SDK (evaluate, defineEval, graders, defineAssertion, defineScriptGrader) -npm install @agentv/sdk - -# Core configuration helpers (defineConfig) -npm install @agentv/core +npm install agentv ``` ## Migrating from `@agentv/eval` -Use `@agentv/sdk` for all new TypeScript SDK code: +Use `agentv` for all new TypeScript SDK code: ```bash npm uninstall @agentv/eval -npm install @agentv/sdk +npm install agentv ``` ```typescript -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; ``` -The general policy is hard convergence for same-week or unreleased surface names: use the correct package, field, or wire name instead of carrying aliases. `@agentv/eval` was already published, then deprecated on npm, and has been removed from this repository. New docs, examples, scaffolds, and skills should use `@agentv/sdk` directly. +The general policy is hard convergence for same-week or unreleased surface names: use the correct package, field, or wire name instead of carrying aliases. `@agentv/eval` was already published, then deprecated on npm, and has been removed from this repository. New docs, examples, scaffolds, and skills should use `agentv` directly. ## Choose a Surface @@ -89,11 +81,11 @@ This is example-local/repo-local guidance, not a promise of a published Python p ## TypeScript Eval Config Authoring -Use `EvalConfig` from `@agentv/sdk` when you want TypeScript ergonomics without creating a second eval vocabulary. AgentV accepts explicit `*.eval.ts` and `*.eval.mts` files as eval configs. Each TypeScript config uses a default export and lowers back to the canonical snake_case eval object contract when AgentV loads the file. +Use `EvalConfig` from `agentv` when you want TypeScript ergonomics without creating a second eval vocabulary. AgentV accepts explicit `*.eval.ts` and `*.eval.mts` files as eval configs. Each TypeScript config uses a default export and lowers back to the canonical snake_case eval object contract when AgentV loads the file. ```typescript // evals/greeting.eval.ts -import { graders, type EvalConfig } from '@agentv/sdk'; +import { graders, type EvalConfig } from 'agentv'; const config: EvalConfig = { name: 'hello-suite', @@ -139,10 +131,10 @@ TypeScript eval configs use the same provider surface as YAML: top-level `provid ## Built-In Grader Helpers -`@agentv/sdk` includes a small `graders` catalog for common deterministic and LLM-backed grader configs. These helpers return ordinary `assert` entries and serialize to the same canonical YAML you could write by hand. +`agentv` includes a small `graders` catalog for common deterministic and LLM-backed grader configs. These helpers return ordinary `assert` entries and serialize to the same canonical YAML you could write by hand. ```typescript -import { graders, type EvalConfig } from '@agentv/sdk'; +import { graders, type EvalConfig } from 'agentv'; const config: EvalConfig = { name: 'grader-helper-suite', @@ -178,7 +170,7 @@ The catalog covers `contains`, `equals`/`exact`, `regex`, `is-json`/`json`, `llm If you are coming from Braintrust `scores` or DeepEval metrics, keep the reusable logic AgentV-native: write helper factories that return `graders.*` configs, then let the TypeScript eval config lower them to ordinary `assert` entries. ```typescript -import { graders, type EvalConfig } from '@agentv/sdk'; +import { graders, type EvalConfig } from 'agentv'; function ragFaithfulness() { return graders.llmRubric(undefined, { @@ -226,7 +218,7 @@ assert: ## Custom Assertions -Use `defineAssertion` from `@agentv/sdk` to create reusable assertion types. Place them in `.agentv/assertions/` — they're auto-discovered by filename. +Use `defineAssertion` from `agentv` to create reusable assertion types. Place them in `.agentv/assertions/` — they're auto-discovered by filename. This is the custom assertion path, not the custom grader path. It matches Promptfoo's assertion terminology for normal eval checks, while extending Promptfoo's fixed custom logic types (`javascript`, `python`, `ruby`, `webhook`) with arbitrary discovered AgentV type names. @@ -234,7 +226,7 @@ This is the custom assertion path, not the custom grader path. It matches Prompt ```typescript // .agentv/assertions/min-words.ts -import { defineAssertion } from '@agentv/sdk'; +import { defineAssertion } from 'agentv'; export default defineAssertion(({ output }) => { const wordCount = (output ?? '').trim().split(/\s+/).filter(Boolean).length; @@ -253,7 +245,7 @@ Return a `score` (0–1) instead of `pass` for graded evaluation: ```typescript // .agentv/assertions/efficiency.ts -import { defineAssertion } from '@agentv/sdk'; +import { defineAssertion } from 'agentv'; export default defineAssertion(({ output, traceSummary }) => { const hasContent = (output ?? '').length > 0 ? 0.5 : 0; @@ -287,10 +279,10 @@ assert: ## Script Graders -Use `defineScriptGrader` from `@agentv/sdk` for full control over scoring with optional per-check results: +Use `defineScriptGrader` from `agentv` for full control over scoring with optional per-check results: ```typescript -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; export default defineScriptGrader(({ output, traceSummary }) => ({ pass: (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 5, @@ -334,7 +326,7 @@ assert: Use `defineWorkspaceGrader` only for tiny one-off file checks or custom score shaping: ```typescript -import { defineWorkspaceGrader } from '@agentv/sdk'; +import { defineWorkspaceGrader } from 'agentv'; export default defineWorkspaceGrader(async ({ workspace }) => [ await workspace.file('app/page.tsx').contains('Status: All systems ready'), @@ -350,7 +342,7 @@ For detailed patterns, input/output contracts, and language-agnostic examples, s ## Wire Format vs SDK Format -Raw grader stdin uses `snake_case` because it crosses a process boundary and may be consumed by Python, shell, jq, or external dashboards. The `@agentv/sdk` package converts that payload to idiomatic TypeScript `camelCase` before calling your handler. +Raw grader stdin uses `snake_case` because it crosses a process boundary and may be consumed by Python, shell, jq, or external dashboards. The `agentv` package converts that payload to idiomatic TypeScript `camelCase` before calling your handler. | Raw stdin | SDK handler field | |-----------|-------------------| @@ -366,12 +358,12 @@ Raw grader stdin uses `snake_case` because it crosses a process boundary and may ## Programmatic API -Use `evaluate()` from `@agentv/sdk` to run evaluations as a library. The implementation is owned by `@agentv/core`, but the SDK re-exports it as the user-facing entrypoint. The most portable pattern is still to keep the suite in YAML and point `specFile` at it; inline tests are best when the eval is tightly coupled to application code. +Use `evaluate()` from `agentv` to run evaluations as a library. The most portable pattern is still to keep the suite in YAML and point `specFile` at it; inline tests are best when the eval is tightly coupled to application code. ### Inline Test Definitions ```typescript -import { evaluate } from '@agentv/sdk'; +import { evaluate } from 'agentv'; const { results, summary } = await evaluate({ tests: [ @@ -390,7 +382,7 @@ console.log(`${summary.passed}/${summary.total} passed`); A strict OR is easy with inline assertion handlers: ```typescript -import { evaluate } from '@agentv/sdk'; +import { evaluate } from 'agentv'; const { summary } = await evaluate({ tests: [ @@ -420,7 +412,7 @@ Auto-discovers the `default` provider from `.agentv/providers.yaml` and `.env` c Point to an existing YAML eval instead of inlining tests: ```typescript -import { evaluate } from '@agentv/sdk'; +import { evaluate } from 'agentv'; const { results, summary } = await evaluate({ specFile: './evals/my-eval.eval.yaml', @@ -431,10 +423,10 @@ This is the recommended bridge when you want SDK control without creating a sepa ## Typed Configuration -Create `agentv.config.ts` at your project root for type-safe, validated configuration using `defineConfig()` from `@agentv/core`: +Create `agentv.config.ts` at your project root for type-safe, validated configuration using `defineConfig()` from `agentv/config`: ```typescript -import { defineConfig } from '@agentv/core'; +import { defineConfig } from 'agentv/config'; export default defineConfig({ execution: { diff --git a/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx b/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx index 968b61490..3ba1c197d 100644 --- a/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx +++ b/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx @@ -32,7 +32,7 @@ AgentV follows that framing for `assert:` entries and `defineAssertion()`. The A ## Installation ```bash -npm install @agentv/sdk +npm install agentv ``` ## Convention-Based Discovery @@ -68,7 +68,7 @@ The simplest pattern returns `pass` (boolean), `reason`, and optional `checks`: ```typescript // .agentv/assertions/min-words.ts -import { defineAssertion } from '@agentv/sdk'; +import { defineAssertion } from 'agentv'; export default defineAssertion(({ output }) => { const wordCount = (output ?? '').trim().split(/\s+/).filter(Boolean).length; @@ -89,7 +89,7 @@ Return a `score` (0 to 1) for granular evaluation instead of binary pass/fail: ```typescript // .agentv/assertions/efficiency.ts -import { defineAssertion } from '@agentv/sdk'; +import { defineAssertion } from 'agentv'; export default defineAssertion(({ output, traceSummary }) => { const hasContent = (output ?? '').length > 0 ? 0.5 : 0; @@ -203,7 +203,7 @@ my-project/ ```typescript // .agentv/assertions/min-words.ts #!/usr/bin/env bun -import { defineAssertion } from '@agentv/sdk'; +import { defineAssertion } from 'agentv'; export default defineAssertion(({ output }) => { const wordCount = (output ?? '').trim().split(/\s+/).filter(Boolean).length; @@ -265,7 +265,7 @@ tests: ### 4. Install and Run ```bash -npm install @agentv/sdk +npm install agentv agentv eval evals/suite.yaml ``` diff --git a/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx b/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx index 273050425..503492905 100644 --- a/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx +++ b/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx @@ -199,11 +199,11 @@ Each `provider:` value must match a provider `label` or backend `id` from ### TypeScript Template -For dynamic prompt generation, use the `definePromptTemplate` function from `@agentv/sdk`: +For dynamic prompt generation, use the `definePromptTemplate` function from `agentv`: ```typescript #!/usr/bin/env bun -import { definePromptTemplate } from '@agentv/sdk'; +import { definePromptTemplate } from 'agentv'; function textFromMessages(messages: Array<{ content?: unknown }>): string { return messages diff --git a/apps/web/src/content/docs/docs/next/graders/script-graders.mdx b/apps/web/src/content/docs/docs/next/graders/script-graders.mdx index 79160f72c..cb655d283 100644 --- a/apps/web/src/content/docs/docs/next/graders/script-graders.mdx +++ b/apps/web/src/content/docs/docs/next/graders/script-graders.mdx @@ -31,7 +31,7 @@ Script graders receive eval context via stdin JSON and return a result via stdou } ``` -Raw grader stdin is a process-boundary wire format, so keys are `snake_case`. TypeScript and JavaScript graders that use `@agentv/sdk` receive the same payload converted to `camelCase`. The repo-local Python helper in `examples/features/sdk-python/` keeps the same `snake_case` field names. +Raw grader stdin is a process-boundary wire format, so keys are `snake_case`. TypeScript and JavaScript graders that use `agentv` receive the same payload converted to `camelCase`. The repo-local Python helper in `examples/features/sdk-python/` keeps the same `snake_case` field names. | Raw stdin key | TypeScript SDK field | Meaning | |---------------|----------------------|---------| @@ -204,11 +204,11 @@ assert: ## TypeScript SDK -The `@agentv/sdk` package provides a declarative API with automatic stdin/stdout handling. Use `defineScriptGrader` to skip protocol boilerplate: +The `agentv` package provides a declarative API with automatic stdin/stdout handling. Use `defineScriptGrader` to skip protocol boilerplate: ```typescript #!/usr/bin/env bun -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; export default defineScriptGrader(({ output, criteria }) => { const outputText = output ?? ''; @@ -274,7 +274,7 @@ For tiny one-off file checks, `defineWorkspaceGrader` can resolve the workspace ```typescript #!/usr/bin/env bun -import { defineWorkspaceGrader } from '@agentv/sdk'; +import { defineWorkspaceGrader } from 'agentv'; export default defineWorkspaceGrader(async ({ workspace }) => [ await workspace.file('app/page.tsx').contains('Status: All systems ready'), @@ -311,7 +311,7 @@ Use `createTargetClient` from the SDK: ```typescript #!/usr/bin/env bun -import { createTargetClient, defineScriptGrader } from '@agentv/sdk'; +import { createTargetClient, defineScriptGrader } from 'agentv'; export default defineScriptGrader(async ({ input, output }) => { const inputText = input diff --git a/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx b/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx index ef10edd2d..2f7d3b467 100644 --- a/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx +++ b/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx @@ -55,7 +55,7 @@ implements equivalent semantics directly. | Surface | Promptfoo shape | AgentV shape | Decision | Notes | | --- | --- | --- | --- | --- | -| TypeScript config file | `promptfooconfig.ts` default export typed as Promptfoo `UnifiedConfig`. | Explicit `*.eval.ts` or `*.eval.mts` default export typed as `EvalConfig` from `@agentv/sdk`. | Keep AgentV divergence | AgentV does not accept `promptfooconfig.ts` by default because the config semantics intentionally diverge beyond provider declarations. Directory discovery includes explicit `*.eval.ts` and `*.eval.mts` config files, but not arbitrary helper `.ts` files. | +| TypeScript config file | `promptfooconfig.ts` default export typed as Promptfoo `UnifiedConfig`. | Explicit `*.eval.ts` or `*.eval.mts` default export typed as `EvalConfig` from `agentv`. | Keep AgentV divergence | AgentV does not accept `promptfooconfig.ts` by default because the config semantics intentionally diverge beyond provider declarations. Directory discovery includes explicit `*.eval.ts` and `*.eval.mts` config files, but not arbitrary helper `.ts` files. | | Prompt matrix | Top-level `prompts` rendered with each test's `vars`. | Top-level `prompts` rendered with `tests[].vars` and `default_test.vars`. | Align with Promptfoo | This is the canonical Promptfoo-compatible input shape in AgentV. Prompt entries can be inline strings, chat arrays, files, or generated prompt functions. | | Test rows | `tests` can be inline rows or a case-file reference; rows carry `vars`, `assert`, metadata, prompt/provider filters, and expected data through vars or assertion values. | `tests` can be inline rows or a raw-case path; rows carry `vars`, `assert`, metadata, optional environment overrides, and run overrides. | Align with Promptfoo | AgentV uses field-local file refs such as `tests: file://...`, `prompts: file://...`, and `default_test: file://...`; coding-agent testbeds use `environment: file://...`. There is no separate imports table. | | Variables | `tests[].vars` plus `defaultTest.vars`; prompt templates can reference top-level var names. | `tests[].vars` plus `default_test.vars`; templates can use `{{ name }}` or `{{ vars.name }}`. | Align with Promptfoo | Per-test vars override default vars by key. | diff --git a/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx b/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx index 1e48888b3..6fba3cc06 100644 --- a/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx +++ b/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx @@ -10,7 +10,7 @@ Custom providers let you implement evaluation targets in TypeScript instead of s ## Provider Interface -Every provider must implement the `Provider` interface from `@agentv/core`: +Every provider must implement the `Provider` interface from `agentv/provider`: ```typescript interface Provider { @@ -68,7 +68,7 @@ import { type Provider, type ProviderRequest, type ProviderResponse, -} from '@agentv/core'; +} from 'agentv/provider'; const registry = createBuiltinProviderRegistry(); @@ -109,7 +109,7 @@ import { type ProviderRequest, type ProviderResponse, type ResolvedProviderBackend, -} from '@agentv/core'; +} from 'agentv/provider'; class HttpAgentProvider implements Provider { readonly id: string; diff --git a/examples/README.md b/examples/README.md index 53f96627d..53226e819 100644 --- a/examples/README.md +++ b/examples/README.md @@ -93,11 +93,11 @@ example-name/ ├── scripts/ # Helper scripts (optional) ├── .agentv/ │ └── providers.yaml # Target configuration (optional) -├── package.json # Dependencies (if using @agentv/sdk) +├── package.json # Dependencies (if using agentv) └── README.md # Example documentation ``` -### Using `@agentv/sdk` +### Using `agentv` For TypeScript script graders, add a `package.json`: @@ -107,7 +107,7 @@ For TypeScript script graders, add a `package.json`: "private": true, "type": "module", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } ``` @@ -116,7 +116,7 @@ Then write type-safe script graders: ```typescript #!/usr/bin/env bun -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; export default defineScriptGrader(({ output }) => ({ pass: (output ?? '').includes('expected'), diff --git a/examples/features/README.md b/examples/features/README.md index ed4d7159a..ce1fdd9dc 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -39,7 +39,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the ### Write custom graders in code | Example | Description | |---------|-------------| -| [script-grader-sdk](script-grader-sdk/) | TypeScript script graders using `defineScriptGrader()` from `@agentv/sdk` | +| [script-grader-sdk](script-grader-sdk/) | TypeScript script graders using `defineScriptGrader()` from `agentv` | | [script-grader-with-llm-calls](script-grader-with-llm-calls/) | script graders that make LLM calls via a target proxy | | [eval-assert-demo](eval-assert-demo/) | script graders runnable both in a suite and individually via `agentv eval assert` | | [functional-grading](functional-grading/) | Install dependencies, compile, and run tests against agent-generated code | diff --git a/examples/features/batch-cli/bun.lock b/examples/features/batch-cli/bun.lock deleted file mode 100644 index fbac5f435..000000000 --- a/examples/features/batch-cli/bun.lock +++ /dev/null @@ -1,17 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "agentv-example-batch-cli", - "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk", - }, - }, - }, - "packages": { - "@agentv/sdk": ["@agentv/sdk@file:../../../packages/sdk", { "dependencies": { "zod": "^3.23.8" } }], - - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - } -} diff --git a/examples/features/batch-cli/graders/check-batch-cli-output.ts b/examples/features/batch-cli/graders/check-batch-cli-output.ts index 2370d58e3..6be490b2c 100644 --- a/examples/features/batch-cli/graders/check-batch-cli-output.ts +++ b/examples/features/batch-cli/graders/check-batch-cli-output.ts @@ -5,7 +5,7 @@ * Validates that the batch CLI runner produces the expected decision * by comparing candidate output against expected_output or input. */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; function isObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); diff --git a/examples/features/batch-cli/package.json b/examples/features/batch-cli/package.json index b5fddd25b..e13dedb9e 100644 --- a/examples/features/batch-cli/package.json +++ b/examples/features/batch-cli/package.json @@ -3,6 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/features/copilot-transcript-replay/README.md b/examples/features/copilot-transcript-replay/README.md index 41622a2ec..6fff1c5fb 100644 --- a/examples/features/copilot-transcript-replay/README.md +++ b/examples/features/copilot-transcript-replay/README.md @@ -71,7 +71,7 @@ Checks whether the `csv-analyzer` skill was (or was not) invoked. Inspects tool call names and skill invocation events in the transcript. ### transcript-quality (script-grader) -Custom grader using `defineScriptGrader` from `@agentv/sdk`. Validates: +Custom grader using `defineScriptGrader` from `agentv`. Validates: 1. Transcript contains assistant messages 2. Tool calls were recorded (inspects `Message[].toolCalls`) 3. Response addresses the CSV analysis question diff --git a/examples/features/copilot-transcript-replay/graders/transcript-quality.ts b/examples/features/copilot-transcript-replay/graders/transcript-quality.ts index 3621e0c34..10d8b821b 100644 --- a/examples/features/copilot-transcript-replay/graders/transcript-quality.ts +++ b/examples/features/copilot-transcript-replay/graders/transcript-quality.ts @@ -15,7 +15,7 @@ * type: script * command: ["bun", "run", "../graders/transcript-quality.ts"] */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; function getMessageText( messages: readonly { role: string; content?: unknown }[], diff --git a/examples/features/deterministic-graders/README.md b/examples/features/deterministic-graders/README.md index da52ca10f..863471e88 100644 --- a/examples/features/deterministic-graders/README.md +++ b/examples/features/deterministic-graders/README.md @@ -31,7 +31,7 @@ Set `negated: true` in config to invert any assertion. ## Files -- `graders/assertions.ts` — Parameterised script grader using `defineScriptGrader` from `@agentv/sdk` +- `graders/assertions.ts` — Parameterised script grader using `defineScriptGrader` from `agentv` - `evals/suite.yaml` — Example tests covering every assertion type ## Setup diff --git a/examples/features/deterministic-graders/graders/assertions.ts b/examples/features/deterministic-graders/graders/assertions.ts index 827e7c8a2..bb513eb06 100644 --- a/examples/features/deterministic-graders/graders/assertions.ts +++ b/examples/features/deterministic-graders/graders/assertions.ts @@ -10,7 +10,7 @@ * value – expected substring, pattern, or prefix (not used for is-json) * negated – when true, inverts the assertion (default: false) */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; type AssertionType = 'contains' | 'icontains' | 'equals' | 'regex' | 'starts-with' | 'is-json'; diff --git a/examples/features/deterministic-graders/package.json b/examples/features/deterministic-graders/package.json index 780b5632b..4703fb3fe 100644 --- a/examples/features/deterministic-graders/package.json +++ b/examples/features/deterministic-graders/package.json @@ -3,6 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/features/eval-assert-demo/README.md b/examples/features/eval-assert-demo/README.md index 6ffdb7814..5db4c066b 100644 --- a/examples/features/eval-assert-demo/README.md +++ b/examples/features/eval-assert-demo/README.md @@ -9,7 +9,7 @@ Demonstrates script graders that can be run both as part of an eval suite and in | `.agentv/graders/keyword-check.ts` | Checks answer contains expected keywords (Paris, France) | | `.agentv/graders/length-check.ts` | Validates answer word count is between 5 and 50 | -Both graders use `defineScriptGrader` from `@agentv/sdk`. +Both graders use `defineScriptGrader` from `agentv`. ## Running the Full Eval diff --git a/examples/features/execution-metrics/bun.lock b/examples/features/execution-metrics/bun.lock deleted file mode 100644 index 8e8f1429d..000000000 --- a/examples/features/execution-metrics/bun.lock +++ /dev/null @@ -1,17 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "agentv-example-execution-metrics", - "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk", - }, - }, - }, - "packages": { - "@agentv/sdk": ["@agentv/sdk@file:../../../packages/sdk", { "dependencies": { "zod": "^3.23.8" } }], - - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - } -} diff --git a/examples/features/execution-metrics/package.json b/examples/features/execution-metrics/package.json index 50578ad0c..4422c27cf 100644 --- a/examples/features/execution-metrics/package.json +++ b/examples/features/execution-metrics/package.json @@ -3,6 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/features/execution-metrics/scripts/check-efficiency.ts b/examples/features/execution-metrics/scripts/check-efficiency.ts index d73093f3b..9497c517d 100644 --- a/examples/features/execution-metrics/scripts/check-efficiency.ts +++ b/examples/features/execution-metrics/scripts/check-efficiency.ts @@ -5,7 +5,7 @@ * Demonstrates how to evaluate agent efficiency using execution metrics * available in the trace payload. */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; // Configurable thresholds const THRESHOLDS = { diff --git a/examples/features/execution-metrics/scripts/check-metrics-present.ts b/examples/features/execution-metrics/scripts/check-metrics-present.ts index b472b3dc7..f7d31b8a0 100644 --- a/examples/features/execution-metrics/scripts/check-metrics-present.ts +++ b/examples/features/execution-metrics/scripts/check-metrics-present.ts @@ -11,7 +11,7 @@ * type: script * command: ["bun", "run", "../scripts/check-metrics-present.ts"] */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; export default defineScriptGrader(({ trace, tokenUsage, costUsd, durationMs }) => { const assertions: Array<{ text: string; passed: boolean }> = []; diff --git a/examples/features/import-claude/README.md b/examples/features/import-claude/README.md index 67240aa69..f8222601d 100644 --- a/examples/features/import-claude/README.md +++ b/examples/features/import-claude/README.md @@ -54,7 +54,7 @@ The import pipeline: ### transcript-quality (script-grader) -Custom grader using `defineScriptGrader` from `@agentv/sdk`. Validates: +Custom grader using `defineScriptGrader` from `agentv`. Validates: 1. Transcript contains at least one assistant message 2. Tool calls were recorded with outputs 3. No empty assistant messages diff --git a/examples/features/import-claude/graders/transcript-quality.ts b/examples/features/import-claude/graders/transcript-quality.ts index d02358e50..e2bc8f510 100644 --- a/examples/features/import-claude/graders/transcript-quality.ts +++ b/examples/features/import-claude/graders/transcript-quality.ts @@ -1,5 +1,5 @@ #!/usr/bin/env bun -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; export default defineScriptGrader(({ output }) => { const assertions: Array<{ text: string; passed: boolean }> = []; diff --git a/examples/features/nlp-metrics/README.md b/examples/features/nlp-metrics/README.md index 0707ad480..733356bba 100644 --- a/examples/features/nlp-metrics/README.md +++ b/examples/features/nlp-metrics/README.md @@ -11,7 +11,7 @@ Demonstrates how to implement common NLP evaluation metrics as AgentV `script_gr | `graders/similarity.ts` | Cosine + Jaccard | Paraphrasing — token-overlap similarity | | `graders/levenshtein.ts` | Levenshtein distance | Extraction — character-level edit distance | -Each grader is a standalone TypeScript file that uses `defineScriptGrader` from `@agentv/sdk`. Scores are normalised to the 0–1 range expected by AgentV. +Each grader is a standalone TypeScript file that uses `defineScriptGrader` from `agentv`. Scores are normalised to the 0–1 range expected by AgentV. ## Running diff --git a/examples/features/nlp-metrics/graders/bleu.ts b/examples/features/nlp-metrics/graders/bleu.ts index 8260bb4a0..21f7f5764 100644 --- a/examples/features/nlp-metrics/graders/bleu.ts +++ b/examples/features/nlp-metrics/graders/bleu.ts @@ -6,7 +6,7 @@ * BLEU (Bilingual Evaluation Understudy) measures n-gram precision with a * brevity penalty, commonly used for translation evaluation. */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; function tokenize(text: string): string[] { return text diff --git a/examples/features/nlp-metrics/graders/levenshtein.ts b/examples/features/nlp-metrics/graders/levenshtein.ts index fdf5095ac..fe8ad1f38 100644 --- a/examples/features/nlp-metrics/graders/levenshtein.ts +++ b/examples/features/nlp-metrics/graders/levenshtein.ts @@ -6,7 +6,7 @@ * The score is 1 - (distance / maxLength), so identical strings score 1.0 * and completely different strings score close to 0. */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; function levenshteinDistance(a: string, b: string): number { const m = a.length; diff --git a/examples/features/nlp-metrics/graders/rouge.ts b/examples/features/nlp-metrics/graders/rouge.ts index ae364648a..ae1799e3a 100644 --- a/examples/features/nlp-metrics/graders/rouge.ts +++ b/examples/features/nlp-metrics/graders/rouge.ts @@ -6,7 +6,7 @@ * ROUGE (Recall-Oriented Understudy for Gisting Evaluation) measures n-gram * overlap, commonly used for summarisation evaluation. */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; function tokenize(text: string): string[] { return text diff --git a/examples/features/nlp-metrics/graders/similarity.ts b/examples/features/nlp-metrics/graders/similarity.ts index a92c00434..e1071439d 100644 --- a/examples/features/nlp-metrics/graders/similarity.ts +++ b/examples/features/nlp-metrics/graders/similarity.ts @@ -6,7 +6,7 @@ * token-overlap (bag-of-words) vectors. This is a lightweight alternative to * embedding-based similarity that requires no external dependencies. */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; function tokenize(text: string): string[] { return text diff --git a/examples/features/prompt-template-sdk/README.md b/examples/features/prompt-template-sdk/README.md index 6cfd102f7..294ec2832 100644 --- a/examples/features/prompt-template-sdk/README.md +++ b/examples/features/prompt-template-sdk/README.md @@ -1,6 +1,6 @@ # Prompt Template SDK -This example demonstrates using TypeScript files for custom LLM grader prompts using the `definePromptTemplate` helper from `@agentv/sdk`. +This example demonstrates using TypeScript files for custom LLM grader prompts using the `definePromptTemplate` helper from `agentv`. ## Features @@ -14,7 +14,7 @@ This example demonstrates using TypeScript files for custom LLM grader prompts u Instead of static text files with `{{variable}}` placeholders, you can use TypeScript files that export a prompt template: ```typescript -import { definePromptTemplate } from '@agentv/sdk'; +import { definePromptTemplate } from 'agentv'; function textFromMessages(messages) { return messages diff --git a/examples/features/prompt-template-sdk/bun.lock b/examples/features/prompt-template-sdk/bun.lock deleted file mode 100644 index 630bb9e0b..000000000 --- a/examples/features/prompt-template-sdk/bun.lock +++ /dev/null @@ -1,17 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "agentv-example-prompt-template-sdk", - "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk", - }, - }, - }, - "packages": { - "@agentv/sdk": ["@agentv/sdk@file:../../../packages/sdk", { "dependencies": { "zod": "^3.23.8" } }], - - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - } -} diff --git a/examples/features/prompt-template-sdk/package.json b/examples/features/prompt-template-sdk/package.json index db336fda4..fcfbeaa8b 100644 --- a/examples/features/prompt-template-sdk/package.json +++ b/examples/features/prompt-template-sdk/package.json @@ -3,6 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/features/prompt-template-sdk/prompts/custom-evaluator.ts b/examples/features/prompt-template-sdk/prompts/custom-evaluator.ts index e8f035fab..6c265ea11 100644 --- a/examples/features/prompt-template-sdk/prompts/custom-evaluator.ts +++ b/examples/features/prompt-template-sdk/prompts/custom-evaluator.ts @@ -5,7 +5,7 @@ * Uses the declarative definePromptTemplate helper to generate * a custom evaluation prompt with full TypeScript support. */ -import { definePromptTemplate } from '@agentv/sdk'; +import { definePromptTemplate } from 'agentv'; function getMessageText( messages: readonly { role: string; content?: unknown }[], diff --git a/examples/features/script-grader-sdk/README.md b/examples/features/script-grader-sdk/README.md index c5256f7bd..d952a1dfe 100644 --- a/examples/features/script-grader-sdk/README.md +++ b/examples/features/script-grader-sdk/README.md @@ -1,6 +1,6 @@ # script grader SDK Helper -Demonstrates how a TypeScript script grader can use `defineScriptGrader` from `@agentv/sdk` for a declarative, low-boilerplate approach while still consuming the canonical AgentV wire format. +Demonstrates how a TypeScript script grader can use `defineScriptGrader` from `agentv` for a declarative, low-boilerplate approach while still consuming the canonical AgentV wire format. Use this pattern for command-backed graders referenced with `type: script`. For reusable assertion types discovered from `.agentv/assertions/`, use `defineAssertion()` instead. @@ -16,7 +16,7 @@ From repository root: ```bash bun install # Links workspace dependencies -bun run build # Builds @agentv/core package +bun run build # Builds the local agentv package ``` ## Run @@ -58,7 +58,7 @@ The `defineScriptGrader` helper: - Handles errors gracefully ```typescript -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; export default defineScriptGrader(({ output, criteria }) => ({ pass: (output ?? '').includes(criteria), diff --git a/examples/features/script-grader-sdk/bun.lock b/examples/features/script-grader-sdk/bun.lock deleted file mode 100644 index 4b99380a9..000000000 --- a/examples/features/script-grader-sdk/bun.lock +++ /dev/null @@ -1,17 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "agentv-example-script-grader-sdk", - "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk", - }, - }, - }, - "packages": { - "@agentv/sdk": ["@agentv/sdk@file:../../../packages/sdk", { "dependencies": { "zod": "^3.23.8" } }], - - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - } -} diff --git a/examples/features/script-grader-sdk/package.json b/examples/features/script-grader-sdk/package.json index 7cc81ab6b..c0640c6d1 100644 --- a/examples/features/script-grader-sdk/package.json +++ b/examples/features/script-grader-sdk/package.json @@ -3,6 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/features/script-grader-sdk/scripts/verify-attachments.ts b/examples/features/script-grader-sdk/scripts/verify-attachments.ts index 53dffa113..4fcc8b130 100755 --- a/examples/features/script-grader-sdk/scripts/verify-attachments.ts +++ b/examples/features/script-grader-sdk/scripts/verify-attachments.ts @@ -5,7 +5,7 @@ * Uses the declarative defineScriptGrader helper to verify attachments * are referenced in the candidate output. */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; function fileName(path: string): string { const parts = path.split('/'); diff --git a/examples/features/script-grader-with-llm-calls/README.md b/examples/features/script-grader-with-llm-calls/README.md index 445c6e51c..b69869301 100644 --- a/examples/features/script-grader-with-llm-calls/README.md +++ b/examples/features/script-grader-with-llm-calls/README.md @@ -168,7 +168,7 @@ graders: ## Usage in Code ```typescript -import { createTargetClient, defineScriptGrader } from '@agentv/sdk'; +import { createTargetClient, defineScriptGrader } from 'agentv'; export default defineScriptGrader(async ({ question, config }) => { const target = createTargetClient(); diff --git a/examples/features/script-grader-with-llm-calls/bun.lock b/examples/features/script-grader-with-llm-calls/bun.lock deleted file mode 100644 index 80b50f2db..000000000 --- a/examples/features/script-grader-with-llm-calls/bun.lock +++ /dev/null @@ -1,17 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "script-grader-with-llm-calls-example", - "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk", - }, - }, - }, - "packages": { - "@agentv/sdk": ["@agentv/sdk@file:../../../packages/sdk", { "dependencies": { "zod": "^3.23.8" } }], - - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - } -} diff --git a/examples/features/script-grader-with-llm-calls/package.json b/examples/features/script-grader-with-llm-calls/package.json index d758a38e7..4af72a99f 100644 --- a/examples/features/script-grader-with-llm-calls/package.json +++ b/examples/features/script-grader-with-llm-calls/package.json @@ -3,6 +3,6 @@ "type": "module", "private": true, "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/features/script-grader-with-llm-calls/scripts/contextual-precision.ts b/examples/features/script-grader-with-llm-calls/scripts/contextual-precision.ts index b69f68e2c..33f634838 100644 --- a/examples/features/script-grader-with-llm-calls/scripts/contextual-precision.ts +++ b/examples/features/script-grader-with-llm-calls/scripts/contextual-precision.ts @@ -15,7 +15,7 @@ * Requires `target: { max_calls: N }` in the grader YAML config, * where N >= number of retrieval context nodes to evaluate. */ -import { createTargetClient, defineScriptGrader } from '@agentv/sdk'; +import { createTargetClient, defineScriptGrader } from 'agentv'; import { extractRetrievalContext } from './utils.js'; interface RelevanceResult { diff --git a/examples/features/script-grader-with-llm-calls/scripts/contextual-recall.ts b/examples/features/script-grader-with-llm-calls/scripts/contextual-recall.ts index 013a317d4..41d89b5c6 100644 --- a/examples/features/script-grader-with-llm-calls/scripts/contextual-recall.ts +++ b/examples/features/script-grader-with-llm-calls/scripts/contextual-recall.ts @@ -19,7 +19,7 @@ * Requires `target: { max_calls: N }` in the grader YAML config, * where N >= 2 (one for statement extraction + one for attribution check). */ -import { createTargetClient, defineScriptGrader } from '@agentv/sdk'; +import { createTargetClient, defineScriptGrader } from 'agentv'; import { extractRetrievalContext } from './utils.js'; interface StatementExtractionResult { diff --git a/examples/features/script-grader-with-llm-calls/scripts/utils.ts b/examples/features/script-grader-with-llm-calls/scripts/utils.ts index 3ff1a6df3..20f544b47 100644 --- a/examples/features/script-grader-with-llm-calls/scripts/utils.ts +++ b/examples/features/script-grader-with-llm-calls/scripts/utils.ts @@ -1,4 +1,4 @@ -import type { Message } from '@agentv/sdk'; +import type { Message } from 'agentv'; /** * Extract retrieval context from expectedOutput tool calls. diff --git a/examples/features/sdk-config-file/README.md b/examples/features/sdk-config-file/README.md index e8795c6c4..4b6429028 100644 --- a/examples/features/sdk-config-file/README.md +++ b/examples/features/sdk-config-file/README.md @@ -1,6 +1,6 @@ # SDK Example: Config File -Demonstrates using `defineConfig()` from `@agentv/core` for typed project-level configuration. +Demonstrates using `defineConfig()` from `agentv/config` for typed project-level configuration. ## What It Does diff --git a/examples/features/sdk-config-file/agentv.config.ts b/examples/features/sdk-config-file/agentv.config.ts index d94cff828..7645e9016 100644 --- a/examples/features/sdk-config-file/agentv.config.ts +++ b/examples/features/sdk-config-file/agentv.config.ts @@ -1,4 +1,4 @@ -import { defineConfig } from '@agentv/core'; +import { defineConfig } from 'agentv/config'; export default defineConfig({ execution: { diff --git a/examples/features/sdk-config-file/package.json b/examples/features/sdk-config-file/package.json index 47e6ec462..186814bd1 100644 --- a/examples/features/sdk-config-file/package.json +++ b/examples/features/sdk-config-file/package.json @@ -3,7 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/core": "file:../../../packages/core", - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/features/sdk-custom-assertion/README.md b/examples/features/sdk-custom-assertion/README.md index 289389baf..af33638e8 100644 --- a/examples/features/sdk-custom-assertion/README.md +++ b/examples/features/sdk-custom-assertion/README.md @@ -1,6 +1,6 @@ # SDK Example: Custom Assertion -Demonstrates creating a custom assertion type using `defineAssertion()` from `@agentv/sdk` and convention-based discovery from `.agentv/assertions/`. +Demonstrates creating a custom assertion type using `defineAssertion()` from `agentv` and convention-based discovery from `.agentv/assertions/`. ## What It Does diff --git a/examples/features/sdk-custom-assertion/bun.lock b/examples/features/sdk-custom-assertion/bun.lock deleted file mode 100644 index f86352d9c..000000000 --- a/examples/features/sdk-custom-assertion/bun.lock +++ /dev/null @@ -1,17 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "agentv-example-sdk-custom-assertion", - "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk", - }, - }, - }, - "packages": { - "@agentv/sdk": ["@agentv/sdk@file:../../../packages/sdk", { "dependencies": { "zod": "^3.23.8" } }], - - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - } -} diff --git a/examples/features/sdk-custom-assertion/package.json b/examples/features/sdk-custom-assertion/package.json index 7e7bff09a..1d29530ea 100644 --- a/examples/features/sdk-custom-assertion/package.json +++ b/examples/features/sdk-custom-assertion/package.json @@ -3,6 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/features/sdk-eval-authoring/README.md b/examples/features/sdk-eval-authoring/README.md index 6d5bc1512..57ed4c7ff 100644 --- a/examples/features/sdk-eval-authoring/README.md +++ b/examples/features/sdk-eval-authoring/README.md @@ -1,6 +1,6 @@ # SDK Example: TypeScript Eval Config Authoring -Demonstrates authoring an explicit `*.eval.ts` file with the `EvalConfig` type from `@agentv/sdk`. +Demonstrates authoring an explicit `*.eval.ts` file with the `EvalConfig` type from `agentv`. ## What It Shows diff --git a/examples/features/sdk-eval-authoring/evals/greeting.eval.ts b/examples/features/sdk-eval-authoring/evals/greeting.eval.ts index 97caa675c..9fe818757 100644 --- a/examples/features/sdk-eval-authoring/evals/greeting.eval.ts +++ b/examples/features/sdk-eval-authoring/evals/greeting.eval.ts @@ -1,8 +1,8 @@ -import { type EvalConfig, graders } from '@agentv/sdk'; +import { type EvalConfig, graders } from 'agentv'; const config: EvalConfig = { name: 'sdk-eval-authoring', - description: 'TypeScript eval config authoring with @agentv/sdk', + description: 'TypeScript eval config authoring with agentv', target: 'mock-sdk', environment: { type: 'host', diff --git a/examples/features/sdk-eval-authoring/package.json b/examples/features/sdk-eval-authoring/package.json index 4c3ba8acb..717ef5d44 100644 --- a/examples/features/sdk-eval-authoring/package.json +++ b/examples/features/sdk-eval-authoring/package.json @@ -3,6 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/features/sdk-programmatic-api-advanced/evaluate.ts b/examples/features/sdk-programmatic-api-advanced/evaluate.ts index 7fdf05338..b37bdd3b8 100644 --- a/examples/features/sdk-programmatic-api-advanced/evaluate.ts +++ b/examples/features/sdk-programmatic-api-advanced/evaluate.ts @@ -6,7 +6,7 @@ * * Run: bun run evaluate.ts */ -import { evaluate } from '@agentv/sdk'; +import { evaluate } from 'agentv'; const { results, summary } = await evaluate({ // Run a setup command before the suite starts diff --git a/examples/features/sdk-programmatic-api-advanced/package.json b/examples/features/sdk-programmatic-api-advanced/package.json index e222f311d..90d329894 100644 --- a/examples/features/sdk-programmatic-api-advanced/package.json +++ b/examples/features/sdk-programmatic-api-advanced/package.json @@ -3,7 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/core": "file:../../../packages/core", - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/features/sdk-programmatic-api/README.md b/examples/features/sdk-programmatic-api/README.md index 35e9f4ee6..d8a27f943 100644 --- a/examples/features/sdk-programmatic-api/README.md +++ b/examples/features/sdk-programmatic-api/README.md @@ -1,10 +1,10 @@ # SDK Example: Programmatic API -Demonstrates using `evaluate()` from `@agentv/sdk` to run evaluations as a library when the eval definition belongs in TypeScript. The config mirrors the canonical YAML surface, with TypeScript-friendly names such as `expectedOutput`. +Demonstrates using `evaluate()` from `agentv` to run evaluations as a library when the eval definition belongs in TypeScript. The config mirrors the canonical YAML surface, with TypeScript-friendly names such as `expectedOutput`. ## What It Does -1. Imports `evaluate()` from `@agentv/sdk` +1. Imports `evaluate()` from `agentv` 2. Defines tests inline with `assert` 3. Runs the evaluation and prints summary statistics 4. Writes canonical AgentV run artifacts under `.agentv/results/...` diff --git a/examples/features/sdk-programmatic-api/evaluate.ts b/examples/features/sdk-programmatic-api/evaluate.ts index a8523a322..354ed2086 100644 --- a/examples/features/sdk-programmatic-api/evaluate.ts +++ b/examples/features/sdk-programmatic-api/evaluate.ts @@ -1,13 +1,13 @@ /** * Programmatic API Example * - * Uses evaluate() from @agentv/sdk to run evaluations as a library. + * Uses evaluate() from agentv to run evaluations as a library. * The inline config mirrors the canonical YAML surface with TypeScript-friendly names. * * Run: bun run evaluate.ts * (Uses 'default' target from .agentv/providers.yaml and .env credentials) */ -import { evaluate } from '@agentv/sdk'; +import { evaluate } from 'agentv'; const { results, summary } = await evaluate({ tests: [ diff --git a/examples/features/sdk-programmatic-api/package.json b/examples/features/sdk-programmatic-api/package.json index f0040259d..cf993228c 100644 --- a/examples/features/sdk-programmatic-api/package.json +++ b/examples/features/sdk-programmatic-api/package.json @@ -3,7 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/core": "file:../../../packages/core", - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/features/tool-evaluation-plugins/graders/tool-args-f1.ts b/examples/features/tool-evaluation-plugins/graders/tool-args-f1.ts index eb3967356..0ce7cd9d7 100644 --- a/examples/features/tool-evaluation-plugins/graders/tool-args-f1.ts +++ b/examples/features/tool-evaluation-plugins/graders/tool-args-f1.ts @@ -22,7 +22,7 @@ * args: { query: "weather" } * - tool: fetch */ -import { type ScriptGraderInput, defineScriptGrader } from '@agentv/sdk'; +import { type ScriptGraderInput, defineScriptGrader } from 'agentv'; interface ExpectedTool { tool: string; diff --git a/examples/features/tool-evaluation-plugins/graders/tool-call-f1.ts b/examples/features/tool-evaluation-plugins/graders/tool-call-f1.ts index da7e258ce..a86f29aa9 100644 --- a/examples/features/tool-evaluation-plugins/graders/tool-call-f1.ts +++ b/examples/features/tool-evaluation-plugins/graders/tool-call-f1.ts @@ -20,7 +20,7 @@ * command: ["bun", "run", "../graders/tool-call-f1.ts"] * expected_tools: ["search", "fetch"] */ -import { type ScriptGraderInput, defineScriptGrader } from '@agentv/sdk'; +import { type ScriptGraderInput, defineScriptGrader } from 'agentv'; function extractActualTools(input: ScriptGraderInput): string[] { const tools: string[] = []; diff --git a/examples/features/trace-evaluation/graders/error-spans.ts b/examples/features/trace-evaluation/graders/error-spans.ts index 25a89724e..7d7d779c0 100644 --- a/examples/features/trace-evaluation/graders/error-spans.ts +++ b/examples/features/trace-evaluation/graders/error-spans.ts @@ -5,7 +5,7 @@ * Detects errors in agent traces by inspecting trace.errorCount * and optionally checking for specific tool failures. */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; export default defineScriptGrader(({ trace, config }) => { if (!trace) { diff --git a/examples/features/trace-evaluation/graders/span-count.ts b/examples/features/trace-evaluation/graders/span-count.ts index 83f939314..ec2cfaa74 100644 --- a/examples/features/trace-evaluation/graders/span-count.ts +++ b/examples/features/trace-evaluation/graders/span-count.ts @@ -5,7 +5,7 @@ * Validates that the number of LLM calls and tool executions stays * within configurable thresholds using trace data. */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; const DEFAULTS = { maxLlmCalls: 10, diff --git a/examples/features/trace-evaluation/graders/span-duration.ts b/examples/features/trace-evaluation/graders/span-duration.ts index 471d04aec..f4bfde822 100644 --- a/examples/features/trace-evaluation/graders/span-duration.ts +++ b/examples/features/trace-evaluation/graders/span-duration.ts @@ -5,7 +5,7 @@ * Validates that no individual tool execution exceeds a time threshold * using trace.toolDurations data. */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; const DEFAULT_MAX_SPAN_MS = 5000; diff --git a/examples/features/trace-evaluation/package.json b/examples/features/trace-evaluation/package.json index 6c88e652d..abf4bb86b 100644 --- a/examples/features/trace-evaluation/package.json +++ b/examples/features/trace-evaluation/package.json @@ -3,6 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/features/trial-output-consistency/graders/trial-consistency.ts b/examples/features/trial-output-consistency/graders/trial-consistency.ts index 26b86c0a6..501512a76 100644 --- a/examples/features/trial-output-consistency/graders/trial-consistency.ts +++ b/examples/features/trial-output-consistency/graders/trial-consistency.ts @@ -15,7 +15,7 @@ * 1 trial → score 1.0 (perfect consistency by definition) * 2+ trials → average pairwise cosine similarity */ -import { createTargetClient, defineScriptGrader, z } from '@agentv/sdk'; +import { createTargetClient, defineScriptGrader, z } from 'agentv'; const ConfigSchema = z.object({ trialOutputs: z.array(z.string()), diff --git a/examples/features/vitest-workspace-grader/bun.lock b/examples/features/vitest-workspace-grader/bun.lock deleted file mode 100644 index 6a055e088..000000000 --- a/examples/features/vitest-workspace-grader/bun.lock +++ /dev/null @@ -1,163 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "agentv-example-vitest-workspace-grader", - "devDependencies": { - "vitest": "^4.0.0", - }, - }, - }, - "packages": { - "@emnapi/core": ["@emnapi/core@1.10.0", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" } }, "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw=="], - - "@emnapi/runtime": ["@emnapi/runtime@1.10.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA=="], - - "@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w=="], - - "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], - - "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.5", "", { "dependencies": { "@tybys/wasm-util": "^0.10.2" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q=="], - - "@oxc-project/types": ["@oxc-project/types@0.133.0", "", {}, "sha512-KzkdCd6Uxqnf6l3HOw1xfatAlUURA0g14cvBYFyJ5SaNOQbOUvBr9PKArcPcrNIeRsBdgcUzOGrhKveVpvOIGA=="], - - "@rolldown/binding-android-arm64": ["@rolldown/binding-android-arm64@1.0.3", "", { "os": "android", "cpu": "arm64" }, "sha512-454rs7jHngixp/NMxd5srYD57OnzSlZ/eFTETjORQHLwJG1lRtmNOJcBerZlfu4GjKqeq8aCCIQrMdHyhI51Hw=="], - - "@rolldown/binding-darwin-arm64": ["@rolldown/binding-darwin-arm64@1.0.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-PcAhP+ynjURNyy8SKGl5DQP94aGuB/7JrXJb/t7P+hanXvQVMWzUvRRhBAcg/lNRadBhoUPqSoP4xw5tR/KBEA=="], - - "@rolldown/binding-darwin-x64": ["@rolldown/binding-darwin-x64@1.0.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-9YpfeUvSE2RS7wysJ81uOZkXJz7f7Q55H2Gvp3VEw/EsahqDtrphrZ0EwDLK5vvKOzaCrBsjF8JmnMLcUt78Gg=="], - - "@rolldown/binding-freebsd-x64": ["@rolldown/binding-freebsd-x64@1.0.3", "", { "os": "freebsd", "cpu": "x64" }, "sha512-yB1IlAsSNHncV6SCTL27/MVGR5htvQsoGxIv5KMGXALp+Ll1wYsn+x98M9MW7qa+NdSbvrrY7ANI4wLJ0n1e6g=="], - - "@rolldown/binding-linux-arm-gnueabihf": ["@rolldown/binding-linux-arm-gnueabihf@1.0.3", "", { "os": "linux", "cpu": "arm" }, "sha512-Yi30IVAAfLUCy2MseFjbB1jAMDl1VMCAas5StnYp8da9+CKvMd2H2cbEjWcw5NPaPqzvYkVIaF1nNUG+b7u/sw=="], - - "@rolldown/binding-linux-arm64-gnu": ["@rolldown/binding-linux-arm64-gnu@1.0.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-jsO7R8To+AdlYgUmN5sHSCZbfhtMBkO0WUx8iORQnPcMMdgr7qM2DQmMwgabs3GhNztdmoKkMKQFHD6DTMCIQw=="], - - "@rolldown/binding-linux-arm64-musl": ["@rolldown/binding-linux-arm64-musl@1.0.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-VWkUHwWriDciit80wleYwKILoR/KMvxh/IdwS/paX+ZgpuRpCrKLUdadJbc0NpBEiyhpYawsJ73j9aCvOH+f7Q=="], - - "@rolldown/binding-linux-ppc64-gnu": ["@rolldown/binding-linux-ppc64-gnu@1.0.3", "", { "os": "linux", "cpu": "ppc64" }, "sha512-5f1laC0SlIR0yDbFCd8acUhvJIag6N3zC5P7oUPN6wX0aOma+uKJ0wBDH5aq7I1PVI2ttTlhJwzwRIBnLiSGEg=="], - - "@rolldown/binding-linux-s390x-gnu": ["@rolldown/binding-linux-s390x-gnu@1.0.3", "", { "os": "linux", "cpu": "s390x" }, "sha512-Iq4ko0r4XsgbrF/LunNgHtAGLRRVE2kXonAXQ/MV0mC6jQpMOhW1SvtZja2EhC/kd05++bP78dsqBeIQyYJ6Yg=="], - - "@rolldown/binding-linux-x64-gnu": ["@rolldown/binding-linux-x64-gnu@1.0.3", "", { "os": "linux", "cpu": "x64" }, "sha512-B8m6tD5+/N5FeNQFbKlLA/2yVq9ycQP1SeedyEYYKWBNR3ZQbkvIUcNnDNM03lO1l5F2roiiFJGgvoLLyZXtSg=="], - - "@rolldown/binding-linux-x64-musl": ["@rolldown/binding-linux-x64-musl@1.0.3", "", { "os": "linux", "cpu": "x64" }, "sha512-pSdpdUJHkuCxun9LE7jvgUB9qsRgaiyNNCX7m/AvHTcq67AiT/Yhoxvw5zPfhrM8k/BfP8ce/hMOpthKDpEUow=="], - - "@rolldown/binding-openharmony-arm64": ["@rolldown/binding-openharmony-arm64@1.0.3", "", { "os": "none", "cpu": "arm64" }, "sha512-OXXS3RKJgX2uLwM+gYyuH5omcH8fL1LJs96pZGgtetVCahON57+d4SJHzTgZiOjxgGkSnpXpOsWuPDGAKAigEg=="], - - "@rolldown/binding-wasm32-wasi": ["@rolldown/binding-wasm32-wasi@1.0.3", "", { "dependencies": { "@emnapi/core": "1.10.0", "@emnapi/runtime": "1.10.0", "@napi-rs/wasm-runtime": "^1.1.4" }, "cpu": "none" }, "sha512-JTtb8BWFynicNSoPrehsCzBtOKjZ6jhMiPFEmOiuXg1Fl8dn2KHQob+GuPSGR0dryQa1PQJbzjF3dqO/whhjLg=="], - - "@rolldown/binding-win32-arm64-msvc": ["@rolldown/binding-win32-arm64-msvc@1.0.3", "", { "os": "win32", "cpu": "arm64" }, "sha512-gEdFFEN70A/jxb2svrWsN3aDL7OUtmvlOy+6fa2jxG8K0wQ1ZbdeLGnidov6Yu5/733dI5ySfzFlQ/cb0bSz1g=="], - - "@rolldown/binding-win32-x64-msvc": ["@rolldown/binding-win32-x64-msvc@1.0.3", "", { "os": "win32", "cpu": "x64" }, "sha512-eXB7CHuaQdqmJcc3koCNtNPmT/bj2gc999kUFgBxG8Ac0NdgXc4rkCHhqrgrhN3zddvvvrgzj1e90SuSfmyIXA=="], - - "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.1", "", {}, "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw=="], - - "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], - - "@tybys/wasm-util": ["@tybys/wasm-util@0.10.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg=="], - - "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], - - "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="], - - "@types/estree": ["@types/estree@1.0.9", "", {}, "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg=="], - - "@vitest/expect": ["@vitest/expect@4.1.9", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", "@vitest/spy": "4.1.9", "@vitest/utils": "4.1.9", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" } }, "sha512-vl/rYsUKcBr3SnQn166+XR5ZQcgMx3DQhFWdfli/cWpLnLUmbxZvyrJZotLFUryib+LtArYMSTJ5RbQ57ZqrlA=="], - - "@vitest/mocker": ["@vitest/mocker@4.1.9", "", { "dependencies": { "@vitest/spy": "4.1.9", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "optionalPeers": ["msw", "vite"] }, "sha512-EVkXzBjrPGM+cK8/ANWgBrkUCfJfb38/EfTSO8h7pWvKkyPkpWxvR7BkD2MyItMF62C97zAEoqdpUixwR/e+Rw=="], - - "@vitest/pretty-format": ["@vitest/pretty-format@4.1.9", "", { "dependencies": { "tinyrainbow": "^3.1.0" } }, "sha512-s0iufns3iIFitdgm+YR7g1whCAaGtXz459VS9/PqyKDEEFgYIhsHOQmXgIgDuYCt7DeQmiZT0Qe2OA2p4ZPu5A=="], - - "@vitest/runner": ["@vitest/runner@4.1.9", "", { "dependencies": { "@vitest/utils": "4.1.9", "pathe": "^2.0.3" } }, "sha512-KXLMDtc7oe70+3mJfGrPUWPesswH+3sTxAMAMl8DG7I8IUQT4XW718dY5ID3vPUcmlu27CcKfY4P3h3I29SLJg=="], - - "@vitest/snapshot": ["@vitest/snapshot@4.1.9", "", { "dependencies": { "@vitest/pretty-format": "4.1.9", "@vitest/utils": "4.1.9", "magic-string": "^0.30.21", "pathe": "^2.0.3" } }, "sha512-Jc7RKGNBo8Z28WYIm0Niej4xdSPByRf6mU58VpHQkd6Zh05rlnA+twjbK5HyeIGHxrzsc3mJgS43uM0CZKzaIA=="], - - "@vitest/spy": ["@vitest/spy@4.1.9", "", {}, "sha512-fHpsS6mIi+PiEW+vcRVOMkX1oSaPKne3VOclSFICPcGOmfKgXPU5iAah+wcNcj2xPrCCmfq99IDGf+EojhhvhA=="], - - "@vitest/utils": ["@vitest/utils@4.1.9", "", { "dependencies": { "@vitest/pretty-format": "4.1.9", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" } }, "sha512-A51o8ymO5PpqlWNnBP9ZHPXDIpuMtTLlGSjN7la4US+LJzoUMyhwjA5QXlm39JexgwHKW4Xjs8Z2d3dLCXOeuA=="], - - "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="], - - "chai": ["chai@6.2.2", "", {}, "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg=="], - - "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], - - "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], - - "es-module-lexer": ["es-module-lexer@2.1.0", "", {}, "sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ=="], - - "estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], - - "expect-type": ["expect-type@1.3.0", "", {}, "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA=="], - - "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], - - "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], - - "lightningcss": ["lightningcss@1.32.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.32.0", "lightningcss-darwin-arm64": "1.32.0", "lightningcss-darwin-x64": "1.32.0", "lightningcss-freebsd-x64": "1.32.0", "lightningcss-linux-arm-gnueabihf": "1.32.0", "lightningcss-linux-arm64-gnu": "1.32.0", "lightningcss-linux-arm64-musl": "1.32.0", "lightningcss-linux-x64-gnu": "1.32.0", "lightningcss-linux-x64-musl": "1.32.0", "lightningcss-win32-arm64-msvc": "1.32.0", "lightningcss-win32-x64-msvc": "1.32.0" } }, "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ=="], - - "lightningcss-android-arm64": ["lightningcss-android-arm64@1.32.0", "", { "os": "android", "cpu": "arm64" }, "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg=="], - - "lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.32.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ=="], - - "lightningcss-darwin-x64": ["lightningcss-darwin-x64@1.32.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w=="], - - "lightningcss-freebsd-x64": ["lightningcss-freebsd-x64@1.32.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig=="], - - "lightningcss-linux-arm-gnueabihf": ["lightningcss-linux-arm-gnueabihf@1.32.0", "", { "os": "linux", "cpu": "arm" }, "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw=="], - - "lightningcss-linux-arm64-gnu": ["lightningcss-linux-arm64-gnu@1.32.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ=="], - - "lightningcss-linux-arm64-musl": ["lightningcss-linux-arm64-musl@1.32.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg=="], - - "lightningcss-linux-x64-gnu": ["lightningcss-linux-x64-gnu@1.32.0", "", { "os": "linux", "cpu": "x64" }, "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA=="], - - "lightningcss-linux-x64-musl": ["lightningcss-linux-x64-musl@1.32.0", "", { "os": "linux", "cpu": "x64" }, "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg=="], - - "lightningcss-win32-arm64-msvc": ["lightningcss-win32-arm64-msvc@1.32.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw=="], - - "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.32.0", "", { "os": "win32", "cpu": "x64" }, "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q=="], - - "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], - - "nanoid": ["nanoid@3.3.15", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA=="], - - "obug": ["obug@2.1.3", "", {}, "sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg=="], - - "pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], - - "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], - - "picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="], - - "postcss": ["postcss@8.5.15", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A=="], - - "rolldown": ["rolldown@1.0.3", "", { "dependencies": { "@oxc-project/types": "=0.133.0", "@rolldown/pluginutils": "^1.0.0" }, "optionalDependencies": { "@rolldown/binding-android-arm64": "1.0.3", "@rolldown/binding-darwin-arm64": "1.0.3", "@rolldown/binding-darwin-x64": "1.0.3", "@rolldown/binding-freebsd-x64": "1.0.3", "@rolldown/binding-linux-arm-gnueabihf": "1.0.3", "@rolldown/binding-linux-arm64-gnu": "1.0.3", "@rolldown/binding-linux-arm64-musl": "1.0.3", "@rolldown/binding-linux-ppc64-gnu": "1.0.3", "@rolldown/binding-linux-s390x-gnu": "1.0.3", "@rolldown/binding-linux-x64-gnu": "1.0.3", "@rolldown/binding-linux-x64-musl": "1.0.3", "@rolldown/binding-openharmony-arm64": "1.0.3", "@rolldown/binding-wasm32-wasi": "1.0.3", "@rolldown/binding-win32-arm64-msvc": "1.0.3", "@rolldown/binding-win32-x64-msvc": "1.0.3" }, "bin": { "rolldown": "./bin/cli.mjs" } }, "sha512-i00lAJ2ks1BYr7rjNjKC7BcqAS7nVfiT3QX1SI5aY+AFHblCmaUf9OE9dbdzDvW6dJxbi2ZCZiy9v3CcwOiX3g=="], - - "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], - - "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], - - "stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="], - - "std-env": ["std-env@4.1.0", "", {}, "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ=="], - - "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], - - "tinyexec": ["tinyexec@1.2.4", "", {}, "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg=="], - - "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="], - - "tinyrainbow": ["tinyrainbow@3.1.0", "", {}, "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw=="], - - "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], - - "vite": ["vite@8.0.16", "", { "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4", "postcss": "^8.5.15", "rolldown": "1.0.3", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.1.18", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw=="], - - "vitest": ["vitest@4.1.9", "", { "dependencies": { "@vitest/expect": "4.1.9", "@vitest/mocker": "4.1.9", "@vitest/pretty-format": "4.1.9", "@vitest/runner": "4.1.9", "@vitest/snapshot": "4.1.9", "@vitest/spy": "4.1.9", "@vitest/utils": "4.1.9", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", "obug": "^2.1.1", "pathe": "^2.0.3", "picomatch": "^4.0.3", "std-env": "^4.0.0-rc.1", "tinybench": "^2.9.0", "tinyexec": "^1.0.2", "tinyglobby": "^0.2.15", "tinyrainbow": "^3.1.0", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", "@vitest/browser-playwright": "4.1.9", "@vitest/browser-preview": "4.1.9", "@vitest/browser-webdriverio": "4.1.9", "@vitest/coverage-istanbul": "4.1.9", "@vitest/coverage-v8": "4.1.9", "@vitest/ui": "4.1.9", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@opentelemetry/api", "@types/node", "@vitest/browser-playwright", "@vitest/browser-preview", "@vitest/browser-webdriverio", "@vitest/coverage-istanbul", "@vitest/coverage-v8", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "./vitest.mjs" } }, "sha512-nE3/LEyc0z87uHYLZebqCUOaJr2hdtuPp7BQ4BosVFnfltxgAvMG08NyrSGlPpOUWvR27c5flSmYFTNr78L9GQ=="], - - "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], - } -} diff --git a/examples/showcase/cross-repo-sync/bun.lock b/examples/showcase/cross-repo-sync/bun.lock deleted file mode 100644 index 3b8fe551f..000000000 --- a/examples/showcase/cross-repo-sync/bun.lock +++ /dev/null @@ -1,80 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "cross-repo-sync-showcase", - "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk", - "tsx": "^4.19.0", - }, - }, - }, - "packages": { - "@agentv/sdk": ["@agentv/sdk@file:../../../packages/sdk", { "dependencies": { "zod": "^3.23.8" } }], - - "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.27.3", "", { "os": "aix", "cpu": "ppc64" }, "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg=="], - - "@esbuild/android-arm": ["@esbuild/android-arm@0.27.3", "", { "os": "android", "cpu": "arm" }, "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA=="], - - "@esbuild/android-arm64": ["@esbuild/android-arm64@0.27.3", "", { "os": "android", "cpu": "arm64" }, "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg=="], - - "@esbuild/android-x64": ["@esbuild/android-x64@0.27.3", "", { "os": "android", "cpu": "x64" }, "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ=="], - - "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.27.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg=="], - - "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.27.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg=="], - - "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.27.3", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w=="], - - "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.27.3", "", { "os": "freebsd", "cpu": "x64" }, "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA=="], - - "@esbuild/linux-arm": ["@esbuild/linux-arm@0.27.3", "", { "os": "linux", "cpu": "arm" }, "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw=="], - - "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.27.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg=="], - - "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.27.3", "", { "os": "linux", "cpu": "ia32" }, "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg=="], - - "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.27.3", "", { "os": "linux", "cpu": "none" }, "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA=="], - - "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.27.3", "", { "os": "linux", "cpu": "none" }, "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw=="], - - "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.27.3", "", { "os": "linux", "cpu": "ppc64" }, "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA=="], - - "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.27.3", "", { "os": "linux", "cpu": "none" }, "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ=="], - - "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.27.3", "", { "os": "linux", "cpu": "s390x" }, "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw=="], - - "@esbuild/linux-x64": ["@esbuild/linux-x64@0.27.3", "", { "os": "linux", "cpu": "x64" }, "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA=="], - - "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.27.3", "", { "os": "none", "cpu": "arm64" }, "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA=="], - - "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.27.3", "", { "os": "none", "cpu": "x64" }, "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA=="], - - "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.27.3", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw=="], - - "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.27.3", "", { "os": "openbsd", "cpu": "x64" }, "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ=="], - - "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.27.3", "", { "os": "none", "cpu": "arm64" }, "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g=="], - - "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.27.3", "", { "os": "sunos", "cpu": "x64" }, "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA=="], - - "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.27.3", "", { "os": "win32", "cpu": "arm64" }, "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA=="], - - "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.27.3", "", { "os": "win32", "cpu": "ia32" }, "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q=="], - - "@esbuild/win32-x64": ["@esbuild/win32-x64@0.27.3", "", { "os": "win32", "cpu": "x64" }, "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA=="], - - "esbuild": ["esbuild@0.27.3", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.27.3", "@esbuild/android-arm": "0.27.3", "@esbuild/android-arm64": "0.27.3", "@esbuild/android-x64": "0.27.3", "@esbuild/darwin-arm64": "0.27.3", "@esbuild/darwin-x64": "0.27.3", "@esbuild/freebsd-arm64": "0.27.3", "@esbuild/freebsd-x64": "0.27.3", "@esbuild/linux-arm": "0.27.3", "@esbuild/linux-arm64": "0.27.3", "@esbuild/linux-ia32": "0.27.3", "@esbuild/linux-loong64": "0.27.3", "@esbuild/linux-mips64el": "0.27.3", "@esbuild/linux-ppc64": "0.27.3", "@esbuild/linux-riscv64": "0.27.3", "@esbuild/linux-s390x": "0.27.3", "@esbuild/linux-x64": "0.27.3", "@esbuild/netbsd-arm64": "0.27.3", "@esbuild/netbsd-x64": "0.27.3", "@esbuild/openbsd-arm64": "0.27.3", "@esbuild/openbsd-x64": "0.27.3", "@esbuild/openharmony-arm64": "0.27.3", "@esbuild/sunos-x64": "0.27.3", "@esbuild/win32-arm64": "0.27.3", "@esbuild/win32-ia32": "0.27.3", "@esbuild/win32-x64": "0.27.3" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg=="], - - "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], - - "get-tsconfig": ["get-tsconfig@4.13.6", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-shZT/QMiSHc/YBLxxOkMtgSid5HFoauqCE3/exfsEcwg1WkeqjG+V40yBbBrsD+jW2HDXcs28xOfcbm2jI8Ddw=="], - - "resolve-pkg-maps": ["resolve-pkg-maps@1.0.0", "", {}, "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw=="], - - "tsx": ["tsx@4.21.0", "", { "dependencies": { "esbuild": "~0.27.0", "get-tsconfig": "^4.7.5" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw=="], - - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - } -} diff --git a/examples/showcase/cross-repo-sync/package.json b/examples/showcase/cross-repo-sync/package.json index 4746ff11f..75b876618 100644 --- a/examples/showcase/cross-repo-sync/package.json +++ b/examples/showcase/cross-repo-sync/package.json @@ -3,7 +3,7 @@ "private": true, "description": "Cross-repo sync workspace evaluation showcase", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" }, "devDependencies": { "tsx": "^4.19.0" diff --git a/examples/showcase/cross-repo-sync/scripts/validate-sync.ts b/examples/showcase/cross-repo-sync/scripts/validate-sync.ts index cc9f82c85..fa1ef4bf1 100644 --- a/examples/showcase/cross-repo-sync/scripts/validate-sync.ts +++ b/examples/showcase/cross-repo-sync/scripts/validate-sync.ts @@ -11,7 +11,7 @@ * - ground_truth: string — path to the ground truth diff file (from metadata) */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; defineScriptGrader(({ fileChanges, config }) => { const assertions: Array<{ text: string; passed: boolean }> = []; diff --git a/examples/showcase/export-screening/bun.lock b/examples/showcase/export-screening/bun.lock deleted file mode 100644 index eecac6893..000000000 --- a/examples/showcase/export-screening/bun.lock +++ /dev/null @@ -1,17 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "agentv-example-export-screening", - "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk", - }, - }, - }, - "packages": { - "@agentv/sdk": ["@agentv/sdk@file:../../../packages/sdk", { "dependencies": { "zod": "^3.23.8" } }], - - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - } -} diff --git a/examples/showcase/export-screening/evals/validate_risk_output.ts b/examples/showcase/export-screening/evals/validate_risk_output.ts index f77b005b5..ed595b197 100644 --- a/examples/showcase/export-screening/evals/validate_risk_output.ts +++ b/examples/showcase/export-screening/evals/validate_risk_output.ts @@ -7,7 +7,7 @@ * * Returns structured output that enables post-processing for metrics. */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; const VALID_RISK_LEVELS = new Set(['High', 'Medium', 'Low']); const REQUIRED_KEYS = ['riskLevel', 'reasoning']; diff --git a/examples/showcase/export-screening/package.json b/examples/showcase/export-screening/package.json index ea152ccee..e19280170 100644 --- a/examples/showcase/export-screening/package.json +++ b/examples/showcase/export-screening/package.json @@ -3,6 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/showcase/grader-conformance/bun.lock b/examples/showcase/grader-conformance/bun.lock deleted file mode 100644 index c3ea1080e..000000000 --- a/examples/showcase/grader-conformance/bun.lock +++ /dev/null @@ -1,20 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "agentv-example-grader-conformance", - "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk", - "yaml": "^2.7.1", - }, - }, - }, - "packages": { - "@agentv/sdk": ["@agentv/sdk@file:../../../packages/sdk", { "dependencies": { "zod": "^3.23.8" } }], - - "yaml": ["yaml@2.8.2", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A=="], - - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - } -} diff --git a/examples/showcase/grader-conformance/graders/keyword-grader.ts b/examples/showcase/grader-conformance/graders/keyword-grader.ts index a2a840113..926df2f44 100644 --- a/examples/showcase/grader-conformance/graders/keyword-grader.ts +++ b/examples/showcase/grader-conformance/graders/keyword-grader.ts @@ -6,7 +6,7 @@ * appear in the candidate output. Produces stable scores for unambiguous * cases and variable scores for partial matches. */ -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; function getMessageText( messages: readonly { role: string; content?: unknown }[], diff --git a/examples/showcase/grader-conformance/package.json b/examples/showcase/grader-conformance/package.json index fc6814633..ee7fbf638 100644 --- a/examples/showcase/grader-conformance/package.json +++ b/examples/showcase/grader-conformance/package.json @@ -3,7 +3,7 @@ "private": true, "type": "module", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk", + "agentv": "file:../../../apps/cli", "yaml": "^2.8.3" } } diff --git a/examples/showcase/tool-evaluation-plugins/bun.lock b/examples/showcase/tool-evaluation-plugins/bun.lock deleted file mode 100644 index 5c7b61e91..000000000 --- a/examples/showcase/tool-evaluation-plugins/bun.lock +++ /dev/null @@ -1,17 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "agentv-example-tool-evaluation-plugins", - "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk", - }, - }, - }, - "packages": { - "@agentv/sdk": ["@agentv/sdk@file:../../../packages/sdk", { "dependencies": { "zod": "^3.23.8" } }], - - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], - } -} diff --git a/examples/showcase/tool-evaluation-plugins/package.json b/examples/showcase/tool-evaluation-plugins/package.json index 8a0f68557..a2be4b940 100644 --- a/examples/showcase/tool-evaluation-plugins/package.json +++ b/examples/showcase/tool-evaluation-plugins/package.json @@ -3,6 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/examples/showcase/tool-evaluation-plugins/scripts/efficiency-scorer.ts b/examples/showcase/tool-evaluation-plugins/scripts/efficiency-scorer.ts index 6b65d39d9..0af16d9a3 100644 --- a/examples/showcase/tool-evaluation-plugins/scripts/efficiency-scorer.ts +++ b/examples/showcase/tool-evaluation-plugins/scripts/efficiency-scorer.ts @@ -19,7 +19,7 @@ * type: script * command: ["bun", "run", "scripts/efficiency-scorer.ts"] */ -import { type TraceSummary, defineScriptGrader } from '@agentv/sdk'; +import { type TraceSummary, defineScriptGrader } from 'agentv'; // Configurable thresholds (customize for your domain) const THRESHOLDS = { diff --git a/examples/showcase/tool-evaluation-plugins/scripts/pairwise-tool-compare.ts b/examples/showcase/tool-evaluation-plugins/scripts/pairwise-tool-compare.ts index 1635ce958..20b326d8f 100644 --- a/examples/showcase/tool-evaluation-plugins/scripts/pairwise-tool-compare.ts +++ b/examples/showcase/tool-evaluation-plugins/scripts/pairwise-tool-compare.ts @@ -17,7 +17,7 @@ * type: script * command: ["bun", "run", "scripts/pairwise-tool-compare.ts"] */ -import { type Message, defineScriptGrader } from '@agentv/sdk'; +import { type Message, defineScriptGrader } from 'agentv'; interface ToolSummary { tools: string[]; diff --git a/examples/showcase/tool-evaluation-plugins/scripts/tool-selection-grader.ts b/examples/showcase/tool-evaluation-plugins/scripts/tool-selection-grader.ts index ef366a57a..eb7da353c 100644 --- a/examples/showcase/tool-evaluation-plugins/scripts/tool-selection-grader.ts +++ b/examples/showcase/tool-evaluation-plugins/scripts/tool-selection-grader.ts @@ -17,7 +17,7 @@ * type: script * command: ["bun", "run", "scripts/tool-selection-grader.ts"] */ -import { type Message, defineScriptGrader } from '@agentv/sdk'; +import { type Message, defineScriptGrader } from 'agentv'; interface ExtractedToolCall { tool: string; diff --git a/examples/showcase/trace-evaluation/graders/recovery-check.ts b/examples/showcase/trace-evaluation/graders/recovery-check.ts index 104281988..8273834cc 100644 --- a/examples/showcase/trace-evaluation/graders/recovery-check.ts +++ b/examples/showcase/trace-evaluation/graders/recovery-check.ts @@ -1,5 +1,5 @@ #!/usr/bin/env bun -import { type Message, type ToolCall, defineScriptGrader } from '@agentv/sdk'; +import { type Message, type ToolCall, defineScriptGrader } from 'agentv'; function allToolCalls(output: readonly Message[] | null | undefined): ToolCall[] { return (output ?? []).flatMap((message) => [...(message.toolCalls ?? [])]); diff --git a/examples/showcase/trace-evaluation/graders/replay-proof.ts b/examples/showcase/trace-evaluation/graders/replay-proof.ts index 901236f09..77f589ab5 100644 --- a/examples/showcase/trace-evaluation/graders/replay-proof.ts +++ b/examples/showcase/trace-evaluation/graders/replay-proof.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun import { appendFileSync } from 'node:fs'; -import { defineScriptGrader } from '@agentv/sdk'; +import { defineScriptGrader } from 'agentv'; export default defineScriptGrader( ({ trace, tokenUsage, costUsd, durationMs, messages, config }) => { diff --git a/examples/showcase/trace-evaluation/package.json b/examples/showcase/trace-evaluation/package.json index 2202aac6f..826f6b246 100644 --- a/examples/showcase/trace-evaluation/package.json +++ b/examples/showcase/trace-evaluation/package.json @@ -3,6 +3,6 @@ "private": true, "type": "module", "dependencies": { - "@agentv/sdk": "file:../../../packages/sdk" + "agentv": "file:../../../apps/cli" } } diff --git a/packages/core/README.md b/packages/core/README.md index 8387fcc07..dbb5a6870 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -1,77 +1,18 @@ # @agentv/core -Core evaluation engine and runtime primitives for AgentV - a TypeScript-based AI agent evaluation and optimization framework. +Internal workspace package for AgentV's evaluation engine, provider contracts, +artifact readers/writers, trace/session read models, project registry, and +shared runtime implementation. -## Overview - -This package provides the foundational components for building and evaluating AI agents: - -- **Provider Abstraction**: Unified interface for Azure OpenAI, Anthropic, Google Gemini, VS Code Copilot, and mock providers -- **Evaluation Engine**: YAML-based test specification and execution -- **Quality Grading**: AI-powered scoring system for comparing expected vs. actual outputs -- **Target Management**: Flexible configuration for different execution environments - -## Installation - -```bash -npm install @agentv/core -``` - -## Usage - -This is a low-level package primarily used by the [agentv](https://www.npmjs.com/package/agentv) CLI. Most users should install the CLI package instead: - -```bash -npm install -g agentv -``` - -For programmatic usage or custom integrations, you can import core components: +This package is private and is not published to npm. Public user-facing +programmatic imports are exposed by the `agentv` package: ```typescript -import { createProvider, runEvaluation } from '@agentv/core'; +import { evaluate } from 'agentv'; +import { defineConfig } from 'agentv/config'; +import type { Provider, ProviderRequest, ProviderResponse } from 'agentv/provider'; ``` -## Features - -### Multi-Provider Support - -- **Azure OpenAI**: Enterprise-grade deployment support -- **Anthropic Claude**: Latest Claude models including Sonnet 4.5 -- **Google Gemini**: Gemini 2.0 Flash and other models -- **VS Code Copilot**: Programmatic integration via subagent -- **Mock Provider**: Testing without API calls - -### Evaluation Framework - -- YAML-based test specifications -- Code block extraction and structured prompting -- Automatic retry handling for timeouts -- Detailed scoring with hit/miss analysis -- Multiple output formats (JSONL, YAML) - -### Quality Grading - -- AI-powered aspect extraction and comparison -- Normalized scoring (0.0 to 1.0) -- Detailed reasoning and analysis -- Configurable grading models - -## Architecture - -Built on modern TypeScript tooling: - -- **Vercel AI SDK**: Direct Azure OpenAI, Anthropic, and Google Gemini integrations -- **Zod**: Runtime type validation -- **YAML**: Configuration and test specifications - -## Documentation - -For complete documentation, examples, and CLI usage, see the [agentv](https://www.npmjs.com/package/agentv) package. - -## Repository - -[https://github.com/EntityProcess/agentv](https://github.com/EntityProcess/agentv) - -## License - -MIT License - see [LICENSE](../../LICENSE) for details. +CLI, Dashboard, tests, and private workspace packages may import `@agentv/core` +directly. Public docs and examples should use `agentv`, `agentv/config`, or +`agentv/provider` unless they are explicitly describing repository internals. diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 89c4b48ed..967e7368c 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -1,305 +1,15 @@ # @agentv/sdk -Public lightweight SDK for AgentV - run evaluations programmatically, build YAML-aligned eval suites, and write custom assertions, script graders, and prompt templates around the canonical AgentV eval model. +Internal workspace package for AgentV TypeScript SDK helper implementation. -## Installation - -```bash -npm install @agentv/sdk -``` - -## Migrating from `@agentv/eval` - -Use `@agentv/sdk` for new code: - -```bash -npm uninstall @agentv/eval -npm install @agentv/sdk -``` - -```typescript -import { defineScriptGrader } from '@agentv/sdk'; -``` - -`@agentv/eval` was a temporary deprecated compatibility package for this SDK. It is no longer published from this repository. Use `@agentv/sdk` directly. - -## Quick Start - -### evaluate (programmatic runs) - -```typescript -import { evaluate } from '@agentv/sdk'; - -const { results, summary } = await evaluate({ - prompts: ['{{ input }}'], - tests: [ - { - id: 'greeting', - vars: { input: 'Say hello' }, - expectedOutput: 'Hello there!', - assert: [{ type: 'contains', value: 'Hello' }], - }, - ], - task: async (input) => `Hello from: ${input}`, -}); - -console.log(`${summary.passed}/${summary.total} passed`); -``` - -Use `specFile` when you want library control around an existing YAML suite: - -```typescript -import { evaluate } from '@agentv/sdk'; - -const { summary } = await evaluate({ - specFile: './evals/my-eval.eval.yaml', -}); -``` - -The `evaluate()` implementation is owned by `@agentv/core`; `@agentv/sdk` re-exports it as the user-facing SDK entrypoint. - -### defineAssertion (simplest way) - -```typescript -#!/usr/bin/env bun -import { defineAssertion } from '@agentv/sdk'; - -export default defineAssertion(({ output }) => ({ - pass: (output ?? '').toLowerCase().includes('hello'), - score: (output ?? '').toLowerCase().includes('hello') ? 1 : 0, - reason: 'Checks for greeting', -})); -``` - -Checks support `pass: boolean` for simple checks and `score: number` (0-1) for granular scoring. -`checks` is an SDK/script convenience shape; public `grading.json` artifacts -normalize checks into recursive `component_results` with `pass`, `score`, and -`reason`. - -### defineScriptGrader (full control) - -```typescript -#!/usr/bin/env bun -import { defineScriptGrader } from '@agentv/sdk'; - -export default defineScriptGrader(({ output, traceSummary }) => ({ - pass: (output ?? '').length > 0 && traceSummary !== null, - score: (output ?? '').length > 0 ? 1.0 : 0.0, - reason: 'Checks output presence and trace availability', - checks: [ - { - text: 'Output received', - pass: (output ?? '').length > 0, - reason: (output ?? '').length > 0 ? 'Output is non-empty' : 'Output is empty', - }, - { - text: 'Trace summary available', - pass: traceSummary !== null, - reason: traceSummary !== null ? 'Trace summary is present' : 'Trace summary is missing', - }, - ], -})); -``` - -Both functions handle stdin/stdout parsing, snake_case conversion, Zod validation, and error handling automatically. Use `defineAssertion()` for reusable assertion types discovered from `.agentv/assertions/`; use `defineScriptGrader()` for command-backed graders referenced with `type: script` and `command:`. - -### Vitest workspace verifiers (preferred deterministic workspace checks) - -Use normal Vitest tests when deterministic workspace checks can be expressed with `expect(...)`: - -```typescript -// graders/welcome-banner.test.ts -import { readFileSync } from 'node:fs'; -import { expect, it } from 'vitest'; - -it('links to the dashboard', () => { - const page = readFileSync('app/page.tsx', 'utf8'); - expect(page).toMatch(/href=["']\/dashboard["']/); -}); -``` - -Then reference the verifier directly from eval YAML through AgentV's built-in script-grader adapter: - -```yaml -assert: - - metric: vitest-welcome-banner - type: script - command: [agentv, eval, graders/welcome-banner.test.ts] -``` - -The command reads the normal script-grader stdin payload, runs Vitest in `workspace_path`, maps each Vitest test to an AgentV check, and computes score as `passed / total`. - -Use the explicit `agentv eval vitest` subcommand when you need adapter options such as `--cwd`, `--in-workspace`, or `--vitest-command`. Use `defineVitestWorkspaceGrader` when embedding this adapter in a custom script: - -```typescript -#!/usr/bin/env bun -import { defineVitestWorkspaceGrader } from '@agentv/sdk'; - -export default defineVitestWorkspaceGrader({ - testFile: 'graders/welcome-banner.test.ts', - copyTestFilesToWorkspace: true, -}); -``` - -### defineWorkspaceGrader (small file checks) - -Use `defineWorkspaceGrader` when a deterministic grader needs to inspect files in the evaluated workspace: - -```typescript -#!/usr/bin/env bun -import { defineWorkspaceGrader } from '@agentv/sdk'; - -export default defineWorkspaceGrader(async ({ workspace }) => [ - await workspace.file('app/page.tsx').contains('Status: All systems ready'), - await workspace.file('app/page.tsx').contains('Open dashboard'), - await workspace.file('app/page.tsx').matches(/href=["']\/dashboard["']/), - await workspace.file('app/page.tsx').notMatches(/TODO/i), -]); -``` - -The helper resolves `workspace_path` or `AGENTV_WORKSPACE_PATH`, reads files relative to the workspace, returns AgentV check objects, and computes `score` as passed checks divided by total checks. Prefer Vitest verifiers for checks that naturally fit a test file; use this lower-level helper for tiny one-off graders or custom score shaping. - -### TypeScript eval config authoring +This package is private and is not published to npm. User-facing TypeScript +imports are exposed by the public `agentv` package: ```typescript -// evals/greeting.eval.ts -import { graders, type EvalConfig } from '@agentv/sdk'; - -const config: EvalConfig = { - name: 'hello-suite', - providers: [ - { id: 'mock', label: 'mock-sdk', config: { response: 'Hello from the mock provider' } }, - { id: 'openai:gpt-5-mini', label: 'grader-provider' }, - ], - defaults: { - provider: 'mock-sdk', - grader: 'grader-provider', - }, - defaultTest: { - options: { - provider: 'grader-provider', - }, - }, - prompts: ['{{ input }}'], - tests: [ - { - id: 'hello', - vars: { input: 'Say hello' }, - expectedOutput: 'Hello from the mock provider', - options: { - provider: 'grader-provider', - }, - assert: [graders.contains('Hello')], - }, - ], -}; - -export default config; +import { evaluate, defineScriptGrader, defineAssertion, defineEval, graders } from 'agentv'; +import { defineConfig } from 'agentv/config'; +import type { Provider } from 'agentv/provider'; ``` -AgentV loads explicit `*.eval.ts` and `*.eval.mts` files through the same core loader used for YAML evals. The supported TypeScript contract is a default-exported `EvalConfig`. `defineEval(config)` is available as a thin optional helper over the same shape; plain typed default exports are the recommended path. - -TypeScript eval configs use the same provider surface as YAML: author systems under test and reusable grader providers in top-level `providers`, use `id` for the backend/spec, use `label` as the stable AgentV identity, and select defaults with `defaults.provider` and `defaults.grader`. Per-test grader provider selection belongs in `defaultTest.options.provider`, `tests[].options.provider`, or assertion-level `provider`. - -### Grader helpers - -Use the `graders` catalog when you want TypeScript helpers for common AgentV grader configs without creating a new eval vocabulary: - -```typescript -import { graders, type EvalConfig } from '@agentv/sdk'; - -const config: EvalConfig = { - name: 'grader-helper-suite', - prompts: ['{{ input }}'], - tests: [ - { - id: 'json-greeting', - vars: { input: 'Return a JSON greeting.' }, - assert: [ - graders.contains('Hello', { metric: 'mentions-hello' }), - graders.regex(/"message"\s*:/, { metric: 'message-key' }), - graders.json({ metric: 'valid-json', required: true }), - graders.llmRubric(['Greets the user'], { metric: 'rubric-review' }), - graders.llmRubric(undefined, { - metric: 'llm-review', - prompt: 'Grade whether the answer is useful.', - provider: 'grader-provider', - }), - graders.scriptGrader(['bun', 'run', 'graders/check.ts'], { metric: 'scripted-check' }), - ], - }, - ], -}; - -export default config; -``` - -The helpers return ordinary `assert` entries such as `type: contains`, `type: llm-rubric`, and `type: script`. Use the shared `transform` option for assertion-level output shaping. CamelCase SDK options such as `minScore` and `maxSteps` lower to canonical YAML keys such as `min_score` and `max_steps`. - -If you are coming from Braintrust `scores` or DeepEval metrics, model reusable checks as small AgentV-native helper factories that return these grader configs. They still lower to the same YAML/runtime contract: - -```typescript -import { graders, type EvalConfig } from '@agentv/sdk'; - -function ragFaithfulness() { - return graders.llmRubric(undefined, { - metric: 'rag-faithfulness', - provider: 'grader-provider', - prompt: 'Grade whether the answer is supported by the provided context.', - }); -} - -const config: EvalConfig = { - name: 'rag-suite', - prompts: ['{{ input }}'], - tests: [ - { - id: 'grounded-answer', - vars: { input: 'Answer using the retrieved context.' }, - assert: [ragFaithfulness()], - }, - ], -}; - -export default config; -``` - -Python workflows should emit canonical YAML/JSONL or implement script graders over the stdin/stdout contract. The repo-local helper under `examples/features/sdk-python/` is an example, not a promised published Python package. - -## Exports - -- `evaluate(config)` - Run evaluations programmatically from inline tests or an eval spec file -- `defineAssertion(handler)` - Define a custom assertion type (pass/fail + optional score) -- `defineScriptGrader(handler)` - Define a script grader (command-backed full score control) -- `defineVitestWorkspaceGrader(options)` - Embed the Vitest workspace verifier adapter in a custom script -- `defineWorkspaceGrader(handler)` - Define a workspace-aware script grader with file assertion helpers -- `definePromptTemplate(handler)` - Define a dynamic prompt template -- `defineEval(config)` - Optional helper for a default-exported TypeScript `EvalConfig` -- `graders` - Catalog of built-in AgentV grader config helpers -- `containsGrader`, `equalsGrader`, `exactGrader`, `regexGrader`, `isJsonGrader`, `jsonGrader`, `llmRubricGrader`, `scriptGrader` - Named grader helper functions -- `toEvalYamlObject(definition)` / `serializeEvalYaml(definition)` - Lower or serialize canonical eval YAML -- `EvalConfig` - TypeScript eval config authoring type -- `EvalRunResult`, `EvalSummary`, `EvalTestInput`, `EvalAssertionInput` - Programmatic evaluation types -- `AssertionContext`, `AssertionScore` - Assertion types -- `ScriptGraderInput`, `ScriptGraderResult`, `Workspace`, `WorkspaceAssertion` - Grader types -- `TraceSummary`, `Message`, `ToolCall` - Trace data types -- `createTargetClient()` - Runtime target proxy for script graders that explicitly opt into target access; this is not eval authoring syntax. -- `z` - Re-exported Zod for custom config schemas - -## Documentation - -For complete documentation including: -- Full grader request/result schemas -- Typed config examples -- Execution metrics usage -- Best practices - -See the docs site guides under `apps/web/src/content/docs/docs/next/graders/` or run `agentv skills get agentv-eval-writer`. - -## Repository - -[https://github.com/EntityProcess/agentv](https://github.com/EntityProcess/agentv) - -## License - -MIT License - see [LICENSE](../../LICENSE) for details. +Keep CLI and Dashboard internals importing private workspace layers directly. +Do not make internal code depend on the public `agentv` facade. From 4c722406110b116bb20e6f05020eda6398f7c376 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 15:39:16 +0200 Subject: [PATCH 3/6] docs(package): record sdk surface decision --- docs/plans/sdk-package-surface-audit.md | 115 ++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 docs/plans/sdk-package-surface-audit.md diff --git a/docs/plans/sdk-package-surface-audit.md b/docs/plans/sdk-package-surface-audit.md new file mode 100644 index 000000000..f6b947ed9 --- /dev/null +++ b/docs/plans/sdk-package-surface-audit.md @@ -0,0 +1,115 @@ +# SDK Package Surface Audit + +Date: 2026-07-07 +Bead: `av-97sv` + +## Recommendation + +`@agentv/sdk` should not remain a separate canonical authoring package. Public TypeScript authoring and programmatic APIs belong on the main `agentv` package facade, following Promptfoo's single obvious public package pattern where practical. + +Do not fold the SDK surface directly into `@agentv/core` as the primary user-facing import. `@agentv/core` should stay the engine and contract workspace layer; adding executable grader wrappers, prompt template wrappers, workspace helper ergonomics, and facade docs there would make the core/package boundary less honest. Because there are no production users to preserve, prefer a breaking cleanup now: make `@agentv/core` private unless a specific lower-level contract needs to stay public, and expose that contract through the `agentv` facade or explicit `agentv/*` subpaths. + +The settled target shape is: + +- `agentv`: default install and user-facing facade for CLI plus Node/TypeScript APIs such as `evaluate`, `defineScriptGrader`, `defineAssertion`, `definePromptTemplate`, `defineEval`, `graders`, and typed config helpers. +- private internal `@agentv/core` workspace package: lower-level runtime, evaluation engine, schemas/contracts, provider interfaces, artifact readers/writers, and implementation APIs used by CLI, Dashboard, and tests. +- no canonical public `@agentv/sdk` authoring package. At most, keep it as temporary compatibility re-exports during migration; first-party docs/examples/scaffolds should import from `agentv`. + +After operator clarification on 2026-07-07, this is no longer an open decision. Branch `av-97sv-sdk-surface` implements the package-surface direction in commits `5e7d733d` and `f7e4b7b3`; this report records the rationale and remaining cleanup. + +## Promptfoo Comparison + +Promptfoo currently publishes one primary package, `promptfoo`, that is both CLI and Node API facade. Its `package.json` exposes the root package as `dist/src/index.js`, exposes CLI binaries `promptfoo` and `pf`, and has only one explicit subpath export, `./contracts` (`/home/entity/projects/promptfoo/promptfoo@origin/main:package.json`, commit `786b2bd`, lines 12-28 and 51-54). + +Promptfoo's public facade exports `evaluate`, provider loading, assertions, cache, guardrails, redteam, and types from `src/index.ts` rather than from a separate SDK package (`src/index.ts`, lines 1-24 and 63-73). Its Node API docs teach imports from `promptfoo`, not `@promptfoo/sdk` (`site/docs/usage/node-api-quick-reference.md`, lines 12-39; `site/docs/usage/node-api-reference.md`, lines 27-69). + +Promptfoo is also not pretending the single package is architecturally clean. Its architecture docs explicitly say it still publishes one package while modeling future private layers, with `src/index.ts` as the public compatibility facade and `src/contracts` as the first leaf-safe surface (`docs/architecture/packages.md`, lines 1-23 and 39-60). Its multi-package proposal recommends keeping `promptfoo` as the default full install, creating private workspace packages first, and publishing only boundaries that prove useful (`docs/plans/2026-05-02-multi-package-system-proposal.md`, lines 3-23 and 57-65). + +Pattern to copy: one obvious default package/facade, narrow leaf contracts, private boundaries before public promises, and helper names that match the product's ergonomic vocabulary. AgentV should preserve `defineAssertion()` and `defineScriptGrader()` while aligning surrounding import/package shape with Promptfoo. + +Pattern to avoid: importing through the public facade from internal modules. Promptfoo explicitly bans internal imports of `src/index.ts` because that hides cycles and makes package extraction harder (`docs/architecture/packages.md`, lines 25-37). + +Important interpretation: Promptfoo's single published package is not a reason to collapse AgentV core, Dashboard, and CLI source into one internal package. It is evidence for a single obvious user-facing install/facade. Promptfoo's own docs point the other way internally: keep a facade while modeling private `contracts`, `core`, `node`, `cli`, and `app` layers. For AgentV, that means the Dashboard can remain a private workspace app bundled into the `agentv` npm tarball, while `@agentv/core` remains the shared non-CLI layer used by the CLI server, Dashboard client read models, and lower-level integrations. + +## AgentV Findings + +AgentV currently has three published package surfaces: `agentv`, `@agentv/core`, and `@agentv/sdk` (`.agents/publish.md`, lines 26-30). The CLI package is described as the CLI entry point and has only a `bin` entry today, while SDK docs make `@agentv/sdk` the user-facing package for `evaluate`, typed eval authoring, custom assertions, script graders, prompt templates, target-client helpers, and Zod (`apps/cli/package.json`, lines 14-17; `packages/sdk/README.md`, lines 1-4 and 269-287). + +That makes `@agentv/sdk` broader than a small grader helper package. Its root export re-exports core `evaluate`, defines eval authoring helpers, grader config factories, target-client helpers, workspace/Vitest helpers, prompt template and assertion wrappers, Zod schemas, and many trace/content types (`packages/sdk/src/index.ts`, lines 66-248 and 268-410). Public docs reinforce the split by telling users to install `@agentv/sdk` for authoring and `@agentv/core` for config (`apps/web/src/content/docs/docs/next/evaluation/sdk.mdx`, lines 9-24 and 432-450). + +`@agentv/core` is not just an SDK dependency. The Dashboard depends on it today. The dashboard app declares `@agentv/core` as a workspace dependency (`apps/dashboard/package.json`, lines 12-19), imports core trace/session read-model functions and types in the browser bundle (`apps/dashboard/src/lib/trace-read-model.ts`, lines 1-10; `apps/dashboard/src/lib/types.ts`, lines 1-14), and the Dashboard Hono server inside the CLI imports core for run/result/project/trace operations (`apps/cli/src/commands/results/serve.ts`, lines 51-73). That is a strong reason to keep a non-CLI core workspace layer. It is not a reason to publish `@agentv/core` publicly, and the current branch marks `@agentv/core` private while exposing deliberate user-facing contracts through `agentv/*` subpaths. + +The split is also leaky: + +- The CLI imports SDK runtime helpers for the built-in `agentv eval vitest` adapter (`apps/cli/src/commands/eval/commands/vitest.ts`, lines 1-4 and 44-62). That makes CLI command code depend on the public SDK facade. +- Core knows about SDK branding symbols when loading TypeScript eval files (`packages/core/src/evaluation/loaders/ts-eval-loader.ts`, lines 18-19 and 110-116). That is a hidden package contract. +- SDK duplicates TypeScript eval lowering keys that core also owns (`packages/sdk/src/eval.ts`, lines 3-45 and 280-320; `packages/core/src/evaluation/loaders/ts-eval-loader.ts`, lines 18-53). +- SDK `AssertionType` duplicates core grader kind vocabulary and carries compatibility/deprecation entries separately from `GRADER_KIND_VALUES` (`packages/sdk/src/assertion.ts`, lines 38-74; `packages/core/src/evaluation/types.ts`, lines 179-216). +- SDK grader helper types duplicate a subset of core `GraderConfig` shapes, but with camelCase authoring names that lower to snake_case later (`packages/sdk/src/graders.ts`, lines 14-138 and 140-269; `packages/core/src/evaluation/types.ts`, lines 415-498 and 693-988). +- SDK trace/message schemas partly parallel core trace/content contracts (`packages/sdk/src/schemas.ts`, lines 28-47 and 248-360; `packages/core/src/evaluation/types.ts`, lines 1-21). +- `createTargetClient()` keeps target vocabulary in the SDK despite the product boundary moving to provider terminology; the file itself documents that it is historical runtime vocabulary, not eval authoring syntax (`packages/sdk/src/target-client.ts`, lines 1-13 and 101-162). + +There is real build and maintenance cost. Before the branch cleanup, the root build ran core, SDK, dashboard, and CLI; SDK built core first; CLI built SDK first, which built core again (`package.json`, lines 8-17; `packages/sdk/package.json`, lines 23-32; `apps/cli/package.json`, lines 18-28). A lightweight Bun inspection showed SDK contained 11 source files and about 94 KB of TypeScript source, with 3 runtime dependencies, while `@agentv/core` and `agentv` each carried their own dependency/optional dependency surfaces. The current branch removes the recursive package-local build chain and publishes only the `agentv` package. + +## Rationale + +Folding the public helpers into `agentv` matches the installed product name and the Promptfoo precedent: users install one package for the CLI and can import the same package for programmatic usage. This also lets docs say "install `agentv`" instead of choosing among `agentv`, `@agentv/sdk`, and `@agentv/core`. + +Keeping implementation in lower layers still matters. The `agentv` facade should not become the internal import path. Internals should import direct core/adapter modules, and the facade should only re-export stable user-facing APIs. This preserves the Promptfoo lesson: public facade outside, private layers inside. + +Folding directly into the core layer would reduce package count but worsens the core boundary. SDK helpers are mostly executable authoring ergonomics: stdin/stdout wrappers, assertion/script runtime wrappers, Vitest adapter wrapping, prompt template execution, workspace convenience APIs, and Zod re-exports. Those are not pure core primitives. + +Publishing `@agentv/core` as a broad public package is also questionable. The current root barrel exposes many implementation-heavy modules and even a stub `createAgentKernel()` (`packages/core/src/index.ts`, lines 1-268). A Promptfoo-aligned package strategy would make `agentv` the public facade, keep core as a private workspace layer, and expose only deliberate lower-level contracts through stable facade subpaths such as `agentv/contracts`, `agentv/provider`, or `agentv/core` if those names are accepted. + +Keeping `@agentv/sdk` forever is also not ideal. It forces users to learn a second first-party package, creates a CLI -> SDK -> core chain for CLI-owned behavior, and encourages a parallel type/schema layer that must stay in sync with core. + +## Risks and Migration Notes + +- Install weight: importing helpers from `agentv` means a local project dependency on the CLI package. This mirrors Promptfoo, but it may be heavier than the current SDK package for users who only want script helper imports. Mitigation: provide leaf subpath exports, e.g. `agentv/sdk` and possibly `agentv/contracts`, that do not import CLI command modules. +- Compatibility: the operator confirmed there are no production users, so this should be a breaking change. First-party docs/examples/scaffolds should move directly to the new `agentv` surface instead of carrying long-lived `@agentv/sdk` or `@agentv/core` aliases. +- Internal cycles: do not let core or CLI internals import from `agentv` facade. Add a package graph test similar to the existing core/sdk package graph test (`packages/sdk/test/package-graph.test.ts`, lines 29-62). +- Type ownership: choose a single source for eval authoring keys, grader kind values, trace/message schemas, and boundary Zod contracts before changing imports. +- Docs cleanup: README still shows stale `target` authoring in a TypeScript SDK example even though current guidance rejects target/targets in favor of providers (`README.md`, lines 245-253; promptfoo parity docs, lines 68-69 and 87). + +## Simplification Opportunities + +### Reuse + +- Replace SDK-local eval key lowering with a shared core contract/lowering utility. Today SDK and core both carry known camelCase-to-snake_case maps. +- Derive SDK `AssertionType` from core grader kind values, or export a shared assertion-kind contract, rather than maintaining a second string union. +- Reuse core grader config contracts for TypeScript helper return types where possible; keep camelCase authoring options as thin helper input types only. +- Move the Vitest adapter implementation to CLI/core internal code and let SDK expose only a wrapper if needed. The CLI should not import the public SDK facade for its own command implementation. +- Create a small contracts subpath for trace/message/script-grader schemas if multiple packages need them, matching Promptfoo's `./contracts` pattern. + +### Quality + +- Make `agentv` the documented primary programmatic facade, and make `@agentv/core` explicitly lower-level. This removes the current docs split where `evaluate()` is owned by core but taught through SDK. +- Keep `@agentv/core` private unless a specific lower-level contract earns a public subpath. Expose such contracts through `agentv/*` subpaths rather than broadening the root facade. +- Rename or isolate `createTargetClient()` vocabulary. The public helper should not keep legacy target wording while provider terminology is canonical. +- Reduce root `@agentv/core` barrel sprawl. `packages/core/src/index.ts` exports implementation-heavy artifact, registry, provider, project, and runtime helpers from one root barrel, plus a stub `createAgentKernel()` (lines 1-268). Prefer explicit subpath exports for non-facade internals. +- Remove stale package migration framing from durable docs once the new package decision lands. Current SDK docs still foreground `@agentv/eval` migration history (`packages/sdk/README.md`, lines 11-24; SDK docs, lines 26-39). + +### Efficiency + +- Fix recursive build scripts so root/CLI builds do not rebuild core multiple times through SDK. Let the root build orchestrate package order, and keep package-local builds package-local where possible. +- If `agentv` gains SDK exports, make those subpath exports leaf-safe so importing `defineScriptGrader` does not load command registration, update checks, dashboard server code, or optional provider SDKs. +- Consolidate example package dependencies after the package decision. At least 16 example package manifests depend on `@agentv/sdk`, some also depend on `@agentv/core`, and 11 example lockfiles exist. One documented local dependency pattern would reduce churn. +- Avoid schema/type generation duplication by deciding whether script-grader schemas live in the contracts layer or are generated from core boundary schemas. + +## Follow-Up Work + +Follow-up Beads created or updated: + +1. `av-qxxr`: expose the `agentv` authoring facade and make `@agentv/sdk` compatibility-only/private, not canonical. +2. `av-b6u5`: make `@agentv/core` private and expose only deliberate lower-level contracts through `agentv/*` subpaths. +3. `av-3c0o`: remove the CLI's dependency on `@agentv/sdk` for `agentv eval vitest` and simplify recursive build scripts. +4. `av-zi7k`: update public docs, examples, scaffolds, and package READMEs to teach `agentv` imports and provider terminology. +5. `av-krn3`: consolidate SDK/core eval authoring, grader kind, and script-grader schema contracts behind a single source of truth. + +## Validation + +Initial audit validation was research-only: `git fetch origin`, `git status --short --branch`, local file reads, DeepWiki orientation for `promptfoo/promptfoo`, `git -C /home/entity/projects/promptfoo/promptfoo fetch origin`, `git show origin/main:` for Promptfoo evidence, and one `bun -e` manifest/source-size inspection. + +Implementation validation on branch `av-97sv-sdk-surface`: `git diff --check`; `bun --filter @agentv/core typecheck`; `bun --filter @agentv/sdk typecheck`; `bun --filter agentv typecheck`; `bun --filter agentv build`; `bun run build`; focused Bun tests for TypeScript eval loader, SDK package graph/evaluate/Vitest, CLI package exports/create assertion/eval Vitest; and Biome checks on changed TypeScript/docs-support files. + +Known validation gap: `bun run validate:examples` fails on unchanged pre-existing `examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml` object-shaped `evaluate_options.repeat`; `origin/main` has the same issue. No live eval dogfood was run because the branch changes package/import/docs plumbing rather than eval execution or grader behavior. From 31c54cfc4d017c0b3f158947833cbd04c851c60d Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 15:45:48 +0200 Subject: [PATCH 4/6] fix(ci): build workspace packages before typecheck --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a7b64f7c5..b778a49ef 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "scripts": { "build": "bun --filter @agentv/core build && bun --filter @agentv/sdk build && bun --filter @agentv/dashboard build && bun --filter agentv build", "verify": "bun run build && bun run typecheck && bun run lint && bun run test", - "typecheck": "bun --filter @agentv/core typecheck && bun --filter @agentv/sdk typecheck && bun --filter agentv typecheck", + "typecheck": "bun --filter @agentv/core build && bun --filter @agentv/sdk build && bun --filter @agentv/core typecheck && bun --filter @agentv/sdk typecheck && bun --filter agentv typecheck", "typecheck:workspace": "tsc -b tsconfig.build.json", "typecheck:watch": "bun --filter @agentv/core typecheck -- --watch & bun --filter agentv typecheck -- --watch", "lint": "biome check .", From d7f8561d90513a727f910d24eec4df1372101ad3 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 16:10:01 +0200 Subject: [PATCH 5/6] fix(ci): build packages before root tests --- apps/cli/test/unit/preprocess-argv.test.ts | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/cli/test/unit/preprocess-argv.test.ts b/apps/cli/test/unit/preprocess-argv.test.ts index 71ee3d55f..239364584 100644 --- a/apps/cli/test/unit/preprocess-argv.test.ts +++ b/apps/cli/test/unit/preprocess-argv.test.ts @@ -5,7 +5,7 @@ import { preprocessArgv, shouldRunBeforeSessionHook, usesDeprecatedStudioAlias, -} from '../../src/index.js'; +} from '../../src/cli-app.js'; describe('preprocessArgv', () => { describe('--eval-id convenience alias', () => { diff --git a/package.json b/package.json index b778a49ef..c8c771998 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "lint": "biome check .", "format": "biome format --write .", "fix": "biome check --write .", - "test": "bun --filter @agentv/core test && bun --filter @agentv/sdk test && bun --filter agentv test && bun --filter @agentv/dashboard test", + "test": "bun --filter @agentv/core build && bun --filter @agentv/sdk build && bun --filter @agentv/core test && bun --filter @agentv/sdk test && bun --filter agentv test && bun --filter @agentv/dashboard test", "test:watch": "bun --filter @agentv/core test:watch & bun --filter agentv test:watch", "agentv": "bun apps/cli/src/cli.ts", "agentv:buildrun": "bun run build && bun apps/cli/dist/cli.js", From 047bd82f355e6a0cf005abd6f33b88e60f4e15ec Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 16:28:37 +0200 Subject: [PATCH 6/6] fix(ci): build agentv before root tests --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c8c771998..ba2b946e2 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "lint": "biome check .", "format": "biome format --write .", "fix": "biome check --write .", - "test": "bun --filter @agentv/core build && bun --filter @agentv/sdk build && bun --filter @agentv/core test && bun --filter @agentv/sdk test && bun --filter agentv test && bun --filter @agentv/dashboard test", + "test": "bun --filter @agentv/core build && bun --filter @agentv/sdk build && bun --filter agentv build && bun --filter @agentv/core test && bun --filter @agentv/sdk test && bun --filter agentv test && bun --filter @agentv/dashboard test", "test:watch": "bun --filter @agentv/core test:watch & bun --filter agentv test:watch", "agentv": "bun apps/cli/src/cli.ts", "agentv:buildrun": "bun run build && bun apps/cli/dist/cli.js",