diff --git a/src/core/eval.tsx b/src/core/eval.tsx index ea59625cd..aa8323191 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -58,14 +58,19 @@ import { type UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import { + DeleteRecommendationCommand, EvaluateCommand, GetABTestCommand, ListABTestsCommand, UpdateABTestCommand, DeleteABTestCommand, GetBatchEvaluationCommand, + GetRecommendationCommand, ListBatchEvaluationsCommand, + ListRecommendationsCommand, StartBatchEvaluationCommand, + StartRecommendationCommand, + type DeleteRecommendationResponse, type EvaluationReferenceInput, type EvaluationResultContent, type EvaluationTarget, @@ -75,8 +80,12 @@ import { type ABTestExecutionStatus, type UpdateABTestResponse, type DeleteABTestResponse, + type GetRecommendationResponse, type ListBatchEvaluationsResponse, + type ListRecommendationsResponse, + type RecommendationStatus, type StartBatchEvaluationResponse, + type StartRecommendationResponse, type DataSourceConfig as DataPlaneDataSourceConfig, type CloudWatchFilterConfig, } from "@aws-sdk/client-bedrock-agentcore"; @@ -133,6 +142,7 @@ import type { SpanRecord, StartBatchInsightsInput, StartBatchEvaluationInput, + StartRecommendationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; @@ -342,6 +352,39 @@ export class EvalClient implements CoreEvalClient { .send(new DeleteEvaluatorCommand({ evaluatorId: id })); } + async startRecommendation( + input: StartRecommendationInput, + options: CoreOptions, + ): Promise { + return this.clients.data(toClientConfig(options)).send(new StartRecommendationCommand(input)); + } + + async getRecommendation(id: string, options: CoreOptions): Promise { + return this.clients + .data(toClientConfig(options)) + .send(new GetRecommendationCommand({ recommendationId: id })); + } + + async listRecommendations( + nextToken: string | undefined, + maxResults: number | undefined, + statusFilter: RecommendationStatus | undefined, + options: CoreOptions, + ): Promise { + return this.clients + .data(toClientConfig(options)) + .send(new ListRecommendationsCommand({ nextToken, maxResults, statusFilter })); + } + + async deleteRecommendation( + id: string, + options: CoreOptions, + ): Promise { + return this.clients + .data(toClientConfig(options)) + .send(new DeleteRecommendationCommand({ recommendationId: id })); + } + // getBatchEvaluation returns the service-side job (status + evaluator summaries // + CloudWatch output config) and, by default, the per-session results read from // the job's CloudWatch stream once it is terminal. Batch evaluation lives on the diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index c28aaa503..0c610ed32 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -12,6 +12,7 @@ import { createBatchInsightsHandler } from "./batch-insights"; import { createOnDemandHandler } from "./ondemand"; import { createConfigBundleHandler } from "./config-bundle"; import { createAbTestHandler } from "./ab-test"; +import { createRecommendationHandler } from "./recommendation"; export function createEvalHandler(core: Core, io: AppIO): Router { return new Router("eval", "evaluate and optimize AgentCore agents") @@ -25,7 +26,8 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .handler(createBatchInsightsHandler(core, io)) .handler(createOnDemandHandler(core, io)) .handler(createConfigBundleHandler(core, io)) - .handler(createAbTestHandler(core, io)); + .handler(createAbTestHandler(core, io)) + .handler(createRecommendationHandler(core, io)); } export { EvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/recommendation/__fixtures__/DeleteRecommendationCommand.117a4cc7f54d3397.json b/src/handlers/eval/recommendation/__fixtures__/DeleteRecommendationCommand.117a4cc7f54d3397.json new file mode 100644 index 000000000..e8cddf557 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/DeleteRecommendationCommand.117a4cc7f54d3397.json @@ -0,0 +1,4 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/GetRecommendationCommand.117a4cc7f54d3397.json b/src/handlers/eval/recommendation/__fixtures__/GetRecommendationCommand.117a4cc7f54d3397.json new file mode 100644 index 000000000..78499ac27 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/GetRecommendationCommand.117a4cc7f54d3397.json @@ -0,0 +1,87 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E", + "name": "agentcore_cli_recommendation_fixture", + "type": "SYSTEM_PROMPT_RECOMMENDATION", + "recommendationConfig": { + "systemPromptRecommendationConfig": { + "systemPrompt": { + "text": "You are a concise support assistant. Answer the user's question directly." + }, + "agentTraces": { + "sessionSpans": [ + { + "traceId": "0123456789abcdef0123456789abcdef", + "endTimeUnixNano": 1750000001000000000, + "resource": { + "attributes": { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent" + } + }, + "kind": "INTERNAL", + "flags": 256, + "durationNano": 1000000000, + "startTimeUnixNano": 1750000000000000000, + "body": { + "output": { + "messages": [ + { + "content": [ + { + "text": "Two plus two is four." + } + ], + "role": "assistant" + } + ] + }, + "input": { + "messages": [ + { + "content": [ + { + "text": "What is two plus two?" + } + ], + "role": "user" + } + ] + } + }, + "spanId": "0123456789abcdef", + "scope": { + "name": "agentcore-cli-fixture" + }, + "name": "invoke_agent AgentCore CLI fixture", + "attributes": { + "gen_ai.agent.name": "AgentCore CLI fixture", + "aws.genai.span_kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.system": "strands-agents" + }, + "status": { + "code": "OK" + } + } + ] + }, + "evaluationConfig": { + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness" + } + ] + } + } + }, + "status": "IN_PROGRESS", + "createdAt": { + "$date": "2026-08-26T14:53:00.323Z" + }, + "updatedAt": { + "$date": "2026-08-26T14:53:00.933Z" + }, + "description": "Golden recommendation fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/ListRecommendationsCommand.ec1869545cd08c49.json b/src/handlers/eval/recommendation/__fixtures__/ListRecommendationsCommand.ec1869545cd08c49.json new file mode 100644 index 000000000..fa5f4d49b --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/ListRecommendationsCommand.ec1869545cd08c49.json @@ -0,0 +1,3 @@ +{ + "recommendationSummaries": [] +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/StartRecommendationCommand.d33be02c79c64275.json b/src/handlers/eval/recommendation/__fixtures__/StartRecommendationCommand.d33be02c79c64275.json new file mode 100644 index 000000000..5273fad79 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/StartRecommendationCommand.d33be02c79c64275.json @@ -0,0 +1,87 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E", + "name": "agentcore_cli_recommendation_fixture", + "type": "SYSTEM_PROMPT_RECOMMENDATION", + "recommendationConfig": { + "systemPromptRecommendationConfig": { + "systemPrompt": { + "text": "You are a concise support assistant. Answer the user's question directly." + }, + "agentTraces": { + "sessionSpans": [ + { + "traceId": "0123456789abcdef0123456789abcdef", + "endTimeUnixNano": 1750000001000000000, + "resource": { + "attributes": { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent" + } + }, + "kind": "INTERNAL", + "flags": 256, + "durationNano": 1000000000, + "startTimeUnixNano": 1750000000000000000, + "body": { + "output": { + "messages": [ + { + "content": [ + { + "text": "Two plus two is four." + } + ], + "role": "assistant" + } + ] + }, + "input": { + "messages": [ + { + "content": [ + { + "text": "What is two plus two?" + } + ], + "role": "user" + } + ] + } + }, + "spanId": "0123456789abcdef", + "scope": { + "name": "agentcore-cli-fixture" + }, + "name": "invoke_agent AgentCore CLI fixture", + "attributes": { + "gen_ai.agent.name": "AgentCore CLI fixture", + "aws.genai.span_kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.system": "strands-agents" + }, + "status": { + "code": "OK" + } + } + ] + }, + "evaluationConfig": { + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness" + } + ] + } + } + }, + "status": "PENDING", + "createdAt": { + "$date": "2026-08-26T14:53:00.323Z" + }, + "updatedAt": { + "$date": "2026-08-26T14:53:00.323Z" + }, + "description": "Golden recommendation fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/delete.golden.json b/src/handlers/eval/recommendation/__fixtures__/delete.golden.json new file mode 100644 index 000000000..e8cddf557 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/delete.golden.json @@ -0,0 +1,4 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/get.golden.json b/src/handlers/eval/recommendation/__fixtures__/get.golden.json new file mode 100644 index 000000000..26577d77b --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/get.golden.json @@ -0,0 +1,83 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E", + "name": "agentcore_cli_recommendation_fixture", + "type": "SYSTEM_PROMPT_RECOMMENDATION", + "recommendationConfig": { + "systemPromptRecommendationConfig": { + "systemPrompt": { + "text": "You are a concise support assistant. Answer the user's question directly." + }, + "agentTraces": { + "sessionSpans": [ + { + "traceId": "0123456789abcdef0123456789abcdef", + "endTimeUnixNano": 1750000001000000000, + "resource": { + "attributes": { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent" + } + }, + "kind": "INTERNAL", + "flags": 256, + "durationNano": 1000000000, + "startTimeUnixNano": 1750000000000000000, + "body": { + "output": { + "messages": [ + { + "content": [ + { + "text": "Two plus two is four." + } + ], + "role": "assistant" + } + ] + }, + "input": { + "messages": [ + { + "content": [ + { + "text": "What is two plus two?" + } + ], + "role": "user" + } + ] + } + }, + "spanId": "0123456789abcdef", + "scope": { + "name": "agentcore-cli-fixture" + }, + "name": "invoke_agent AgentCore CLI fixture", + "attributes": { + "gen_ai.agent.name": "AgentCore CLI fixture", + "aws.genai.span_kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.system": "strands-agents" + }, + "status": { + "code": "OK" + } + } + ] + }, + "evaluationConfig": { + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness" + } + ] + } + } + }, + "status": "IN_PROGRESS", + "createdAt": "2026-08-26T14:53:00.323Z", + "updatedAt": "2026-08-26T14:53:00.933Z", + "description": "Golden recommendation fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/list.golden.json b/src/handlers/eval/recommendation/__fixtures__/list.golden.json new file mode 100644 index 000000000..fa5f4d49b --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/list.golden.json @@ -0,0 +1,3 @@ +{ + "recommendationSummaries": [] +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/start.golden.json b/src/handlers/eval/recommendation/__fixtures__/start.golden.json new file mode 100644 index 000000000..66b4dc097 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/start.golden.json @@ -0,0 +1,83 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E", + "name": "agentcore_cli_recommendation_fixture", + "type": "SYSTEM_PROMPT_RECOMMENDATION", + "recommendationConfig": { + "systemPromptRecommendationConfig": { + "systemPrompt": { + "text": "You are a concise support assistant. Answer the user's question directly." + }, + "agentTraces": { + "sessionSpans": [ + { + "traceId": "0123456789abcdef0123456789abcdef", + "endTimeUnixNano": 1750000001000000000, + "resource": { + "attributes": { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent" + } + }, + "kind": "INTERNAL", + "flags": 256, + "durationNano": 1000000000, + "startTimeUnixNano": 1750000000000000000, + "body": { + "output": { + "messages": [ + { + "content": [ + { + "text": "Two plus two is four." + } + ], + "role": "assistant" + } + ] + }, + "input": { + "messages": [ + { + "content": [ + { + "text": "What is two plus two?" + } + ], + "role": "user" + } + ] + } + }, + "spanId": "0123456789abcdef", + "scope": { + "name": "agentcore-cli-fixture" + }, + "name": "invoke_agent AgentCore CLI fixture", + "attributes": { + "gen_ai.agent.name": "AgentCore CLI fixture", + "aws.genai.span_kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.system": "strands-agents" + }, + "status": { + "code": "OK" + } + } + ] + }, + "evaluationConfig": { + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness" + } + ] + } + } + }, + "status": "PENDING", + "createdAt": "2026-08-26T14:53:00.323Z", + "updatedAt": "2026-08-26T14:53:00.323Z", + "description": "Golden recommendation fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/delete/index.tsx b/src/handlers/eval/recommendation/delete/index.tsx new file mode 100644 index 000000000..2eedde534 --- /dev/null +++ b/src/handlers/eval/recommendation/delete/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createDeleteRecommendationHandler = (core: Core) => + createHandler({ + name: "delete", + description: "delete a recommendation by id", + flags: [flag("id", "the ID of the recommendation to delete", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.deleteRecommendation(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/recommendation/get/index.tsx b/src/handlers/eval/recommendation/get/index.tsx new file mode 100644 index 000000000..af1d2a622 --- /dev/null +++ b/src/handlers/eval/recommendation/get/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetRecommendationHandler = (core: Core) => + createHandler({ + name: "get", + description: "get a recommendation by id", + flags: [flag("id", "the ID of the recommendation", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.getRecommendation(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/recommendation/index.tsx b/src/handlers/eval/recommendation/index.tsx new file mode 100644 index 000000000..917664075 --- /dev/null +++ b/src/handlers/eval/recommendation/index.tsx @@ -0,0 +1,16 @@ +import type { AppIO } from "../../../io"; +import { Router } from "../../../router"; +import type { Core } from "../../types"; +import { createDeleteRecommendationHandler } from "./delete"; +import { createGetRecommendationHandler } from "./get"; +import { createListRecommendationsHandler } from "./list"; +import { createStartRecommendationHandler } from "./start"; + +export function createRecommendationHandler(core: Core, io: AppIO): Router { + return new Router("recommendation", "manage AgentCore recommendations") + .supportedTuiCommands() + .handler(createStartRecommendationHandler(core, io)) + .handler(createGetRecommendationHandler(core)) + .handler(createListRecommendationsHandler(core)) + .handler(createDeleteRecommendationHandler(core)); +} diff --git a/src/handlers/eval/recommendation/list/index.tsx b/src/handlers/eval/recommendation/list/index.tsx new file mode 100644 index 000000000..3f2a86386 --- /dev/null +++ b/src/handlers/eval/recommendation/list/index.tsx @@ -0,0 +1,37 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +const RECOMMENDATION_STATUSES = [ + "PENDING", + "IN_PROGRESS", + "COMPLETED", + "FAILED", + "DELETING", +] as const; + +export const createListRecommendationsHandler = (core: Core) => + createHandler({ + name: "list", + description: "list recommendations", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + flag( + "status-filter", + `return only recommendations with this status (${RECOMMENDATION_STATUSES.join(" | ")})`, + z.enum(RECOMMENDATION_STATUSES).optional(), + ), + ], + handle: async (ctx, flags) => { + const response = await core.eval.listRecommendations( + flags["next-token"], + flags["max-results"], + flags["status-filter"], + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/recommendation/recommendation.fixture.test.tsx b/src/handlers/eval/recommendation/recommendation.fixture.test.tsx new file mode 100644 index 000000000..746c5fe95 --- /dev/null +++ b/src/handlers/eval/recommendation/recommendation.fixture.test.tsx @@ -0,0 +1,289 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { + DeleteRecommendationCommand, + GetRecommendationCommand, + ListRecommendationsCommand, + type BedrockAgentCoreClient, + type RecommendationConfig, + type RecommendationStatus, +} from "@aws-sdk/client-bedrock-agentcore"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { createDataClient } from "../../../core/factories"; +import { + createSilentLogger, + fixtureFactories, + isRecording, + matchGolden, + settle, + TestGlobalConfigAccessor, + testIO, + waitFor, + WaitForTimeoutError, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); +const RECOMMENDATION_NAME = "agentcore_cli_recommendation_fixture"; +const RECORDING_TIMEOUT_MS = 10 * 60_000; +const POLL_INTERVAL_MS = 5_000; +const CONFIG: RecommendationConfig = { + systemPromptRecommendationConfig: { + systemPrompt: { + text: "You are a concise support assistant. Answer the user's question directly.", + }, + agentTraces: { + sessionSpans: [ + { + resource: { + attributes: { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent", + }, + }, + traceId: "0123456789abcdef0123456789abcdef", + spanId: "0123456789abcdef", + flags: 256, + name: "invoke_agent AgentCore CLI fixture", + kind: "INTERNAL", + startTimeUnixNano: 1_750_000_000_000_000_000, + endTimeUnixNano: 1_750_000_001_000_000_000, + durationNano: 1_000_000_000, + scope: { name: "agentcore-cli-fixture" }, + attributes: { + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.operation.name": "invoke_agent", + "gen_ai.agent.name": "AgentCore CLI fixture", + "gen_ai.system": "strands-agents", + "aws.genai.span_kind": "AGENT", + }, + status: { code: "OK" }, + body: { + input: { + messages: [{ role: "user", content: [{ text: "What is two plus two?" }] }], + }, + output: { + messages: [{ role: "assistant", content: [{ text: "Two plus two is four." }] }], + }, + }, + }, + ], + }, + evaluationConfig: { + evaluators: [ + { + evaluatorArn: "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + }, + ], + }, + }, +}; + +// Record with: +// RECORD=1 bun test src/handlers/eval/recommendation/recommendation.fixture.test.tsx +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + }); +} + +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route(["bun", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +let recommendationId: string | undefined; +let deleted = false; + +function requireRecommendationId(): string { + if (!recommendationId) { + throw new Error("start fixture did not return a recommendation id"); + } + return recommendationId; +} + +function isNotFound(error: unknown): boolean { + return (error as Error).name === "ResourceNotFoundException"; +} + +async function waitForTerminal( + client: BedrockAgentCoreClient, + id: string, +): Promise { + let status: RecommendationStatus | undefined; + try { + await waitFor( + async () => { + status = (await client.send(new GetRecommendationCommand({ recommendationId: id }))).status; + return status === "COMPLETED" || status === "FAILED" || status === "DELETING"; + }, + RECORDING_TIMEOUT_MS, + POLL_INTERVAL_MS, + ); + } catch (error) { + if (!(error instanceof WaitForTimeoutError)) throw error; + throw new Error(`timed out waiting for recommendation ${id} to reach a terminal state`, { + cause: error, + }); + } + return status; +} + +async function waitUntilDeleted(client: BedrockAgentCoreClient, id: string): Promise { + try { + await waitFor( + async () => { + try { + await client.send(new GetRecommendationCommand({ recommendationId: id })); + return false; + } catch (error) { + if (isNotFound(error)) return true; + throw error; + } + }, + RECORDING_TIMEOUT_MS, + POLL_INTERVAL_MS, + ); + } catch (error) { + if (!(error instanceof WaitForTimeoutError)) throw error; + throw new Error(`timed out waiting for recommendation ${id} to be deleted`, { cause: error }); + } +} + +async function cleanupRecommendation(client: BedrockAgentCoreClient, id: string): Promise { + let status: RecommendationStatus | undefined; + try { + status = (await client.send(new GetRecommendationCommand({ recommendationId: id }))).status; + } catch (error) { + if (isNotFound(error)) return; + throw error; + } + + if (status === "PENDING" || status === "IN_PROGRESS") { + status = await waitForTerminal(client, id); + } + if (status !== "DELETING") { + await client.send(new DeleteRecommendationCommand({ recommendationId: id })); + } + await waitUntilDeleted(client, id); +} + +beforeAll(async () => { + if (!isRecording()) return; + + const client = createDataClient({ region: REGION }); + const ids: string[] = []; + let nextToken: string | undefined; + do { + const page = await client.send(new ListRecommendationsCommand({ maxResults: 100, nextToken })); + ids.push( + ...(page.recommendationSummaries ?? []) + .filter((summary) => summary.name === RECOMMENDATION_NAME) + .flatMap((summary) => (summary.recommendationId ? [summary.recommendationId] : [])), + ); + nextToken = page.nextToken; + } while (nextToken); + + for (const id of ids) await cleanupRecommendation(client, id); +}, RECORDING_TIMEOUT_MS); + +afterAll(async () => { + if (!isRecording() || !recommendationId || deleted) return; + + try { + await cleanupRecommendation(createDataClient({ region: REGION }), recommendationId); + } catch (error) { + if (!isNotFound(error)) { + console.error(`could not clean up fixture recommendation ${recommendationId}:`, error); + } + } +}, RECORDING_TIMEOUT_MS); + +describe("eval recommendation against recorded responses", () => { + test("starts a recommendation", async () => { + const stdout = await run([ + "eval", + "recommendation", + "start", + "--name", + RECOMMENDATION_NAME, + "--description", + "Golden recommendation fixture", + "--type", + "SYSTEM_PROMPT_RECOMMENDATION", + "--recommendation-config", + JSON.stringify(CONFIG), + "--tags", + "suite=golden", + ]); + + matchGolden(FIXTURES, "start.golden.json", stdout); + const response = JSON.parse(stdout); + recommendationId = response.recommendationId; + expect(recommendationId).toBeString(); + expect(response.name).toBe(RECOMMENDATION_NAME); + expect(response.type).toBe("SYSTEM_PROMPT_RECOMMENDATION"); + }); + + test("gets the recommendation", async () => { + await settle(2_000); + const id = requireRecommendationId(); + + const stdout = await run(["eval", "recommendation", "get", "--id", id]); + + matchGolden(FIXTURES, "get.golden.json", stdout); + const response = JSON.parse(stdout); + expect(response.recommendationId).toBe(id); + expect(response.name).toBe(RECOMMENDATION_NAME); + expect(response.recommendationConfig).toEqual(CONFIG); + }, 60_000); + + test("lists recommendations with API pagination and filtering", async () => { + const stdout = await run([ + "eval", + "recommendation", + "list", + "--max-results", + "5", + "--status-filter", + "COMPLETED", + ]); + + matchGolden(FIXTURES, "list.golden.json", stdout); + expect(JSON.parse(stdout).recommendationSummaries).toBeArray(); + }); + + test( + "deletes the recommendation", + async () => { + const id = requireRecommendationId(); + if (isRecording()) { + await waitForTerminal(createDataClient({ region: REGION }), id); + } + + const stdout = await run(["eval", "recommendation", "delete", "--id", id]); + + matchGolden(FIXTURES, "delete.golden.json", stdout); + expect(JSON.parse(stdout)).toMatchObject({ + recommendationId: id, + status: "DELETING", + }); + deleted = true; + }, + RECORDING_TIMEOUT_MS, + ); +}); diff --git a/src/handlers/eval/recommendation/start/index.tsx b/src/handlers/eval/recommendation/start/index.tsx new file mode 100644 index 000000000..d69a5b5b7 --- /dev/null +++ b/src/handlers/eval/recommendation/start/index.tsx @@ -0,0 +1,80 @@ +import type { RecommendationConfig } from "@aws-sdk/client-bedrock-agentcore"; +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { SourceResolver, type AppIO } from "../../../../io"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag, parseTags } from "../../../utils"; + +const RECOMMENDATION_TYPES = [ + "SYSTEM_PROMPT_RECOMMENDATION", + "TOOL_DESCRIPTION_RECOMMENDATION", +] as const; + +const REQUIRED_FLAGS = ["name", "type", "recommendation-config"] as const; +type RequiredFlag = (typeof REQUIRED_FLAGS)[number]; + +function assertRequiredFlags( + flags: Partial>, +): asserts flags is Record { + for (const name of REQUIRED_FLAGS) { + if (!flags[name]) { + throw new InputValidationError(`required option '--${name} <${name}>' not specified`); + } + } +} + +export const createStartRecommendationHandler = (core: Core, io: AppIO) => + createHandler({ + name: "start", + description: "start an asynchronous recommendation", + flags: [ + flag("name", "the name of the recommendation", z.string().optional()), + flag( + "type", + `the recommendation type (${RECOMMENDATION_TYPES.join(" | ")})`, + z.enum(RECOMMENDATION_TYPES).optional(), + ), + flag( + "recommendation-config", + "recommendation configuration (JSON inline, file://, or - for stdin)", + z.string().optional(), + { sensitive: true }, + ), + flag("description", "a description of the recommendation", z.string().optional()), + flag( + "kms-key-arn", + "customer managed KMS key ARN for recommendation data", + z.string().optional(), + ), + flag("tags", "tags as key=value (repeatable) or JSON object", z.array(z.string()).optional()), + ], + handle: async (ctx, flags) => { + assertRequiredFlags(flags); + + const source = new SourceResolver({ stdin: io.stdin }); + const recommendationConfig = parseJsonFlag( + "recommendation-config", + await source.resolveText("recommendation-config", flags["recommendation-config"]), + ); + if (!recommendationConfig) { + throw new InputValidationError( + "required option '--recommendation-config ' not specified", + ); + } + + const response = await core.eval.startRecommendation( + { + name: flags["name"], + type: flags["type"], + recommendationConfig, + description: flags["description"], + kmsKeyArn: flags["kms-key-arn"], + tags: parseTags(flags["tags"]), + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 252dac569..6bcb6447f 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -35,8 +35,15 @@ import type { ABTestExecutionStatus, UpdateABTestResponse, DeleteABTestResponse, + DeleteRecommendationResponse, GetBatchEvaluationResponse, + GetRecommendationResponse, + ListRecommendationsResponse, ListBatchEvaluationsResponse, + RecommendationConfig, + RecommendationStatus, + RecommendationType, + StartRecommendationResponse, StartBatchEvaluationResponse, SessionMetadataShape, InlineGroundTruth, @@ -225,6 +232,14 @@ export type RoleScopeWarning = { }; export type CreateDatasetInput = CreateDatasetRequest; +export type StartRecommendationInput = { + name: string; + description?: string; + type: RecommendationType; + recommendationConfig: RecommendationConfig; + kmsKeyArn?: string; + tags?: Record; +}; export type CreateConfigurationBundleInput = Pick< CreateConfigurationBundleRequest, "bundleName" | "components" | "branchName" | "commitMessage" | "kmsKeyArn" @@ -365,6 +380,19 @@ export interface CoreEvalClient { ): Promise; deleteEvaluator(id: string, options: CoreOptions): Promise; + startRecommendation( + input: StartRecommendationInput, + options: CoreOptions, + ): Promise; + getRecommendation(id: string, options: CoreOptions): Promise; + listRecommendations( + nextToken: string | undefined, + maxResults: number | undefined, + statusFilter: RecommendationStatus | undefined, + options: CoreOptions, + ): Promise; + deleteRecommendation(id: string, options: CoreOptions): Promise; + // getBatchEvaluation returns the service-side job and, unless `includeResults` // is false, the per-session results read from its per-job CloudWatch stream once // terminal. A CloudWatch read failure is returned as `resultsError` (never diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 5414837c2..ef5064c21 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -78,7 +78,9 @@ import type { ABTestExecutionStatus, UpdateABTestResponse, DeleteABTestResponse, + DeleteRecommendationResponse, GetBatchEvaluationResponse, + GetRecommendationResponse, GetEventInput, GetEventOutput, GetMemoryRecordInput, @@ -96,9 +98,12 @@ import type { ListEventsOutput, ListMemoryRecordsInput, ListMemoryRecordsOutput, + ListRecommendationsResponse, ListSessionsInput, ListSessionsOutput, + RecommendationStatus, StartBatchEvaluationResponse, + StartRecommendationResponse, } from "@aws-sdk/client-bedrock-agentcore"; import type { Core } from "../handlers/types"; import type { CoreHarnessClient, CreateHarnessInput } from "../handlers/harness/types"; @@ -148,6 +153,7 @@ import type { SessionTrace, StartBatchInsightsInput, StartBatchEvaluationInput, + StartRecommendationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; @@ -279,6 +285,12 @@ const DEFAULT_START_BATCH_EVAL_RESPONSE = { batchEvaluationId: "batch-eval-test", status: "RUNNING", } as unknown as StartBatchEvaluationResponse; +const DEFAULT_START_RECOMMENDATION_RESPONSE = {} as StartRecommendationResponse; +const DEFAULT_GET_RECOMMENDATION_RESPONSE = {} as GetRecommendationResponse; +const DEFAULT_LIST_RECOMMENDATIONS_RESPONSE: ListRecommendationsResponse = { + recommendationSummaries: [], +}; +const DEFAULT_DELETE_RECOMMENDATION_RESPONSE = {} as DeleteRecommendationResponse; const DEFAULT_UPDATE_DATASET_RESULT: DatasetUpdateResult = { datasetId: "dataset-orders-abc123", added: 0, @@ -1370,6 +1382,13 @@ export class TestEvalClient implements CoreEvalClient { private updateResponse: UpdateEvaluatorResponse = DEFAULT_UPDATE_EVALUATOR_RESPONSE; private getResponse: GetEvaluatorResponse = DEFAULT_GET_EVALUATOR_RESPONSE; private deleteResponse: DeleteEvaluatorResponse = DEFAULT_DELETE_EVALUATOR_RESPONSE; + private startRecommendationResponse: StartRecommendationResponse = + DEFAULT_START_RECOMMENDATION_RESPONSE; + private getRecommendationResponse: GetRecommendationResponse = + DEFAULT_GET_RECOMMENDATION_RESPONSE; + private recommendationListResponses = new Map(); + private deleteRecommendationResponse: DeleteRecommendationResponse = + DEFAULT_DELETE_RECOMMENDATION_RESPONSE; // Online-eval responses, keyed the same way: listOnlineEvaluationConfigs pages // by nextToken, the rest are single canned values. private onlineEvalListResponses = new Map< @@ -1467,6 +1486,29 @@ export class TestEvalClient implements CoreEvalClient { return this; } + setStartRecommendationResponse(response: StartRecommendationResponse): this { + this.startRecommendationResponse = response; + return this; + } + + setGetRecommendationResponse(response: GetRecommendationResponse): this { + this.getRecommendationResponse = response; + return this; + } + + setListRecommendationsResponse( + response: ListRecommendationsResponse, + forNextToken?: string, + ): this { + this.recommendationListResponses.set(forNextToken, response); + return this; + } + + setDeleteRecommendationResponse(response: DeleteRecommendationResponse): this { + this.deleteRecommendationResponse = response; + return this; + } + // setOnlineEvalListResponse sets what listOnlineEvaluationConfigs resolves to // (when not erroring). Pass `forNextToken` to serve a later page. setOnlineEvalListResponse( @@ -1710,6 +1752,48 @@ export class TestEvalClient implements CoreEvalClient { return this.deleteResponse; } + async startRecommendation( + input: StartRecommendationInput, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "startRecommendation", args: [input, options] }); + if (this.error) throw this.error; + return this.startRecommendationResponse; + } + + async getRecommendation(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "getRecommendation", args: [id, options] }); + if (this.error) throw this.error; + return this.getRecommendationResponse; + } + + async listRecommendations( + nextToken: string | undefined, + maxResults: number | undefined, + statusFilter: RecommendationStatus | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ + method: "listRecommendations", + args: [nextToken, maxResults, statusFilter, options], + }); + if (this.error) throw this.error; + return ( + this.recommendationListResponses.get(nextToken) ?? + this.recommendationListResponses.get(undefined) ?? + DEFAULT_LIST_RECOMMENDATIONS_RESPONSE + ); + } + + async deleteRecommendation( + id: string, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "deleteRecommendation", args: [id, options] }); + if (this.error) throw this.error; + return this.deleteRecommendationResponse; + } + async getBatchEvaluation( id: string, options: CoreOptions, diff --git a/src/testing/index.tsx b/src/testing/index.tsx index fc313b4f5..548d3fd16 100644 --- a/src/testing/index.tsx +++ b/src/testing/index.tsx @@ -1,7 +1,7 @@ export { parse, stringify } from "./serialization"; export { fixtureFactories, fixtureFetch, isRecording, matchGolden, settle } from "./fixtures"; export { testIO, type TestIO } from "./testIO"; -export { tick, waitFor } from "./timing"; +export { tick, waitFor, WaitForTimeoutError } from "./timing"; export { TestCoreClient, TestGatewayClient, diff --git a/src/testing/timing.tsx b/src/testing/timing.tsx index 2d7b62494..52f392575 100644 --- a/src/testing/timing.tsx +++ b/src/testing/timing.tsx @@ -2,6 +2,13 @@ export function tick(ms = 0): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +export class WaitForTimeoutError extends Error { + constructor() { + super("waitFor: condition not met before timeout"); + this.name = "WaitForTimeoutError"; + } +} + /** * Polls {@link predicate} until it returns true or the timeout elapses, * ticking between attempts so pending renders/queries flush. @@ -11,17 +18,18 @@ export function tick(ms = 0): Promise { * * @param predicate - Condition to wait for. * @param timeoutMs - Maximum time to wait in milliseconds (default 1000). + * @param intervalMs - Delay between attempts in milliseconds (default 5). * @throws If the predicate is not satisfied before the timeout. */ export async function waitFor( predicate: () => boolean | Promise, timeoutMs = 1000, + intervalMs = 5, ): Promise { - const step = 5; let waited = 0; while (!(await predicate())) { - if (waited >= timeoutMs) throw new Error("waitFor: condition not met before timeout"); - await tick(step); - waited += step; + if (waited >= timeoutMs) throw new WaitForTimeoutError(); + await tick(intervalMs); + waited += intervalMs; } }