From 40d9d041acc2f9d5392fd0ec053e1ceb670e01f4 Mon Sep 17 00:00:00 2001 From: Nicolas Borges Date: Wed, 26 Aug 2026 10:55:43 -0400 Subject: [PATCH 1/3] feat: support imperative eval recommendation command --- src/core/eval.tsx | 43 +++ src/core/recommendation.test.tsx | 119 +++++++ src/handlers/eval/index.tsx | 4 +- ...ecommendationCommand.117a4cc7f54d3397.json | 4 + ...ecommendationCommand.117a4cc7f54d3397.json | 87 +++++ ...commendationsCommand.ec1869545cd08c49.json | 3 + ...ecommendationCommand.d33be02c79c64275.json | 87 +++++ .../__fixtures__/delete.golden.json | 4 + .../__fixtures__/get.golden.json | 83 +++++ .../__fixtures__/list.golden.json | 3 + .../__fixtures__/start.golden.json | 83 +++++ .../eval/recommendation/delete/index.tsx | 20 ++ .../eval/recommendation/get/index.tsx | 20 ++ src/handlers/eval/recommendation/index.tsx | 16 + .../eval/recommendation/list/index.tsx | 37 ++ .../recommendation.fixture.test.tsx | 273 ++++++++++++++ .../recommendation/recommendation.test.tsx | 332 ++++++++++++++++++ .../eval/recommendation/start/index.tsx | 77 ++++ src/handlers/eval/types.tsx | 19 + src/testing/TestCoreClient.tsx | 84 +++++ 20 files changed, 1397 insertions(+), 1 deletion(-) create mode 100644 src/core/recommendation.test.tsx create mode 100644 src/handlers/eval/recommendation/__fixtures__/DeleteRecommendationCommand.117a4cc7f54d3397.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/GetRecommendationCommand.117a4cc7f54d3397.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/ListRecommendationsCommand.ec1869545cd08c49.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/StartRecommendationCommand.d33be02c79c64275.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/delete.golden.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/get.golden.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/list.golden.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/start.golden.json create mode 100644 src/handlers/eval/recommendation/delete/index.tsx create mode 100644 src/handlers/eval/recommendation/get/index.tsx create mode 100644 src/handlers/eval/recommendation/index.tsx create mode 100644 src/handlers/eval/recommendation/list/index.tsx create mode 100644 src/handlers/eval/recommendation/recommendation.fixture.test.tsx create mode 100644 src/handlers/eval/recommendation/recommendation.test.tsx create mode 100644 src/handlers/eval/recommendation/start/index.tsx diff --git a/src/core/eval.tsx b/src/core/eval.tsx index ea59625cd..5d10fb6e5 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -58,14 +58,19 @@ import { type UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import { + DeleteRecommendationCommand, EvaluateCommand, GetABTestCommand, ListABTestsCommand, UpdateABTestCommand, DeleteABTestCommand, GetBatchEvaluationCommand, + GetRecommendationCommand, ListBatchEvaluationsCommand, + ListRecommendationsCommand, StartBatchEvaluationCommand, + StartRecommendationCommand, + type DeleteRecommendationResponse, type EvaluationReferenceInput, type EvaluationResultContent, type EvaluationTarget, @@ -75,8 +80,13 @@ import { type ABTestExecutionStatus, type UpdateABTestResponse, type DeleteABTestResponse, + type GetRecommendationResponse, type ListBatchEvaluationsResponse, + type ListRecommendationsResponse, + type RecommendationStatus, type StartBatchEvaluationResponse, + type StartRecommendationRequest, + type StartRecommendationResponse, type DataSourceConfig as DataPlaneDataSourceConfig, type CloudWatchFilterConfig, } from "@aws-sdk/client-bedrock-agentcore"; @@ -342,6 +352,39 @@ export class EvalClient implements CoreEvalClient { .send(new DeleteEvaluatorCommand({ evaluatorId: id })); } + async startRecommendation( + request: StartRecommendationRequest, + options: CoreOptions, + ): Promise { + return this.clients.data(toClientConfig(options)).send(new StartRecommendationCommand(request)); + } + + async getRecommendation(id: string, options: CoreOptions): Promise { + return this.clients + .data(toClientConfig(options)) + .send(new GetRecommendationCommand({ recommendationId: id })); + } + + async listRecommendations( + nextToken: string | undefined, + maxResults: number | undefined, + statusFilter: RecommendationStatus | undefined, + options: CoreOptions, + ): Promise { + return this.clients + .data(toClientConfig(options)) + .send(new ListRecommendationsCommand({ nextToken, maxResults, statusFilter })); + } + + async deleteRecommendation( + id: string, + options: CoreOptions, + ): Promise { + return this.clients + .data(toClientConfig(options)) + .send(new DeleteRecommendationCommand({ recommendationId: id })); + } + // getBatchEvaluation returns the service-side job (status + evaluator summaries // + CloudWatch output config) and, by default, the per-session results read from // the job's CloudWatch stream once it is terminal. Batch evaluation lives on the diff --git a/src/core/recommendation.test.tsx b/src/core/recommendation.test.tsx new file mode 100644 index 000000000..e8499272e --- /dev/null +++ b/src/core/recommendation.test.tsx @@ -0,0 +1,119 @@ +import { describe, expect, test } from "bun:test"; +import { + DeleteRecommendationCommand, + GetRecommendationCommand, + ListRecommendationsCommand, + StartRecommendationCommand, + type DeleteRecommendationResponse, + type GetRecommendationResponse, + type ListRecommendationsResponse, + type RecommendationConfig, + type StartRecommendationRequest, + type StartRecommendationResponse, +} from "@aws-sdk/client-bedrock-agentcore"; +import type { AwsClients, ClientConfig } from "./types"; +import { EvalClient } from "./eval"; + +const options = { + region: "us-west-2", + endpointUrl: "https://agentcore.example.test", +}; +const recommendationConfig: RecommendationConfig = { + systemPromptRecommendationConfig: { + systemPrompt: { text: "You are a support agent." }, + agentTraces: { sessionSpans: [{ traceId: "trace-1", spanId: "span-1" }] }, + }, +}; + +function evalClient(send: (command: unknown) => Promise): { + client: EvalClient; + configs: ClientConfig[]; +} { + const configs: ClientConfig[] = []; + return { + client: new EvalClient({ + data: (config: ClientConfig) => { + configs.push(config); + return { send } as never; + }, + } as unknown as AwsClients), + configs, + }; +} + +describe("EvalClient recommendations", () => { + test("starts a recommendation with the request unchanged on the data plane", async () => { + const request: StartRecommendationRequest = { + name: "support-prompt", + description: "Improve the support prompt", + type: "SYSTEM_PROMPT_RECOMMENDATION", + recommendationConfig, + kmsKeyArn: "arn:aws:kms:us-west-2:123456789012:key/recommendations", + tags: { team: "eval" }, + }; + const response = { + recommendationId: "recommendation-1", + status: "PENDING", + } as StartRecommendationResponse; + const { client, configs } = evalClient(async (command) => { + expect(command).toBeInstanceOf(StartRecommendationCommand); + expect((command as StartRecommendationCommand).input).toEqual(request); + return response; + }); + + await expect(client.startRecommendation(request, options)).resolves.toBe(response); + expect(configs).toEqual([{ region: "us-west-2", endpoint: "https://agentcore.example.test" }]); + }); + + test("gets a recommendation by recommendationId", async () => { + const response = { + recommendationId: "recommendation-1", + status: "COMPLETED", + } as GetRecommendationResponse; + const { client } = evalClient(async (command) => { + expect(command).toBeInstanceOf(GetRecommendationCommand); + expect((command as GetRecommendationCommand).input).toEqual({ + recommendationId: "recommendation-1", + }); + return response; + }); + + await expect(client.getRecommendation("recommendation-1", options)).resolves.toBe(response); + }); + + test("lists recommendations with pagination and statusFilter", async () => { + const response = { + recommendationSummaries: [], + nextToken: "token-2", + } as ListRecommendationsResponse; + const { client } = evalClient(async (command) => { + expect(command).toBeInstanceOf(ListRecommendationsCommand); + expect((command as ListRecommendationsCommand).input).toEqual({ + nextToken: "token-1", + maxResults: 10, + statusFilter: "FAILED", + }); + return response; + }); + + await expect(client.listRecommendations("token-1", 10, "FAILED", options)).resolves.toBe( + response, + ); + }); + + test("deletes a recommendation by recommendationId", async () => { + const response = { + recommendationId: "recommendation-1", + status: "DELETING", + } as DeleteRecommendationResponse; + const { client } = evalClient(async (command) => { + expect(command).toBeInstanceOf(DeleteRecommendationCommand); + expect((command as DeleteRecommendationCommand).input).toEqual({ + recommendationId: "recommendation-1", + }); + return response; + }); + + await expect(client.deleteRecommendation("recommendation-1", options)).resolves.toBe(response); + }); +}); diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index c28aaa503..0c610ed32 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -12,6 +12,7 @@ import { createBatchInsightsHandler } from "./batch-insights"; import { createOnDemandHandler } from "./ondemand"; import { createConfigBundleHandler } from "./config-bundle"; import { createAbTestHandler } from "./ab-test"; +import { createRecommendationHandler } from "./recommendation"; export function createEvalHandler(core: Core, io: AppIO): Router { return new Router("eval", "evaluate and optimize AgentCore agents") @@ -25,7 +26,8 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .handler(createBatchInsightsHandler(core, io)) .handler(createOnDemandHandler(core, io)) .handler(createConfigBundleHandler(core, io)) - .handler(createAbTestHandler(core, io)); + .handler(createAbTestHandler(core, io)) + .handler(createRecommendationHandler(core, io)); } export { EvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/recommendation/__fixtures__/DeleteRecommendationCommand.117a4cc7f54d3397.json b/src/handlers/eval/recommendation/__fixtures__/DeleteRecommendationCommand.117a4cc7f54d3397.json new file mode 100644 index 000000000..e8cddf557 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/DeleteRecommendationCommand.117a4cc7f54d3397.json @@ -0,0 +1,4 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/GetRecommendationCommand.117a4cc7f54d3397.json b/src/handlers/eval/recommendation/__fixtures__/GetRecommendationCommand.117a4cc7f54d3397.json new file mode 100644 index 000000000..78499ac27 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/GetRecommendationCommand.117a4cc7f54d3397.json @@ -0,0 +1,87 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E", + "name": "agentcore_cli_recommendation_fixture", + "type": "SYSTEM_PROMPT_RECOMMENDATION", + "recommendationConfig": { + "systemPromptRecommendationConfig": { + "systemPrompt": { + "text": "You are a concise support assistant. Answer the user's question directly." + }, + "agentTraces": { + "sessionSpans": [ + { + "traceId": "0123456789abcdef0123456789abcdef", + "endTimeUnixNano": 1750000001000000000, + "resource": { + "attributes": { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent" + } + }, + "kind": "INTERNAL", + "flags": 256, + "durationNano": 1000000000, + "startTimeUnixNano": 1750000000000000000, + "body": { + "output": { + "messages": [ + { + "content": [ + { + "text": "Two plus two is four." + } + ], + "role": "assistant" + } + ] + }, + "input": { + "messages": [ + { + "content": [ + { + "text": "What is two plus two?" + } + ], + "role": "user" + } + ] + } + }, + "spanId": "0123456789abcdef", + "scope": { + "name": "agentcore-cli-fixture" + }, + "name": "invoke_agent AgentCore CLI fixture", + "attributes": { + "gen_ai.agent.name": "AgentCore CLI fixture", + "aws.genai.span_kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.system": "strands-agents" + }, + "status": { + "code": "OK" + } + } + ] + }, + "evaluationConfig": { + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness" + } + ] + } + } + }, + "status": "IN_PROGRESS", + "createdAt": { + "$date": "2026-08-26T14:53:00.323Z" + }, + "updatedAt": { + "$date": "2026-08-26T14:53:00.933Z" + }, + "description": "Golden recommendation fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/ListRecommendationsCommand.ec1869545cd08c49.json b/src/handlers/eval/recommendation/__fixtures__/ListRecommendationsCommand.ec1869545cd08c49.json new file mode 100644 index 000000000..fa5f4d49b --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/ListRecommendationsCommand.ec1869545cd08c49.json @@ -0,0 +1,3 @@ +{ + "recommendationSummaries": [] +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/StartRecommendationCommand.d33be02c79c64275.json b/src/handlers/eval/recommendation/__fixtures__/StartRecommendationCommand.d33be02c79c64275.json new file mode 100644 index 000000000..5273fad79 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/StartRecommendationCommand.d33be02c79c64275.json @@ -0,0 +1,87 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E", + "name": "agentcore_cli_recommendation_fixture", + "type": "SYSTEM_PROMPT_RECOMMENDATION", + "recommendationConfig": { + "systemPromptRecommendationConfig": { + "systemPrompt": { + "text": "You are a concise support assistant. Answer the user's question directly." + }, + "agentTraces": { + "sessionSpans": [ + { + "traceId": "0123456789abcdef0123456789abcdef", + "endTimeUnixNano": 1750000001000000000, + "resource": { + "attributes": { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent" + } + }, + "kind": "INTERNAL", + "flags": 256, + "durationNano": 1000000000, + "startTimeUnixNano": 1750000000000000000, + "body": { + "output": { + "messages": [ + { + "content": [ + { + "text": "Two plus two is four." + } + ], + "role": "assistant" + } + ] + }, + "input": { + "messages": [ + { + "content": [ + { + "text": "What is two plus two?" + } + ], + "role": "user" + } + ] + } + }, + "spanId": "0123456789abcdef", + "scope": { + "name": "agentcore-cli-fixture" + }, + "name": "invoke_agent AgentCore CLI fixture", + "attributes": { + "gen_ai.agent.name": "AgentCore CLI fixture", + "aws.genai.span_kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.system": "strands-agents" + }, + "status": { + "code": "OK" + } + } + ] + }, + "evaluationConfig": { + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness" + } + ] + } + } + }, + "status": "PENDING", + "createdAt": { + "$date": "2026-08-26T14:53:00.323Z" + }, + "updatedAt": { + "$date": "2026-08-26T14:53:00.323Z" + }, + "description": "Golden recommendation fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/delete.golden.json b/src/handlers/eval/recommendation/__fixtures__/delete.golden.json new file mode 100644 index 000000000..e8cddf557 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/delete.golden.json @@ -0,0 +1,4 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/get.golden.json b/src/handlers/eval/recommendation/__fixtures__/get.golden.json new file mode 100644 index 000000000..26577d77b --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/get.golden.json @@ -0,0 +1,83 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E", + "name": "agentcore_cli_recommendation_fixture", + "type": "SYSTEM_PROMPT_RECOMMENDATION", + "recommendationConfig": { + "systemPromptRecommendationConfig": { + "systemPrompt": { + "text": "You are a concise support assistant. Answer the user's question directly." + }, + "agentTraces": { + "sessionSpans": [ + { + "traceId": "0123456789abcdef0123456789abcdef", + "endTimeUnixNano": 1750000001000000000, + "resource": { + "attributes": { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent" + } + }, + "kind": "INTERNAL", + "flags": 256, + "durationNano": 1000000000, + "startTimeUnixNano": 1750000000000000000, + "body": { + "output": { + "messages": [ + { + "content": [ + { + "text": "Two plus two is four." + } + ], + "role": "assistant" + } + ] + }, + "input": { + "messages": [ + { + "content": [ + { + "text": "What is two plus two?" + } + ], + "role": "user" + } + ] + } + }, + "spanId": "0123456789abcdef", + "scope": { + "name": "agentcore-cli-fixture" + }, + "name": "invoke_agent AgentCore CLI fixture", + "attributes": { + "gen_ai.agent.name": "AgentCore CLI fixture", + "aws.genai.span_kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.system": "strands-agents" + }, + "status": { + "code": "OK" + } + } + ] + }, + "evaluationConfig": { + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness" + } + ] + } + } + }, + "status": "IN_PROGRESS", + "createdAt": "2026-08-26T14:53:00.323Z", + "updatedAt": "2026-08-26T14:53:00.933Z", + "description": "Golden recommendation fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/list.golden.json b/src/handlers/eval/recommendation/__fixtures__/list.golden.json new file mode 100644 index 000000000..fa5f4d49b --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/list.golden.json @@ -0,0 +1,3 @@ +{ + "recommendationSummaries": [] +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/start.golden.json b/src/handlers/eval/recommendation/__fixtures__/start.golden.json new file mode 100644 index 000000000..66b4dc097 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/start.golden.json @@ -0,0 +1,83 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E", + "name": "agentcore_cli_recommendation_fixture", + "type": "SYSTEM_PROMPT_RECOMMENDATION", + "recommendationConfig": { + "systemPromptRecommendationConfig": { + "systemPrompt": { + "text": "You are a concise support assistant. Answer the user's question directly." + }, + "agentTraces": { + "sessionSpans": [ + { + "traceId": "0123456789abcdef0123456789abcdef", + "endTimeUnixNano": 1750000001000000000, + "resource": { + "attributes": { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent" + } + }, + "kind": "INTERNAL", + "flags": 256, + "durationNano": 1000000000, + "startTimeUnixNano": 1750000000000000000, + "body": { + "output": { + "messages": [ + { + "content": [ + { + "text": "Two plus two is four." + } + ], + "role": "assistant" + } + ] + }, + "input": { + "messages": [ + { + "content": [ + { + "text": "What is two plus two?" + } + ], + "role": "user" + } + ] + } + }, + "spanId": "0123456789abcdef", + "scope": { + "name": "agentcore-cli-fixture" + }, + "name": "invoke_agent AgentCore CLI fixture", + "attributes": { + "gen_ai.agent.name": "AgentCore CLI fixture", + "aws.genai.span_kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.system": "strands-agents" + }, + "status": { + "code": "OK" + } + } + ] + }, + "evaluationConfig": { + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness" + } + ] + } + } + }, + "status": "PENDING", + "createdAt": "2026-08-26T14:53:00.323Z", + "updatedAt": "2026-08-26T14:53:00.323Z", + "description": "Golden recommendation fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/delete/index.tsx b/src/handlers/eval/recommendation/delete/index.tsx new file mode 100644 index 000000000..2eedde534 --- /dev/null +++ b/src/handlers/eval/recommendation/delete/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createDeleteRecommendationHandler = (core: Core) => + createHandler({ + name: "delete", + description: "delete a recommendation by id", + flags: [flag("id", "the ID of the recommendation to delete", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.deleteRecommendation(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/recommendation/get/index.tsx b/src/handlers/eval/recommendation/get/index.tsx new file mode 100644 index 000000000..af1d2a622 --- /dev/null +++ b/src/handlers/eval/recommendation/get/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetRecommendationHandler = (core: Core) => + createHandler({ + name: "get", + description: "get a recommendation by id", + flags: [flag("id", "the ID of the recommendation", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.getRecommendation(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/recommendation/index.tsx b/src/handlers/eval/recommendation/index.tsx new file mode 100644 index 000000000..917664075 --- /dev/null +++ b/src/handlers/eval/recommendation/index.tsx @@ -0,0 +1,16 @@ +import type { AppIO } from "../../../io"; +import { Router } from "../../../router"; +import type { Core } from "../../types"; +import { createDeleteRecommendationHandler } from "./delete"; +import { createGetRecommendationHandler } from "./get"; +import { createListRecommendationsHandler } from "./list"; +import { createStartRecommendationHandler } from "./start"; + +export function createRecommendationHandler(core: Core, io: AppIO): Router { + return new Router("recommendation", "manage AgentCore recommendations") + .supportedTuiCommands() + .handler(createStartRecommendationHandler(core, io)) + .handler(createGetRecommendationHandler(core)) + .handler(createListRecommendationsHandler(core)) + .handler(createDeleteRecommendationHandler(core)); +} diff --git a/src/handlers/eval/recommendation/list/index.tsx b/src/handlers/eval/recommendation/list/index.tsx new file mode 100644 index 000000000..3f2a86386 --- /dev/null +++ b/src/handlers/eval/recommendation/list/index.tsx @@ -0,0 +1,37 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +const RECOMMENDATION_STATUSES = [ + "PENDING", + "IN_PROGRESS", + "COMPLETED", + "FAILED", + "DELETING", +] as const; + +export const createListRecommendationsHandler = (core: Core) => + createHandler({ + name: "list", + description: "list recommendations", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + flag( + "status-filter", + `return only recommendations with this status (${RECOMMENDATION_STATUSES.join(" | ")})`, + z.enum(RECOMMENDATION_STATUSES).optional(), + ), + ], + handle: async (ctx, flags) => { + const response = await core.eval.listRecommendations( + flags["next-token"], + flags["max-results"], + flags["status-filter"], + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/recommendation/recommendation.fixture.test.tsx b/src/handlers/eval/recommendation/recommendation.fixture.test.tsx new file mode 100644 index 000000000..18e457912 --- /dev/null +++ b/src/handlers/eval/recommendation/recommendation.fixture.test.tsx @@ -0,0 +1,273 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { + DeleteRecommendationCommand, + GetRecommendationCommand, + ListRecommendationsCommand, + type BedrockAgentCoreClient, + type RecommendationConfig, + type RecommendationStatus, +} from "@aws-sdk/client-bedrock-agentcore"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { createDataClient } from "../../../core/factories"; +import { + createSilentLogger, + fixtureFactories, + isRecording, + matchGolden, + settle, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); +const RECOMMENDATION_NAME = "agentcore_cli_recommendation_fixture"; +const RECORDING_TIMEOUT_MS = 10 * 60_000; +const POLL_INTERVAL_MS = 5_000; +const CONFIG: RecommendationConfig = { + systemPromptRecommendationConfig: { + systemPrompt: { + text: "You are a concise support assistant. Answer the user's question directly.", + }, + agentTraces: { + sessionSpans: [ + { + resource: { + attributes: { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent", + }, + }, + traceId: "0123456789abcdef0123456789abcdef", + spanId: "0123456789abcdef", + flags: 256, + name: "invoke_agent AgentCore CLI fixture", + kind: "INTERNAL", + startTimeUnixNano: 1_750_000_000_000_000_000, + endTimeUnixNano: 1_750_000_001_000_000_000, + durationNano: 1_000_000_000, + scope: { name: "agentcore-cli-fixture" }, + attributes: { + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.operation.name": "invoke_agent", + "gen_ai.agent.name": "AgentCore CLI fixture", + "gen_ai.system": "strands-agents", + "aws.genai.span_kind": "AGENT", + }, + status: { code: "OK" }, + body: { + input: { + messages: [{ role: "user", content: [{ text: "What is two plus two?" }] }], + }, + output: { + messages: [{ role: "assistant", content: [{ text: "Two plus two is four." }] }], + }, + }, + }, + ], + }, + evaluationConfig: { + evaluators: [ + { + evaluatorArn: "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + }, + ], + }, + }, +}; + +// Record with: +// RECORD=1 bun test src/handlers/eval/recommendation/recommendation.fixture.test.tsx +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + }); +} + +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route(["bun", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +let recommendationId: string | undefined; +let deleted = false; + +function requireRecommendationId(): string { + if (!recommendationId) { + throw new Error("start fixture did not return a recommendation id"); + } + return recommendationId; +} + +function isNotFound(error: unknown): boolean { + return (error as Error).name === "ResourceNotFoundException"; +} + +async function waitForTerminal( + client: BedrockAgentCoreClient, + id: string, +): Promise { + const deadline = Date.now() + RECORDING_TIMEOUT_MS; + while (Date.now() < deadline) { + const response = await client.send(new GetRecommendationCommand({ recommendationId: id })); + if (response.status === "COMPLETED" || response.status === "FAILED") { + return response.status; + } + if (response.status === "DELETING") return response.status; + await Bun.sleep(POLL_INTERVAL_MS); + } + throw new Error(`timed out waiting for recommendation ${id} to reach a terminal state`); +} + +async function waitUntilDeleted(client: BedrockAgentCoreClient, id: string): Promise { + const deadline = Date.now() + RECORDING_TIMEOUT_MS; + while (Date.now() < deadline) { + try { + await client.send(new GetRecommendationCommand({ recommendationId: id })); + } catch (error) { + if (isNotFound(error)) return; + throw error; + } + await Bun.sleep(POLL_INTERVAL_MS); + } + throw new Error(`timed out waiting for recommendation ${id} to be deleted`); +} + +async function cleanupRecommendation(client: BedrockAgentCoreClient, id: string): Promise { + let status: RecommendationStatus | undefined; + try { + status = (await client.send(new GetRecommendationCommand({ recommendationId: id }))).status; + } catch (error) { + if (isNotFound(error)) return; + throw error; + } + + if (status === "PENDING" || status === "IN_PROGRESS") { + status = await waitForTerminal(client, id); + } + if (status !== "DELETING") { + await client.send(new DeleteRecommendationCommand({ recommendationId: id })); + } + await waitUntilDeleted(client, id); +} + +beforeAll(async () => { + if (!isRecording()) return; + + const client = createDataClient({ region: REGION }); + const ids: string[] = []; + let nextToken: string | undefined; + do { + const page = await client.send(new ListRecommendationsCommand({ maxResults: 100, nextToken })); + ids.push( + ...(page.recommendationSummaries ?? []) + .filter((summary) => summary.name === RECOMMENDATION_NAME) + .flatMap((summary) => (summary.recommendationId ? [summary.recommendationId] : [])), + ); + nextToken = page.nextToken; + } while (nextToken); + + for (const id of ids) await cleanupRecommendation(client, id); +}, RECORDING_TIMEOUT_MS); + +afterAll(async () => { + if (!isRecording() || !recommendationId || deleted) return; + + try { + await cleanupRecommendation(createDataClient({ region: REGION }), recommendationId); + } catch (error) { + if (!isNotFound(error)) { + console.error(`could not clean up fixture recommendation ${recommendationId}:`, error); + } + } +}, RECORDING_TIMEOUT_MS); + +describe("eval recommendation against recorded responses", () => { + test("starts a recommendation", async () => { + const stdout = await run([ + "eval", + "recommendation", + "start", + "--name", + RECOMMENDATION_NAME, + "--description", + "Golden recommendation fixture", + "--type", + "SYSTEM_PROMPT_RECOMMENDATION", + "--recommendation-config", + JSON.stringify(CONFIG), + "--tags", + "suite=golden", + ]); + + matchGolden(FIXTURES, "start.golden.json", stdout); + const response = JSON.parse(stdout); + recommendationId = response.recommendationId; + expect(recommendationId).toBeString(); + expect(response.name).toBe(RECOMMENDATION_NAME); + expect(response.type).toBe("SYSTEM_PROMPT_RECOMMENDATION"); + }); + + test("gets the recommendation", async () => { + await settle(2_000); + const id = requireRecommendationId(); + + const stdout = await run(["eval", "recommendation", "get", "--id", id]); + + matchGolden(FIXTURES, "get.golden.json", stdout); + const response = JSON.parse(stdout); + expect(response.recommendationId).toBe(id); + expect(response.name).toBe(RECOMMENDATION_NAME); + expect(response.recommendationConfig).toEqual(CONFIG); + }, 60_000); + + test("lists recommendations with API pagination and filtering", async () => { + const stdout = await run([ + "eval", + "recommendation", + "list", + "--max-results", + "5", + "--status-filter", + "COMPLETED", + ]); + + matchGolden(FIXTURES, "list.golden.json", stdout); + expect(JSON.parse(stdout).recommendationSummaries).toBeArray(); + }); + + test( + "deletes the recommendation", + async () => { + const id = requireRecommendationId(); + if (isRecording()) { + await waitForTerminal(createDataClient({ region: REGION }), id); + } + + const stdout = await run(["eval", "recommendation", "delete", "--id", id]); + + matchGolden(FIXTURES, "delete.golden.json", stdout); + expect(JSON.parse(stdout)).toMatchObject({ + recommendationId: id, + status: "DELETING", + }); + deleted = true; + }, + RECORDING_TIMEOUT_MS, + ); +}); diff --git a/src/handlers/eval/recommendation/recommendation.test.tsx b/src/handlers/eval/recommendation/recommendation.test.tsx new file mode 100644 index 000000000..26d8ca977 --- /dev/null +++ b/src/handlers/eval/recommendation/recommendation.test.tsx @@ -0,0 +1,332 @@ +import { describe, expect, test } from "bun:test"; +import type { + DeleteRecommendationResponse, + GetRecommendationResponse, + ListRecommendationsResponse, + RecommendationConfig, + StartRecommendationResponse, +} from "@aws-sdk/client-bedrock-agentcore"; +import { + createSilentLogger, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const ENDPOINT = "https://agentcore.example.test"; +const CONFIG: RecommendationConfig = { + systemPromptRecommendationConfig: { + systemPrompt: { text: "You are a support agent." }, + agentTraces: { + sessionSpans: [{ traceId: "trace-1", spanId: "span-1" }], + }, + evaluationConfig: { + evaluators: [ + { + evaluatorArn: + "arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/Builtin.Helpfulness", + }, + ], + }, + }, +}; + +function testRecommendationCommand(stdin?: string) { + const core = new TestCoreClient(); + const io = testIO({ stdin }); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + return { + core, + stdout: io.stdout, + route: (args: string[]) => root.route(["bun", "agentcore", ...args, "--region", REGION]), + root, + }; +} + +function callArgs(core: TestCoreClient, method: string): unknown[] { + const call = core.eval.calls.find((candidate) => candidate.method === method); + if (!call) throw new Error(`${method} was not called`); + return call.args; +} + +describe("eval recommendation command hierarchy", () => { + test("registers the imperative API commands and flags", () => { + const { root } = testRecommendationCommand(); + const recommendation = root + .children() + .find((child) => child.name() === "eval") + ?.children() + .find((child) => child.name() === "recommendation"); + + expect(recommendation?.children().map((child) => child.name())).toEqual([ + "start", + "get", + "list", + "delete", + ]); + + const start = recommendation?.children().find((child) => child.name() === "start"); + expect(start?.flags().map((candidate) => candidate.name)).toEqual([ + "name", + "type", + "recommendation-config", + "description", + "kms-key-arn", + "tags", + ]); + expect( + start?.flags().find((candidate) => candidate.name === "recommendation-config")?.sensitive, + ).toBe(true); + expect( + recommendation + ?.children() + .find((child) => child.name() === "list") + ?.flags() + .map((candidate) => candidate.name), + ).toEqual(["next-token", "max-results", "status-filter"]); + }); +}); + +describe("recommendation start", () => { + test("maps the complete API request, tags, Core options, and response", async () => { + const { core, stdout, route } = testRecommendationCommand(); + const response = { + recommendationId: "recommendation-1", + recommendationArn: "arn:recommendation-1", + name: "support-prompt", + type: "SYSTEM_PROMPT_RECOMMENDATION", + status: "PENDING", + createdAt: new Date("2026-08-26T12:00:00.000Z"), + updatedAt: new Date("2026-08-26T12:00:00.000Z"), + } as StartRecommendationResponse; + core.eval.setStartRecommendationResponse(response); + + await route([ + "eval", + "recommendation", + "start", + "--name", + "support-prompt", + "--type", + "SYSTEM_PROMPT_RECOMMENDATION", + "--recommendation-config", + JSON.stringify(CONFIG), + "--description", + "Improve the support prompt", + "--kms-key-arn", + "arn:aws:kms:us-west-2:123456789012:key/recommendations", + "--tags", + "environment=test", + "--tags", + "team=eval", + "--endpoint-url", + ENDPOINT, + ]); + + expect(callArgs(core, "startRecommendation")).toEqual([ + { + name: "support-prompt", + type: "SYSTEM_PROMPT_RECOMMENDATION", + recommendationConfig: CONFIG, + description: "Improve the support prompt", + kmsKeyArn: "arn:aws:kms:us-west-2:123456789012:key/recommendations", + tags: { environment: "test", team: "eval" }, + }, + { region: REGION, endpointUrl: ENDPOINT }, + ]); + expect(JSON.parse(stdout())).toEqual({ + ...response, + createdAt: "2026-08-26T12:00:00.000Z", + updatedAt: "2026-08-26T12:00:00.000Z", + }); + }); + + test("resolves recommendation config from stdin", async () => { + const { core, route } = testRecommendationCommand(JSON.stringify(CONFIG)); + + await route([ + "eval", + "recommendation", + "start", + "--name", + "support-prompt", + "--type", + "SYSTEM_PROMPT_RECOMMENDATION", + "--recommendation-config", + "-", + ]); + + expect(callArgs(core, "startRecommendation")[0]).toMatchObject({ + recommendationConfig: CONFIG, + }); + }); + + test("rejects malformed config JSON before calling Core", async () => { + const { core, route } = testRecommendationCommand(); + + await expect( + route([ + "eval", + "recommendation", + "start", + "--name", + "support-prompt", + "--type", + "SYSTEM_PROMPT_RECOMMENDATION", + "--recommendation-config", + "{not-json", + ]), + ).rejects.toThrow(/Invalid JSON for option '--recommendation-config'/); + expect(core.eval.calls).toHaveLength(0); + }); +}); + +describe("recommendation get, list, and delete", () => { + test("gets by id and renders the API response unchanged", async () => { + const { core, stdout, route } = testRecommendationCommand(); + const response = { + recommendationId: "recommendation-1", + name: "support-prompt", + type: "SYSTEM_PROMPT_RECOMMENDATION", + recommendationConfig: CONFIG, + status: "COMPLETED", + createdAt: new Date("2026-08-26T12:00:00.000Z"), + updatedAt: new Date("2026-08-26T12:05:00.000Z"), + } as GetRecommendationResponse; + core.eval.setGetRecommendationResponse(response); + + await route(["eval", "recommendation", "get", "--id", "recommendation-1"]); + + expect(callArgs(core, "getRecommendation")).toEqual([ + "recommendation-1", + { region: REGION, endpointUrl: undefined }, + ]); + expect(JSON.parse(stdout()).recommendationId).toBe("recommendation-1"); + expect(JSON.parse(stdout()).recommendationConfig).toEqual(CONFIG); + }); + + test("passes pagination and status filtering and renders the response", async () => { + const { core, stdout, route } = testRecommendationCommand(); + const response = { + recommendationSummaries: [ + { + recommendationId: "recommendation-2", + recommendationArn: "arn:recommendation-2", + name: "tool-descriptions", + type: "TOOL_DESCRIPTION_RECOMMENDATION", + status: "IN_PROGRESS", + createdAt: new Date("2026-08-26T13:00:00.000Z"), + updatedAt: new Date("2026-08-26T13:01:00.000Z"), + }, + ], + nextToken: "token-2", + } as ListRecommendationsResponse; + core.eval.setListRecommendationsResponse(response, "token-1"); + + await route([ + "eval", + "recommendation", + "list", + "--next-token", + "token-1", + "--max-results", + "5", + "--status-filter", + "IN_PROGRESS", + ]); + + expect(callArgs(core, "listRecommendations")).toEqual([ + "token-1", + 5, + "IN_PROGRESS", + { region: REGION, endpointUrl: undefined }, + ]); + expect(JSON.parse(stdout())).toMatchObject({ + recommendationSummaries: [{ recommendationId: "recommendation-2" }], + nextToken: "token-2", + }); + }); + + test("deletes by id and renders the API response", async () => { + const { core, stdout, route } = testRecommendationCommand(); + const response = { + recommendationId: "recommendation-1", + status: "DELETING", + } as DeleteRecommendationResponse; + core.eval.setDeleteRecommendationResponse(response); + + await route(["eval", "recommendation", "delete", "--id", "recommendation-1"]); + + expect(callArgs(core, "deleteRecommendation")).toEqual([ + "recommendation-1", + { region: REGION, endpointUrl: undefined }, + ]); + expect(JSON.parse(stdout())).toEqual(response); + }); +}); + +describe("recommendation flag validation", () => { + test.each([ + [["eval", "recommendation", "start"], /--name/], + [ + [ + "eval", + "recommendation", + "start", + "--name", + "support-prompt", + "--recommendation-config", + JSON.stringify(CONFIG), + ], + /--type/, + ], + [ + [ + "eval", + "recommendation", + "start", + "--name", + "support-prompt", + "--type", + "SYSTEM_PROMPT_RECOMMENDATION", + ], + /--recommendation-config/, + ], + [["eval", "recommendation", "get"], /--id/], + [["eval", "recommendation", "delete"], /--id/], + ] as const)("requires API-mandated flags for %p", async (args, expected) => { + const { core, route } = testRecommendationCommand(); + + await expect(route([...args])).rejects.toThrow(expected); + expect(core.eval.calls).toHaveLength(0); + }); + + test.each([ + [["--type", "FUTURE_RECOMMENDATION"], /--type/], + [["--status-filter", "CANCELLED"], /--status-filter/], + ] as const)("rejects unsupported enum input %p", async ([flag, value], expected) => { + const { core, route } = testRecommendationCommand(); + const base = + flag === "--type" + ? [ + "eval", + "recommendation", + "start", + "--name", + "support-prompt", + "--recommendation-config", + JSON.stringify(CONFIG), + ] + : ["eval", "recommendation", "list"]; + + await expect(route([...base, flag, value])).rejects.toThrow(expected); + expect(core.eval.calls).toHaveLength(0); + }); +}); diff --git a/src/handlers/eval/recommendation/start/index.tsx b/src/handlers/eval/recommendation/start/index.tsx new file mode 100644 index 000000000..820a8d184 --- /dev/null +++ b/src/handlers/eval/recommendation/start/index.tsx @@ -0,0 +1,77 @@ +import type { RecommendationConfig } from "@aws-sdk/client-bedrock-agentcore"; +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { SourceResolver, type AppIO } from "../../../../io"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag, parseTags } from "../../../utils"; + +const RECOMMENDATION_TYPES = [ + "SYSTEM_PROMPT_RECOMMENDATION", + "TOOL_DESCRIPTION_RECOMMENDATION", +] as const; + +export const createStartRecommendationHandler = (core: Core, io: AppIO) => + createHandler({ + name: "start", + description: "start an asynchronous recommendation", + flags: [ + flag("name", "the name of the recommendation", z.string().optional()), + flag( + "type", + `the recommendation type (${RECOMMENDATION_TYPES.join(" | ")})`, + z.enum(RECOMMENDATION_TYPES).optional(), + ), + flag( + "recommendation-config", + "recommendation configuration (JSON inline, file://, or - for stdin)", + z.string().optional(), + { sensitive: true }, + ), + flag("description", "a description of the recommendation", z.string().optional()), + flag( + "kms-key-arn", + "customer managed KMS key ARN for recommendation data", + z.string().optional(), + ), + flag("tags", "tags as key=value (repeatable) or JSON object", z.array(z.string()).optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) { + throw new InputValidationError("required option '--name ' not specified"); + } + if (!flags["type"]) { + throw new InputValidationError("required option '--type ' not specified"); + } + if (!flags["recommendation-config"]) { + throw new InputValidationError( + "required option '--recommendation-config ' not specified", + ); + } + + const source = new SourceResolver({ stdin: io.stdin }); + const recommendationConfig = parseJsonFlag( + "recommendation-config", + await source.resolveText("recommendation-config", flags["recommendation-config"]), + ); + if (!recommendationConfig) { + throw new InputValidationError( + "required option '--recommendation-config ' not specified", + ); + } + + const response = await core.eval.startRecommendation( + { + name: flags["name"], + type: flags["type"], + recommendationConfig, + description: flags["description"], + kmsKeyArn: flags["kms-key-arn"], + tags: parseTags(flags["tags"]), + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 252dac569..9022b70ae 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -35,8 +35,14 @@ import type { ABTestExecutionStatus, UpdateABTestResponse, DeleteABTestResponse, + DeleteRecommendationResponse, GetBatchEvaluationResponse, + GetRecommendationResponse, + ListRecommendationsResponse, ListBatchEvaluationsResponse, + RecommendationStatus, + StartRecommendationRequest, + StartRecommendationResponse, StartBatchEvaluationResponse, SessionMetadataShape, InlineGroundTruth, @@ -365,6 +371,19 @@ export interface CoreEvalClient { ): Promise; deleteEvaluator(id: string, options: CoreOptions): Promise; + startRecommendation( + request: StartRecommendationRequest, + options: CoreOptions, + ): Promise; + getRecommendation(id: string, options: CoreOptions): Promise; + listRecommendations( + nextToken: string | undefined, + maxResults: number | undefined, + statusFilter: RecommendationStatus | undefined, + options: CoreOptions, + ): Promise; + deleteRecommendation(id: string, options: CoreOptions): Promise; + // getBatchEvaluation returns the service-side job and, unless `includeResults` // is false, the per-session results read from its per-job CloudWatch stream once // terminal. A CloudWatch read failure is returned as `resultsError` (never diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 5414837c2..c9289c0f1 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -78,7 +78,9 @@ import type { ABTestExecutionStatus, UpdateABTestResponse, DeleteABTestResponse, + DeleteRecommendationResponse, GetBatchEvaluationResponse, + GetRecommendationResponse, GetEventInput, GetEventOutput, GetMemoryRecordInput, @@ -96,9 +98,13 @@ import type { ListEventsOutput, ListMemoryRecordsInput, ListMemoryRecordsOutput, + ListRecommendationsResponse, ListSessionsInput, ListSessionsOutput, + RecommendationStatus, StartBatchEvaluationResponse, + StartRecommendationRequest, + StartRecommendationResponse, } from "@aws-sdk/client-bedrock-agentcore"; import type { Core } from "../handlers/types"; import type { CoreHarnessClient, CreateHarnessInput } from "../handlers/harness/types"; @@ -279,6 +285,12 @@ const DEFAULT_START_BATCH_EVAL_RESPONSE = { batchEvaluationId: "batch-eval-test", status: "RUNNING", } as unknown as StartBatchEvaluationResponse; +const DEFAULT_START_RECOMMENDATION_RESPONSE = {} as StartRecommendationResponse; +const DEFAULT_GET_RECOMMENDATION_RESPONSE = {} as GetRecommendationResponse; +const DEFAULT_LIST_RECOMMENDATIONS_RESPONSE: ListRecommendationsResponse = { + recommendationSummaries: [], +}; +const DEFAULT_DELETE_RECOMMENDATION_RESPONSE = {} as DeleteRecommendationResponse; const DEFAULT_UPDATE_DATASET_RESULT: DatasetUpdateResult = { datasetId: "dataset-orders-abc123", added: 0, @@ -1370,6 +1382,13 @@ export class TestEvalClient implements CoreEvalClient { private updateResponse: UpdateEvaluatorResponse = DEFAULT_UPDATE_EVALUATOR_RESPONSE; private getResponse: GetEvaluatorResponse = DEFAULT_GET_EVALUATOR_RESPONSE; private deleteResponse: DeleteEvaluatorResponse = DEFAULT_DELETE_EVALUATOR_RESPONSE; + private startRecommendationResponse: StartRecommendationResponse = + DEFAULT_START_RECOMMENDATION_RESPONSE; + private getRecommendationResponse: GetRecommendationResponse = + DEFAULT_GET_RECOMMENDATION_RESPONSE; + private recommendationListResponses = new Map(); + private deleteRecommendationResponse: DeleteRecommendationResponse = + DEFAULT_DELETE_RECOMMENDATION_RESPONSE; // Online-eval responses, keyed the same way: listOnlineEvaluationConfigs pages // by nextToken, the rest are single canned values. private onlineEvalListResponses = new Map< @@ -1467,6 +1486,29 @@ export class TestEvalClient implements CoreEvalClient { return this; } + setStartRecommendationResponse(response: StartRecommendationResponse): this { + this.startRecommendationResponse = response; + return this; + } + + setGetRecommendationResponse(response: GetRecommendationResponse): this { + this.getRecommendationResponse = response; + return this; + } + + setListRecommendationsResponse( + response: ListRecommendationsResponse, + forNextToken?: string, + ): this { + this.recommendationListResponses.set(forNextToken, response); + return this; + } + + setDeleteRecommendationResponse(response: DeleteRecommendationResponse): this { + this.deleteRecommendationResponse = response; + return this; + } + // setOnlineEvalListResponse sets what listOnlineEvaluationConfigs resolves to // (when not erroring). Pass `forNextToken` to serve a later page. setOnlineEvalListResponse( @@ -1710,6 +1752,48 @@ export class TestEvalClient implements CoreEvalClient { return this.deleteResponse; } + async startRecommendation( + request: StartRecommendationRequest, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "startRecommendation", args: [request, options] }); + if (this.error) throw this.error; + return this.startRecommendationResponse; + } + + async getRecommendation(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "getRecommendation", args: [id, options] }); + if (this.error) throw this.error; + return this.getRecommendationResponse; + } + + async listRecommendations( + nextToken: string | undefined, + maxResults: number | undefined, + statusFilter: RecommendationStatus | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ + method: "listRecommendations", + args: [nextToken, maxResults, statusFilter, options], + }); + if (this.error) throw this.error; + return ( + this.recommendationListResponses.get(nextToken) ?? + this.recommendationListResponses.get(undefined) ?? + DEFAULT_LIST_RECOMMENDATIONS_RESPONSE + ); + } + + async deleteRecommendation( + id: string, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "deleteRecommendation", args: [id, options] }); + if (this.error) throw this.error; + return this.deleteRecommendationResponse; + } + async getBatchEvaluation( id: string, options: CoreOptions, From f5dcf4e61cf546602678ef46c2e082996bce3646 Mon Sep 17 00:00:00 2001 From: Nicolas Borges Date: Thu, 27 Aug 2026 10:13:24 -0400 Subject: [PATCH 2/3] chore: leverage shared utils + delete stale tests --- .../recommendation.fixture.test.tsx | 54 ++- .../recommendation/recommendation.test.tsx | 332 ------------------ src/testing/TestCoreClient.tsx | 6 +- src/testing/index.tsx | 2 +- src/testing/timing.tsx | 16 +- 5 files changed, 51 insertions(+), 359 deletions(-) delete mode 100644 src/handlers/eval/recommendation/recommendation.test.tsx diff --git a/src/handlers/eval/recommendation/recommendation.fixture.test.tsx b/src/handlers/eval/recommendation/recommendation.fixture.test.tsx index 18e457912..746c5fe95 100644 --- a/src/handlers/eval/recommendation/recommendation.fixture.test.tsx +++ b/src/handlers/eval/recommendation/recommendation.fixture.test.tsx @@ -18,6 +18,8 @@ import { settle, TestGlobalConfigAccessor, testIO, + waitFor, + WaitForTimeoutError, } from "../../../testing"; import { createRootHandler } from "../../index"; @@ -122,30 +124,44 @@ async function waitForTerminal( client: BedrockAgentCoreClient, id: string, ): Promise { - const deadline = Date.now() + RECORDING_TIMEOUT_MS; - while (Date.now() < deadline) { - const response = await client.send(new GetRecommendationCommand({ recommendationId: id })); - if (response.status === "COMPLETED" || response.status === "FAILED") { - return response.status; - } - if (response.status === "DELETING") return response.status; - await Bun.sleep(POLL_INTERVAL_MS); + let status: RecommendationStatus | undefined; + try { + await waitFor( + async () => { + status = (await client.send(new GetRecommendationCommand({ recommendationId: id }))).status; + return status === "COMPLETED" || status === "FAILED" || status === "DELETING"; + }, + RECORDING_TIMEOUT_MS, + POLL_INTERVAL_MS, + ); + } catch (error) { + if (!(error instanceof WaitForTimeoutError)) throw error; + throw new Error(`timed out waiting for recommendation ${id} to reach a terminal state`, { + cause: error, + }); } - throw new Error(`timed out waiting for recommendation ${id} to reach a terminal state`); + return status; } async function waitUntilDeleted(client: BedrockAgentCoreClient, id: string): Promise { - const deadline = Date.now() + RECORDING_TIMEOUT_MS; - while (Date.now() < deadline) { - try { - await client.send(new GetRecommendationCommand({ recommendationId: id })); - } catch (error) { - if (isNotFound(error)) return; - throw error; - } - await Bun.sleep(POLL_INTERVAL_MS); + try { + await waitFor( + async () => { + try { + await client.send(new GetRecommendationCommand({ recommendationId: id })); + return false; + } catch (error) { + if (isNotFound(error)) return true; + throw error; + } + }, + RECORDING_TIMEOUT_MS, + POLL_INTERVAL_MS, + ); + } catch (error) { + if (!(error instanceof WaitForTimeoutError)) throw error; + throw new Error(`timed out waiting for recommendation ${id} to be deleted`, { cause: error }); } - throw new Error(`timed out waiting for recommendation ${id} to be deleted`); } async function cleanupRecommendation(client: BedrockAgentCoreClient, id: string): Promise { diff --git a/src/handlers/eval/recommendation/recommendation.test.tsx b/src/handlers/eval/recommendation/recommendation.test.tsx deleted file mode 100644 index 26d8ca977..000000000 --- a/src/handlers/eval/recommendation/recommendation.test.tsx +++ /dev/null @@ -1,332 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import type { - DeleteRecommendationResponse, - GetRecommendationResponse, - ListRecommendationsResponse, - RecommendationConfig, - StartRecommendationResponse, -} from "@aws-sdk/client-bedrock-agentcore"; -import { - createSilentLogger, - TestCoreClient, - TestGlobalConfigAccessor, - testIO, -} from "../../../testing"; -import { createRootHandler } from "../../index"; - -const REGION = "us-west-2"; -const ENDPOINT = "https://agentcore.example.test"; -const CONFIG: RecommendationConfig = { - systemPromptRecommendationConfig: { - systemPrompt: { text: "You are a support agent." }, - agentTraces: { - sessionSpans: [{ traceId: "trace-1", spanId: "span-1" }], - }, - evaluationConfig: { - evaluators: [ - { - evaluatorArn: - "arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/Builtin.Helpfulness", - }, - ], - }, - }, -}; - -function testRecommendationCommand(stdin?: string) { - const core = new TestCoreClient(); - const io = testIO({ stdin }); - const root = createRootHandler(core, { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - - return { - core, - stdout: io.stdout, - route: (args: string[]) => root.route(["bun", "agentcore", ...args, "--region", REGION]), - root, - }; -} - -function callArgs(core: TestCoreClient, method: string): unknown[] { - const call = core.eval.calls.find((candidate) => candidate.method === method); - if (!call) throw new Error(`${method} was not called`); - return call.args; -} - -describe("eval recommendation command hierarchy", () => { - test("registers the imperative API commands and flags", () => { - const { root } = testRecommendationCommand(); - const recommendation = root - .children() - .find((child) => child.name() === "eval") - ?.children() - .find((child) => child.name() === "recommendation"); - - expect(recommendation?.children().map((child) => child.name())).toEqual([ - "start", - "get", - "list", - "delete", - ]); - - const start = recommendation?.children().find((child) => child.name() === "start"); - expect(start?.flags().map((candidate) => candidate.name)).toEqual([ - "name", - "type", - "recommendation-config", - "description", - "kms-key-arn", - "tags", - ]); - expect( - start?.flags().find((candidate) => candidate.name === "recommendation-config")?.sensitive, - ).toBe(true); - expect( - recommendation - ?.children() - .find((child) => child.name() === "list") - ?.flags() - .map((candidate) => candidate.name), - ).toEqual(["next-token", "max-results", "status-filter"]); - }); -}); - -describe("recommendation start", () => { - test("maps the complete API request, tags, Core options, and response", async () => { - const { core, stdout, route } = testRecommendationCommand(); - const response = { - recommendationId: "recommendation-1", - recommendationArn: "arn:recommendation-1", - name: "support-prompt", - type: "SYSTEM_PROMPT_RECOMMENDATION", - status: "PENDING", - createdAt: new Date("2026-08-26T12:00:00.000Z"), - updatedAt: new Date("2026-08-26T12:00:00.000Z"), - } as StartRecommendationResponse; - core.eval.setStartRecommendationResponse(response); - - await route([ - "eval", - "recommendation", - "start", - "--name", - "support-prompt", - "--type", - "SYSTEM_PROMPT_RECOMMENDATION", - "--recommendation-config", - JSON.stringify(CONFIG), - "--description", - "Improve the support prompt", - "--kms-key-arn", - "arn:aws:kms:us-west-2:123456789012:key/recommendations", - "--tags", - "environment=test", - "--tags", - "team=eval", - "--endpoint-url", - ENDPOINT, - ]); - - expect(callArgs(core, "startRecommendation")).toEqual([ - { - name: "support-prompt", - type: "SYSTEM_PROMPT_RECOMMENDATION", - recommendationConfig: CONFIG, - description: "Improve the support prompt", - kmsKeyArn: "arn:aws:kms:us-west-2:123456789012:key/recommendations", - tags: { environment: "test", team: "eval" }, - }, - { region: REGION, endpointUrl: ENDPOINT }, - ]); - expect(JSON.parse(stdout())).toEqual({ - ...response, - createdAt: "2026-08-26T12:00:00.000Z", - updatedAt: "2026-08-26T12:00:00.000Z", - }); - }); - - test("resolves recommendation config from stdin", async () => { - const { core, route } = testRecommendationCommand(JSON.stringify(CONFIG)); - - await route([ - "eval", - "recommendation", - "start", - "--name", - "support-prompt", - "--type", - "SYSTEM_PROMPT_RECOMMENDATION", - "--recommendation-config", - "-", - ]); - - expect(callArgs(core, "startRecommendation")[0]).toMatchObject({ - recommendationConfig: CONFIG, - }); - }); - - test("rejects malformed config JSON before calling Core", async () => { - const { core, route } = testRecommendationCommand(); - - await expect( - route([ - "eval", - "recommendation", - "start", - "--name", - "support-prompt", - "--type", - "SYSTEM_PROMPT_RECOMMENDATION", - "--recommendation-config", - "{not-json", - ]), - ).rejects.toThrow(/Invalid JSON for option '--recommendation-config'/); - expect(core.eval.calls).toHaveLength(0); - }); -}); - -describe("recommendation get, list, and delete", () => { - test("gets by id and renders the API response unchanged", async () => { - const { core, stdout, route } = testRecommendationCommand(); - const response = { - recommendationId: "recommendation-1", - name: "support-prompt", - type: "SYSTEM_PROMPT_RECOMMENDATION", - recommendationConfig: CONFIG, - status: "COMPLETED", - createdAt: new Date("2026-08-26T12:00:00.000Z"), - updatedAt: new Date("2026-08-26T12:05:00.000Z"), - } as GetRecommendationResponse; - core.eval.setGetRecommendationResponse(response); - - await route(["eval", "recommendation", "get", "--id", "recommendation-1"]); - - expect(callArgs(core, "getRecommendation")).toEqual([ - "recommendation-1", - { region: REGION, endpointUrl: undefined }, - ]); - expect(JSON.parse(stdout()).recommendationId).toBe("recommendation-1"); - expect(JSON.parse(stdout()).recommendationConfig).toEqual(CONFIG); - }); - - test("passes pagination and status filtering and renders the response", async () => { - const { core, stdout, route } = testRecommendationCommand(); - const response = { - recommendationSummaries: [ - { - recommendationId: "recommendation-2", - recommendationArn: "arn:recommendation-2", - name: "tool-descriptions", - type: "TOOL_DESCRIPTION_RECOMMENDATION", - status: "IN_PROGRESS", - createdAt: new Date("2026-08-26T13:00:00.000Z"), - updatedAt: new Date("2026-08-26T13:01:00.000Z"), - }, - ], - nextToken: "token-2", - } as ListRecommendationsResponse; - core.eval.setListRecommendationsResponse(response, "token-1"); - - await route([ - "eval", - "recommendation", - "list", - "--next-token", - "token-1", - "--max-results", - "5", - "--status-filter", - "IN_PROGRESS", - ]); - - expect(callArgs(core, "listRecommendations")).toEqual([ - "token-1", - 5, - "IN_PROGRESS", - { region: REGION, endpointUrl: undefined }, - ]); - expect(JSON.parse(stdout())).toMatchObject({ - recommendationSummaries: [{ recommendationId: "recommendation-2" }], - nextToken: "token-2", - }); - }); - - test("deletes by id and renders the API response", async () => { - const { core, stdout, route } = testRecommendationCommand(); - const response = { - recommendationId: "recommendation-1", - status: "DELETING", - } as DeleteRecommendationResponse; - core.eval.setDeleteRecommendationResponse(response); - - await route(["eval", "recommendation", "delete", "--id", "recommendation-1"]); - - expect(callArgs(core, "deleteRecommendation")).toEqual([ - "recommendation-1", - { region: REGION, endpointUrl: undefined }, - ]); - expect(JSON.parse(stdout())).toEqual(response); - }); -}); - -describe("recommendation flag validation", () => { - test.each([ - [["eval", "recommendation", "start"], /--name/], - [ - [ - "eval", - "recommendation", - "start", - "--name", - "support-prompt", - "--recommendation-config", - JSON.stringify(CONFIG), - ], - /--type/, - ], - [ - [ - "eval", - "recommendation", - "start", - "--name", - "support-prompt", - "--type", - "SYSTEM_PROMPT_RECOMMENDATION", - ], - /--recommendation-config/, - ], - [["eval", "recommendation", "get"], /--id/], - [["eval", "recommendation", "delete"], /--id/], - ] as const)("requires API-mandated flags for %p", async (args, expected) => { - const { core, route } = testRecommendationCommand(); - - await expect(route([...args])).rejects.toThrow(expected); - expect(core.eval.calls).toHaveLength(0); - }); - - test.each([ - [["--type", "FUTURE_RECOMMENDATION"], /--type/], - [["--status-filter", "CANCELLED"], /--status-filter/], - ] as const)("rejects unsupported enum input %p", async ([flag, value], expected) => { - const { core, route } = testRecommendationCommand(); - const base = - flag === "--type" - ? [ - "eval", - "recommendation", - "start", - "--name", - "support-prompt", - "--recommendation-config", - JSON.stringify(CONFIG), - ] - : ["eval", "recommendation", "list"]; - - await expect(route([...base, flag, value])).rejects.toThrow(expected); - expect(core.eval.calls).toHaveLength(0); - }); -}); diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index c9289c0f1..ef5064c21 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -103,7 +103,6 @@ import type { ListSessionsOutput, RecommendationStatus, StartBatchEvaluationResponse, - StartRecommendationRequest, StartRecommendationResponse, } from "@aws-sdk/client-bedrock-agentcore"; import type { Core } from "../handlers/types"; @@ -154,6 +153,7 @@ import type { SessionTrace, StartBatchInsightsInput, StartBatchEvaluationInput, + StartRecommendationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; @@ -1753,10 +1753,10 @@ export class TestEvalClient implements CoreEvalClient { } async startRecommendation( - request: StartRecommendationRequest, + input: StartRecommendationInput, options: CoreOptions, ): Promise { - this.calls.push({ method: "startRecommendation", args: [request, options] }); + this.calls.push({ method: "startRecommendation", args: [input, options] }); if (this.error) throw this.error; return this.startRecommendationResponse; } diff --git a/src/testing/index.tsx b/src/testing/index.tsx index fc313b4f5..548d3fd16 100644 --- a/src/testing/index.tsx +++ b/src/testing/index.tsx @@ -1,7 +1,7 @@ export { parse, stringify } from "./serialization"; export { fixtureFactories, fixtureFetch, isRecording, matchGolden, settle } from "./fixtures"; export { testIO, type TestIO } from "./testIO"; -export { tick, waitFor } from "./timing"; +export { tick, waitFor, WaitForTimeoutError } from "./timing"; export { TestCoreClient, TestGatewayClient, diff --git a/src/testing/timing.tsx b/src/testing/timing.tsx index 2d7b62494..52f392575 100644 --- a/src/testing/timing.tsx +++ b/src/testing/timing.tsx @@ -2,6 +2,13 @@ export function tick(ms = 0): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +export class WaitForTimeoutError extends Error { + constructor() { + super("waitFor: condition not met before timeout"); + this.name = "WaitForTimeoutError"; + } +} + /** * Polls {@link predicate} until it returns true or the timeout elapses, * ticking between attempts so pending renders/queries flush. @@ -11,17 +18,18 @@ export function tick(ms = 0): Promise { * * @param predicate - Condition to wait for. * @param timeoutMs - Maximum time to wait in milliseconds (default 1000). + * @param intervalMs - Delay between attempts in milliseconds (default 5). * @throws If the predicate is not satisfied before the timeout. */ export async function waitFor( predicate: () => boolean | Promise, timeoutMs = 1000, + intervalMs = 5, ): Promise { - const step = 5; let waited = 0; while (!(await predicate())) { - if (waited >= timeoutMs) throw new Error("waitFor: condition not met before timeout"); - await tick(step); - waited += step; + if (waited >= timeoutMs) throw new WaitForTimeoutError(); + await tick(intervalMs); + waited += intervalMs; } } From 278329243c96c49c85b79ad5232f6c9a2be4aa01 Mon Sep 17 00:00:00 2001 From: Nicolas Borges Date: Thu, 27 Aug 2026 10:14:09 -0400 Subject: [PATCH 3/3] fix: create recommendation input interface + update required flag handling --- src/core/eval.tsx | 6 +- src/core/recommendation.test.tsx | 119 ------------------ .../eval/recommendation/start/index.tsx | 25 ++-- src/handlers/eval/types.tsx | 13 +- 4 files changed, 28 insertions(+), 135 deletions(-) delete mode 100644 src/core/recommendation.test.tsx diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 5d10fb6e5..aa8323191 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -85,7 +85,6 @@ import { type ListRecommendationsResponse, type RecommendationStatus, type StartBatchEvaluationResponse, - type StartRecommendationRequest, type StartRecommendationResponse, type DataSourceConfig as DataPlaneDataSourceConfig, type CloudWatchFilterConfig, @@ -143,6 +142,7 @@ import type { SpanRecord, StartBatchInsightsInput, StartBatchEvaluationInput, + StartRecommendationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; @@ -353,10 +353,10 @@ export class EvalClient implements CoreEvalClient { } async startRecommendation( - request: StartRecommendationRequest, + input: StartRecommendationInput, options: CoreOptions, ): Promise { - return this.clients.data(toClientConfig(options)).send(new StartRecommendationCommand(request)); + return this.clients.data(toClientConfig(options)).send(new StartRecommendationCommand(input)); } async getRecommendation(id: string, options: CoreOptions): Promise { diff --git a/src/core/recommendation.test.tsx b/src/core/recommendation.test.tsx deleted file mode 100644 index e8499272e..000000000 --- a/src/core/recommendation.test.tsx +++ /dev/null @@ -1,119 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { - DeleteRecommendationCommand, - GetRecommendationCommand, - ListRecommendationsCommand, - StartRecommendationCommand, - type DeleteRecommendationResponse, - type GetRecommendationResponse, - type ListRecommendationsResponse, - type RecommendationConfig, - type StartRecommendationRequest, - type StartRecommendationResponse, -} from "@aws-sdk/client-bedrock-agentcore"; -import type { AwsClients, ClientConfig } from "./types"; -import { EvalClient } from "./eval"; - -const options = { - region: "us-west-2", - endpointUrl: "https://agentcore.example.test", -}; -const recommendationConfig: RecommendationConfig = { - systemPromptRecommendationConfig: { - systemPrompt: { text: "You are a support agent." }, - agentTraces: { sessionSpans: [{ traceId: "trace-1", spanId: "span-1" }] }, - }, -}; - -function evalClient(send: (command: unknown) => Promise): { - client: EvalClient; - configs: ClientConfig[]; -} { - const configs: ClientConfig[] = []; - return { - client: new EvalClient({ - data: (config: ClientConfig) => { - configs.push(config); - return { send } as never; - }, - } as unknown as AwsClients), - configs, - }; -} - -describe("EvalClient recommendations", () => { - test("starts a recommendation with the request unchanged on the data plane", async () => { - const request: StartRecommendationRequest = { - name: "support-prompt", - description: "Improve the support prompt", - type: "SYSTEM_PROMPT_RECOMMENDATION", - recommendationConfig, - kmsKeyArn: "arn:aws:kms:us-west-2:123456789012:key/recommendations", - tags: { team: "eval" }, - }; - const response = { - recommendationId: "recommendation-1", - status: "PENDING", - } as StartRecommendationResponse; - const { client, configs } = evalClient(async (command) => { - expect(command).toBeInstanceOf(StartRecommendationCommand); - expect((command as StartRecommendationCommand).input).toEqual(request); - return response; - }); - - await expect(client.startRecommendation(request, options)).resolves.toBe(response); - expect(configs).toEqual([{ region: "us-west-2", endpoint: "https://agentcore.example.test" }]); - }); - - test("gets a recommendation by recommendationId", async () => { - const response = { - recommendationId: "recommendation-1", - status: "COMPLETED", - } as GetRecommendationResponse; - const { client } = evalClient(async (command) => { - expect(command).toBeInstanceOf(GetRecommendationCommand); - expect((command as GetRecommendationCommand).input).toEqual({ - recommendationId: "recommendation-1", - }); - return response; - }); - - await expect(client.getRecommendation("recommendation-1", options)).resolves.toBe(response); - }); - - test("lists recommendations with pagination and statusFilter", async () => { - const response = { - recommendationSummaries: [], - nextToken: "token-2", - } as ListRecommendationsResponse; - const { client } = evalClient(async (command) => { - expect(command).toBeInstanceOf(ListRecommendationsCommand); - expect((command as ListRecommendationsCommand).input).toEqual({ - nextToken: "token-1", - maxResults: 10, - statusFilter: "FAILED", - }); - return response; - }); - - await expect(client.listRecommendations("token-1", 10, "FAILED", options)).resolves.toBe( - response, - ); - }); - - test("deletes a recommendation by recommendationId", async () => { - const response = { - recommendationId: "recommendation-1", - status: "DELETING", - } as DeleteRecommendationResponse; - const { client } = evalClient(async (command) => { - expect(command).toBeInstanceOf(DeleteRecommendationCommand); - expect((command as DeleteRecommendationCommand).input).toEqual({ - recommendationId: "recommendation-1", - }); - return response; - }); - - await expect(client.deleteRecommendation("recommendation-1", options)).resolves.toBe(response); - }); -}); diff --git a/src/handlers/eval/recommendation/start/index.tsx b/src/handlers/eval/recommendation/start/index.tsx index 820a8d184..d69a5b5b7 100644 --- a/src/handlers/eval/recommendation/start/index.tsx +++ b/src/handlers/eval/recommendation/start/index.tsx @@ -12,6 +12,19 @@ const RECOMMENDATION_TYPES = [ "TOOL_DESCRIPTION_RECOMMENDATION", ] as const; +const REQUIRED_FLAGS = ["name", "type", "recommendation-config"] as const; +type RequiredFlag = (typeof REQUIRED_FLAGS)[number]; + +function assertRequiredFlags( + flags: Partial>, +): asserts flags is Record { + for (const name of REQUIRED_FLAGS) { + if (!flags[name]) { + throw new InputValidationError(`required option '--${name} <${name}>' not specified`); + } + } +} + export const createStartRecommendationHandler = (core: Core, io: AppIO) => createHandler({ name: "start", @@ -38,17 +51,7 @@ export const createStartRecommendationHandler = (core: Core, io: AppIO) => flag("tags", "tags as key=value (repeatable) or JSON object", z.array(z.string()).optional()), ], handle: async (ctx, flags) => { - if (!flags["name"]) { - throw new InputValidationError("required option '--name ' not specified"); - } - if (!flags["type"]) { - throw new InputValidationError("required option '--type ' not specified"); - } - if (!flags["recommendation-config"]) { - throw new InputValidationError( - "required option '--recommendation-config ' not specified", - ); - } + assertRequiredFlags(flags); const source = new SourceResolver({ stdin: io.stdin }); const recommendationConfig = parseJsonFlag( diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 9022b70ae..6bcb6447f 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -40,8 +40,9 @@ import type { GetRecommendationResponse, ListRecommendationsResponse, ListBatchEvaluationsResponse, + RecommendationConfig, RecommendationStatus, - StartRecommendationRequest, + RecommendationType, StartRecommendationResponse, StartBatchEvaluationResponse, SessionMetadataShape, @@ -231,6 +232,14 @@ export type RoleScopeWarning = { }; export type CreateDatasetInput = CreateDatasetRequest; +export type StartRecommendationInput = { + name: string; + description?: string; + type: RecommendationType; + recommendationConfig: RecommendationConfig; + kmsKeyArn?: string; + tags?: Record; +}; export type CreateConfigurationBundleInput = Pick< CreateConfigurationBundleRequest, "bundleName" | "components" | "branchName" | "commitMessage" | "kmsKeyArn" @@ -372,7 +381,7 @@ export interface CoreEvalClient { deleteEvaluator(id: string, options: CoreOptions): Promise; startRecommendation( - request: StartRecommendationRequest, + input: StartRecommendationInput, options: CoreOptions, ): Promise; getRecommendation(id: string, options: CoreOptions): Promise;