Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions src/core/eval.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,19 @@ import {
type UpdateOnlineEvaluationConfigResponse,
} from "@aws-sdk/client-bedrock-agentcore-control";
import {
DeleteRecommendationCommand,
EvaluateCommand,
GetABTestCommand,
ListABTestsCommand,
UpdateABTestCommand,
DeleteABTestCommand,
GetBatchEvaluationCommand,
GetRecommendationCommand,
ListBatchEvaluationsCommand,
ListRecommendationsCommand,
StartBatchEvaluationCommand,
StartRecommendationCommand,
type DeleteRecommendationResponse,
type EvaluationReferenceInput,
type EvaluationResultContent,
type EvaluationTarget,
Expand All @@ -75,8 +80,12 @@ import {
type ABTestExecutionStatus,
type UpdateABTestResponse,
type DeleteABTestResponse,
type GetRecommendationResponse,
type ListBatchEvaluationsResponse,
type ListRecommendationsResponse,
type RecommendationStatus,
type StartBatchEvaluationResponse,
type StartRecommendationResponse,
type DataSourceConfig as DataPlaneDataSourceConfig,
type CloudWatchFilterConfig,
} from "@aws-sdk/client-bedrock-agentcore";
Expand Down Expand Up @@ -133,6 +142,7 @@ import type {
SpanRecord,
StartBatchInsightsInput,
StartBatchEvaluationInput,
StartRecommendationInput,
UpdateConfigurationBundleInput,
UpdateOnlineEvalInput,
} from "../handlers/eval/types";
Expand Down Expand Up @@ -342,6 +352,39 @@ export class EvalClient implements CoreEvalClient {
.send(new DeleteEvaluatorCommand({ evaluatorId: id }));
}

async startRecommendation(
input: StartRecommendationInput,
options: CoreOptions,
): Promise<StartRecommendationResponse> {
return this.clients.data(toClientConfig(options)).send(new StartRecommendationCommand(input));
}

async getRecommendation(id: string, options: CoreOptions): Promise<GetRecommendationResponse> {
return this.clients
.data(toClientConfig(options))
.send(new GetRecommendationCommand({ recommendationId: id }));
}

async listRecommendations(
nextToken: string | undefined,
maxResults: number | undefined,
statusFilter: RecommendationStatus | undefined,
options: CoreOptions,
): Promise<ListRecommendationsResponse> {
return this.clients
.data(toClientConfig(options))
.send(new ListRecommendationsCommand({ nextToken, maxResults, statusFilter }));
}

async deleteRecommendation(
id: string,
options: CoreOptions,
): Promise<DeleteRecommendationResponse> {
return this.clients
.data(toClientConfig(options))
.send(new DeleteRecommendationCommand({ recommendationId: id }));
}

// getBatchEvaluation returns the service-side job (status + evaluator summaries
// + CloudWatch output config) and, by default, the per-session results read from
// the job's CloudWatch stream once it is terminal. Batch evaluation lives on the
Expand Down
4 changes: 3 additions & 1 deletion src/handlers/eval/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { createBatchInsightsHandler } from "./batch-insights";
import { createOnDemandHandler } from "./ondemand";
import { createConfigBundleHandler } from "./config-bundle";
import { createAbTestHandler } from "./ab-test";
import { createRecommendationHandler } from "./recommendation";

export function createEvalHandler(core: Core, io: AppIO): Router {
return new Router("eval", "evaluate and optimize AgentCore agents")
Expand All @@ -25,7 +26,8 @@ export function createEvalHandler(core: Core, io: AppIO): Router {
.handler(createBatchInsightsHandler(core, io))
.handler(createOnDemandHandler(core, io))
.handler(createConfigBundleHandler(core, io))
.handler(createAbTestHandler(core, io));
.handler(createAbTestHandler(core, io))
.handler(createRecommendationHandler(core, io));
}

export { EvalScreen } from "./screen.tsx";
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E",
"status": "DELETING"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
{
"recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E",
"recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E",
"name": "agentcore_cli_recommendation_fixture",
"type": "SYSTEM_PROMPT_RECOMMENDATION",
"recommendationConfig": {
"systemPromptRecommendationConfig": {
"systemPrompt": {
"text": "You are a concise support assistant. Answer the user's question directly."
},
"agentTraces": {
"sessionSpans": [
{
"traceId": "0123456789abcdef0123456789abcdef",
"endTimeUnixNano": 1750000001000000000,
"resource": {
"attributes": {
"service.name": "agentcore-cli-fixture",
"aws.service.type": "gen_ai_agent"
}
},
"kind": "INTERNAL",
"flags": 256,
"durationNano": 1000000000,
"startTimeUnixNano": 1750000000000000000,
"body": {
"output": {
"messages": [
{
"content": [
{
"text": "Two plus two is four."
}
],
"role": "assistant"
}
]
},
"input": {
"messages": [
{
"content": [
{
"text": "What is two plus two?"
}
],
"role": "user"
}
]
}
},
"spanId": "0123456789abcdef",
"scope": {
"name": "agentcore-cli-fixture"
},
"name": "invoke_agent AgentCore CLI fixture",
"attributes": {
"gen_ai.agent.name": "AgentCore CLI fixture",
"aws.genai.span_kind": "AGENT",
"gen_ai.operation.name": "invoke_agent",
"session.id": "00000000-0000-4000-8000-000000000001",
"gen_ai.system": "strands-agents"
},
"status": {
"code": "OK"
}
}
]
},
"evaluationConfig": {
"evaluators": [
{
"evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness"
}
]
}
}
},
"status": "IN_PROGRESS",
"createdAt": {
"$date": "2026-08-26T14:53:00.323Z"
},
"updatedAt": {
"$date": "2026-08-26T14:53:00.933Z"
},
"description": "Golden recommendation fixture"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"recommendationSummaries": []
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
{
"recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E",
"recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E",
"name": "agentcore_cli_recommendation_fixture",
"type": "SYSTEM_PROMPT_RECOMMENDATION",
"recommendationConfig": {
"systemPromptRecommendationConfig": {
"systemPrompt": {
"text": "You are a concise support assistant. Answer the user's question directly."
},
"agentTraces": {
"sessionSpans": [
{
"traceId": "0123456789abcdef0123456789abcdef",
"endTimeUnixNano": 1750000001000000000,
"resource": {
"attributes": {
"service.name": "agentcore-cli-fixture",
"aws.service.type": "gen_ai_agent"
}
},
"kind": "INTERNAL",
"flags": 256,
"durationNano": 1000000000,
"startTimeUnixNano": 1750000000000000000,
"body": {
"output": {
"messages": [
{
"content": [
{
"text": "Two plus two is four."
}
],
"role": "assistant"
}
]
},
"input": {
"messages": [
{
"content": [
{
"text": "What is two plus two?"
}
],
"role": "user"
}
]
}
},
"spanId": "0123456789abcdef",
"scope": {
"name": "agentcore-cli-fixture"
},
"name": "invoke_agent AgentCore CLI fixture",
"attributes": {
"gen_ai.agent.name": "AgentCore CLI fixture",
"aws.genai.span_kind": "AGENT",
"gen_ai.operation.name": "invoke_agent",
"session.id": "00000000-0000-4000-8000-000000000001",
"gen_ai.system": "strands-agents"
},
"status": {
"code": "OK"
}
}
]
},
"evaluationConfig": {
"evaluators": [
{
"evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness"
}
]
}
}
},
"status": "PENDING",
"createdAt": {
"$date": "2026-08-26T14:53:00.323Z"
},
"updatedAt": {
"$date": "2026-08-26T14:53:00.323Z"
},
"description": "Golden recommendation fixture"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E",
"status": "DELETING"
}
83 changes: 83 additions & 0 deletions src/handlers/eval/recommendation/__fixtures__/get.golden.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
{
"recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E",
"recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E",
"name": "agentcore_cli_recommendation_fixture",
"type": "SYSTEM_PROMPT_RECOMMENDATION",
"recommendationConfig": {
"systemPromptRecommendationConfig": {
"systemPrompt": {
"text": "You are a concise support assistant. Answer the user's question directly."
},
"agentTraces": {
"sessionSpans": [
{
"traceId": "0123456789abcdef0123456789abcdef",
"endTimeUnixNano": 1750000001000000000,
"resource": {
"attributes": {
"service.name": "agentcore-cli-fixture",
"aws.service.type": "gen_ai_agent"
}
},
"kind": "INTERNAL",
"flags": 256,
"durationNano": 1000000000,
"startTimeUnixNano": 1750000000000000000,
"body": {
"output": {
"messages": [
{
"content": [
{
"text": "Two plus two is four."
}
],
"role": "assistant"
}
]
},
"input": {
"messages": [
{
"content": [
{
"text": "What is two plus two?"
}
],
"role": "user"
}
]
}
},
"spanId": "0123456789abcdef",
"scope": {
"name": "agentcore-cli-fixture"
},
"name": "invoke_agent AgentCore CLI fixture",
"attributes": {
"gen_ai.agent.name": "AgentCore CLI fixture",
"aws.genai.span_kind": "AGENT",
"gen_ai.operation.name": "invoke_agent",
"session.id": "00000000-0000-4000-8000-000000000001",
"gen_ai.system": "strands-agents"
},
"status": {
"code": "OK"
}
}
]
},
"evaluationConfig": {
"evaluators": [
{
"evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness"
}
]
}
}
},
"status": "IN_PROGRESS",
"createdAt": "2026-08-26T14:53:00.323Z",
"updatedAt": "2026-08-26T14:53:00.933Z",
"description": "Golden recommendation fixture"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"recommendationSummaries": []
}
Loading
Loading