From 9bd43e303f80ba561888091fb22f0c2cb08bff48 Mon Sep 17 00:00:00 2001 From: gitikavj Date: Thu, 27 Aug 2026 07:31:11 +0000 Subject: [PATCH 1/2] feat(project): add `project add evaluator llm-as-a-judge` Adds a CLI command to attach a custom LLM-as-a-Judge evaluator to a project. The judge is another LLM prompted with scoring instructions and a rating scale, written into spec.evaluators (deployed as an AWS::BedrockAgentCore::Evaluator by the existing CDK constructs). - New `evaluator` subrouter under `project add` with an `llm-as-a-judge` subcommand. - Flags: --name, --level (SESSION|TRACE|TOOL_CALL), --model (Bedrock id/ARN), --instructions (inline/file:///stdin), --rating-scale, --description, --kms-key-arn, --tags. - --rating-scale accepts either a named preset (1-5-quality, 1-3-simple, pass-fail, good-neutral-bad) or an inline JSON rating scale; presets live beside the subcommand and expand into the schema's numerical/categorical shapes. - --tags is parsed via parseJsonFlagWithSchema against TagsSchema. - Instruction placeholder validation is left to the CreateEvaluator service so the CLI never rejects placeholders the service later adds. - Wire the new `evaluator` resource type through AddResourceInput and FsProjectManager.addResource / toProjectSpecKey. Verified end-to-end: preset and inline-JSON rating scales both deploy a real evaluator (CREATE_COMPLETE) and write deployed-state.json. --- src/core/project/manager.tsx | 7 + src/handlers/project/add/evaluator/index.ts | 9 + .../evaluator/llm-as-a-judge/index.test.ts | 349 ++++++++++++++++++ .../add/evaluator/llm-as-a-judge/index.ts | 119 ++++++ .../evaluator/llm-as-a-judge/ratingScales.ts | 85 +++++ src/handlers/project/add/index.ts | 2 + src/handlers/project/types.ts | 5 + 7 files changed, 576 insertions(+) create mode 100644 src/handlers/project/add/evaluator/index.ts create mode 100644 src/handlers/project/add/evaluator/llm-as-a-judge/index.test.ts create mode 100644 src/handlers/project/add/evaluator/llm-as-a-judge/index.ts create mode 100644 src/handlers/project/add/evaluator/llm-as-a-judge/ratingScales.ts diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index 5b65372e8..eb778af01 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -30,6 +30,7 @@ import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project" import { ConfigBundleSchema } from "../../projectSchemas/config-bundle"; import { CredentialSchema } from "../../projectSchemas/credential"; import { MemorySchema } from "../../projectSchemas/memory"; +import { EvaluatorSchema } from "../../projectSchemas/evaluator"; import { OnlineEvalConfigSchema } from "../../projectSchemas/online-eval-config"; import { enclosingProjectRoot } from "./fsUtils"; import { @@ -242,6 +243,10 @@ export class FsProjectManager implements ProjectManager { projectSpec.memories.push(parseResource(MemorySchema, input.resourceConfig)); break; } + case "evaluator": { + projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig)); + break; + } case "gateway": projectSpec.agentCoreGateways.push(input.resourceConfig); break; @@ -451,6 +456,8 @@ function toProjectSpecKey(resourceType: ProjectResource) { return "onlineEvalConfigs"; case "memory": return "memories"; + case "evaluator": + return "evaluators"; case "gateway": case "gateway-target": return "agentCoreGateways"; diff --git a/src/handlers/project/add/evaluator/index.ts b/src/handlers/project/add/evaluator/index.ts new file mode 100644 index 000000000..5dbc413ab --- /dev/null +++ b/src/handlers/project/add/evaluator/index.ts @@ -0,0 +1,9 @@ +import { Router } from "../../../../router"; +import type { AddProjectResourceConfig } from "../types"; +import { createAddLlmAsAJudgeEvaluatorHandler } from "./llm-as-a-judge"; + +export function createAddEvaluatorHandler(config: AddProjectResourceConfig): Router { + const evaluator = new Router("evaluator", "add a custom evaluator to the current project"); + evaluator.handler(createAddLlmAsAJudgeEvaluatorHandler(config)); + return evaluator; +} diff --git a/src/handlers/project/add/evaluator/llm-as-a-judge/index.test.ts b/src/handlers/project/add/evaluator/llm-as-a-judge/index.test.ts new file mode 100644 index 000000000..d7ec605da --- /dev/null +++ b/src/handlers/project/add/evaluator/llm-as-a-judge/index.test.ts @@ -0,0 +1,349 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createRootHandler } from "../../../../index"; +import { + createSilentLogger, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, +} from "../../../../../testing"; +import { DeserializationError, InputValidationError } from "../../../../../errors"; + +const originalCwd = process.cwd(); +const tempDirectories: string[] = []; + +async function inTempDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), "agentcore-evaluator-")); + tempDirectories.push(directory); + process.chdir(directory); + return process.cwd(); +} + +afterEach(async () => { + process.chdir(originalCwd); + await Promise.all( + tempDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +async function run(args: string[], opts?: { core?: TestCoreClient }) { + const io = testIO(); + const core = opts?.core ?? new TestCoreClient(); + const root = createRootHandler(core, { + io: io.io, + globalConfigAccessor: new TestGlobalConfigAccessor(), + logger: createSilentLogger(), + }); + await root.route(["node", "agentcore", "project", ...args]); + return { io, core }; +} + +async function inProject(name = "TestProject"): Promise { + const directory = await inTempDirectory(); + await run(["create", "--name", name, "--skip-install", "--skip-git"]); + const projectRoot = join(directory, name); + process.chdir(projectRoot); + return projectRoot; +} + +const MODEL = "anthropic.claude-3-5-sonnet-20240620-v1:0"; + +describe("project add evaluator llm-as-a-judge", () => { + test("writes a numerical preset evaluator into the spec", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "llm-as-a-judge", + "--name", + "quality", + "--level", + "SESSION", + "--model", + MODEL, + "--instructions", + "Score the helpfulness of the response.", + "--rating-scale", + "1-5-quality", + ]); + + const spec = await Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); + const evaluator = spec.evaluators.find((e: { name: string }) => e.name === "quality"); + expect(evaluator).toMatchObject({ + name: "quality", + level: "SESSION", + config: { + llmAsAJudge: { + model: MODEL, + instructions: "Score the helpfulness of the response.", + }, + }, + }); + expect(evaluator.config.llmAsAJudge.ratingScale.numerical).toHaveLength(5); + expect(evaluator.config.llmAsAJudge.ratingScale.categorical).toBeUndefined(); + }); + + test("writes a categorical preset evaluator", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "llm-as-a-judge", + "--name", + "gate", + "--level", + "TRACE", + "--model", + MODEL, + "--instructions", + "Pass if the answer is grounded.", + "--rating-scale", + "pass-fail", + ]); + + const spec = await Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); + const evaluator = spec.evaluators.find((e: { name: string }) => e.name === "gate"); + expect(evaluator.config.llmAsAJudge.ratingScale.categorical).toEqual([ + { label: "pass", definition: "The response meets the evaluation criteria." }, + { label: "fail", definition: "The response does not meet the evaluation criteria." }, + ]); + }); + + test("reads instructions from a file:// source", async () => { + const projectRoot = await inProject(); + const instructionsPath = join(projectRoot, "instructions.txt"); + await writeFile(instructionsPath, "Evaluate factual accuracy.\n"); + + await run([ + "add", + "evaluator", + "llm-as-a-judge", + "--name", + "accuracy", + "--level", + "TOOL_CALL", + "--model", + MODEL, + "--instructions", + `file://${instructionsPath}`, + "--rating-scale", + "1-3-simple", + ]); + + const spec = await Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); + const evaluator = spec.evaluators.find((e: { name: string }) => e.name === "accuracy"); + expect(evaluator.config.llmAsAJudge.instructions).toBe("Evaluate factual accuracy.\n"); + }); + + test("accepts an inline JSON rating scale on --rating-scale", async () => { + const projectRoot = await inProject(); + + await run([ + "add", + "evaluator", + "llm-as-a-judge", + "--name", + "custom", + "--level", + "SESSION", + "--model", + MODEL, + "--instructions", + "Judge the answer.", + "--rating-scale", + JSON.stringify({ + categorical: [ + { label: "yes", definition: "Meets the bar." }, + { label: "no", definition: "Does not." }, + ], + }), + ]); + + const spec = await Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); + const evaluator = spec.evaluators.find((e: { name: string }) => e.name === "custom"); + expect(evaluator.config.llmAsAJudge.ratingScale.categorical).toEqual([ + { label: "yes", definition: "Meets the bar." }, + { label: "no", definition: "Does not." }, + ]); + }); + + test("persists description, kms key, and tags", async () => { + const projectRoot = await inProject(); + const kms = "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"; + await run([ + "add", + "evaluator", + "llm-as-a-judge", + "--name", + "full", + "--level", + "SESSION", + "--model", + MODEL, + "--instructions", + "Judge the answer.", + "--rating-scale", + "pass-fail", + "--description", + "gate on grounding", + "--kms-key-arn", + kms, + "--tags", + '{"team":"ml"}', + ]); + + const spec = await Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); + const evaluator = spec.evaluators.find((e: { name: string }) => e.name === "full"); + expect(evaluator).toMatchObject({ + description: "gate on grounding", + kmsKeyArn: kms, + tags: { team: "ml" }, + }); + }); + + test("rejects a duplicate evaluator name", async () => { + await inProject(); + const flags = [ + "add", + "evaluator", + "llm-as-a-judge", + "--name", + "dup", + "--level", + "SESSION", + "--model", + MODEL, + "--instructions", + "Judge the answer.", + "--rating-scale", + "pass-fail", + ]; + await run(flags); + await expect(run(flags)).rejects.toBeInstanceOf(InputValidationError); + }); + + test("rejects when the existing spec is invalid", async () => { + const projectRoot = await inProject(); + const specPath = join(projectRoot, "agentcore", "agentcore.json"); + const spec = await Bun.file(specPath).json(); + spec.unknownField = "bad"; + await Bun.write(specPath, JSON.stringify(spec)); + + await expect( + run([ + "add", + "evaluator", + "llm-as-a-judge", + "--name", + "x", + "--level", + "SESSION", + "--model", + MODEL, + "--instructions", + "Judge the answer.", + "--rating-scale", + "pass-fail", + ]), + ).rejects.toBeInstanceOf(DeserializationError); + }); + + test.each<[string, string[]]>([ + [ + "missing --name", + [ + "--level", + "SESSION", + "--model", + MODEL, + "--instructions", + "i", + "--rating-scale", + "pass-fail", + ], + ], + [ + "missing --level", + ["--name", "x", "--model", MODEL, "--instructions", "i", "--rating-scale", "pass-fail"], + ], + [ + "missing --model", + ["--name", "x", "--level", "SESSION", "--instructions", "i", "--rating-scale", "pass-fail"], + ], + [ + "missing --instructions", + ["--name", "x", "--level", "SESSION", "--model", MODEL, "--rating-scale", "pass-fail"], + ], + [ + "invalid --model", + [ + "--name", + "x", + "--level", + "SESSION", + "--model", + "not a model", + "--instructions", + "i", + "--rating-scale", + "pass-fail", + ], + ], + [ + "invalid --level", + [ + "--name", + "x", + "--level", + "NOPE", + "--model", + MODEL, + "--instructions", + "i", + "--rating-scale", + "pass-fail", + ], + ], + [ + "--rating-scale is neither a preset nor JSON", + [ + "--name", + "x", + "--level", + "SESSION", + "--model", + MODEL, + "--instructions", + "i", + "--rating-scale", + "bogus", + ], + ], + [ + "--rating-scale inline JSON fails the schema", + [ + "--name", + "x", + "--level", + "SESSION", + "--model", + MODEL, + "--instructions", + "i", + "--rating-scale", + '{"numerical":[],"categorical":[]}', + ], + ], + [ + "no rating scale", + ["--name", "x", "--level", "SESSION", "--model", MODEL, "--instructions", "i"], + ], + ])("%s", async (_label, flags) => { + await inProject(); + await expect(run(["add", "evaluator", "llm-as-a-judge", ...flags])).rejects.toBeInstanceOf( + InputValidationError, + ); + }); +}); diff --git a/src/handlers/project/add/evaluator/llm-as-a-judge/index.ts b/src/handlers/project/add/evaluator/llm-as-a-judge/index.ts new file mode 100644 index 000000000..027cc2680 --- /dev/null +++ b/src/handlers/project/add/evaluator/llm-as-a-judge/index.ts @@ -0,0 +1,119 @@ +import z from "zod"; +import { createHandler, flag, ProjectKey } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { SourceResolver } from "../../../../../io"; +import { + EvaluatorSchema, + isValidBedrockModelId, + RatingScaleSchema, + type RatingScale, +} from "../../../../../projectSchemas/evaluator"; +import { TagsSchema } from "../../../../../projectSchemas/tags"; +import { parseJsonFlagWithSchema } from "../../../../utils"; +import type { AddProjectResourceConfig } from "../../types"; +import { + isRatingScalePreset, + RATING_SCALE_PRESETS, + RATING_SCALE_PRESET_NAMES, +} from "./ratingScales"; + +export const createAddLlmAsAJudgeEvaluatorHandler = (config: AddProjectResourceConfig) => + createHandler({ + name: "llm-as-a-judge", + description: + "add an LLM-as-a-Judge evaluator — another LLM prompted with instructions on how to score a session", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", "what to score: SESSION, TRACE, or TOOL_CALL", z.string().optional()), + flag( + "model", + "Bedrock model ID or inference-profile/foundation-model ARN for the judge", + z.string().optional(), + ), + flag( + "instructions", + "scoring instructions for the judge (inline text, 'file://', or '-' for stdin); use level placeholders like '{context}'", + z.string().optional(), + ), + flag( + "rating-scale", + `a rating scale preset (${RATING_SCALE_PRESET_NAMES.join(", ")}) or an inline JSON rating scale`, + z.string().optional(), + ), + flag("description", "a description of what this evaluator measures", z.string().optional()), + flag( + "kms-key-arn", + "customer-managed KMS key ARN to encrypt the evaluator", + z.string().optional(), + ), + flag("tags", "tags to apply (JSON object of key/value strings)", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (!flags["level"]) + throw new InputValidationError("required option '--level ' not specified"); + if (!flags["model"]) + throw new InputValidationError("required option '--model ' not specified"); + if (flags["instructions"] === undefined) + throw new InputValidationError( + "required option '--instructions ' not specified", + ); + if (flags["rating-scale"] === undefined) + throw new InputValidationError( + "required option '--rating-scale ' not specified", + ); + + if (!isValidBedrockModelId(flags["model"])) + throw new InputValidationError( + `invalid --model "${flags["model"]}": expected a Bedrock model ID (e.g. anthropic.claude-3-5-sonnet-20240620-v1:0) or an inference-profile/foundation-model ARN`, + ); + + const ratingScale = resolveRatingScale(flags["rating-scale"]); + + const resolver = new SourceResolver({ stdin: config.io.stdin }); + const instructions = await resolver.resolveText("instructions", flags["instructions"]); + + const candidate = { + name: flags["name"], + level: flags["level"], + description: flags["description"], + config: { + llmAsAJudge: { + model: flags["model"], + instructions, + ratingScale, + }, + }, + kmsKeyArn: flags["kms-key-arn"], + tags: parseJsonFlagWithSchema("tags", flags["tags"], TagsSchema), + }; + + const parsed = EvaluatorSchema.safeParse(candidate); + if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); + + const project = ctx.require(ProjectKey); + for await (const event of config.projectManager.addResource(project, { + resourceType: "evaluator", + resourceConfig: parsed.data, + })) { + config.io.stderr.write(`${event.message}\n`); + } + + config.io.stderr.write(`added evaluator '${flags["name"]}' to '${project.name}'\n`); + }, + }); + +// A preset name expands to a fresh copy of the shared table; anything else is +// treated as an inline JSON rating scale and validated against the schema. +function resolveRatingScale(value: string): RatingScale { + if (isRatingScalePreset(value)) { + return structuredClone(RATING_SCALE_PRESETS[value]) as RatingScale; + } + if (!value.trim().startsWith("{")) { + throw new InputValidationError( + `invalid --rating-scale "${value}": expected a preset (${RATING_SCALE_PRESET_NAMES.join(", ")}) or an inline JSON rating scale`, + ); + } + return parseJsonFlagWithSchema("rating-scale", value, RatingScaleSchema)!; +} diff --git a/src/handlers/project/add/evaluator/llm-as-a-judge/ratingScales.ts b/src/handlers/project/add/evaluator/llm-as-a-judge/ratingScales.ts new file mode 100644 index 000000000..c75d18dc5 --- /dev/null +++ b/src/handlers/project/add/evaluator/llm-as-a-judge/ratingScales.ts @@ -0,0 +1,85 @@ +import type { RatingScale } from "../../../../../projectSchemas/evaluator"; + +/** + * Named rating-scale presets for `add evaluator llm-as-a-judge --rating-scale`. + * + * Each preset expands into the {@link RatingScale} shape the schema expects: + * numerical scales carry an integer `value` per rung, categorical scales do not. + * The `definition` text is surfaced to the judge model, so it must describe the + * rung precisely enough for the model to choose between adjacent options. + */ +export const RATING_SCALE_PRESETS = { + "1-5-quality": { + numerical: [ + { + value: 1, + label: "Very Poor", + definition: "The response fails to address the task and contains significant errors.", + }, + { + value: 2, + label: "Poor", + definition: + "The response partially addresses the task but has notable gaps or inaccuracies.", + }, + { + value: 3, + label: "Fair", + definition: "The response addresses the task adequately, with minor issues.", + }, + { + value: 4, + label: "Good", + definition: "The response addresses the task well and is accurate and helpful.", + }, + { + value: 5, + label: "Excellent", + definition: "The response fully addresses the task and is accurate, complete, and helpful.", + }, + ], + }, + "1-3-simple": { + numerical: [ + { + value: 1, + label: "Poor", + definition: "The response does not meet expectations.", + }, + { + value: 2, + label: "Acceptable", + definition: "The response meets basic expectations, with some shortcomings.", + }, + { + value: 3, + label: "Good", + definition: "The response fully meets expectations.", + }, + ], + }, + "pass-fail": { + categorical: [ + { label: "pass", definition: "The response meets the evaluation criteria." }, + { label: "fail", definition: "The response does not meet the evaluation criteria." }, + ], + }, + "good-neutral-bad": { + categorical: [ + { label: "good", definition: "The response is helpful and meets the evaluation criteria." }, + { label: "neutral", definition: "The response is neither clearly good nor clearly bad." }, + { + label: "bad", + definition: "The response is unhelpful or violates the evaluation criteria.", + }, + ], + }, +} as const satisfies Record; + +export type RatingScalePreset = keyof typeof RATING_SCALE_PRESETS; + +export const RATING_SCALE_PRESET_NAMES = Object.keys(RATING_SCALE_PRESETS) as RatingScalePreset[]; + +export function isRatingScalePreset(value: string): value is RatingScalePreset { + return value in RATING_SCALE_PRESETS; +} diff --git a/src/handlers/project/add/index.ts b/src/handlers/project/add/index.ts index f65600d59..8a503b3aa 100644 --- a/src/handlers/project/add/index.ts +++ b/src/handlers/project/add/index.ts @@ -7,6 +7,7 @@ import { createAddMemoryHandler } from "./memory"; import { createAddRuntimeHandler } from "./runtime"; import { createAddOnlineEvalHandler } from "./online-eval"; import { createAddOnlineInsightHandler } from "./online-insight"; +import { createAddEvaluatorHandler } from "./evaluator"; import { createAddGatewayHandler } from "./gateway"; import { createAddGatewayTargetHandler } from "./gateway-target"; import { createAddGatewayConnectorHandler } from "./gateway-connector"; @@ -21,6 +22,7 @@ export function createAddProjectResourceHandler(config: AddProjectResourceConfig projectAdd.handler(createAddRuntimeHandler(config)); projectAdd.handler(createAddOnlineEvalHandler(config)); projectAdd.handler(createAddOnlineInsightHandler(config)); + projectAdd.handler(createAddEvaluatorHandler(config)); projectAdd.handler(createAddCredentialsHandler(config)); projectAdd.handler(createAddGatewayHandler(config)); projectAdd.handler(createAddGatewayTargetHandler(config)); diff --git a/src/handlers/project/types.ts b/src/handlers/project/types.ts index ad60bac73..803beb7a9 100644 --- a/src/handlers/project/types.ts +++ b/src/handlers/project/types.ts @@ -2,6 +2,7 @@ import { HarnessSpecSchema } from "../../projectSchemas/harness"; import type { CredentialSchema } from "../../projectSchemas/credential"; import type { ConfigBundleSchema } from "../../projectSchemas/config-bundle"; import type { MemorySchema } from "../../projectSchemas/memory"; +import type { EvaluatorSchema } from "../../projectSchemas/evaluator"; import type { ProjectSpecSchema } from "../../projectSchemas/project"; import z from "zod"; import type { RuntimeResourceConfig } from "./add/runtime/types"; @@ -171,6 +172,10 @@ export type AddResourceInput = resourceType: "memory"; resourceConfig: z.input; } + | { + resourceType: "evaluator"; + resourceConfig: z.input; + } | { resourceType: "gateway"; resourceConfig: AgentCoreGateway; From 963be2ddad1f114d37816c7f296044a594def065 Mon Sep 17 00:00:00 2001 From: gitikavj Date: Thu, 27 Aug 2026 20:59:20 +0000 Subject: [PATCH 2/2] fix(project): accept foundation-model ARNs without an account segment Bedrock foundation-model ARNs omit the account id (arn:aws:bedrock:::foundation-model/) while inference-profile ARNs include it. Make the account segment optional in the evaluator model-id ARN validator so a valid foundation-model ARN is not rejected. --- src/projectSchemas/evaluator.test.ts | 10 ++++++++++ src/projectSchemas/evaluator.ts | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/projectSchemas/evaluator.test.ts b/src/projectSchemas/evaluator.test.ts index 7336d871c..070c4a6bd 100644 --- a/src/projectSchemas/evaluator.test.ts +++ b/src/projectSchemas/evaluator.test.ts @@ -31,6 +31,16 @@ describe("evaluator custom validation", () => { }); it("validates model identifiers and KMS key ARNs through owned helpers", () => { expect(isValidBedrockModelId("anthropic.claude-v2:1")).toBe(true); + expect(isValidBedrockModelId("us.anthropic.claude-sonnet-4-5-20250929-v1:0")).toBe(true); + // foundation-model ARNs omit the account segment; inference-profile ARNs carry it. + expect( + isValidBedrockModelId("arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-v2"), + ).toBe(true); + expect( + isValidBedrockModelId( + "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-v2", + ), + ).toBe(true); expect(isValidBedrockModelId("not a model")).toBe(false); expect( isValidKmsKeyArn( diff --git a/src/projectSchemas/evaluator.ts b/src/projectSchemas/evaluator.ts index 0906a7744..bb1cf25ea 100644 --- a/src/projectSchemas/evaluator.ts +++ b/src/projectSchemas/evaluator.ts @@ -36,8 +36,11 @@ export const RatingScaleSchema = z ); export type RatingScale = z.infer; const BEDROCK_MODEL_ID_PATTERN = /^[a-z][a-z0-9-]*\.[a-zA-Z0-9._-]+(:[0-9]+)?$/; +// The account segment is optional: foundation-model ARNs omit it +// (arn:aws:bedrock:us-east-1::foundation-model/...), while inference-profile +// ARNs carry it (arn:aws:bedrock:us-east-1:123456789012:inference-profile/...). const BEDROCK_ARN_PATTERN = - /^arn:aws[a-z-]*:bedrock:[a-z0-9-]+:\d{12}:(inference-profile|foundation-model)\/.+$/; + /^arn:aws[a-z-]*:bedrock:[a-z0-9-]+:(\d{12})?:(inference-profile|foundation-model)\/.+$/; export function isValidBedrockModelId(value: string): boolean { return BEDROCK_MODEL_ID_PATTERN.test(value) || BEDROCK_ARN_PATTERN.test(value); }