feat(eval): ab-test config-bundle run (create)

src/core/abTestExecutionRole.test.ts

@@ -0,0 +1,100 @@
+import { test, expect, describe } from "bun:test";
+import { CreateRoleCommand, GetRoleCommand, type IAMClient } from "@aws-sdk/client-iam";
+import {
+  abTestExecutionRoleName,
+  accountIdFromArn,
+  provisionAbTestRole,
+} from "./abTestExecutionRole";
+
+const GATEWAY_ARN = "arn:aws:bedrock-agentcore:us-west-2:123456789012:gateway/orders-gw";
+
+type Sent = { name: string; input: unknown };
+
+function fakeIam(onGet: "found" | "missing"): { iam: IAMClient; sent: Sent[] } {
+  const sent: Sent[] = [];
+  const iam = {
+    send: async (command: { constructor: { name: string }; input: unknown }) => {
+      sent.push({ name: command.constructor.name, input: command.input });
+      if (command instanceof GetRoleCommand) {
+        if (onGet === "missing") {
+          throw Object.assign(new Error("no such entity"), { name: "NoSuchEntityException" });
+        }
+        return {
+          Role: {
+            Arn: `arn:aws:iam::123456789012:role/${(command.input as { RoleName: string }).RoleName}`,
+          },
+        };
+      }
+      if (command instanceof CreateRoleCommand) {
+        return {
+          Role: {
+            Arn: `arn:aws:iam::123456789012:role/${(command.input as { RoleName: string }).RoleName}`,
+          },
+        };
+      }
+      return {};
+    },
+  } as unknown as IAMClient;
+  return { iam, sent };
+}
+
+describe("abTestExecutionRoleName", () => {
+  test("stays within IAM's 64-char limit and is deterministic", () => {
+    const long = abTestExecutionRoleName("x".repeat(120));
+    expect(long.length).toBeLessThanOrEqual(64);
+    expect(abTestExecutionRoleName("orders")).toBe(abTestExecutionRoleName("orders"));
+  });
+
+  test("distinct names for distinct tests", () => {
+    expect(abTestExecutionRoleName("a")).not.toBe(abTestExecutionRoleName("b"));
+  });
+});
+
+describe("accountIdFromArn", () => {
+  test("extracts the account segment", () => {
+    expect(accountIdFromArn(GATEWAY_ARN)).toBe("123456789012");
+  });
+  test("throws on a malformed ARN", () => {
+    expect(() => accountIdFromArn("not-an-arn")).toThrow(/account id/);
+  });
+});
+
+describe("provisionAbTestRole", () => {
+  test("creates the role + inline policy and reports created=true", async () => {
+    const { iam, sent } = fakeIam("missing");
+    const result = await provisionAbTestRole(iam, "orders-v2", GATEWAY_ARN, "us-west-2");
+
+    expect(result.created).toBe(true);
+    expect(result.roleArn).toContain(":role/");
+    expect(sent.map((s) => s.name)).toEqual([
+      "GetRoleCommand",
+      "CreateRoleCommand",
+      "PutRolePolicyCommand",
+    ]);
+
+    const create = sent.find((s) => s.name === "CreateRoleCommand")!.input as {
+      AssumeRolePolicyDocument: string;
+    };
+    const trust = JSON.parse(create.AssumeRolePolicyDocument);
+    expect(trust.Statement[0].Principal.Service).toBe("bedrock-agentcore.amazonaws.com");
+    expect(trust.Statement[0].Condition.StringEquals["aws:SourceAccount"]).toBe("123456789012");
+    expect(trust.Statement[0].Condition.ArnLike["aws:SourceArn"]).toContain(":ab-test/*");
+
+    const policy = sent.find((s) => s.name === "PutRolePolicyCommand")!.input as {
+      PolicyDocument: string;
+    };
+    const doc = JSON.parse(policy.PolicyDocument);
+    const actions = doc.Statement.flatMap((s: { Action: string[] }) => s.Action);
+    expect(actions).toContain("bedrock-agentcore:GetGateway");
+    expect(actions).toContain("bedrock-agentcore:GetConfigurationBundleVersion");
+    expect(actions).toContain("bedrock-agentcore:GetOnlineEvaluationConfig");
+  });
+
+  test("reuses an existing role and reports created=false", async () => {
+    const { iam, sent } = fakeIam("found");
+    const result = await provisionAbTestRole(iam, "orders-v2", GATEWAY_ARN, "us-west-2");
+
+    expect(result.created).toBe(false);
+    expect(sent.map((s) => s.name)).toEqual(["GetRoleCommand", "PutRolePolicyCommand"]);
+  });
+});

src/core/abTestExecutionRole.tsx

@@ -0,0 +1,155 @@
+import {
+  CreateRoleCommand,
+  GetRoleCommand,
+  PutRolePolicyCommand,
+  DeleteRoleCommand,
+  DeleteRolePolicyCommand,
+  type IAMClient,
+} from "@aws-sdk/client-iam";
+import { createHash } from "node:crypto";
+
+const AB_TEST_POLICY_NAME = "ABTestExecutionPolicy";
+
+export function abTestExecutionRoleName(testName: string): string {
+  const hash = createHash("sha256").update(`ab-test:${testName}`).digest("hex").slice(0, 8);
+  const base = `AgentCoreABTest-${testName}`;
+  return `${base.slice(0, 55)}-${hash}`;
+}
+
+export function roleNameFromArn(roleArn: string): string {
+  const parts = roleArn.split("/");
+  return parts[parts.length - 1] ?? roleArn;
+}
+
+export function accountIdFromArn(arn: string): string {
+  const accountId = arn.split(":")[4];
+  if (!accountId) throw new Error(`could not extract account id from ARN: ${arn}`);
+  return accountId;
+}
+
+function trustPolicy(accountId: string, region: string): string {
+  return JSON.stringify({
+    Version: "2012-10-17",
+    Statement: [
+      {
+        Effect: "Allow",
+        Principal: { Service: "bedrock-agentcore.amazonaws.com" },
+        Action: "sts:AssumeRole",
+        Condition: {
+          StringEquals: { "aws:SourceAccount": accountId },
+          ArnLike: {
+            "aws:SourceArn": `arn:aws:bedrock-agentcore:${region}:${accountId}:ab-test/*`,
+          },
+        },
+      },
+    ],
+  });
+}
+
+function executionPolicy(accountId: string, region: string): string {
+  return JSON.stringify({
+    Version: "2012-10-17",
+    Statement: [
+      {
+        Sid: "AgentCoreResources",
+        Effect: "Allow",
+        Action: [
+          "bedrock-agentcore:GetGateway",
+          "bedrock-agentcore:GetGatewayTarget",
+          "bedrock-agentcore:ListGatewayTargets",
+          "bedrock-agentcore:CreateGatewayRule",
+          "bedrock-agentcore:UpdateGatewayRule",
+          "bedrock-agentcore:GetGatewayRule",
+          "bedrock-agentcore:DeleteGatewayRule",
+          "bedrock-agentcore:ListGatewayRules",
+          "bedrock-agentcore:GetOnlineEvaluationConfig",
+          "bedrock-agentcore:GetEvaluator",
+          "bedrock-agentcore:GetConfigurationBundle",
+          "bedrock-agentcore:GetConfigurationBundleVersion",
+          "bedrock-agentcore:ListConfigurationBundleVersions",
+        ],
+        Resource: `arn:aws:bedrock-agentcore:${region}:${accountId}:*`,
+        Condition: { StringEquals: { "aws:ResourceAccount": accountId } },
+      },
+      {
+        Sid: "CloudWatchLogsDescribe",
+        Effect: "Allow",
+        Action: ["logs:DescribeLogGroups"],
+        Resource: "*",
+      },
+      {
+        Sid: "CloudWatchLogs",
+        Effect: "Allow",
+        Action: [
+          "logs:DescribeIndexPolicies",
+          "logs:PutIndexPolicy",
+          "logs:StartQuery",
+          "logs:GetQueryResults",
+          "logs:StopQuery",
+          "logs:FilterLogEvents",
+          "logs:GetLogEvents",
+        ],
+        Resource: [
+          `arn:aws:logs:${region}:${accountId}:log-group:/aws/bedrock-agentcore/evaluations/*`,
+          `arn:aws:logs:${region}:${accountId}:log-group:/aws/bedrock-agentcore/runtimes/*`,
+          `arn:aws:logs:${region}:${accountId}:log-group:aws/spans`,
+          `arn:aws:logs:${region}:${accountId}:log-group:aws/spans:*`,
+        ],
+      },
+    ],
+  });
+}
+
+export async function provisionAbTestRole(
+  iam: IAMClient,
+  testName: string,
+  gatewayArn: string,
+  region: string,
+): Promise<{ roleArn: string; created: boolean }> {
+  const accountId = accountIdFromArn(gatewayArn);
+  const roleName = abTestExecutionRoleName(testName);
+
+  let roleArn: string;
+  let created = false;
+  try {
+    const existing = await iam.send(new GetRoleCommand({ RoleName: roleName }));
+    roleArn = existing.Role!.Arn!;
+  } catch (error) {
+    if ((error as Error).name !== "NoSuchEntityException") throw error;
+    const result = await iam.send(
+      new CreateRoleCommand({
+        RoleName: roleName,
+        AssumeRolePolicyDocument: trustPolicy(accountId, region),
+        Description: `Execution role for AgentCore A/B test "${testName}" (created by agentcore CLI)`,
+      }),
+    );
+    roleArn = result.Role!.Arn!;
+    created = true;
+  }
+
+  await iam.send(
+    new PutRolePolicyCommand({
+      RoleName: roleName,
+      PolicyName: AB_TEST_POLICY_NAME,
+      PolicyDocument: executionPolicy(accountId, region),
+    }),
+  );
+
+  return { roleArn, created };
+}
+
+export async function deleteAbTestRole(iam: IAMClient, roleArn: string): Promise<void> {
+  const roleName = roleNameFromArn(roleArn);
+  try {
+    await iam.send(
+      new DeleteRolePolicyCommand({ RoleName: roleName, PolicyName: AB_TEST_POLICY_NAME }),
+    );
+  } catch {
+    void 0;
+  }
+  try {
+    await iam.send(new DeleteRoleCommand({ RoleName: roleName }));
+  } catch {
+    void 0;
+  }
+}