From 1854eb7e957d1a391da63a517dbc3b74fa3b7133 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 26 Oct 2026 22:49:26 -0700 Subject: [PATCH] test(server): fail when agent context grows past main Co-Authored-By: Claude Opus 5.5 --- .github/workflows/agent-context-budget.yml | 55 ++++++++ apps/server/src/mcp/McpHttpServer.ts | 7 +- .../src/provider/AgentContextBudget.test.ts | 127 ++++++++++++++++++ .../src/provider/agentContextBudget.json | 5 + 4 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/agent-context-budget.yml create mode 100644 apps/server/src/provider/AgentContextBudget.test.ts create mode 100644 apps/server/src/provider/agentContextBudget.json diff --git a/.github/workflows/agent-context-budget.yml b/.github/workflows/agent-context-budget.yml new file mode 100644 index 0000000000000..9396d8e3e7a63 --- /dev/null +++ b/.github/workflows/agent-context-budget.yml @@ -0,0 +1,55 @@ +name: Agent Context Budget + +# Fails when a PR raises a value in agentContextBudget.json above main's. AgentContextBudget.test.ts +# keeps that file equal to the measured context, so the context cannot grow by editing the file. +# To accept growth on purpose, a maintainer adds the agent-context-increase label. +on: + pull_request: + types: [opened, synchronize, reopened, labeled, unlabeled] + +permissions: + contents: read + +concurrency: + group: agent-context-budget-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + budget: + name: Agent context budget + runs-on: ubuntu-24.04 + timeout-minutes: 5 + env: + BUDGET: apps/server/src/provider/agentContextBudget.json + ALLOW_INCREASE: ${{ contains(github.event.pull_request.labels.*.name, 'agent-context-increase') }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + sparse-checkout: apps/server/src/provider/agentContextBudget.json + sparse-checkout-cone-mode: false + + - name: Compare with main + run: | + git fetch --quiet --depth=1 origin main + if ! git cat-file -e "FETCH_HEAD:$BUDGET" 2>/dev/null; then + echo "main has no agent context budget yet." + exit 0 + fi + git show "FETCH_HEAD:$BUDGET" > "$RUNNER_TEMP/main.json" + grown=$(jq -r --slurpfile main "$RUNNER_TEMP/main.json" ' + to_entries[] + | select($main[0][.key] != null and .value > $main[0][.key]) + | "\(.key): \($main[0][.key]) -> \(.value) (+\(.value - $main[0][.key]))" + ' "$BUDGET") + if [ -z "$grown" ]; then + echo "Agent context is not larger than main." + exit 0 + fi + printf 'Agent context grew compared to main:\n%s\n' "$grown" + if [ "$ALLOW_INCREASE" = "true" ]; then + echo "Accepted by the agent-context-increase label." + exit 0 + fi + echo "::error file=$BUDGET::Agent context grew compared to main. Cut the added text, or ask a maintainer to add the agent-context-increase label." + exit 1 diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 179d82dc998c2..044a43baeab1e 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -1055,7 +1055,8 @@ export const layerMcpTransport = McpServer.layerHttp({ allowSessionTermination: true, }).pipe(Layer.provide(layerMcpAuthMiddleware)); -export const layer = Layer.mergeAll( +/** Every tool the T3 MCP server advertises. AgentContextBudget.test.ts measures this set. */ +export const layerTools = Layer.mergeAll( layerPreviewToolkit, layerOrchestratorToolkit, layerThreadToolkit, @@ -1067,4 +1068,6 @@ export const layer = Layer.mergeAll( layerPullRequestsToolkit, layerDeviceToolkit, layerHtmlToolkit, -).pipe(Layer.provideMerge(layerMcpTransport)); +); + +export const layer = layerTools.pipe(Layer.provideMerge(layerMcpTransport)); diff --git a/apps/server/src/provider/AgentContextBudget.test.ts b/apps/server/src/provider/AgentContextBudget.test.ts new file mode 100644 index 0000000000000..09d2fb89ba65a --- /dev/null +++ b/apps/server/src/provider/AgentContextBudget.test.ts @@ -0,0 +1,127 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { ProviderInstanceId } from "@t3tools/contracts"; +import { McpProviderSessions } from "@t3tools/provider-core/server/McpProviderSessions"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { McpServer } from "effect/ai"; + +import { ServerSecretStore } from "../auth/ServerSecretStore.ts"; +import { ServerConfig } from "../config.ts"; +import { DeviceService } from "../device/DeviceService.ts"; +import { ServerEnvironment } from "../environment/ServerEnvironment.ts"; +import { GitWorkflowService } from "../git/GitWorkflowService.ts"; +import * as McpHttpServer from "../mcp/McpHttpServer.ts"; +import { PreviewAutomationBroker } from "../mcp/PreviewAutomationBroker.ts"; +import { + CLAUDE_PROVIDER, + makeClaudeQueryOptions, +} from "../orchestration-v2/Adapters/ClaudeAdapterV2.ts"; +import { OrchestratorV2 } from "../orchestration-v2/Orchestrator.ts"; +import { ProviderAdapterRegistryV2 } from "../orchestration-v2/ProviderAdapterRegistry.ts"; +import { ThreadLaunchService } from "../orchestration-v2/ThreadLaunchService.ts"; +import { ThreadManagementService } from "../orchestration-v2/ThreadManagementService.ts"; +import { ThreadSearch } from "../orchestration-v2/ThreadSearch.ts"; +import { PreviewManager } from "../preview/Manager.ts"; +import { PreviewBrowser } from "../preview/PreviewBrowser.ts"; +import { ManagedProjectFolders } from "../project/ManagedProjectFolders.ts"; +import { ProjectService } from "../project/ProjectService.ts"; +import { ProjectSetupScriptRunner } from "../project/ProjectSetupScriptRunner.ts"; +import { PullRequestReviewGuideService } from "../pullRequest/PullRequestReviewGuideService.ts"; +import { ScheduledTaskService } from "../scheduledTasks/ScheduledTaskService.ts"; +import { SecretRequests } from "../secrets/SecretRequests.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; +import { SourceControlRepositoryService } from "../sourceControl/SourceControlRepositoryService.ts"; +import { GitVcsDriver } from "../vcs/GitVcsDriver.ts"; +import { VcsStatusBroadcaster } from "../vcs/VcsStatusBroadcaster.ts"; +import { buildCodexAdditionalContext } from "./CodexDeveloperInstructions.ts"; +import { DocumentText } from "./DocumentText.ts"; +import { ProviderRegistry } from "./ProviderRegistry.ts"; +import budget from "./agentContextBudget.json" with { type: "json" }; + +// Registration only stores tool definitions, so the handlers' services are never called. +const toolHandlerServices = Layer.mergeAll( + Layer.mock(DeviceService)({}), + Layer.mock(DocumentText)({}), + Layer.mock(GitVcsDriver)({}), + Layer.mock(GitWorkflowService)({}), + Layer.mock(ManagedProjectFolders)({ namedProjectsRoot: "" }), + Layer.mock(McpProviderSessions)({}), + Layer.mock(OrchestratorV2)({}), + Layer.mock(PreviewAutomationBroker)({}), + Layer.mock(PreviewBrowser)({}), + Layer.mock(PreviewManager)({}), + Layer.mock(ProjectService)({}), + Layer.mock(ProjectSetupScriptRunner)({}), + Layer.mock(ProviderAdapterRegistryV2)({}), + Layer.mock(ProviderRegistry)({}), + Layer.mock(PullRequestReviewGuideService)({}), + Layer.mock(ScheduledTaskService)({}), + Layer.mock(SecretRequests)({}), + ServerConfig.layerTest(process.cwd(), { prefix: "t3-agent-context-budget-" }), + Layer.mock(ServerEnvironment)({}), + Layer.mock(ServerSecretStore)({}), + Layer.mock(ServerSettingsService)({}), + Layer.mock(SourceControlRepositoryService)({}), + Layer.mock(ThreadLaunchService)({}), + Layer.mock(ThreadManagementService)({}), + Layer.mock(ThreadSearch)({}), + Layer.mock(VcsStatusBroadcaster)({}), +).pipe(Layer.provideMerge(NodeServices.layer)); + +const utf8Bytes = (text: string) => Buffer.byteLength(text, "utf8"); + +const claudeSystemAppend = () => { + const { systemPrompt } = makeClaudeQueryOptions({ + modelSelection: { + instanceId: ProviderInstanceId.make(CLAUDE_PROVIDER), + model: "claude-opus-5-5", + }, + nativeThreadId: "context-budget", + resume: false, + cwd: null, + mcpServers: { "t3-code": { type: "http", url: "http://127.0.0.1/mcp" } }, + }); + return typeof systemPrompt === "object" && "append" in systemPrompt ? systemPrompt.append : ""; +}; + +const codexAdditionalContext = () => + Object.values( + buildCodexAdditionalContext( + { model: "gpt-6.1-sol", modelName: "GPT-6.1 Sol", reasoningEffort: "high" }, + { browser: true, device: true }, + ), + ) + .map((entry) => entry.value) + .join(""); + +// CI rejects a PR that raises any value above main's agentContextBudget.json, so growth +// needs the agent-context-increase label. Lower the values when the text shrinks. +it.effect("agent context matches the committed budget", () => + Effect.gen(function* () { + const mcp = yield* McpServer.McpServer; + const measured = { + claudeSystemAppendBytes: utf8Bytes(claudeSystemAppend()), + codexAdditionalContextBytes: utf8Bytes(codexAdditionalContext()), + // The model sees these fields. outputSchema and annotations are protocol metadata. + mcpToolListJsonBytes: utf8Bytes( + JSON.stringify( + mcp.tools.map(({ tool }) => ({ + name: tool.name, + description: tool.description, + inputSchema: tool.inputSchema, + })), + ), + ), + }; + + expect(measured).toEqual(budget); + }).pipe( + Effect.provide( + McpHttpServer.layerTools.pipe( + Layer.provideMerge(McpServer.McpServer.layer), + Layer.provide(toolHandlerServices), + ), + ), + ), +); diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json new file mode 100644 index 0000000000000..e3272bfdf8021 --- /dev/null +++ b/apps/server/src/provider/agentContextBudget.json @@ -0,0 +1,5 @@ +{ + "claudeSystemAppendBytes": 4857, + "codexAdditionalContextBytes": 5409, + "mcpToolListJsonBytes": 54964 +}