Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 20 additions & 8 deletions apps/server/src/mcp/McpHttpServer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -220,10 +220,15 @@ it.effect.each([{}, { includeImage: false }])(
);

it.effect.each([
{ mode: "default", input: {}, images: false },
{ mode: "explicit image", input: { includeImage: true }, images: true },
{ mode: "text only", input: { includeImage: false }, images: false },
])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images }) =>
{ mode: "default", input: {}, images: false, hostInput: { screenshot: false } },
{ mode: "explicit image", input: { includeImage: true }, images: true, hostInput: {} },
{
mode: "text only",
input: { includeImage: false },
images: false,
hostInput: { screenshot: false },
},
])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images, hostInput }) =>
Effect.scoped(
Effect.gen(function* () {
const server = yield* McpServer.McpServer;
Expand Down Expand Up @@ -254,6 +259,7 @@ it.effect.each([
};
const screenshot = { mimeType: "image/png", width: 1, height: 1 };
let requests = 0;
const hostInputs: Array<unknown> = [];
const events = yield* broker.connect({ clientId: "mcp-image-option-client", environmentId });
yield* Stream.runForEach(events, (event) => {
if (event.type === "connected") return Deferred.succeed(connected, undefined);
Expand All @@ -263,16 +269,19 @@ it.effect.each([
tabId: alternateTabId,
threadId,
});
expect(event.request.input).toEqual({});
hostInputs.push(event.request.input);
return broker.respond({
clientId: "mcp-image-option-client",
connectionId: event.connectionId,
requestId: event.request.requestId,
ok: true,
// Like the server browser, the host captures only when it is asked to.
result: {
...page,
title: `Snapshot ${requests}`,
screenshot: { ...screenshot, data: png },
...((event.request.input as { screenshot?: boolean }).screenshot === false
? {}
: { screenshot: { ...screenshot, data: png } }),
},
});
}).pipe(Effect.forkScoped);
Expand All @@ -288,7 +297,7 @@ it.effect.each([
Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
Effect.provideService(McpSchema.McpServerClient, client),
);
const metadata = { ...page, title: `Snapshot ${call}`, screenshot };
const metadata = { ...page, title: `Snapshot ${call}`, ...(images ? { screenshot } : {}) };
const { accessibilityTree: _tree, ...boundedMetadata } = metadata;
expect(snapshot.isError).toBe(false);
expect(snapshot.structuredContent).toEqual({
Expand Down Expand Up @@ -327,9 +336,12 @@ it.effect.each([
Effect.provideService(McpSchema.McpServerClient, client),
);
expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "text", "text"]);
expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7", screenshot });
expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7" });
expect(nextDefault.structuredContent).not.toHaveProperty("screenshot");
expect(nextDefault.structuredContent).not.toHaveProperty("accessibilityTree");
expect(requests).toBe(7);
// Only a call that shows or saves the image asks the host to capture one.
expect(hostInputs).toEqual([...Array(6).fill(hostInput), { screenshot: false }]);
}),
).pipe(Effect.provide(layerTest)),
);
Expand Down
26 changes: 16 additions & 10 deletions apps/server/src/mcp/McpHttpServer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -478,17 +478,19 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
Effect.gen(function* () {
const snapshot = encodedResult as SnapshotMetadata & {
readonly url: string;
readonly screenshot: {
readonly screenshot?: {
readonly mimeType: "image/png";
readonly data: string;
readonly width: number;
readonly height: number;
};
};
const { screenshot, ...page } = snapshot;
const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));
const png = screenshot && new Uint8Array(Buffer.from(screenshot.data, "base64"));
const screenshotPath =
payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined;
png && payload?.save === true
? yield* saveScreenshot(snapshot.url, png)
: undefined;
// Providers replay images on later requests, and some reject them; include images only when requested.
const includeImage = payload?.includeImage === true;
if (screenshotPath !== undefined && !includeImage) {
Expand All @@ -504,11 +506,15 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
}
const metadata = {
...page,
screenshot: {
mimeType: screenshot.mimeType,
width: screenshot.width,
height: screenshot.height,
},
...(screenshot === undefined
? {}
: {
screenshot: {
mimeType: screenshot.mimeType,
width: screenshot.width,
height: screenshot.height,
},
}),
...(screenshotPath === undefined ? {} : { screenshotPath }),
};
const bounded = boundSnapshotMetadata(metadata);
Expand All @@ -534,8 +540,8 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`,
},
]),
...(includeImage
? [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]
...(includeImage && png
? [{ type: "image" as const, data: png, mimeType: "image/png" }]
: []),
],
});
Expand Down
10 changes: 8 additions & 2 deletions apps/server/src/mcp/toolkits/preview/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,8 +202,14 @@ const handlers = {
invokeTargeted<PreviewAutomationSetColorSchemeResult>("setColorScheme", input),
),
preview_snapshot: McpToolAccess.readsAsCaller((input) => {
const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {};
return invokeTargeted<PreviewAutomationSnapshot>("snapshot", operationInput);
const { includeImage, save, ...operationInput } = input ?? {};
// A capture makes a hidden desktop tab paint until it answers; skip one nobody reads.
return invokeTargeted<PreviewAutomationSnapshot>(
"snapshot",
includeImage === true || save === true
? operationInput
: { ...operationInput, screenshot: false },
);
}),
preview_click: McpToolAccess.actsAsCaller((input) =>
invokeTargeted<object>("click", input, input.timeoutMs),
Expand Down
2 changes: 1 addition & 1 deletion apps/server/src/mcp/toolkits/preview/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ const PreviewSetAppearanceTool = safeBrowserTool(
export const PreviewSnapshotTool = safeBrowserTool(
Tool.make("preview_snapshot", {
description:
"Use it only when the current request asks for browser work, including checking an edit in the browser. Never use it unasked to verify your own edit, or to repeat a snapshot from an earlier turn. Works on a tab the user is driving. Returns page state, elements, diagnostics, and screenshot size. Target one element with `locator='aria-ref=<ref>'` from `accessibilityTree`, even in frames. Refs die on navigation, a new snapshot, or handoff. Text caps near 20 KB; on your own or an unclaimed tab, `preview_evaluate` reads more. To show a screenshot: `save=true`, then embed ![alt](screenshotPath).",
"Use it only when the current request asks for browser work, including checking an edit in the browser. Never use it unasked to verify your own edit, or to repeat a snapshot from an earlier turn. Works on a tab the user is driving. Returns page state, elements, and diagnostics, plus screenshot size with `includeImage`. Target one element with `locator='aria-ref=<ref>'` from `accessibilityTree`, even in frames. Refs die on navigation, a new snapshot, or handoff. Text caps near 20 KB; on your own or an unclaimed tab, `preview_evaluate` reads more. To show a screenshot: `save=true`, then embed ![alt](screenshotPath).",
parameters: Schema.Struct({
...PreviewAutomationTabTargetInput.fields,
includeImage: Schema.optional(
Expand Down
8 changes: 5 additions & 3 deletions apps/server/src/preview/ServerBrowser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1958,12 +1958,14 @@ const make = Effect.gen(function* () {
return { tabId: tab.tabId, colorScheme };
}
case "snapshot": {
return withScreencastsPaused(tab, () =>
const screenshot = (input as { readonly screenshot?: boolean }).screenshot !== false;
const take = () =>
ServerBrowserPage.snapshot({
...tab,
renderScale: tab.desktop ? "display" : RENDER_SCALE,
}),
);
screenshot,
});
return screenshot ? withScreencastsPaused(tab, take) : take();
}
case "click": {
const clickInput = input as PreviewAutomationClickInput;
Expand Down
26 changes: 26 additions & 0 deletions apps/server/src/preview/ServerBrowserPage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,32 @@ describe("server browser element refs", () => {
expect(tree).not.toContain("PAYLOAD");
});

it("reads a page that draws no frames when no screenshot is wanted", async () => {
await page.setContent("<button>continue</button>");
const methods: Array<string> = [];
// Stands in for a hidden tab that never paints, so Chromium never answers its screenshot.
const drawsNothing = {
send: (...[method, params]: Parameters<CDPSession["send"]>) => {
methods.push(method);
return method === "Page.captureScreenshot"
? new Promise<never>(() => {})
: cdp.send(method, params);
},
} as CDPSession;
const result = await ServerBrowserPage.snapshot({
page,
cdp: drawsNothing,
renderScale: 1,
screenshot: false,
consoleEntries: [],
networkEntries: [],
actionTimeline: [],
});
buttonLocator(result.accessibilityTree, "continue");
expect(result.screenshot).toBeUndefined();
expect(methods).not.toContain("Page.captureScreenshot");
});

it("rejects refs from another tab", async () => {
await page.setContent("<button>same label</button>");
const locator = buttonLocator((await takeSnapshot()).accessibilityTree, "same label");
Expand Down
14 changes: 8 additions & 6 deletions apps/server/src/preview/ServerBrowserPage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,8 @@ type SnapshotInput = {
readonly cdp: CDPSession;
/** Headless pages render at the scale the server set; a desktop page at its display's. */
readonly renderScale: number | "display";
/** A screenshot waits for the page's next frame, which a hidden desktop tab must paint. */
readonly screenshot?: boolean;
readonly consoleEntries: ReadonlyArray<PreviewAutomationConsoleEntry>;
readonly networkEntries: ReadonlyArray<PreviewAutomationNetworkEntry>;
readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"];
Expand Down Expand Up @@ -536,7 +538,9 @@ const capture = async (
"url" | "title" | "loading" | "visibleText" | "interactiveElements"
>
>,
captureViewport(input.page, input.cdp, { format: "png", scale, viewport, offset }),
input.screenshot === false
? undefined
: captureViewport(input.page, input.cdp, { format: "png", scale, viewport, offset }),
]);
void pageAndScreenshot.catch(constVoid);
// Leaves time for the second read; a frame that never loads holds the tree until then.
Expand Down Expand Up @@ -581,11 +585,9 @@ const capture = async (
consoleEntries: [...input.consoleEntries],
networkEntries: [...input.networkEntries],
actionTimeline: [...input.actionTimeline],
screenshot: {
mimeType: "image/png",
data,
...pngSize(data),
},
...(data === undefined
? {}
: { screenshot: { mimeType: "image/png", data, ...pngSize(data) } }),
};
};

Expand Down
15 changes: 9 additions & 6 deletions packages/contracts/src/previewAutomation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -500,12 +500,15 @@ export const PreviewAutomationSnapshot = Schema.Struct({
consoleEntries: Schema.Array(PreviewAutomationConsoleEntry),
networkEntries: Schema.Array(PreviewAutomationNetworkEntry),
actionTimeline: Schema.Array(PreviewAutomationActionEvent),
screenshot: Schema.Struct({
mimeType: Schema.Literal("image/png"),
data: Schema.String,
width: Schema.Int,
height: Schema.Int,
}),
/** Absent when the caller asked for none. */
screenshot: Schema.optional(
Schema.Struct({
mimeType: Schema.Literal("image/png"),
data: Schema.String,
width: Schema.Int,
height: Schema.Int,
}),
),
});
export type PreviewAutomationSnapshot = typeof PreviewAutomationSnapshot.Type;

Expand Down
Loading