From 58eaebede0161f7313209b8bfa55eba2ab2181eb Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Fri, 16 Oct 2026 16:04:31 +0200 Subject: [PATCH] test(e2e): Cover Vercel AI decisions in node-openai-decisions-send-to-sentry Co-Authored-By: Claude Opus 5.5 --- .../package.json | 9 ++- .../src/app.ts | 26 ++++++++ .../tests/send-to-sentry.test.ts | 65 +++++++++++++++++++ 3 files changed, 97 insertions(+), 3 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json index bd9ada7f04f1..8f6ee984c8a1 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json @@ -1,6 +1,6 @@ { "name": "node-openai-decisions-send-to-sentry", - "description": "The OpenAI integration for the Decisions API as a user runs it: a plain Sentry.init on an express app, real decisions.create requests against OpenAI, and the gen_ai.decide spans read back from a real Sentry project", + "description": "The OpenAI Decisions API as a user runs it: a plain Sentry.init on an express app, real decisions requests against OpenAI through the openai client and the Vercel AI SDK, and the gen_ai.decide spans read back from a real Sentry project", "version": "1.0.0", "private": true, "type": "module", @@ -10,13 +10,16 @@ "test": "playwright test", "clean": "npx rimraf node_modules dist pnpm-lock.yaml", "test:build": "pnpm install && pnpm build", - "test:build-latest": "pnpm install && pnpm add openai@latest && pnpm build", + "test:build-latest": "pnpm install && pnpm add openai@latest ai@latest @ai-sdk/openai@latest && pnpm build", "test:assert": "pnpm test" }, "dependencies": { + "@ai-sdk/openai": "4.0.95", "@sentry/node": "file:../../packed/sentry-node-packed.tgz", + "ai": "7.0.142", "express": "^4.21.2", - "openai": "7.32.0" + "openai": "7.32.0", + "zod": "^4.1.8" }, "devDependencies": { "@playwright/test": "1.64.1", diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts index f37d3198784c..20c7ba9a96c7 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts @@ -1,5 +1,7 @@ // `instrument.ts` is preloaded via `node --import`, so Sentry is already initialised here. +import { createOpenAI } from '@ai-sdk/openai'; import * as Sentry from '@sentry/node'; +import { experimental_decide } from 'ai'; import express from 'express'; import OpenAI from 'openai'; @@ -11,6 +13,7 @@ if (!apiKey) { } const client = new OpenAI({ apiKey }); +const aiSdkOpenAI = createOpenAI({ apiKey }); const MODEL = 'gpt-6-luna'; @@ -74,6 +77,29 @@ app.get('/decide-image', async (_req, res, next) => { } }); +// The same decision through the Vercel AI SDK, which calls the Decisions API itself rather than through +// the `openai` client, so the Vercel AI integration records it. +app.get('/vercel-ai-decide', async (_req, res, next) => { + try { + const { answers } = await experimental_decide({ + model: aiSdkOpenAI.decisionModel(MODEL), + state: 'I was charged twice this month and want a refund.', + questions: { + wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' }, + department: { + type: 'choice', + instructions: 'Which team should handle this?', + criteria: { billing: 'Charges and refunds', other: 'Anything else' }, + }, + urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'] }, + }, + }); + res.send({ traceId: Sentry.getActiveSpan()?.spanContext().traceId, answers }); + } catch (error) { + next(error); + } +}); + app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => { res.status(500).send({ message: error.message }); }); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts index 5116ba2ebe5b..7b0ae9c56e8f 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -157,3 +157,68 @@ test('records an image decision with its image data', async ({ baseURL }) => { { type: 'evaluation', answers: { is_red: { type: 'predicate', probability: expect.any(Number) } } }, ]); }); + +test('records a Vercel AI decision with the confidence of its answers', async ({ baseURL }) => { + const response = await fetch(`${baseURL}/vercel-ai-decide`); + const body = await response.json(); + expect(response.status, JSON.stringify(body)).toBe(200); + const { traceId }: { traceId: string } = body; + + console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); + + let spanId: string | undefined; + await expect + .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) + .toBeDefined(); + + let attributes: Record | undefined; + await expect + .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) + .toMatchObject({ + 'span.name': `decide ${MODEL}`, + [SEARCH_SPAN__OP]: GEN_AI_DECIDE, + [SEARCH_SPAN__STATUS]: 'ok', + [SEARCH_ORIGIN]: 'auto.vercelai.channel', + [SEARCH_GEN_AI__OPERATION__NAME]: 'decide', + [SEARCH_GEN_AI__PROVIDER__NAME]: 'openai.decision', + [SEARCH_GEN_AI__REQUEST__MODEL]: MODEL, + }); + + expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([ + { + type: 'evaluation', + state: 'I was charged twice this month and want a refund.', + questions: { + wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' }, + department: { + type: 'choice', + instructions: 'Which team should handle this?', + criteria: { billing: 'Charges and refunds', other: 'Anything else' }, + }, + urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'] }, + }, + }, + ]); + + // The OpenAI provider moves `confidence` into its provider metadata; the span must keep it on the answers. + expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([ + { + type: 'evaluation', + answers: { + wantsRefund: { type: 'boolean', probability: expect.any(Number) }, + department: { + type: 'choice', + choice: expect.stringMatching(/^(billing|other)$/), + probabilities: { billing: expect.any(Number), other: expect.any(Number) }, + confidence: expect.any(Number), + }, + urgency: { + type: 'score', + score: expect.any(Number), + probabilities: { 0: expect.any(Number), 1: expect.any(Number) }, + confidence: expect.any(Number), + }, + }, + }, + ]); +});