From 367ece8189c78a21066024c279b9c8f866ed7277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Mon, 5 Oct 2026 10:47:55 -0400 Subject: [PATCH 1/5] test(evals): Add search agent evals for regex log search Covers log searches that need key://pattern// regex, plus wildcard searches where regex isn't the better choice. The regex cases fail until the search agent's prompt teaches the syntax. Refs LOGS-1013 --- .../evals/search-events-agent-regex.eval.ts | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts new file mode 100644 index 000000000..953f3d161 --- /dev/null +++ b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts @@ -0,0 +1,120 @@ +import { SentryApiService } from "@sentry/mcp-core/api-client"; +import { searchEventsAgent } from "@sentry/mcp-core/tools/search-events/agent"; +import { describeEval, ToolCallScorer } from "vitest-evals"; +import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; +import "../setup-env"; + +function messageRegexPattern(query: unknown): string | undefined { + if (typeof query !== "string") { + return undefined; + } + return query.match(/(?:^|[\s(])!?message:\/\/(.+?)\/\/(?=[\s)]|$)/)?.[1]; +} + +describeEval("search-events-agent-regex", { + data: async () => [ + { + input: + "Show logs from the last day whose message matches the regex `timeout after \\d+ms`", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => { + const pattern = messageRegexPattern(value); + return ( + pattern !== undefined && + pattern.includes("timeout after") && + pattern.includes("\\d") + ); + }, + timeRange: { statsPeriod: "24h" }, + }, + }, + { + input: "Find logs whose message contains an IPv4 address", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => { + const pattern = messageRegexPattern(value); + return ( + pattern !== undefined && + /\\d|\[0-9\]/.test(pattern) && + pattern.includes("\\.") + ); + }, + }, + }, + { + input: + "Find error logs whose message ends with a 5xx status code, like 'upstream returned status=503'", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => { + const pattern = messageRegexPattern(value); + return ( + /\b(severity|level):error\b/.test(String(value)) && + pattern !== undefined && + pattern.includes("5") && + pattern.endsWith("$") + ); + }, + }, + }, + { + input: + "Show logs whose message does not look like 'job completed'", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => + typeof value === "string" && + value.includes("!message://") && + messageRegexPattern(value)?.includes("completed") === true, + }, + }, + { + input: "Show me error logs about database", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => + typeof value === "string" && + value.includes("*database*") && + !value.includes("://"), + }, + }, + { + input: "Find warning logs that mention memory", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => + typeof value === "string" && + value.includes("*memory*") && + !value.includes("://"), + }, + }, + ], + task: async (input) => { + const apiService = new SentryApiService({ accessToken: "test-token" }); + const agentResult = await searchEventsAgent({ + query: input, + organizationSlug: "sentry-mcp-evals", + apiService, + }); + + return { + result: JSON.stringify(agentResult.result), + toolCalls: agentResult.toolCalls.map((call) => ({ + name: call.toolName, + arguments: + typeof call.args === "object" && call.args !== null + ? { ...call.args } + : {}, + })), + }; + }, + scorers: [ToolCallScorer(), StructuredOutputScorer({ match: "fuzzy" })], +}); From 8436c921dd56ebf2dd4b5cb32e37aa2cf9d29716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Mon, 5 Oct 2026 14:17:00 -0400 Subject: [PATCH 2/5] test(evals): Cover case-insensitive, UUID, wildcard, and non-logs regex cases Adds evals for (?i) patterns, UUIDs within the 64-character limit, prefix and either-or searches that should stay wildcards, and spans or errors searches that must not get a regex. Refs LOGS-1013 Co-Authored-By: Claude Opus 5.5 --- .../evals/search-events-agent-regex.eval.ts | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts index 953f3d161..2243a2769 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts @@ -11,6 +11,14 @@ function messageRegexPattern(query: unknown): string | undefined { return query.match(/(?:^|[\s(])!?message:\/\/(.+?)\/\/(?=[\s)]|$)/)?.[1]; } +function regexPatternLength(pattern: string): number { + return pattern.replace(/\\./g, "_").length; +} + +function hasRegexFilter(query: unknown): boolean { + return typeof query === "string" && /:\/\/.+\/\/(?=[\s)]|$)/.test(query); +} + describeEval("search-events-agent-regex", { data: async () => [ { @@ -74,6 +82,82 @@ describeEval("search-events-agent-regex", { messageRegexPattern(value)?.includes("completed") === true, }, }, + { + input: + "Find logs whose message looks like 'connection refused on port ', ignoring case", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => { + const pattern = messageRegexPattern(value); + return ( + pattern !== undefined && + pattern.startsWith("(?i)") && + pattern.toLowerCase().includes("connection refused") && + pattern.includes("\\d") + ); + }, + }, + }, + { + input: "Find logs whose message contains a UUID", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => { + const pattern = messageRegexPattern(value); + return ( + pattern !== undefined && + pattern.includes("-") && + /\{(4|8|12|36)\}/.test(pattern) && + regexPatternLength(pattern) <= 64 + ); + }, + }, + }, + { + input: "Find logs whose message starts with 'Worker shutting down'", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => + typeof value === "string" && + value.includes("Worker shutting down*") && + !value.includes("*Worker shutting down") && + !hasRegexFilter(value), + }, + }, + { + input: "Find logs that mention either 'cache miss' or 'cache evicted'", + expectedTools: [], + expected: { + dataset: "logs", + query: (value: unknown) => + typeof value === "string" && + value.includes("*cache miss*") && + value.includes("*cache evicted*") && + !hasRegexFilter(value), + }, + }, + { + input: + "Find spans whose description looks like 'GET /api/users/'", + expectedTools: [], + expected: { + dataset: "spans", + query: (value: unknown) => + typeof value === "string" && !hasRegexFilter(value), + }, + }, + { + input: "Find errors whose message looks like 'timeout after ms'", + expectedTools: [], + expected: { + dataset: "errors", + query: (value: unknown) => + typeof value === "string" && !hasRegexFilter(value), + }, + }, { input: "Show me error logs about database", expectedTools: [], From 6f2342aaa326040f0599513d04a83833ef53f70d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Mon, 5 Oct 2026 15:21:31 -0400 Subject: [PATCH 3/5] test(evals): Make the regex evals reject queries Sentry would refuse Regex matchers now reject over-long, lookaround, backreference, and over-escaped patterns, the non-logs cases must keep the user's text, and the plain-text guards no longer copy prompt examples. Refs LOGS-1013 Co-Authored-By: Claude Opus 5.5 --- .../evals/search-events-agent-regex.eval.ts | 49 +++++++++++++------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts index 2243a2769..a3b702aef 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts @@ -8,13 +8,26 @@ function messageRegexPattern(query: unknown): string | undefined { if (typeof query !== "string") { return undefined; } - return query.match(/(?:^|[\s(])!?message:\/\/(.+?)\/\/(?=[\s)]|$)/)?.[1]; + const pattern = query.match( + /(?:^|[\s(])!?(?:message|log\.body):\/\/(.+?)\/\/(?=[\s)]|$)/, + )?.[1]; + return pattern !== undefined && isSentryRegex(pattern) ? pattern : undefined; } function regexPatternLength(pattern: string): number { return pattern.replace(/\\./g, "_").length; } +function isSentryRegex(pattern: string): boolean { + return ( + regexPatternLength(pattern) <= 64 && + !/\(\? { const pattern = messageRegexPattern(value); return ( - /\b(severity|level):error\b/.test(String(value)) && + /\bseverity:error\b/.test(String(value)) && pattern !== undefined && pattern.includes("5") && pattern.endsWith("$") @@ -78,7 +91,7 @@ describeEval("search-events-agent-regex", { dataset: "logs", query: (value: unknown) => typeof value === "string" && - value.includes("!message://") && + /!(?:message|log\.body):\/\//.test(value) && messageRegexPattern(value)?.includes("completed") === true, }, }, @@ -92,8 +105,11 @@ describeEval("search-events-agent-regex", { const pattern = messageRegexPattern(value); return ( pattern !== undefined && - pattern.startsWith("(?i)") && - pattern.toLowerCase().includes("connection refused") && + /^\(\?i[):]/.test(pattern) && + pattern + .toLowerCase() + .replace(/\\s[+*]?/g, " ") + .includes("connection refused") && pattern.includes("\\d") ); }, @@ -109,8 +125,7 @@ describeEval("search-events-agent-regex", { return ( pattern !== undefined && pattern.includes("-") && - /\{(4|8|12|36)\}/.test(pattern) && - regexPatternLength(pattern) <= 64 + /\{(4|8|12|36)\}/.test(pattern) ); }, }, @@ -146,7 +161,9 @@ describeEval("search-events-agent-regex", { expected: { dataset: "spans", query: (value: unknown) => - typeof value === "string" && !hasRegexFilter(value), + typeof value === "string" && + value.includes("/api/users/") && + !hasRegexFilter(value), }, }, { @@ -155,29 +172,31 @@ describeEval("search-events-agent-regex", { expected: { dataset: "errors", query: (value: unknown) => - typeof value === "string" && !hasRegexFilter(value), + typeof value === "string" && + value.includes("timeout after") && + !hasRegexFilter(value), }, }, { - input: "Show me error logs about database", + input: "Show me error logs about payments", expectedTools: [], expected: { dataset: "logs", query: (value: unknown) => typeof value === "string" && - value.includes("*database*") && - !value.includes("://"), + value.includes("*payment") && + !hasRegexFilter(value), }, }, { - input: "Find warning logs that mention memory", + input: "Find warning logs that mention disk space", expectedTools: [], expected: { dataset: "logs", query: (value: unknown) => typeof value === "string" && - value.includes("*memory*") && - !value.includes("://"), + value.includes("*disk") && + !hasRegexFilter(value), }, }, ], From c0133c565c85fdb83c4078620574458a1d9422d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Mon, 5 Oct 2026 15:37:47 -0400 Subject: [PATCH 4/5] test(evals): Reject more invalid RE2 and accept [0-9] for ignoring case The regex check now also rejects possessive quantifiers, atomic groups, and escapes RE2 refuses, and the ignoring-case case accepts [0-9] as well as \d. Refs LOGS-1013 Co-Authored-By: Claude Opus 5.5 --- .../src/evals/search-events-agent-regex.eval.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts index a3b702aef..ce8014352 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts @@ -11,18 +11,19 @@ function messageRegexPattern(query: unknown): string | undefined { const pattern = query.match( /(?:^|[\s(])!?(?:message|log\.body):\/\/(.+?)\/\/(?=[\s)]|$)/, )?.[1]; - return pattern !== undefined && isSentryRegex(pattern) ? pattern : undefined; + return pattern !== undefined && isLikelySentryRegex(pattern) ? pattern : undefined; } function regexPatternLength(pattern: string): number { return pattern.replace(/\\./g, "_").length; } -function isSentryRegex(pattern: string): boolean { +function isLikelySentryRegex(pattern: string): boolean { return ( regexPatternLength(pattern) <= 64 && - !/\(\?)/.test(pattern) && + !/(? Date: Mon, 5 Oct 2026 16:13:02 -0400 Subject: [PATCH 5/5] test(evals): Match RE2 more closely in the regex check and format the file The possessive check no longer trips on character classes or \p{...}+, more escapes RE2 refuses are rejected, and the file is biome-formatted. Refs LOGS-1013 Co-Authored-By: Claude Opus 5.5 --- .../src/evals/search-events-agent-regex.eval.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts index ce8014352..2c127f319 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts @@ -11,7 +11,9 @@ function messageRegexPattern(query: unknown): string | undefined { const pattern = query.match( /(?:^|[\s(])!?(?:message|log\.body):\/\/(.+?)\/\/(?=[\s)]|$)/, )?.[1]; - return pattern !== undefined && isLikelySentryRegex(pattern) ? pattern : undefined; + return pattern !== undefined && isLikelySentryRegex(pattern) + ? pattern + : undefined; } function regexPatternLength(pattern: string): number { @@ -22,8 +24,8 @@ function isLikelySentryRegex(pattern: string): boolean { return ( regexPatternLength(pattern) <= 64 && !/\(\?(?:)/.test(pattern) && - !/(?