From e7527e4a73edacfa90da5da3055878a32bf751c0 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 12 Oct 2026 23:25:40 -0700 Subject: [PATCH 01/12] improvement(search): any-term recall fallback, passage previews, batched MCP reads --- .../v2/files/uploads/[uploadId]/route.test.ts | 3 +- apps/sim/lib/api/contracts/knowledge/mcp.ts | 34 +++++- .../api/server/routes/v2-json-route.test.ts | 6 +- .../lib/api/server/routes/v2-json-route.ts | 9 +- .../sim/lib/core/rate-limiter/rate-limiter.ts | 12 +- apps/sim/lib/knowledge/mcp/server.test.ts | 62 ++++++++++ apps/sim/lib/knowledge/mcp/server.ts | 111 ++++++++++++------ apps/sim/lib/knowledge/mcp/tool-runner.ts | 4 +- .../lib/sim-search/live/account-session.ts | 31 +++-- .../lib/sim-search/live/application.test.ts | 81 +++++++++++++ apps/sim/lib/sim-search/live/application.ts | 39 +++++- apps/sim/lib/sim-search/live/atlassian.ts | 18 ++- apps/sim/lib/sim-search/live/google.ts | 23 +++- apps/sim/lib/sim-search/live/providers.ts | 29 ++++- apps/sim/lib/sim-search/live/scoped-search.ts | 8 +- .../lib/sim-search/live/service-session.ts | 6 +- .../lib/sim-search/live/term-fallback.test.ts | 105 +++++++++++++++++ apps/sim/lib/sim-search/live/term-fallback.ts | 75 ++++++++++++ .../lib/sim-search/live/text-match.test.ts | 31 +++++ apps/sim/lib/sim-search/live/text-match.ts | 89 ++++++++++++++ apps/sim/lib/sim-search/live/types.ts | 3 + 21 files changed, 700 insertions(+), 79 deletions(-) create mode 100644 apps/sim/lib/sim-search/live/term-fallback.test.ts create mode 100644 apps/sim/lib/sim-search/live/term-fallback.ts create mode 100644 apps/sim/lib/sim-search/live/text-match.test.ts create mode 100644 apps/sim/lib/sim-search/live/text-match.ts diff --git a/apps/sim/app/api/v2/files/uploads/[uploadId]/route.test.ts b/apps/sim/app/api/v2/files/uploads/[uploadId]/route.test.ts index 9638d18ff15..7a90a0f3ece 100644 --- a/apps/sim/app/api/v2/files/uploads/[uploadId]/route.test.ts +++ b/apps/sim/app/api/v2/files/uploads/[uploadId]/route.test.ts @@ -88,7 +88,8 @@ describe('GET /api/v2/files/uploads/[uploadId]', () => { expect(v2RouteMocks.operationRate).toHaveBeenCalledWith( expect.stringContaining('files.upload.read'), - expect.anything() + expect.anything(), + 1 ) expect(v2RouteMocks.operationRate).not.toHaveBeenCalledWith( expect.stringContaining('files.upload.cancel'), diff --git a/apps/sim/lib/api/contracts/knowledge/mcp.ts b/apps/sim/lib/api/contracts/knowledge/mcp.ts index 9a3d279959a..c0e381a8b2d 100644 --- a/apps/sim/lib/api/contracts/knowledge/mcp.ts +++ b/apps/sim/lib/api/contracts/knowledge/mcp.ts @@ -17,7 +17,39 @@ export const organizationKnowledgeMcpContract = defineRouteContract({ }) export const liveSearchMcpSchema = searchWorkspaceInputSchema.strict() -export const readLiveDocumentMcpSchema = readDocumentInputSchema.strict() +/** Documents one read_document call may read; each counts as one read against the rate limit. */ +export const MAX_MCP_READ_DOCUMENTS = 5 + +export const readLiveDocumentMcpSchema = readDocumentInputSchema + .extend({ + documentId: readDocumentInputSchema.shape.documentId.optional(), + documentIds: z + .array(readDocumentInputSchema.shape.documentId) + .min(1) + .max(MAX_MCP_READ_DOCUMENTS) + .optional() + .describe( + `Up to ${MAX_MCP_READ_DOCUMENTS} document IDs returned by search, read together from the beginning. Use instead of documentId.` + ), + }) + .strict() + .superRefine((input, context) => { + if (Boolean(input.documentId) === Boolean(input.documentIds)) + context.addIssue({ + code: 'custom', + path: ['documentId'], + message: 'Pass either documentId or documentIds.', + }) + if ( + input.documentIds && + (input.startChunkIndex !== undefined || input.startOffset !== undefined) + ) + context.addIssue({ + code: 'custom', + path: ['documentIds'], + message: 'Continue a document with documentId and its next offsets, not documentIds.', + }) + }) export const chatSearchMcpSchema = workspaceSearchFiltersSchema .extend({ diff --git a/apps/sim/lib/api/server/routes/v2-json-route.test.ts b/apps/sim/lib/api/server/routes/v2-json-route.test.ts index c287431dfc5..b58799290af 100644 --- a/apps/sim/lib/api/server/routes/v2-json-route.test.ts +++ b/apps/sim/lib/api/server/routes/v2-json-route.test.ts @@ -278,11 +278,13 @@ describe('defineV2JsonRoute', () => { expect(v2RouteMocks.operationRate).toHaveBeenCalledTimes(2) expect(v2RouteMocks.operationRate).toHaveBeenCalledWith( 'v2:widgets.update:api-key:key-1', - expect.objectContaining({ maxTokens: 100 }) + expect.objectContaining({ maxTokens: 100 }), + 1 ) expect(v2RouteMocks.operationRate).toHaveBeenCalledWith( 'v2:widgets.update:user:user-1', - expect.objectContaining({ maxTokens: 100 }) + expect.objectContaining({ maxTokens: 100 }), + 1 ) expect(response.headers.get('X-RateLimit-Limit')).toBe('100') expect(response.headers.get('X-RateLimit-Remaining')).toBe('12') diff --git a/apps/sim/lib/api/server/routes/v2-json-route.ts b/apps/sim/lib/api/server/routes/v2-json-route.ts index 69f219a6d37..ffab2e745f6 100644 --- a/apps/sim/lib/api/server/routes/v2-json-route.ts +++ b/apps/sim/lib/api/server/routes/v2-json-route.ts @@ -83,17 +83,19 @@ export const v2ApiKeyAuth = { export interface V2RateLimitPolicy { readonly kind: 'public_api' + /** `cost` charges a request that performs several operations, such as a batched read. */ enforce( request: NextRequest, auth: V2ApiKeyAuthContext, - operation: ApplicationOperation + operation: ApplicationOperation, + cost?: number ): Promise } export const v2RateLimits = { publicApi: { kind: 'public_api', - async enforce(request, auth, operation) { + async enforce(request, auth, operation, cost = 1) { const plan = (auth.rateLimitSubscription?.plan ?? 'free') as SubscriptionPlan const config = getRateLimit(plan, 'api-endpoint') const buckets = await Promise.all( @@ -101,7 +103,8 @@ export const v2RateLimits = { try { return await rateLimiter.checkRateLimitDirectOrThrow( `v2:${operation.id}:${subjectId}`, - config + config, + cost ) } catch (error) { throw new V2RouteInfrastructureError('rate_limit', error) diff --git a/apps/sim/lib/core/rate-limiter/rate-limiter.ts b/apps/sim/lib/core/rate-limiter/rate-limiter.ts index d2e132ecf1e..838ced01b68 100644 --- a/apps/sim/lib/core/rate-limiter/rate-limiter.ts +++ b/apps/sim/lib/core/rate-limiter/rate-limiter.ts @@ -227,15 +227,17 @@ export class RateLimiter { } /** - * Consume one token from an already-namespaced bucket and propagate storage - * failures. Declarative API adapters use this for credential/workspace - * buckets so an unavailable limiter is never mistaken for spare capacity. + * Consume tokens (one by default) from an already-namespaced bucket and + * propagate storage failures. Declarative API adapters use this for + * credential/workspace buckets so an unavailable limiter is never mistaken + * for spare capacity. */ async checkRateLimitDirectOrThrow( storageKey: string, - config: { maxTokens: number; refillRate: number; refillIntervalMs: number } + config: { maxTokens: number; refillRate: number; refillIntervalMs: number }, + tokens = 1 ): Promise { - const result = await this.storage.consumeTokens(storageKey, 1, config) + const result = await this.storage.consumeTokens(storageKey, tokens, config) if (!result.allowed) { logger.info('Rate limit exceeded', { storageKey, tokensRemaining: result.tokensRemaining }) } diff --git a/apps/sim/lib/knowledge/mcp/server.test.ts b/apps/sim/lib/knowledge/mcp/server.test.ts index 087fd8b2bd1..f7ec4449a40 100644 --- a/apps/sim/lib/knowledge/mcp/server.test.ts +++ b/apps/sim/lib/knowledge/mcp/server.test.ts @@ -163,6 +163,68 @@ describe('live organization Search MCP', () => { ) }) +describe('batched document reads', () => { + const readResult = (documentId: string) => ({ + documentId, + knowledgeBaseId: '', + knowledgeBaseName: 'Drive', + documentName: documentId, + sourceUrl: `https://example.com/${documentId}`, + connectorType: 'google_drive', + chunks: [], + hasMore: false, + next: null, + }) + const parsedBody = (result: CallToolResult) => { + const [part] = result.content + if (part?.type !== 'text') throw new Error('Expected a text result') + return JSON.parse(part.text) as { documents: Record[] } + } + + it('reports an unreadable document in its place and returns the others', async () => { + create() + mocks.liveRead.mockImplementation(async ({ input }: { input: { documentId: string } }) => { + if (input.documentId === 'gone') throw new OrchestrationError('not_found', 'Document moved') + if (input.documentId === 'broken') throw new Error('private backend detail') + return readResult(input.documentId) + }) + const result = await call('read_document', { documentIds: ['a', 'gone', 'broken', 'b'] }) + expect(result.isError).toBeUndefined() + expect(parsedBody(result).documents).toEqual([ + expect.objectContaining({ documentId: 'a', title: 'a' }), + { documentId: 'gone', error: 'Document moved' }, + { documentId: 'broken', error: 'Unable to read this document. Please try again.' }, + expect.objectContaining({ documentId: 'b', title: 'b' }), + ]) + }) + + it('charges the rate limit once per distinct document', async () => { + create() + mocks.liveRead.mockImplementation(async ({ input }: { input: { documentId: string } }) => + readResult(input.documentId) + ) + await call('read_document', { documentIds: ['a', 'b', 'a'] }) + expect(mocks.rateLimit).toHaveBeenCalledWith(request, auth, expect.anything(), 2) + expect(mocks.liveRead).toHaveBeenCalledTimes(2) + }) + + it('reads nothing when the batch exceeds the remaining rate limit', async () => { + create() + mocks.rateLimit.mockResolvedValueOnce(new Response(null, { status: 429 })) + expect((await call('read_document', { documentIds: ['a', 'b'] })).isError).toBe(true) + expect(mocks.liveRead).not.toHaveBeenCalled() + }) + + it.each([ + [{ documentId: 'a', documentIds: ['b'] }], + [{}], + [{ documentIds: ['a'], startOffset: 8000 }], + ])('rejects ambiguous read input %j', (input) => { + create() + expect(() => mocks.configs.get('read_document')?.inputSchema.parse(input)).toThrow() + }) +}) + describe('organization Search MCP tools', () => { it('does not return content denied by the canonical document ACL operation', async () => { create() diff --git a/apps/sim/lib/knowledge/mcp/server.ts b/apps/sim/lib/knowledge/mcp/server.ts index 449b1989d58..3e0d3a6c465 100644 --- a/apps/sim/lib/knowledge/mcp/server.ts +++ b/apps/sim/lib/knowledge/mcp/server.ts @@ -31,6 +31,28 @@ import { v2CaughtOrchestrationError } from '@/app/api/v2/lib/response' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const logger = createLogger('KnowledgeMcp') + +/** The message a caller may see for an expected failure; undefined for an internal error. */ +async function callerErrorMessage(error: unknown): Promise { + if (error instanceof LiveReadError) return error.message + const response = v2CaughtOrchestrationError(error) + if (!response) return undefined + const body: unknown = await response.json() + return isPlainRecord(body) && isPlainRecord(body.error) && typeof body.error.message === 'string' + ? body.error.message + : undefined +} + +/** One read_document result in the shape the tool returns it. */ +function projectReadResult(result: Awaited>) { + const { knowledgeBaseId: _knowledgeBaseId, knowledgeBaseName: _name, ...document } = result + return { + ...document, + title: result.documentName, + citationId: liveCitationId(result.documentId), + citationUrl: result.sourceUrl, + } +} interface KnowledgeMcpContext { organizationId: string request: NextRequest @@ -43,13 +65,13 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe const principal = auth.principal const server = new McpServer({ name: 'Sim Search', version: '1.0.0' }) - const execute: KnowledgeMcpToolRunner = async (toolName, operation, toolSignal, run) => { + const execute: KnowledgeMcpToolRunner = async (toolName, operation, toolSignal, run, cost) => { const startedAt = performance.now() const signal = AbortSignal.any([request.signal, toolSignal]) let outcome: SearchMcpActivityInput['outcome'] = 'error' try { signal.throwIfAborted() - const limited = await v2RateLimits.publicApi.enforce(request, auth, operation) + const limited = await v2RateLimits.publicApi.enforce(request, auth, operation, cost) signal.throwIfAborted() if (limited) { outcome = 'rate_limited' @@ -68,18 +90,8 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe return result } catch (error) { if (signal.aborted) outcome = 'cancelled' - if (error instanceof LiveReadError) return toolError(error.message) - const response = v2CaughtOrchestrationError(error) - if (response) { - const body: unknown = await response.json() - if ( - isPlainRecord(body) && - isPlainRecord(body.error) && - typeof body.error.message === 'string' - ) { - return toolError(body.error.message) - } - } + const message = await callerErrorMessage(error) + if (message) return toolError(message) logger.error('Knowledge MCP operation failed', { operation: operation.id, error }) return toolError('Unable to complete this operation. Please try again.') } finally { @@ -111,7 +123,7 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe { title: 'Search', description: - 'Search this organization’s sources through their live APIs, within your access and the admin’s source settings. Use source and date filters to narrow results, or nativeQueries for provider queries and pagination. Inspect live.accounts for provider status and continuation cursors, and live.guidance for query syntax. Results are candidates, not proof of complete coverage. Use read_document with the exact returned documentId for context and cite citationUrl when available.', + 'Search this organization’s sources through their live APIs, within your access and the admin’s source settings. A plain query searches every source; use source and date filters to narrow results, and nativeQueries only for a provider’s own operators (exact phrases, fields such as a Jira project or Confluence space, OR across alternatives) or a continuation cursor. Each result carries a passage around its match. Lower topK on follow-up searches to keep results compact. Inspect live.accounts for provider status and continuation cursors, and live.guidance for query syntax. Results are candidates, not proof of complete coverage. When a passage does not answer the question, use read_document with the exact returned documentId, and cite citationUrl when available.', inputSchema: liveSearchMcpSchema, annotations: KNOWLEDGE_MCP_READ_ONLY, }, @@ -159,38 +171,59 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe { title: 'Read document', description: - 'Read a live document using the exact documentId returned by search. Access and the admin’s source settings are checked again on every read. When hasMore is true, pass next.startChunkIndex and next.startOffset with the same documentId to continue. Cite citationUrl when available.', + 'Read live documents using the exact document IDs returned by search: documentId for one, or documentIds to read several in one call rather than one call each. Access and the admin’s source settings are checked again on every read. When a document’s hasMore is true, pass its next.startChunkIndex and next.startOffset with the same documentId to continue. Cite citationUrl when available.', inputSchema: readLiveDocumentMcpSchema, annotations: KNOWLEDGE_MCP_READ_ONLY, }, - async (raw: unknown, extra: { signal: AbortSignal }) => - execute( + async (raw: unknown, extra: { signal: AbortSignal }) => { + const parsed = readLiveDocumentMcpSchema.safeParse(raw) + const batch = + parsed.success && parsed.data.documentIds ? [...new Set(parsed.data.documentIds)] : [] + return execute( 'read_document', knowledgeOperations.readDocument, extra.signal, async (registry, signal) => { - const input = readLiveDocumentMcpSchema.parse(raw) - const result = await readLiveDocument.execute({ - principal, - input: { ...input, organizationId, resultSecretRegistry: registry, signal }, - request, - }) - const { - knowledgeBaseId: _knowledgeBaseId, - knowledgeBaseName: _name, - ...document - } = result - return projectResult( - { - ...document, - title: result.documentName, - citationId: liveCitationId(result.documentId), - citationUrl: result.sourceUrl, - }, - registry + if (!parsed.success) throw parsed.error + const { documentId, documentIds: _documentIds, ...options } = parsed.data + const read = (id: string) => + readLiveDocument.execute({ + principal, + input: { + ...options, + documentId: id, + organizationId, + resultSecretRegistry: registry, + signal, + }, + request, + }) + if (documentId) return projectResult(projectReadResult(await read(documentId)), registry) + /** One unreadable document leaves the others; each failure is reported in its place. */ + const documents = await Promise.all( + batch.map(async (id) => { + try { + return projectReadResult(await read(id)) + } catch (error) { + signal.throwIfAborted() + const message = await callerErrorMessage(error) + if (!message) + logger.error('Knowledge MCP batched read failed', { + operation: knowledgeOperations.readDocument.id, + error, + }) + return { + documentId: id, + error: message ?? 'Unable to read this document. Please try again.', + } + } + }) ) - } + return projectResult({ documents }, registry) + }, + Math.max(1, batch.length) ) + } ) server.registerTool( @@ -198,7 +231,7 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe { title: 'Chat', description: - 'Ask the Sim Assistant to answer a question using your accessible organization sources. Returns an answer with citations and starts a new private conversation. Use source, modifiedAfter, or documentIds to narrow the evidence. No web search or source changes. Use search instead when you need raw passages.', + 'Ask the Sim Assistant to research a question in your accessible organization sources and write a cited answer. It runs its own multi-step search, so it usually takes 30 to 60 seconds and returns a synthesized answer rather than the sources. To answer a question yourself, use search and read_document, which return sources in seconds; use chat only when the user asks for the Sim Assistant’s answer. Starts a new private conversation. Use source, modifiedAfter, or documentIds to narrow the evidence. No web search or source changes.', inputSchema: chatSearchMcpSchema, annotations: { readOnlyHint: false, diff --git a/apps/sim/lib/knowledge/mcp/tool-runner.ts b/apps/sim/lib/knowledge/mcp/tool-runner.ts index 8b03011a7fa..a3828accab6 100644 --- a/apps/sim/lib/knowledge/mcp/tool-runner.ts +++ b/apps/sim/lib/knowledge/mcp/tool-runner.ts @@ -22,7 +22,9 @@ export type KnowledgeMcpToolRunner = ( toolName: SearchMcpActivityInput['toolName'], operation: ApplicationOperation, toolSignal: AbortSignal, - run: (registry: ResolvedSecretTraceRegistry, signal: AbortSignal) => Promise + run: (registry: ResolvedSecretTraceRegistry, signal: AbortSignal) => Promise, + /** Operations the call performs for the rate limit, such as each document a batched read reads. */ + cost?: number ) => Promise /** A tool result projected for its resolved secrets and bounded in size. */ diff --git a/apps/sim/lib/sim-search/live/account-session.ts b/apps/sim/lib/sim-search/live/account-session.ts index b230a4aca56..0092b4cff55 100644 --- a/apps/sim/lib/sim-search/live/account-session.ts +++ b/apps/sim/lib/sim-search/live/account-session.ts @@ -22,9 +22,15 @@ import type { LiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' import { livePolicyFor, loadLiveSearchPolicies } from '@/lib/sim-search/live/policy-store' import { LIVE_SEARCH_PROVIDER_CATALOG } from '@/lib/sim-search/live/provider-catalog' import { isSearchProviderEnabled } from '@/lib/sim-search/live/provider-rollout' -import { readNativeProvider, searchNativeProvider } from '@/lib/sim-search/live/providers' +import { + previewNativeText, + readNativeProvider, + requiresEveryTerm, + searchNativeProvider, +} from '@/lib/sim-search/live/providers' import { searchWithinPolicy } from '@/lib/sim-search/live/scoped-search' import { createLiveServiceSession } from '@/lib/sim-search/live/service-session' +import { searchWithAnyTermFallback } from '@/lib/sim-search/live/term-fallback' import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types' import { readZoomMcp, searchZoomMcp } from '@/lib/sim-search/live/zoom-mcp' @@ -45,6 +51,8 @@ export interface LiveAccountSession { /** True only when the document is inside the source boundary and the member may read it. */ verify(document: Reference): Promise read(reference: Reference, filters?: WorkspaceSearchFilters): Promise + /** Full text of a verified search result whose provider returned only an excerpt or metadata. */ + previewText(document: NativeDocument): Promise /** Checks a fetched document against the source settings saved now, not at session start. */ verifyCurrent(document: NativeDocument): Promise } @@ -189,12 +197,17 @@ export async function openLiveAccountSession( policy: boundary.policy, servicePartial: boundary.partial, async search(search) { - const scoped = boundary.scopeSearch ? boundary.scopeSearch(search) : search - if (scoped === null) return { documents: [] } - if (admin) return admin.search(scoped) - return searchWithinPolicy(provider, client, scoped, (request) => - client ? searchNativeProvider(provider, client, request) : searchMcp(request) - ) + const searchOnce = async (request: NativeSearchInput): Promise => { + const scoped = boundary.scopeSearch ? boundary.scopeSearch(request) : request + if (scoped === null) return { documents: [] } + if (admin) return admin.search(scoped) + return searchWithinPolicy(provider, client, scoped, (native) => + client ? searchNativeProvider(provider, client, native) : searchMcp(native) + ) + } + return client && requiresEveryTerm(provider) + ? searchWithAnyTermFallback(searchOnce, search) + : searchOnce(search) }, async verify(document) { if (!(await boundary.verify(document))) return false @@ -211,6 +224,10 @@ export async function openLiveAccountSession( }) return readMcp(reference) }, + async previewText(document) { + if (admin || !client) return undefined + return previewNativeText(provider, client, document) + }, async verifyCurrent(document) { if (!(await isSearchProviderEnabled(provider, resourceScopeFromOwner(owner)))) return false const current = await sourceBoundary( diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index d30e97119a7..2dbda040586 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -26,6 +26,8 @@ const mocks = vi.hoisted(() => ({ json: vi.fn(), service: vi.fn(), destroyPool: vi.fn(), + requiresEveryTerm: vi.fn(), + preview: vi.fn(), })) vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) vi.mock('@/lib/sim-search/live/service-session', () => ({ @@ -65,6 +67,8 @@ vi.mock('@/lib/sim-search/live/providers', () => ({ liveSearchGuidance: (providers: string[]) => `Live coverage: ${[...new Set(providers)]}`, searchNativeProvider: mocks.search, readNativeProvider: mocks.read, + requiresEveryTerm: mocks.requiresEveryTerm, + previewNativeText: mocks.preview, })) vi.mock('@/executor/utils/resolved-secret-content-projection', () => ({ projectResolvedSecretModelContent: (value: string) => ({ safe: true, value }), @@ -136,6 +140,8 @@ describe('authorized live retrieval', () => { documents: [{ ...document, id: 'src/a.ts', container: '42', kind: 'code' }], }) mocks.adminVerify.mockResolvedValue(true) + mocks.requiresEveryTerm.mockReset().mockReturnValue(false) + mocks.preview.mockReset().mockResolvedValue(undefined) }) describe('Zoom continuation integrity', () => { const connected = { @@ -611,6 +617,81 @@ describe('authorized live retrieval', () => { expect(result.live?.accounts.map((status) => status.queryIndex)).toEqual([0, 1]) expect(result.live?.guidance).toBe('Live coverage: slack') }) + describe('result previews', () => { + const excerpts = ['a', 'b', 'c', 'd'].map((id) => ({ + ...document, + id, + url: `https://docs.google.com/document/d/${id}/edit`, + content: `Excerpt ${id}`, + })) + const contents = (result: Awaited>) => + result.results.map((row) => row.content) + + it('replaces the excerpts of an account’s top results with their full text', async () => { + mocks.search.mockResolvedValue({ documents: excerpts }) + mocks.preview.mockImplementation(async (_provider, _client, { id }: { id: string }) => + id === 'a' ? undefined : `Full ${id} launch text` + ) + const result = await searchLiveKnowledge.execute({ principal, input }) + expect(contents(result)).toEqual([ + 'Excerpt a', + 'Full b launch text', + 'Full c launch text', + 'Excerpt d', + ]) + expect(mocks.preview).toHaveBeenCalledTimes(3) + }) + + it('keeps the provider excerpt when a preview read fails', async () => { + mocks.search.mockResolvedValue({ documents: excerpts.slice(0, 2) }) + mocks.preview + .mockRejectedValueOnce(new NativeSearchError('rate_limited', 'Slow down')) + .mockResolvedValueOnce('Full b launch text') + const result = await searchLiveKnowledge.execute({ principal, input }) + expect(contents(result)).toEqual(['Excerpt a', 'Full b launch text']) + expect(result.live?.accounts[0]?.status).toBe('ok') + }) + + it('keeps the provider excerpt when a preview read misses its deadline', async () => { + vi.useFakeTimers() + try { + mocks.search.mockResolvedValue({ documents: excerpts.slice(0, 1) }) + mocks.preview.mockReturnValue(new Promise(() => {})) + const pending = searchLiveKnowledge.execute({ principal, input }) + await vi.advanceTimersByTimeAsync(2500) + expect(contents(await pending)).toEqual(['Excerpt a']) + } finally { + vi.useRealTimers() + } + }) + + it('does not read previews for a listing without search terms', async () => { + mocks.search.mockResolvedValue({ documents: excerpts }) + await searchLiveKnowledge.execute({ + principal, + input: { ...input, query: '', filters: { sortBy: 'newest' } }, + }) + expect(mocks.preview).not.toHaveBeenCalled() + }) + }) + + it('retries an underfilled search with any term for providers that require every term', async () => { + mocks.requiresEveryTerm.mockReturnValue(true) + const loose = { ...document, id: 'loose', url: 'https://docs.google.com/document/d/loose/edit' } + mocks.search.mockImplementation(async (_provider, _client, search) => ({ + documents: search.textMatch === 'any' ? [loose, document] : [document], + })) + const result = await searchLiveKnowledge.execute({ + principal, + input: { ...input, query: 'launch checklist' }, + }) + expect(result.results.map((row) => decodeLiveReference(row.documentId).id)).toEqual([ + 'doc', + 'loose', + ]) + expect(mocks.search).toHaveBeenCalledTimes(2) + }) + it('checks the service source before returning member-visible candidates', async () => { const verify = vi.fn().mockResolvedValue(false) mocks.service.mockResolvedValue({ policy: defaultLiveSearchPolicy(), verify, partial: false }) diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 65ca860549c..796edbf7b34 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -2,6 +2,7 @@ import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js' import { requirePrincipalSubjectUserId } from '@sim/auth/principal' import { safeCompare } from '@sim/security/compare' import { hmacSha256Hex } from '@sim/security/hmac' +import { sleep } from '@sim/utils/helpers' import { compareStrings } from '@sim/utils/string' import { z } from 'zod' import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge' @@ -307,6 +308,36 @@ async function verifyCandidates(session: LiveAccountSession, candidates: LiveCan } } +/** Top results per account read for a passage when their provider returns only an excerpt. */ +const PREVIEW_READS = 3 +/** A slow preview read keeps the provider's excerpt rather than delaying the search. */ +const PREVIEW_DEADLINE_MS = 2500 + +/** + * Replaces the excerpt or metadata of an account's top results with their full text, so their + * preview is a passage around the match like other providers' results rather than a prompt to + * read. A read that fails or misses the deadline keeps the provider's own preview. + */ +async function withPreviewText( + session: LiveAccountSession, + candidates: LiveCandidate[] +): Promise { + const texts = await Promise.all( + candidates + .slice(0, PREVIEW_READS) + .map((candidate) => + Promise.race([ + session.previewText(candidate.document).catch(() => undefined), + sleep(PREVIEW_DEADLINE_MS).then(() => undefined), + ]) + ) + ) + return candidates.map((candidate, index) => { + const text = texts[index]?.trim() + return text ? { ...candidate, document: { ...candidate.document, content: text } } : candidate + }) +} + /** Keeps the first of candidates a provider marked as the same item, such as a shared meeting. */ function firstOfEachDocument(candidates: LiveCandidate[]): LiveCandidate[] { const seen = new Set() @@ -507,6 +538,10 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ matchesLiveFilters(document, documentId, account.provider, queryFilters) ) ) + const previewQuery = native?.query || input.query + const previewed = previewQuery.trim() + ? await measureSearchStage('live.preview', () => withPreviewText(session, matching)) + : matching const undatedExcluded = permitted.some(({ document }) => lacksFilterDate(document, account.provider, queryFilters) ) @@ -571,12 +606,12 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ }) : page.nextCursor, }, - results: matching.map((candidate, index) => { + results: previewed.map((candidate, index) => { const result = resultFor( candidate, account, index + 1, - native?.query || input.query, + previewQuery, input.resultSecretRegistry ) /** A provider's dedupe key names one item across its collections, within its account. */ diff --git a/apps/sim/lib/sim-search/live/atlassian.ts b/apps/sim/lib/sim-search/live/atlassian.ts index 5c554666a04..d21014204fc 100644 --- a/apps/sim/lib/sim-search/live/atlassian.ts +++ b/apps/sim/lib/sim-search/live/atlassian.ts @@ -9,6 +9,7 @@ import { } from '@/lib/sim-search/live/http' import { interleaveByRank } from '@/lib/sim-search/live/pages' import { providerText } from '@/lib/sim-search/live/text' +import { atlassianTextClause } from '@/lib/sim-search/live/text-match' import type { NativeClient, NativeDocument, @@ -16,9 +17,6 @@ import type { NativeSearchInput, } from '@/lib/sim-search/live/types' -export const escapeSearchPhrase = (value: string) => - JSON.stringify(value.replace(/[+\-&|!(){}[\]^"~*?:\\/]/g, '\\$&')) - /** Place trusted scope before ORDER BY while respecting quoted JQL/CQL values. */ export function scopeAtlassianQuery(query: string, scope?: string, order?: string): string { if (!scope && !order) return query @@ -163,7 +161,7 @@ export async function searchAtlassian( await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, { body: { jql: scopeAtlassianQuery( - input.native?.query || (text ? `text ~ ${escapeSearchPhrase(text)}` : ''), + input.native?.query || (text ? atlassianTextClause(text, input.textMatch) : ''), scope, order ), @@ -184,7 +182,7 @@ export async function searchAtlassian( query: { cql: scopeAtlassianQuery( input.native?.query || - `type IN (page, blogpost)${text ? ` AND text ~ ${escapeSearchPhrase(text)}` : ''}`, + `type IN (page, blogpost)${text ? ` AND ${atlassianTextClause(text, input.textMatch)}` : ''}`, scope, order ), @@ -246,6 +244,16 @@ export async function readAtlassian( return readConfluence(client, cloudId, string(site.url), id, kind) } +/** Page and blog post text for a search preview: one v2 read, reusing the search's site list. */ +export async function previewConfluence( + client: NativeClient, + document: NativeDocument +): Promise { + if (document.kind !== 'page' && document.kind !== 'blogpost') return undefined + return (await readAtlassian(client, 'confluence', document.id, document.container, document.kind)) + .content +} + /** * Reads through the v2 API, whose page, blog post, and space endpoints need only the granular * read scopes a Search connection grants; v1 content reads also need read:content-details. diff --git a/apps/sim/lib/sim-search/live/google.ts b/apps/sim/lib/sim-search/live/google.ts index 408ee7828bc..c0ae80fc9b9 100644 --- a/apps/sim/lib/sim-search/live/google.ts +++ b/apps/sim/lib/sim-search/live/google.ts @@ -14,6 +14,7 @@ import { array, NativeSearchError, object, segment, string } from '@/lib/sim-sea import { interleaveByRank } from '@/lib/sim-search/live/pages' import { permitsResources } from '@/lib/sim-search/live/policy' import { providerText } from '@/lib/sim-search/live/text' +import { driveTextClause } from '@/lib/sim-search/live/text-match' import type { NativeClient, NativeDocument, @@ -24,8 +25,6 @@ import type { const DRIVE_FIELDS = 'id,name,mimeType,webViewLink,description,modifiedTime,owners(displayName,emailAddress)' -export const escapeDriveLiteral = (value: string) => - value.replace(/\\/g, '\\\\').replace(/'/g, "\\'") function driveDocument(row: Record): NativeDocument { return { @@ -48,8 +47,7 @@ export async function searchDrive( ): Promise { const text = nativeText(input) const query = - input.native?.query || - (text ? `fullText contains '${escapeDriveLiteral(text)}'` : 'trashed = false') + input.native?.query || (text ? driveTextClause(text, input.textMatch) : 'trashed = false') const dates = nativeDateBounds(input) const data = object( await client.json('/drive/v3/files', { @@ -81,10 +79,25 @@ export async function searchDrive( nextCursor: string(data.nextPageToken) || undefined, partial: data.incompleteSearch === true, message: - 'Drive searches file names and full text, including Docs, Sheets and Slides. Previews are file metadata; read a result for its content.', + 'Drive searches file names and full text, including Docs, Sheets and Slides. The top Docs and Slides results preview their text; other previews are file metadata, so read a result for its content.', } } +/** Docs and Slides export as plain text in one request; other kinds need a download and parse. */ +const PREVIEW_EXPORT_KINDS = new Set([ + 'application/vnd.google-apps.document', + 'application/vnd.google-apps.presentation', +]) + +/** Plain text of a Doc or Slides file for a search preview; Drive search returns only metadata. */ +export async function previewDrive( + client: NativeClient, + document: NativeDocument +): Promise { + if (!PREVIEW_EXPORT_KINDS.has(document.kind ?? '')) return undefined + return client.text(`/drive/v3/files/${segment(document.id)}/export`, { mimeType: 'text/plain' }) +} + export async function readDrive( client: NativeClient, id: string, diff --git a/apps/sim/lib/sim-search/live/providers.ts b/apps/sim/lib/sim-search/live/providers.ts index 1f74acef8f5..59bb82570db 100644 --- a/apps/sim/lib/sim-search/live/providers.ts +++ b/apps/sim/lib/sim-search/live/providers.ts @@ -1,9 +1,10 @@ import { MAX_NATIVE_QUERIES_PER_ACCOUNT } from '@/lib/api/contracts/mothership-assistant-tools' -import { readAtlassian, searchAtlassian } from '@/lib/sim-search/live/atlassian' +import { previewConfluence, readAtlassian, searchAtlassian } from '@/lib/sim-search/live/atlassian' import { readCoda, searchCoda } from '@/lib/sim-search/live/coda' import { readGitHub, searchGitHub } from '@/lib/sim-search/live/github' import { readGitLab, searchGitLab } from '@/lib/sim-search/live/gitlab' import { + previewDrive, readCalendar, readDrive, readGmail, @@ -45,6 +46,13 @@ interface NativeQueryGuide { interface NativeProvider { guide: NativeQueryGuide + /** Plain queries must contain every word, so an underfilled search retries with any term. */ + requiresEveryTerm?: true + /** + * Full text for a search preview, for providers whose search returns only an excerpt or + * metadata. At most one request; undefined for kinds without a cheap text form. + */ + previewText?(client: NativeClient, document: NativeDocument): Promise search(client: NativeClient, input: NativeSearchInput): Promise read( client: NativeClient, @@ -82,6 +90,8 @@ export const LIVE_SEARCH_PROVIDERS = { avoid: 'bare words without a term and operator, which Drive rejects, and trashed or modifiedTime clauses, which the server adds from startDate/endDate. Drive search does not search comments or replies; find the file by title/content, then read it to retrieve its discussion. PDF and DOCX reads extract text within download and parsing limits; scanned PDFs need OCR and unsupported binaries provide metadata only. Saved Google Meet transcripts and generated notes are Google Docs: use Drive fullText search and read their content. Drive date filters use file modification time, not meeting time.', }, + requiresEveryTerm: true, + previewText: previewDrive, search: searchDrive, read: (client, reference, options) => readDrive(client, reference.id, options.signal), }, @@ -167,6 +177,7 @@ export const LIVE_SEARCH_PROVIDERS = { avoid: 'JQL with only an ORDER BY clause (Jira rejects unbounded queries), and identifying users other than currentUser() by display name or email; use their account ID.', }, + requiresEveryTerm: true, search: (client, input) => searchAtlassian(client, 'jira', input), read: (client, reference) => readAtlassian(client, 'jira', reference.id, reference.container), }, @@ -180,6 +191,8 @@ export const LIVE_SEARCH_PROVIDERS = { avoid: 'starting the query with a negative clause (NOT, !=, !~ or NOT IN), which CQL rejects; lead with a positive clause such as type IN (page, blogpost).', }, + requiresEveryTerm: true, + previewText: previewConfluence, search: (client, input) => searchAtlassian(client, 'confluence', input), read: (client, reference) => readAtlassian(client, 'confluence', reference.id, reference.container, reference.kind), @@ -301,6 +314,20 @@ export function searchNativeProvider( return adapter.search(client, input) } +export function requiresEveryTerm(provider: LiveSearchProviderId): boolean { + const adapter = LIVE_SEARCH_PROVIDERS[provider] + return 'requiresEveryTerm' in adapter && adapter.requiresEveryTerm === true +} + +export async function previewNativeText( + provider: LiveSearchProviderId, + client: NativeClient, + document: NativeDocument +): Promise { + const adapter = LIVE_SEARCH_PROVIDERS[provider] + return 'previewText' in adapter ? adapter.previewText(client, document) : undefined +} + export function readNativeProvider( provider: LiveSearchProviderId, client: NativeClient, diff --git a/apps/sim/lib/sim-search/live/scoped-search.ts b/apps/sim/lib/sim-search/live/scoped-search.ts index 9bda4dbe251..2b2b6144d2b 100644 --- a/apps/sim/lib/sim-search/live/scoped-search.ts +++ b/apps/sim/lib/sim-search/live/scoped-search.ts @@ -1,8 +1,8 @@ import type { LiveSearchProvider } from '@/lib/api/contracts/mothership-assistant-tools' import { parseCodaResourceUri } from '@/lib/sim-search/live/coda-uri' -import { escapeDriveLiteral } from '@/lib/sim-search/live/google' import { array, object, string } from '@/lib/sim-search/live/http' import { collectNativePages } from '@/lib/sim-search/live/pages' +import { driveTextClause, escapeDriveLiteral } from '@/lib/sim-search/live/text-match' import type { NativeClient, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types' /** Provider restrictions improve recall; the independent verifier remains authoritative. */ @@ -51,9 +51,7 @@ export async function searchWithinPolicy( provider, query: input.native?.query ?? - (input.query - ? `fullText contains '${escapeDriveLiteral(input.query)}'` - : 'trashed = false'), + (input.query ? driveTextClause(input.query, input.textMatch) : 'trashed = false'), ...input.native, project: id, }, @@ -109,7 +107,7 @@ export async function searchWithinPolicy( } const text = input.native?.query ?? - (input.query ? `fullText contains '${escapeDriveLiteral(input.query)}'` : 'trashed = false') + (input.query ? driveTextClause(input.query, input.textMatch) : 'trashed = false') const result = await search( withQuery( `(${text}) and (${[...folders].map((id) => `'${escapeDriveLiteral(id)}' in parents`).join(' or ')})` diff --git a/apps/sim/lib/sim-search/live/service-session.ts b/apps/sim/lib/sim-search/live/service-session.ts index 9006ff86227..9702e70bfca 100644 --- a/apps/sim/lib/sim-search/live/service-session.ts +++ b/apps/sim/lib/sim-search/live/service-session.ts @@ -5,7 +5,7 @@ import { resolveConnectorAccessToken, resolveConnectorTokenUserId, } from '@/lib/knowledge/connectors/access-token' -import { escapeSearchPhrase, scopeAtlassianQuery } from '@/lib/sim-search/live/atlassian' +import { scopeAtlassianQuery } from '@/lib/sim-search/live/atlassian' import type { CodaMcpClient } from '@/lib/sim-search/live/coda-mcp' import { createCodaServiceVerifier } from '@/lib/sim-search/live/coda-service' import { createGitHubServiceVerifier } from '@/lib/sim-search/live/github-service' @@ -23,6 +23,7 @@ import type { LiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' import { LIVE_SEARCH_PROVIDER_CATALOG } from '@/lib/sim-search/live/provider-catalog' import { loadLiveGitHubSources, loadLiveServiceSource } from '@/lib/sim-search/live/service-sources' import { liveSourcePolicy } from '@/lib/sim-search/live/source-policy' +import { atlassianTextClause } from '@/lib/sim-search/live/text-match' import type { NativeClient, NativeDocument, NativeSearchInput } from '@/lib/sim-search/live/types' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' @@ -141,7 +142,8 @@ export async function createLiveServiceSession(input: { : []), ].join(' AND ') const query = - search.native?.query ?? (search.query ? `text ~ ${escapeSearchPhrase(search.query)}` : '') + search.native?.query ?? + (search.query ? atlassianTextClause(search.query, search.textMatch) : '') return { ...search, native: { provider, ...search.native, query: scopeAtlassianQuery(query, scope) }, diff --git a/apps/sim/lib/sim-search/live/term-fallback.test.ts b/apps/sim/lib/sim-search/live/term-fallback.test.ts new file mode 100644 index 00000000000..6198ab0e49c --- /dev/null +++ b/apps/sim/lib/sim-search/live/term-fallback.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it, vi } from 'vitest' +import { searchWithAnyTermFallback } from '@/lib/sim-search/live/term-fallback' +import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types' + +const input: NativeSearchInput = { query: 'contractor offboarding runbook', limit: 4, scopes: [] } + +function page(...documents: Partial[]): NativePage { + return { + documents: documents.map((document, index) => ({ + id: `doc-${index}`, + title: '', + url: '', + content: '', + ...document, + })), + } +} + +function provider(strict: NativePage, relaxed: NativePage | Error) { + return vi.fn(async (request: NativeSearchInput) => { + if (request.textMatch !== 'any') return strict + if (relaxed instanceof Error) throw relaxed + return relaxed + }) +} + +describe('searchWithAnyTermFallback', () => { + it('follows an exhausted all-terms page with any-term matches, most terms first', async () => { + const search = provider( + page({ id: 'exact', title: 'Contractor offboarding runbook' }), + page( + { id: 'one-term', title: 'Runbook index' }, + { id: 'exact', title: 'Contractor offboarding runbook' }, + { id: 'two-terms', title: 'Contractor offboarding form' }, + { id: 'one-term-later', title: 'Contractor directory' }, + { id: 'over-limit', title: 'Runbook archive' } + ) + ) + const result = await searchWithAnyTermFallback(search, input) + expect(result.documents.map((document) => document.id)).toEqual([ + 'exact', + 'two-terms', + 'one-term', + 'one-term-later', + ]) + expect(result.nextCursor).toBeUndefined() + expect(result.message).toContain('Only 1 result contained every search term') + }) + + it('finds the best partial match beyond the provider’s loose top results', async () => { + const loose = Array.from({ length: 30 }, (_, index) => ({ + id: `loose-${index}`, + title: index === 20 ? 'Contractor offboarding form' : `Runbook ${index}`, + })) + const search = vi.fn(async (request: NativeSearchInput) => + request.textMatch === 'any' + ? { documents: page(...loose).documents.slice(0, request.limit) } + : page() + ) + const result = await searchWithAnyTermFallback(search, { ...input, limit: 2 }) + expect(result.documents.map((document) => document.title)).toEqual([ + 'Contractor offboarding form', + 'Runbook 0', + ]) + }) + + it('never relaxes a native query, which is the caller’s exact syntax', async () => { + const search = provider(page(), page({ id: 'loose' })) + const native = { provider: 'confluence' as const, query: 'text ~ "runbook"' } + expect((await searchWithAnyTermFallback(search, { ...input, native })).documents).toEqual([]) + expect(search).toHaveBeenCalledTimes(1) + }) + + it('keeps a continuable page so its cursor continues the query that produced it', async () => { + const strict = { ...page({ id: 'exact' }), nextCursor: 'page-2' } + const search = provider(strict, page({ id: 'loose' })) + expect(await searchWithAnyTermFallback(search, input)).toBe(strict) + expect(search).toHaveBeenCalledTimes(1) + }) + + it('does not relax a page that is already full', async () => { + const strict = page({}, {}, {}, {}) + const search = provider(strict, page({ id: 'loose' })) + expect(await searchWithAnyTermFallback(search, input)).toBe(strict) + expect(search).toHaveBeenCalledTimes(1) + }) + + it('does not relax a query with fewer than two distinctive terms', async () => { + const search = provider(page(), page({ id: 'loose' })) + await searchWithAnyTermFallback(search, { ...input, query: 'what is the runbook' }) + expect(search).toHaveBeenCalledTimes(1) + }) + + it('returns the all-terms page when the any-term search fails', async () => { + const strict = page({ id: 'exact' }) + const search = provider(strict, new Error('rate limited')) + expect(await searchWithAnyTermFallback(search, input)).toBe(strict) + }) + + it('leaves the page untouched when the any-term search adds nothing new', async () => { + const strict = page({ id: 'exact' }) + const search = provider(strict, page({ id: 'exact' })) + expect(await searchWithAnyTermFallback(search, input)).toBe(strict) + }) +}) diff --git a/apps/sim/lib/sim-search/live/term-fallback.ts b/apps/sim/lib/sim-search/live/term-fallback.ts new file mode 100644 index 00000000000..46b9867ad00 --- /dev/null +++ b/apps/sim/lib/sim-search/live/term-fallback.ts @@ -0,0 +1,75 @@ +import { joinMessages } from '@/lib/sim-search/live/pages' +import { significantTerms, termCoverage } from '@/lib/sim-search/live/text-match' +import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types' + +/** + * Candidates the any-term search requests. Providers rank OR queries loosely, so the page is + * chosen from a wider pool by how many terms each candidate contains. + */ +const ANY_TERM_CANDIDATES = 25 + +const exactMatches = (count: number) => + count === 0 ? 'No result contained' : `Only ${count} result${count === 1 ? '' : 's'} contained` + +/** One provider item, however many queries returned it. */ +const documentKey = ({ container, kind, id }: NativeDocument) => + JSON.stringify([container ?? '', kind ?? '', id]) + +/** + * Searches with every query word required, then, when that exhausts the provider's matches + * without filling the page, with any of the query's significant terms. A natural-language query + * often contains one word the right page lacks, which an all-terms search turns into no result. + * + * All-terms matches keep their provider order ahead of the rest; the added matches are ordered by + * how many terms they contain, then how many their title contains, then provider order. The fallback runs only for a plain query + * (native queries are the caller's exact syntax) and never replaces a continuable page, so a + * cursor always continues the query that produced it. A failed fallback leaves the all-terms page. + */ +export async function searchWithAnyTermFallback( + search: (input: NativeSearchInput) => Promise, + input: NativeSearchInput +): Promise { + const strict = await search(input) + const terms = significantTerms(input.query) + if ( + input.native || + terms.length < 2 || + strict.nextCursor || + strict.hasMore || + strict.documents.length >= input.limit + ) + return strict + const relaxed = await search({ + ...input, + limit: Math.max(input.limit, ANY_TERM_CANDIDATES), + textMatch: 'any', + }).catch(() => undefined) + if (!relaxed) return strict + const seen = new Set(strict.documents.map(documentKey)) + const additions = relaxed.documents + .filter((document) => !seen.has(documentKey(document))) + .map((document, rank) => ({ + document, + rank, + coverage: termCoverage(`${document.title}\n${document.content}`, terms), + titleCoverage: termCoverage(document.title, terms), + })) + .sort( + (left, right) => + right.coverage - left.coverage || + right.titleCoverage - left.titleCoverage || + left.rank - right.rank + ) + .slice(0, input.limit - strict.documents.length) + .map(({ document }) => document) + if (!additions.length) return strict + return { + ...strict, + documents: [...strict.documents, ...additions], + partial: strict.partial || relaxed.partial, + message: joinMessages([ + strict.message, + `${exactMatches(strict.documents.length)} every search term, so results containing some of the terms follow, ordered by how many they contain.`, + ]), + } +} diff --git a/apps/sim/lib/sim-search/live/text-match.test.ts b/apps/sim/lib/sim-search/live/text-match.test.ts new file mode 100644 index 00000000000..3f44d4d17c2 --- /dev/null +++ b/apps/sim/lib/sim-search/live/text-match.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from 'vitest' +import { atlassianTextClause, driveTextClause } from '@/lib/sim-search/live/text-match' + +describe('text match clauses', () => { + it('keeps one all-terms clause by default', () => { + expect(atlassianTextClause('release process')).toBe('text ~ "release process"') + expect(driveTextClause("q3 plan's owner")).toBe("fullText contains 'q3 plan\\'s owner'") + }) + + it('joins distinctive terms with OR, leaving out stopwords', () => { + expect(atlassianTextClause('what is the release process', 'any')).toBe( + '(text ~ "release" OR text ~ "process")' + ) + expect(driveTextClause('roadmap for planning', 'any')).toBe( + "(fullText contains 'planning' or fullText contains 'roadmap')" + ) + }) + + it('escapes every term so a term cannot add query syntax', () => { + expect(atlassianTextClause('ci/cd say"hi rollout', 'any')).toBe( + String.raw`(text ~ "rollout" OR text ~ "say\\\"hi" OR text ~ "ci\\/cd")` + ) + expect(driveTextClause(String.raw`o'brien back\slash rollout`, 'any')).toBe( + String.raw`(fullText contains 'back\\slash' or fullText contains 'o\'brien' or fullText contains 'rollout')` + ) + }) + + it('falls back to the all-terms clause when fewer than two terms are distinctive', () => { + expect(atlassianTextClause('how does it work', 'any')).toBe('text ~ "how does it work"') + }) +}) diff --git a/apps/sim/lib/sim-search/live/text-match.ts b/apps/sim/lib/sim-search/live/text-match.ts new file mode 100644 index 00000000000..dca9aa84c41 --- /dev/null +++ b/apps/sim/lib/sim-search/live/text-match.ts @@ -0,0 +1,89 @@ +import { findTermMatches, queryTerms } from '@/lib/knowledge/search/snippet' + +/** + * How a plain query's words must match a document. Confluence, Jira and Drive full-text search + * require every word by default, so one word a page lacks hides it; `any` matches pages containing + * at least one of the query's significant terms. + */ +export type TextMatch = 'all' | 'any' + +/** Terms an any-term query sends; more would make the provider query long without adding recall. */ +const MAX_ANY_TERMS = 8 + +/** Words that match nearly every page, so they would make an any-term query return noise. */ +const STOPWORDS = new Set([ + 'about', + 'all', + 'and', + 'any', + 'are', + 'can', + 'could', + 'did', + 'does', + 'for', + 'from', + 'had', + 'has', + 'have', + 'how', + 'into', + 'its', + 'not', + 'our', + 'should', + 'that', + 'the', + 'their', + 'there', + 'this', + 'was', + 'were', + 'what', + 'when', + 'where', + 'which', + 'who', + 'why', + 'will', + 'with', + 'would', + 'you', + 'your', +]) + +/** + * The query's distinctive terms, longest first. Fewer than two means an any-term query would + * match the same pages as the all-terms one. + */ +export function significantTerms(query: string): string[] { + return queryTerms(query) + .filter((term) => !STOPWORDS.has(term.toLowerCase())) + .slice(0, MAX_ANY_TERMS) +} + +/** Escapes a value for a quoted JQL or CQL string, treating Lucene operators as literal text. */ +const escapeSearchPhrase = (value: string) => + JSON.stringify(value.replace(/[+\-&|!(){}[\]^"~*?:\\/]/g, '\\$&')) + +/** The `text ~` clause for a plain query in Jira JQL or Confluence CQL. */ +export function atlassianTextClause(text: string, match: TextMatch = 'all'): string { + const terms = match === 'any' ? significantTerms(text) : [] + if (terms.length < 2) return `text ~ ${escapeSearchPhrase(text)}` + return `(${terms.map((term) => `text ~ ${escapeSearchPhrase(term)}`).join(' OR ')})` +} + +export const escapeDriveLiteral = (value: string) => + value.replace(/\\/g, '\\\\').replace(/'/g, "\\'") + +/** The `fullText contains` clause for a plain query in Drive's q language. */ +export function driveTextClause(text: string, match: TextMatch = 'all'): string { + const terms = match === 'any' ? significantTerms(text) : [] + if (terms.length < 2) return `fullText contains '${escapeDriveLiteral(text)}'` + return `(${terms.map((term) => `fullText contains '${escapeDriveLiteral(term)}'`).join(' or ')})` +} + +/** How many distinct terms occur as whole words in the text. */ +export function termCoverage(text: string, terms: readonly string[]): number { + return terms.filter((term) => findTermMatches(text, [term]).length > 0).length +} diff --git a/apps/sim/lib/sim-search/live/types.ts b/apps/sim/lib/sim-search/live/types.ts index 7f582aabfa5..24a06185e85 100644 --- a/apps/sim/lib/sim-search/live/types.ts +++ b/apps/sim/lib/sim-search/live/types.ts @@ -4,6 +4,7 @@ import type { NativeSearchQuery, } from '@/lib/api/contracts/mothership-assistant-tools' import type { LiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' +import type { TextMatch } from '@/lib/sim-search/live/text-match' export interface LiveAccount { id: string @@ -94,4 +95,6 @@ export interface NativeSearchInput { native?: NativeSearchQuery limit: number scopes: readonly string[] + /** How a plain query's words must match; providers without full-text search ignore it. */ + textMatch?: TextMatch } From e8c904ca994da937d81ca12d3db102f44cb95b03 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 12 Oct 2026 23:37:56 -0700 Subject: [PATCH 02/12] fix(search): register preview stage, unexport read cap, update limiter assertion --- .../app/api/v2/workflows/[workflowId]/execute/route.test.ts | 6 ++++-- apps/sim/lib/api/contracts/knowledge/mcp.ts | 2 +- apps/sim/lib/knowledge/search/diagnostics.ts | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.test.ts b/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.test.ts index 92cd87f0dbb..bef2b85433c 100644 --- a/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.test.ts +++ b/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.test.ts @@ -500,11 +500,13 @@ describe('POST /api/v2/workflows/[workflowId]/execute', () => { expect(mockCheckOperationRate).toHaveBeenCalledTimes(2) expect(mockCheckOperationRate).toHaveBeenCalledWith( 'v2:workflows.execute:api-key:key-1', - expect.anything() + expect.anything(), + 1 ) expect(mockCheckOperationRate).toHaveBeenCalledWith( 'v2:workflows.execute:workspace:workspace-1', - expect.anything() + expect.anything(), + 1 ) expect(mockPreprocessExecution).toHaveBeenCalledWith( expect.objectContaining({ rateLimitCounter: 'sync' }) diff --git a/apps/sim/lib/api/contracts/knowledge/mcp.ts b/apps/sim/lib/api/contracts/knowledge/mcp.ts index c0e381a8b2d..814a4e55409 100644 --- a/apps/sim/lib/api/contracts/knowledge/mcp.ts +++ b/apps/sim/lib/api/contracts/knowledge/mcp.ts @@ -18,7 +18,7 @@ export const organizationKnowledgeMcpContract = defineRouteContract({ export const liveSearchMcpSchema = searchWorkspaceInputSchema.strict() /** Documents one read_document call may read; each counts as one read against the rate limit. */ -export const MAX_MCP_READ_DOCUMENTS = 5 +const MAX_MCP_READ_DOCUMENTS = 5 export const readLiveDocumentMcpSchema = readDocumentInputSchema .extend({ diff --git a/apps/sim/lib/knowledge/search/diagnostics.ts b/apps/sim/lib/knowledge/search/diagnostics.ts index 2468262b182..81a13db8188 100644 --- a/apps/sim/lib/knowledge/search/diagnostics.ts +++ b/apps/sim/lib/knowledge/search/diagnostics.ts @@ -53,6 +53,7 @@ export type SearchStage = | 'live.session' | 'live.search' | 'live.verify' + | 'live.preview' | 'live.read' /** Fixed, content-free fields. Never pass queries, filters, document identities, SQL, or errors. */ From c70460886002111ba4f01d988a8818cee3cccbd2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 00:13:20 -0700 Subject: [PATCH 03/12] fix(search): keep cancellation, skip fallback for date-sorted and folder-walking Drive searches, preview plain queries only --- apps/sim/lib/knowledge/mcp/server.ts | 3 ++- .../lib/sim-search/live/account-session.ts | 13 ++++++++-- .../lib/sim-search/live/application.test.ts | 24 ++++++++++++++++++ apps/sim/lib/sim-search/live/application.ts | 25 +++++++++++-------- .../lib/sim-search/live/term-fallback.test.ts | 18 ++++++++++--- apps/sim/lib/sim-search/live/term-fallback.ts | 21 ++++++++++------ .../lib/sim-search/live/text-match.test.ts | 5 ---- 7 files changed, 79 insertions(+), 30 deletions(-) diff --git a/apps/sim/lib/knowledge/mcp/server.ts b/apps/sim/lib/knowledge/mcp/server.ts index 3e0d3a6c465..a08847b6b3c 100644 --- a/apps/sim/lib/knowledge/mcp/server.ts +++ b/apps/sim/lib/knowledge/mcp/server.ts @@ -53,6 +53,7 @@ function projectReadResult(result: Awaited { } }) + it('does not read previews for a native query, whose operators would anchor the passage', async () => { + mocks.search.mockResolvedValue({ documents: excerpts }) + await searchLiveKnowledge.execute({ + principal, + input: { + ...input, + query: '', + nativeQueries: [{ provider: 'google_drive' as const, query: "fullText contains 'launch'" }], + }, + }) + expect(mocks.preview).not.toHaveBeenCalled() + }) + it('does not read previews for a listing without search terms', async () => { mocks.search.mockResolvedValue({ documents: excerpts }) await searchLiveKnowledge.execute({ @@ -692,6 +705,17 @@ describe('authorized live retrieval', () => { expect(mocks.search).toHaveBeenCalledTimes(2) }) + it('does not spend a second search on a Drive source whose verification walks folders', async () => { + mocks.requiresEveryTerm.mockReturnValue(true) + vi.mocked(livePolicyFor).mockReturnValue({ + ...defaultLiveSearchPolicy(), + fileTypes: ['application/pdf'], + }) + mocks.search.mockResolvedValue({ documents: [] }) + await searchLiveKnowledge.execute({ principal, input: { ...input, query: 'launch checklist' } }) + expect(mocks.search).toHaveBeenCalledTimes(1) + }) + it('checks the service source before returning member-visible candidates', async () => { const verify = vi.fn().mockResolvedValue(false) mocks.service.mockResolvedValue({ policy: defaultLiveSearchPolicy(), verify, partial: false }) diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 796edbf7b34..4c296f169b3 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -2,7 +2,6 @@ import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js' import { requirePrincipalSubjectUserId } from '@sim/auth/principal' import { safeCompare } from '@sim/security/compare' import { hmacSha256Hex } from '@sim/security/hmac' -import { sleep } from '@sim/utils/helpers' import { compareStrings } from '@sim/utils/string' import { z } from 'zod' import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge' @@ -312,6 +311,8 @@ async function verifyCandidates(session: LiveAccountSession, candidates: LiveCan const PREVIEW_READS = 3 /** A slow preview read keeps the provider's excerpt rather than delaying the search. */ const PREVIEW_DEADLINE_MS = 2500 +/** Text kept from a preview read; the passage is cut from it, so a long document stays bounded. */ +const PREVIEW_SOURCE_CHARACTERS = 100_000 /** * Replaces the excerpt or metadata of an account's top results with their full text, so their @@ -322,18 +323,19 @@ async function withPreviewText( session: LiveAccountSession, candidates: LiveCandidate[] ): Promise { + const deadline = AbortSignal.timeout(PREVIEW_DEADLINE_MS) + const expired = new Promise((resolve) => + deadline.addEventListener('abort', () => resolve(undefined), { once: true }) + ) const texts = await Promise.all( candidates .slice(0, PREVIEW_READS) .map((candidate) => - Promise.race([ - session.previewText(candidate.document).catch(() => undefined), - sleep(PREVIEW_DEADLINE_MS).then(() => undefined), - ]) + Promise.race([session.previewText(candidate.document).catch(() => undefined), expired]) ) ) return candidates.map((candidate, index) => { - const text = texts[index]?.trim() + const text = texts[index]?.trim().slice(0, PREVIEW_SOURCE_CHARACTERS) return text ? { ...candidate, document: { ...candidate.document, content: text } } : candidate }) } @@ -538,10 +540,11 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ matchesLiveFilters(document, documentId, account.provider, queryFilters) ) ) - const previewQuery = native?.query || input.query - const previewed = previewQuery.trim() - ? await measureSearchStage('live.preview', () => withPreviewText(session, matching)) - : matching + /** A native query's operators would anchor the passage, so only plain queries preview. */ + const previewed = + !native && input.query.trim() + ? await measureSearchStage('live.preview', () => withPreviewText(session, matching)) + : matching const undatedExcluded = permitted.some(({ document }) => lacksFilterDate(document, account.provider, queryFilters) ) @@ -611,7 +614,7 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ candidate, account, index + 1, - previewQuery, + native?.query || input.query, input.resultSecretRegistry ) /** A provider's dedupe key names one item across its collections, within its account. */ diff --git a/apps/sim/lib/sim-search/live/term-fallback.test.ts b/apps/sim/lib/sim-search/live/term-fallback.test.ts index 6198ab0e49c..beb77941b48 100644 --- a/apps/sim/lib/sim-search/live/term-fallback.test.ts +++ b/apps/sim/lib/sim-search/live/term-fallback.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from 'vitest' +import { NativeSearchError } from '@/lib/sim-search/live/http' import { searchWithAnyTermFallback } from '@/lib/sim-search/live/term-fallback' import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types' @@ -44,7 +45,7 @@ describe('searchWithAnyTermFallback', () => { 'one-term-later', ]) expect(result.nextCursor).toBeUndefined() - expect(result.message).toContain('Only 1 result contained every search term') + expect(result.message).toContain('Few results contained every search term') }) it('finds the best partial match beyond the provider’s loose top results', async () => { @@ -91,12 +92,23 @@ describe('searchWithAnyTermFallback', () => { expect(search).toHaveBeenCalledTimes(1) }) - it('returns the all-terms page when the any-term search fails', async () => { + it('returns the all-terms page when the provider refuses the any-term search', async () => { const strict = page({ id: 'exact' }) - const search = provider(strict, new Error('rate limited')) + const search = provider(strict, new NativeSearchError('rate_limited', 'Slow down')) expect(await searchWithAnyTermFallback(search, input)).toBe(strict) }) + it('propagates cancellation during the any-term search instead of returning a page', async () => { + const search = provider(page({ id: 'exact' }), new DOMException('Timed out', 'TimeoutError')) + await expect(searchWithAnyTermFallback(search, input)).rejects.toThrow('Timed out') + }) + + it('does not relax a date-sorted search, whose page is reordered by date', async () => { + const search = provider(page(), page({ id: 'loose' })) + await searchWithAnyTermFallback(search, { ...input, filters: { sortBy: 'newest' } }) + expect(search).toHaveBeenCalledTimes(1) + }) + it('leaves the page untouched when the any-term search adds nothing new', async () => { const strict = page({ id: 'exact' }) const search = provider(strict, page({ id: 'exact' })) diff --git a/apps/sim/lib/sim-search/live/term-fallback.ts b/apps/sim/lib/sim-search/live/term-fallback.ts index 46b9867ad00..ae68d9fb47e 100644 --- a/apps/sim/lib/sim-search/live/term-fallback.ts +++ b/apps/sim/lib/sim-search/live/term-fallback.ts @@ -1,3 +1,5 @@ +import { dateSortDirection } from '@/lib/sim-search/live/dates' +import { NativeSearchError } from '@/lib/sim-search/live/http' import { joinMessages } from '@/lib/sim-search/live/pages' import { significantTerms, termCoverage } from '@/lib/sim-search/live/text-match' import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types' @@ -8,9 +10,6 @@ import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-se */ const ANY_TERM_CANDIDATES = 25 -const exactMatches = (count: number) => - count === 0 ? 'No result contained' : `Only ${count} result${count === 1 ? '' : 's'} contained` - /** One provider item, however many queries returned it. */ const documentKey = ({ container, kind, id }: NativeDocument) => JSON.stringify([container ?? '', kind ?? '', id]) @@ -21,9 +20,11 @@ const documentKey = ({ container, kind, id }: NativeDocument) => * often contains one word the right page lacks, which an all-terms search turns into no result. * * All-terms matches keep their provider order ahead of the rest; the added matches are ordered by - * how many terms they contain, then how many their title contains, then provider order. The fallback runs only for a plain query - * (native queries are the caller's exact syntax) and never replaces a continuable page, so a - * cursor always continues the query that produced it. A failed fallback leaves the all-terms page. + * how many terms their title and preview contain, then how many their title contains, then + * provider order. The fallback runs only for a plain relevance-ordered query: native queries are + * the caller's exact syntax, and a date-sorted page is reordered by date. It never replaces a + * continuable page, so a cursor always continues the query that produced it. A provider refusal + * leaves the all-terms page; cancellation and transport failures propagate like the first search. */ export async function searchWithAnyTermFallback( search: (input: NativeSearchInput) => Promise, @@ -33,6 +34,7 @@ export async function searchWithAnyTermFallback( const terms = significantTerms(input.query) if ( input.native || + dateSortDirection(input.filters) || terms.length < 2 || strict.nextCursor || strict.hasMore || @@ -43,7 +45,10 @@ export async function searchWithAnyTermFallback( ...input, limit: Math.max(input.limit, ANY_TERM_CANDIDATES), textMatch: 'any', - }).catch(() => undefined) + }).catch((error: unknown) => { + if (error instanceof NativeSearchError) return undefined + throw error + }) if (!relaxed) return strict const seen = new Set(strict.documents.map(documentKey)) const additions = relaxed.documents @@ -69,7 +74,7 @@ export async function searchWithAnyTermFallback( partial: strict.partial || relaxed.partial, message: joinMessages([ strict.message, - `${exactMatches(strict.documents.length)} every search term, so results containing some of the terms follow, ordered by how many they contain.`, + 'Few results contained every search term, so results containing some of the terms follow, ordered by how many they contain.', ]), } } diff --git a/apps/sim/lib/sim-search/live/text-match.test.ts b/apps/sim/lib/sim-search/live/text-match.test.ts index 3f44d4d17c2..88c417ac38f 100644 --- a/apps/sim/lib/sim-search/live/text-match.test.ts +++ b/apps/sim/lib/sim-search/live/text-match.test.ts @@ -2,11 +2,6 @@ import { describe, expect, it } from 'vitest' import { atlassianTextClause, driveTextClause } from '@/lib/sim-search/live/text-match' describe('text match clauses', () => { - it('keeps one all-terms clause by default', () => { - expect(atlassianTextClause('release process')).toBe('text ~ "release process"') - expect(driveTextClause("q3 plan's owner")).toBe("fullText contains 'q3 plan\\'s owner'") - }) - it('joins distinctive terms with OR, leaving out stopwords', () => { expect(atlassianTextClause('what is the release process', 'any')).toBe( '(text ~ "release" OR text ~ "process")' From d2db24cd5929fcd8fa8ac9142ba6346a706ce1f3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 00:18:17 -0700 Subject: [PATCH 04/12] style(search): format application test --- apps/sim/lib/sim-search/live/application.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index 7ddb2045a7c..9dc80f267f9 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -672,7 +672,9 @@ describe('authorized live retrieval', () => { input: { ...input, query: '', - nativeQueries: [{ provider: 'google_drive' as const, query: "fullText contains 'launch'" }], + nativeQueries: [ + { provider: 'google_drive' as const, query: "fullText contains 'launch'" }, + ], }, }) expect(mocks.preview).not.toHaveBeenCalled() From 10db1dac8fcedd12256dc4ae3d3efc8486dc89b8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 00:26:40 -0700 Subject: [PATCH 05/12] fix(search): keep excerpts for textless pages, anchor long previews on the match, report refused fallbacks --- .../sim/lib/sim-search/live/application.test.ts | 7 +++++++ apps/sim/lib/sim-search/live/application.ts | 15 ++++++++++----- apps/sim/lib/sim-search/live/atlassian.test.ts | 17 ++++++++++++++++- apps/sim/lib/sim-search/live/atlassian.ts | 11 +++++++++-- .../lib/sim-search/live/term-fallback.test.ts | 12 ++++++++---- apps/sim/lib/sim-search/live/term-fallback.ts | 8 +++++--- 6 files changed, 55 insertions(+), 15 deletions(-) diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index 9dc80f267f9..46711b27ae8 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -642,6 +642,13 @@ describe('authorized live retrieval', () => { expect(mocks.preview).toHaveBeenCalledTimes(3) }) + it('previews the passage around a match deep inside a long document', async () => { + mocks.search.mockResolvedValue({ documents: excerpts.slice(0, 1) }) + mocks.preview.mockResolvedValue(`${'filler '.repeat(30_000)}the launch checklist`) + const [row] = (await searchLiveKnowledge.execute({ principal, input })).results + expect(row?.content).toContain('launch checklist') + }) + it('keeps the provider excerpt when a preview read fails', async () => { mocks.search.mockResolvedValue({ documents: excerpts.slice(0, 2) }) mocks.preview diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 4c296f169b3..6c92a86a95c 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -311,7 +311,7 @@ async function verifyCandidates(session: LiveAccountSession, candidates: LiveCan const PREVIEW_READS = 3 /** A slow preview read keeps the provider's excerpt rather than delaying the search. */ const PREVIEW_DEADLINE_MS = 2500 -/** Text kept from a preview read; the passage is cut from it, so a long document stays bounded. */ +/** Text kept around the match from a preview read, so a long document stays bounded. */ const PREVIEW_SOURCE_CHARACTERS = 100_000 /** @@ -321,7 +321,8 @@ const PREVIEW_SOURCE_CHARACTERS = 100_000 */ async function withPreviewText( session: LiveAccountSession, - candidates: LiveCandidate[] + candidates: LiveCandidate[], + query: string ): Promise { const deadline = AbortSignal.timeout(PREVIEW_DEADLINE_MS) const expired = new Promise((resolve) => @@ -335,8 +336,10 @@ async function withPreviewText( ) ) return candidates.map((candidate, index) => { - const text = texts[index]?.trim().slice(0, PREVIEW_SOURCE_CHARACTERS) - return text ? { ...candidate, document: { ...candidate.document, content: text } } : candidate + const text = texts[index]?.trim() + if (!text) return candidate + const content = matchPassage(text, query, PREVIEW_SOURCE_CHARACTERS).content + return { ...candidate, document: { ...candidate.document, content } } }) } @@ -543,7 +546,9 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ /** A native query's operators would anchor the passage, so only plain queries preview. */ const previewed = !native && input.query.trim() - ? await measureSearchStage('live.preview', () => withPreviewText(session, matching)) + ? await measureSearchStage('live.preview', () => + withPreviewText(session, matching, input.query) + ) : matching const undatedExcluded = permitted.some(({ document }) => lacksFilterDate(document, account.provider, queryFilters) diff --git a/apps/sim/lib/sim-search/live/atlassian.test.ts b/apps/sim/lib/sim-search/live/atlassian.test.ts index d6a9376c80b..d0d0816ddd5 100644 --- a/apps/sim/lib/sim-search/live/atlassian.test.ts +++ b/apps/sim/lib/sim-search/live/atlassian.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest' -import { readAtlassian, searchAtlassian } from '@/lib/sim-search/live/atlassian' +import { previewConfluence, readAtlassian, searchAtlassian } from '@/lib/sim-search/live/atlassian' import { NativeSearchError } from '@/lib/sim-search/live/http' import type { NativeClient } from '@/lib/sim-search/live/types' @@ -181,4 +181,19 @@ describe('Confluence live documents', () => { expect(page.message).toContain('cannot be read') expect(page.partial).toBe(true) }) + + it('keeps the search excerpt when a previewed page body has no text', async () => { + const page = (value: string) => + client({ [`${v2}/pages/7`]: { id: '7', title: 'Diagram', body: { view: { value } } } }) + const document = { + id: '7', + kind: 'page', + container: 'cloud', + title: 'Diagram', + url: '', + content: '', + } + expect(await previewConfluence(page(''), document)).toBeUndefined() + expect(await previewConfluence(page('

Rollout steps

'), document)).toBe('Rollout steps') + }) }) diff --git a/apps/sim/lib/sim-search/live/atlassian.ts b/apps/sim/lib/sim-search/live/atlassian.ts index d21014204fc..40d1017f6b2 100644 --- a/apps/sim/lib/sim-search/live/atlassian.ts +++ b/apps/sim/lib/sim-search/live/atlassian.ts @@ -250,8 +250,15 @@ export async function previewConfluence( document: NativeDocument ): Promise { if (document.kind !== 'page' && document.kind !== 'blogpost') return undefined - return (await readAtlassian(client, 'confluence', document.id, document.container, document.kind)) - .content + const page = await readAtlassian( + client, + 'confluence', + document.id, + document.container, + document.kind + ) + /** A body without text reads as its title, which says less than the search excerpt. */ + return page.content === page.title ? undefined : page.content } /** diff --git a/apps/sim/lib/sim-search/live/term-fallback.test.ts b/apps/sim/lib/sim-search/live/term-fallback.test.ts index beb77941b48..b655134596e 100644 --- a/apps/sim/lib/sim-search/live/term-fallback.test.ts +++ b/apps/sim/lib/sim-search/live/term-fallback.test.ts @@ -92,10 +92,14 @@ describe('searchWithAnyTermFallback', () => { expect(search).toHaveBeenCalledTimes(1) }) - it('returns the all-terms page when the provider refuses the any-term search', async () => { - const strict = page({ id: 'exact' }) - const search = provider(strict, new NativeSearchError('rate_limited', 'Slow down')) - expect(await searchWithAnyTermFallback(search, input)).toBe(strict) + it('keeps the all-terms page as partial when the provider refuses the any-term search', async () => { + const search = provider( + page({ id: 'exact' }), + new NativeSearchError('rate_limited', 'Slow down') + ) + const result = await searchWithAnyTermFallback(search, input) + expect(result.documents.map((document) => document.id)).toEqual(['exact']) + expect(result).toMatchObject({ partial: true, message: 'Slow down' }) }) it('propagates cancellation during the any-term search instead of returning a page', async () => { diff --git a/apps/sim/lib/sim-search/live/term-fallback.ts b/apps/sim/lib/sim-search/live/term-fallback.ts index ae68d9fb47e..4478f43aa05 100644 --- a/apps/sim/lib/sim-search/live/term-fallback.ts +++ b/apps/sim/lib/sim-search/live/term-fallback.ts @@ -24,7 +24,8 @@ const documentKey = ({ container, kind, id }: NativeDocument) => * provider order. The fallback runs only for a plain relevance-ordered query: native queries are * the caller's exact syntax, and a date-sorted page is reordered by date. It never replaces a * continuable page, so a cursor always continues the query that produced it. A provider refusal - * leaves the all-terms page; cancellation and transport failures propagate like the first search. + * keeps the all-terms page, reported as partial with the provider's message; cancellation and + * transport failures propagate like the first search. */ export async function searchWithAnyTermFallback( search: (input: NativeSearchInput) => Promise, @@ -46,10 +47,11 @@ export async function searchWithAnyTermFallback( limit: Math.max(input.limit, ANY_TERM_CANDIDATES), textMatch: 'any', }).catch((error: unknown) => { - if (error instanceof NativeSearchError) return undefined + if (error instanceof NativeSearchError) return error throw error }) - if (!relaxed) return strict + if (relaxed instanceof NativeSearchError) + return { ...strict, partial: true, message: joinMessages([strict.message, relaxed.message]) } const seen = new Set(strict.documents.map(documentKey)) const additions = relaxed.documents .filter((document) => !seen.has(documentKey(document))) From 32ea9a4143b37ce7ddffd8aad3db09af94ee2a3c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 01:14:03 -0700 Subject: [PATCH 06/12] fix(search): drop the topK hint that made clients shrink their first search --- apps/sim/lib/knowledge/mcp/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/sim/lib/knowledge/mcp/server.ts b/apps/sim/lib/knowledge/mcp/server.ts index a08847b6b3c..d0cf03d02af 100644 --- a/apps/sim/lib/knowledge/mcp/server.ts +++ b/apps/sim/lib/knowledge/mcp/server.ts @@ -124,7 +124,7 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe { title: 'Search', description: - 'Search this organization’s sources through their live APIs, within your access and the admin’s source settings. A plain query searches every source; use source and date filters to narrow results, and nativeQueries only for a provider’s own operators (exact phrases, fields such as a Jira project or Confluence space, OR across alternatives) or a continuation cursor. Results carry a passage around their match where the source provides text. Lower topK on follow-up searches to keep results compact. Inspect live.accounts for provider status and continuation cursors, and live.guidance for query syntax. Results are candidates, not proof of complete coverage. When a passage does not answer the question, use read_document with the exact returned documentId, and cite citationUrl when available.', + 'Search this organization’s sources through their live APIs, within your access and the admin’s source settings. A plain query searches every source; use source and date filters to narrow results, and nativeQueries only for a provider’s own operators (exact phrases, fields such as a Jira project or Confluence space, OR across alternatives) or a continuation cursor. Results carry a passage around their match where the source provides text. Inspect live.accounts for provider status and continuation cursors, and live.guidance for query syntax. Results are candidates, not proof of complete coverage. When a passage does not answer the question, use read_document with the exact returned documentId, and cite citationUrl when available.', inputSchema: liveSearchMcpSchema, annotations: KNOWLEDGE_MCP_READ_ONLY, }, From cbbf96e6062d4e02cdd05ecf37eec334fe88821a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 02:42:58 -0700 Subject: [PATCH 07/12] fix(search): relax Confluence through site search and keep partial matches few and relevant --- apps/sim/lib/sim-search/live/atlassian.ts | 6 +-- .../lib/sim-search/live/service-session.ts | 4 +- .../lib/sim-search/live/term-fallback.test.ts | 44 +++++++++++++++---- apps/sim/lib/sim-search/live/term-fallback.ts | 13 ++++-- .../lib/sim-search/live/text-match.test.ts | 22 ++++++++-- apps/sim/lib/sim-search/live/text-match.ts | 17 +++++-- 6 files changed, 80 insertions(+), 26 deletions(-) diff --git a/apps/sim/lib/sim-search/live/atlassian.ts b/apps/sim/lib/sim-search/live/atlassian.ts index 40d1017f6b2..b7e0cfc93e8 100644 --- a/apps/sim/lib/sim-search/live/atlassian.ts +++ b/apps/sim/lib/sim-search/live/atlassian.ts @@ -9,7 +9,7 @@ import { } from '@/lib/sim-search/live/http' import { interleaveByRank } from '@/lib/sim-search/live/pages' import { providerText } from '@/lib/sim-search/live/text' -import { atlassianTextClause } from '@/lib/sim-search/live/text-match' +import { confluenceTextClause, jiraTextClause } from '@/lib/sim-search/live/text-match' import type { NativeClient, NativeDocument, @@ -161,7 +161,7 @@ export async function searchAtlassian( await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, { body: { jql: scopeAtlassianQuery( - input.native?.query || (text ? atlassianTextClause(text, input.textMatch) : ''), + input.native?.query || (text ? jiraTextClause(text, input.textMatch) : ''), scope, order ), @@ -182,7 +182,7 @@ export async function searchAtlassian( query: { cql: scopeAtlassianQuery( input.native?.query || - `type IN (page, blogpost)${text ? ` AND ${atlassianTextClause(text, input.textMatch)}` : ''}`, + `type IN (page, blogpost)${text ? ` AND ${confluenceTextClause(text, input.textMatch)}` : ''}`, scope, order ), diff --git a/apps/sim/lib/sim-search/live/service-session.ts b/apps/sim/lib/sim-search/live/service-session.ts index 9702e70bfca..30c8601d79d 100644 --- a/apps/sim/lib/sim-search/live/service-session.ts +++ b/apps/sim/lib/sim-search/live/service-session.ts @@ -23,7 +23,7 @@ import type { LiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' import { LIVE_SEARCH_PROVIDER_CATALOG } from '@/lib/sim-search/live/provider-catalog' import { loadLiveGitHubSources, loadLiveServiceSource } from '@/lib/sim-search/live/service-sources' import { liveSourcePolicy } from '@/lib/sim-search/live/source-policy' -import { atlassianTextClause } from '@/lib/sim-search/live/text-match' +import { confluenceTextClause } from '@/lib/sim-search/live/text-match' import type { NativeClient, NativeDocument, NativeSearchInput } from '@/lib/sim-search/live/types' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' @@ -143,7 +143,7 @@ export async function createLiveServiceSession(input: { ].join(' AND ') const query = search.native?.query ?? - (search.query ? atlassianTextClause(search.query, search.textMatch) : '') + (search.query ? confluenceTextClause(search.query, search.textMatch) : '') return { ...search, native: { provider, ...search.native, query: scopeAtlassianQuery(query, scope) }, diff --git a/apps/sim/lib/sim-search/live/term-fallback.test.ts b/apps/sim/lib/sim-search/live/term-fallback.test.ts index b655134596e..f332a696a0e 100644 --- a/apps/sim/lib/sim-search/live/term-fallback.test.ts +++ b/apps/sim/lib/sim-search/live/term-fallback.test.ts @@ -26,23 +26,22 @@ function provider(strict: NativePage, relaxed: NativePage | Error) { } describe('searchWithAnyTermFallback', () => { - it('follows an exhausted all-terms page with any-term matches, most terms first', async () => { + it('follows an exhausted all-terms page with partial matches, most terms first', async () => { const search = provider( page({ id: 'exact', title: 'Contractor offboarding runbook' }), page( - { id: 'one-term', title: 'Runbook index' }, + { id: 'runbook', title: 'Runbook index' }, { id: 'exact', title: 'Contractor offboarding runbook' }, - { id: 'two-terms', title: 'Contractor offboarding form' }, - { id: 'one-term-later', title: 'Contractor directory' }, - { id: 'over-limit', title: 'Runbook archive' } + { id: 'in-preview', title: 'Offboarding', content: 'Steps for each contractor' }, + { id: 'in-title', title: 'Contractor offboarding form' }, + { id: 'contractors', title: 'Contractor directory' } ) ) const result = await searchWithAnyTermFallback(search, input) expect(result.documents.map((document) => document.id)).toEqual([ 'exact', - 'two-terms', - 'one-term', - 'one-term-later', + 'in-title', + 'in-preview', ]) expect(result.nextCursor).toBeUndefined() expect(result.message).toContain('Few results contained every search term') @@ -61,10 +60,37 @@ describe('searchWithAnyTermFallback', () => { const result = await searchWithAnyTermFallback(search, { ...input, limit: 2 }) expect(result.documents.map((document) => document.title)).toEqual([ 'Contractor offboarding form', - 'Runbook 0', ]) }) + it('keeps a page sharing a few of a long query’s terms, whose preview shows only some', async () => { + const search = provider( + page(), + page({ id: 'enrichments', title: 'Enrichments and Workflow Columns' }) + ) + const result = await searchWithAnyTermFallback(search, { + ...input, + query: 'enrichments vs workflow columns built-in email draft column', + }) + expect(result.documents.map((document) => document.id)).toEqual(['enrichments']) + }) + + it('keeps single-term matches of a two-term query, the closest a partial match gets', async () => { + const search = provider(page(), page({ id: 'contractor', title: 'Contractor directory' })) + const result = await searchWithAnyTermFallback(search, { + ...input, + query: 'contractor offboarding', + }) + expect(result.documents.map((document) => document.id)).toEqual(['contractor']) + }) + + it('adds a few partial matches however large the page', async () => { + const partial = Array.from({ length: 12 }, () => ({ title: 'Contractor offboarding' })) + const search = provider(page(), page(...partial)) + const result = await searchWithAnyTermFallback(search, { ...input, limit: 20 }) + expect(result.documents).toHaveLength(5) + }) + it('never relaxes a native query, which is the caller’s exact syntax', async () => { const search = provider(page(), page({ id: 'loose' })) const native = { provider: 'confluence' as const, query: 'text ~ "runbook"' } diff --git a/apps/sim/lib/sim-search/live/term-fallback.ts b/apps/sim/lib/sim-search/live/term-fallback.ts index 4478f43aa05..0e739e86b3a 100644 --- a/apps/sim/lib/sim-search/live/term-fallback.ts +++ b/apps/sim/lib/sim-search/live/term-fallback.ts @@ -10,6 +10,9 @@ import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-se */ const ANY_TERM_CANDIDATES = 25 +/** Partial matches added to a page, so a loosely matching provider cannot crowd out other sources. */ +const ANY_TERM_RESULTS = 5 + /** One provider item, however many queries returned it. */ const documentKey = ({ container, kind, id }: NativeDocument) => JSON.stringify([container ?? '', kind ?? '', id]) @@ -19,9 +22,9 @@ const documentKey = ({ container, kind, id }: NativeDocument) => * without filling the page, with any of the query's significant terms. A natural-language query * often contains one word the right page lacks, which an all-terms search turns into no result. * - * All-terms matches keep their provider order ahead of the rest; the added matches are ordered by - * how many terms their title and preview contain, then how many their title contains, then - * provider order. The fallback runs only for a plain relevance-ordered query: native queries are + * All-terms matches keep their provider order ahead of the rest. At most a few matches are added, + * each containing at least two of the terms in its title and preview (one, for a two-term query), + * ordered by how many terms those contain, then how many its title contains, then provider order. The fallback runs only for a plain relevance-ordered query: native queries are * the caller's exact syntax, and a date-sorted page is reordered by date. It never replaces a * continuable page, so a cursor always continues the query that produced it. A provider refusal * keeps the all-terms page, reported as partial with the provider's message; cancellation and @@ -53,6 +56,7 @@ export async function searchWithAnyTermFallback( if (relaxed instanceof NativeSearchError) return { ...strict, partial: true, message: joinMessages([strict.message, relaxed.message]) } const seen = new Set(strict.documents.map(documentKey)) + const minCoverage = Math.min(2, terms.length - 1) const additions = relaxed.documents .filter((document) => !seen.has(documentKey(document))) .map((document, rank) => ({ @@ -61,13 +65,14 @@ export async function searchWithAnyTermFallback( coverage: termCoverage(`${document.title}\n${document.content}`, terms), titleCoverage: termCoverage(document.title, terms), })) + .filter(({ coverage }) => coverage >= minCoverage) .sort( (left, right) => right.coverage - left.coverage || right.titleCoverage - left.titleCoverage || left.rank - right.rank ) - .slice(0, input.limit - strict.documents.length) + .slice(0, Math.min(ANY_TERM_RESULTS, input.limit - strict.documents.length)) .map(({ document }) => document) if (!additions.length) return strict return { diff --git a/apps/sim/lib/sim-search/live/text-match.test.ts b/apps/sim/lib/sim-search/live/text-match.test.ts index 88c417ac38f..aa5d9ad2fc9 100644 --- a/apps/sim/lib/sim-search/live/text-match.test.ts +++ b/apps/sim/lib/sim-search/live/text-match.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it } from 'vitest' -import { atlassianTextClause, driveTextClause } from '@/lib/sim-search/live/text-match' +import { + confluenceTextClause, + driveTextClause, + jiraTextClause, +} from '@/lib/sim-search/live/text-match' describe('text match clauses', () => { it('joins distinctive terms with OR, leaving out stopwords', () => { - expect(atlassianTextClause('what is the release process', 'any')).toBe( + expect(jiraTextClause('what is the release process', 'any')).toBe( '(text ~ "release" OR text ~ "process")' ) expect(driveTextClause('roadmap for planning', 'any')).toBe( @@ -11,16 +15,26 @@ describe('text match clauses', () => { ) }) + it('relaxes a Confluence query to its relevance-ranked site search', () => { + expect(confluenceTextClause('release process', 'all')).toBe('text ~ "release process"') + expect(confluenceTextClause('what is the release process', 'any')).toBe( + 'siteSearch ~ "what is the release process"' + ) + }) + it('escapes every term so a term cannot add query syntax', () => { - expect(atlassianTextClause('ci/cd say"hi rollout', 'any')).toBe( + expect(jiraTextClause('ci/cd say"hi rollout', 'any')).toBe( String.raw`(text ~ "rollout" OR text ~ "say\\\"hi" OR text ~ "ci\\/cd")` ) + expect(confluenceTextClause('ci/cd say"hi', 'any')).toBe( + String.raw`siteSearch ~ "ci\\/cd say\\\"hi"` + ) expect(driveTextClause(String.raw`o'brien back\slash rollout`, 'any')).toBe( String.raw`(fullText contains 'back\\slash' or fullText contains 'o\'brien' or fullText contains 'rollout')` ) }) it('falls back to the all-terms clause when fewer than two terms are distinctive', () => { - expect(atlassianTextClause('how does it work', 'any')).toBe('text ~ "how does it work"') + expect(jiraTextClause('how does it work', 'any')).toBe('text ~ "how does it work"') }) }) diff --git a/apps/sim/lib/sim-search/live/text-match.ts b/apps/sim/lib/sim-search/live/text-match.ts index dca9aa84c41..355c3c71f2d 100644 --- a/apps/sim/lib/sim-search/live/text-match.ts +++ b/apps/sim/lib/sim-search/live/text-match.ts @@ -2,8 +2,8 @@ import { findTermMatches, queryTerms } from '@/lib/knowledge/search/snippet' /** * How a plain query's words must match a document. Confluence, Jira and Drive full-text search - * require every word by default, so one word a page lacks hides it; `any` matches pages containing - * at least one of the query's significant terms. + * require every word by default, so one word a page lacks hides it; `any` also matches pages that + * contain only some of the query's words. */ export type TextMatch = 'all' | 'any' @@ -66,13 +66,22 @@ export function significantTerms(query: string): string[] { const escapeSearchPhrase = (value: string) => JSON.stringify(value.replace(/[+\-&|!(){}[\]^"~*?:\\/]/g, '\\$&')) -/** The `text ~` clause for a plain query in Jira JQL or Confluence CQL. */ -export function atlassianTextClause(text: string, match: TextMatch = 'all'): string { +/** The `text ~` clause for a plain query in Jira JQL; `any` joins its significant terms with OR. */ +export function jiraTextClause(text: string, match: TextMatch = 'all'): string { const terms = match === 'any' ? significantTerms(text) : [] if (terms.length < 2) return `text ~ ${escapeSearchPhrase(text)}` return `(${terms.map((term) => `text ~ ${escapeSearchPhrase(term)}`).join(' OR ')})` } +/** + * The text clause for a plain query in Confluence CQL. `any` uses `siteSearch`, Confluence's own + * relevance search, which ranks a page with the query's rarer words above the many pages sharing + * one common word, where a `text ~` OR query ranks them the other way round. + */ +export function confluenceTextClause(text: string, match: TextMatch = 'all'): string { + return `${match === 'any' ? 'siteSearch' : 'text'} ~ ${escapeSearchPhrase(text)}` +} + export const escapeDriveLiteral = (value: string) => value.replace(/\\/g, '\\\\').replace(/'/g, "\\'") From e010ad5672db0cbfd1f4b4bf40033c8540e41ca4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 03:13:24 -0700 Subject: [PATCH 08/12] fix(search): require two terms in the provider's full text for Jira and Drive partial matches --- .../lib/sim-search/live/term-fallback.test.ts | 23 +--------- apps/sim/lib/sim-search/live/term-fallback.ts | 6 +-- .../lib/sim-search/live/text-match.test.ts | 23 +++++++--- apps/sim/lib/sim-search/live/text-match.ts | 42 ++++++++++++++++--- 4 files changed, 58 insertions(+), 36 deletions(-) diff --git a/apps/sim/lib/sim-search/live/term-fallback.test.ts b/apps/sim/lib/sim-search/live/term-fallback.test.ts index f332a696a0e..1284af22a1e 100644 --- a/apps/sim/lib/sim-search/live/term-fallback.test.ts +++ b/apps/sim/lib/sim-search/live/term-fallback.test.ts @@ -42,6 +42,7 @@ describe('searchWithAnyTermFallback', () => { 'exact', 'in-title', 'in-preview', + 'runbook', ]) expect(result.nextCursor).toBeUndefined() expect(result.message).toContain('Few results contained every search term') @@ -60,30 +61,10 @@ describe('searchWithAnyTermFallback', () => { const result = await searchWithAnyTermFallback(search, { ...input, limit: 2 }) expect(result.documents.map((document) => document.title)).toEqual([ 'Contractor offboarding form', + 'Runbook 0', ]) }) - it('keeps a page sharing a few of a long query’s terms, whose preview shows only some', async () => { - const search = provider( - page(), - page({ id: 'enrichments', title: 'Enrichments and Workflow Columns' }) - ) - const result = await searchWithAnyTermFallback(search, { - ...input, - query: 'enrichments vs workflow columns built-in email draft column', - }) - expect(result.documents.map((document) => document.id)).toEqual(['enrichments']) - }) - - it('keeps single-term matches of a two-term query, the closest a partial match gets', async () => { - const search = provider(page(), page({ id: 'contractor', title: 'Contractor directory' })) - const result = await searchWithAnyTermFallback(search, { - ...input, - query: 'contractor offboarding', - }) - expect(result.documents.map((document) => document.id)).toEqual(['contractor']) - }) - it('adds a few partial matches however large the page', async () => { const partial = Array.from({ length: 12 }, () => ({ title: 'Contractor offboarding' })) const search = provider(page(), page(...partial)) diff --git a/apps/sim/lib/sim-search/live/term-fallback.ts b/apps/sim/lib/sim-search/live/term-fallback.ts index 0e739e86b3a..0ceb1b0990b 100644 --- a/apps/sim/lib/sim-search/live/term-fallback.ts +++ b/apps/sim/lib/sim-search/live/term-fallback.ts @@ -23,8 +23,8 @@ const documentKey = ({ container, kind, id }: NativeDocument) => * often contains one word the right page lacks, which an all-terms search turns into no result. * * All-terms matches keep their provider order ahead of the rest. At most a few matches are added, - * each containing at least two of the terms in its title and preview (one, for a two-term query), - * ordered by how many terms those contain, then how many its title contains, then provider order. The fallback runs only for a plain relevance-ordered query: native queries are + * ordered by how many terms their title and preview contain, then how many their title contains, + * then provider order. The fallback runs only for a plain relevance-ordered query: native queries are * the caller's exact syntax, and a date-sorted page is reordered by date. It never replaces a * continuable page, so a cursor always continues the query that produced it. A provider refusal * keeps the all-terms page, reported as partial with the provider's message; cancellation and @@ -56,7 +56,6 @@ export async function searchWithAnyTermFallback( if (relaxed instanceof NativeSearchError) return { ...strict, partial: true, message: joinMessages([strict.message, relaxed.message]) } const seen = new Set(strict.documents.map(documentKey)) - const minCoverage = Math.min(2, terms.length - 1) const additions = relaxed.documents .filter((document) => !seen.has(documentKey(document))) .map((document, rank) => ({ @@ -65,7 +64,6 @@ export async function searchWithAnyTermFallback( coverage: termCoverage(`${document.title}\n${document.content}`, terms), titleCoverage: termCoverage(document.title, terms), })) - .filter(({ coverage }) => coverage >= minCoverage) .sort( (left, right) => right.coverage - left.coverage || diff --git a/apps/sim/lib/sim-search/live/text-match.test.ts b/apps/sim/lib/sim-search/live/text-match.test.ts index aa5d9ad2fc9..b3cbe292d04 100644 --- a/apps/sim/lib/sim-search/live/text-match.test.ts +++ b/apps/sim/lib/sim-search/live/text-match.test.ts @@ -6,7 +6,7 @@ import { } from '@/lib/sim-search/live/text-match' describe('text match clauses', () => { - it('joins distinctive terms with OR, leaving out stopwords', () => { + it('matches either term of a two-term query, leaving out stopwords', () => { expect(jiraTextClause('what is the release process', 'any')).toBe( '(text ~ "release" OR text ~ "process")' ) @@ -15,6 +15,19 @@ describe('text match clauses', () => { ) }) + it('requires two of the terms of a longer query anywhere in the full text', () => { + expect(jiraTextClause('contractor offboarding runbook', 'any')).toBe( + '((text ~ "offboarding" AND text ~ "contractor") OR (text ~ "offboarding" AND text ~ "runbook") OR (text ~ "contractor" AND text ~ "runbook"))' + ) + expect(driveTextClause('contractor offboarding runbook', 'any')).toBe( + "((fullText contains 'offboarding' and fullText contains 'contractor') or (fullText contains 'offboarding' and fullText contains 'runbook') or (fullText contains 'contractor' and fullText contains 'runbook'))" + ) + }) + + it('counts a term once whatever its case, so a pair always holds two different words', () => { + expect(jiraTextClause('API api rollout', 'any')).toBe('(text ~ "rollout" OR text ~ "API")') + }) + it('relaxes a Confluence query to its relevance-ranked site search', () => { expect(confluenceTextClause('release process', 'all')).toBe('text ~ "release process"') expect(confluenceTextClause('what is the release process', 'any')).toBe( @@ -23,14 +36,14 @@ describe('text match clauses', () => { }) it('escapes every term so a term cannot add query syntax', () => { - expect(jiraTextClause('ci/cd say"hi rollout', 'any')).toBe( - String.raw`(text ~ "rollout" OR text ~ "say\\\"hi" OR text ~ "ci\\/cd")` + expect(jiraTextClause('ci/cd say"hi', 'any')).toBe( + String.raw`(text ~ "say\\\"hi" OR text ~ "ci\\/cd")` ) expect(confluenceTextClause('ci/cd say"hi', 'any')).toBe( String.raw`siteSearch ~ "ci\\/cd say\\\"hi"` ) - expect(driveTextClause(String.raw`o'brien back\slash rollout`, 'any')).toBe( - String.raw`(fullText contains 'back\\slash' or fullText contains 'o\'brien' or fullText contains 'rollout')` + expect(driveTextClause(String.raw`o'brien back\slash`, 'any')).toBe( + String.raw`(fullText contains 'back\\slash' or fullText contains 'o\'brien')` ) }) diff --git a/apps/sim/lib/sim-search/live/text-match.ts b/apps/sim/lib/sim-search/live/text-match.ts index 355c3c71f2d..492c9ebc4ed 100644 --- a/apps/sim/lib/sim-search/live/text-match.ts +++ b/apps/sim/lib/sim-search/live/text-match.ts @@ -53,24 +53,49 @@ const STOPWORDS = new Set([ ]) /** - * The query's distinctive terms, longest first. Fewer than two means an any-term query would - * match the same pages as the all-terms one. + * The query's distinctive terms, longest first, each counted once whatever its case. Fewer than + * two means an any-term query would match the same pages as the all-terms one. */ export function significantTerms(query: string): string[] { + const seen = new Set() return queryTerms(query) - .filter((term) => !STOPWORDS.has(term.toLowerCase())) + .filter((term) => { + const key = term.toLowerCase() + if (STOPWORDS.has(key) || seen.has(key)) return false + seen.add(key) + return true + }) .slice(0, MAX_ANY_TERMS) } +/** + * An any-term clause: every pair of terms ORed together, so a match contains at least two of them + * somewhere in its full text, or either term of a two-term query. + */ +function anyTermClause( + terms: readonly string[], + clause: (term: string) => string, + and: string, + or: string +): string { + const groups = + terms.length === 2 + ? terms.map(clause) + : terms.flatMap((first, index) => + terms.slice(index + 1).map((second) => `(${clause(first)} ${and} ${clause(second)})`) + ) + return `(${groups.join(` ${or} `)})` +} + /** Escapes a value for a quoted JQL or CQL string, treating Lucene operators as literal text. */ const escapeSearchPhrase = (value: string) => JSON.stringify(value.replace(/[+\-&|!(){}[\]^"~*?:\\/]/g, '\\$&')) -/** The `text ~` clause for a plain query in Jira JQL; `any` joins its significant terms with OR. */ +/** The `text ~` clause for a plain query in Jira JQL. */ export function jiraTextClause(text: string, match: TextMatch = 'all'): string { const terms = match === 'any' ? significantTerms(text) : [] if (terms.length < 2) return `text ~ ${escapeSearchPhrase(text)}` - return `(${terms.map((term) => `text ~ ${escapeSearchPhrase(term)}`).join(' OR ')})` + return anyTermClause(terms, (term) => `text ~ ${escapeSearchPhrase(term)}`, 'AND', 'OR') } /** @@ -89,7 +114,12 @@ export const escapeDriveLiteral = (value: string) => export function driveTextClause(text: string, match: TextMatch = 'all'): string { const terms = match === 'any' ? significantTerms(text) : [] if (terms.length < 2) return `fullText contains '${escapeDriveLiteral(text)}'` - return `(${terms.map((term) => `fullText contains '${escapeDriveLiteral(term)}'`).join(' or ')})` + return anyTermClause( + terms, + (term) => `fullText contains '${escapeDriveLiteral(term)}'`, + 'and', + 'or' + ) } /** How many distinct terms occur as whole words in the text. */ From ed8859b06308454c1011f2dac7073ff1ee7d4610 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 03:15:25 -0700 Subject: [PATCH 09/12] fix(search): keep Drive on exact matches, relax Jira to two-term matches --- .../lib/sim-search/live/account-session.ts | 13 +---- .../lib/sim-search/live/application.test.ts | 11 ---- apps/sim/lib/sim-search/live/google.ts | 6 ++- apps/sim/lib/sim-search/live/providers.ts | 1 - apps/sim/lib/sim-search/live/scoped-search.ts | 8 +-- .../lib/sim-search/live/text-match.test.ts | 15 +----- apps/sim/lib/sim-search/live/text-match.ts | 54 ++++++------------- 7 files changed, 27 insertions(+), 81 deletions(-) diff --git a/apps/sim/lib/sim-search/live/account-session.ts b/apps/sim/lib/sim-search/live/account-session.ts index 2c2ffd0bf8b..0092b4cff55 100644 --- a/apps/sim/lib/sim-search/live/account-session.ts +++ b/apps/sim/lib/sim-search/live/account-session.ts @@ -18,7 +18,7 @@ import { createManagedSearchMcpClient } from '@/lib/sim-search/live/managed-mcp' import { isManagedSearchMcpProvider } from '@/lib/sim-search/live/managed-mcp-config' import { readNotionMcp, searchNotionMcp } from '@/lib/sim-search/live/notion-mcp' import { createPolicyVerifier } from '@/lib/sim-search/live/policy' -import { type LiveSearchPolicy, requiresScopedRetrieval } from '@/lib/sim-search/live/policy-schema' +import type { LiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' import { livePolicyFor, loadLiveSearchPolicies } from '@/lib/sim-search/live/policy-store' import { LIVE_SEARCH_PROVIDER_CATALOG } from '@/lib/sim-search/live/provider-catalog' import { isSearchProviderEnabled } from '@/lib/sim-search/live/provider-rollout' @@ -190,15 +190,6 @@ export async function openLiveAccountSession( throw error }) - /** - * A Drive source scoped to folders or file types, or verified by a service account, walks - * folders and checks each file's ancestry within the search's request budget; a second search - * would spend the requests that verify its exact matches. - */ - const verifiesByWalkingDrive = - provider === 'google_drive' && - (Boolean(boundary.scopeSearch) || requiresScopedRetrieval(provider, boundary.policy)) - return { async close() { await mcp?.close() @@ -214,7 +205,7 @@ export async function openLiveAccountSession( client ? searchNativeProvider(provider, client, native) : searchMcp(native) ) } - return client && requiresEveryTerm(provider) && !verifiesByWalkingDrive + return client && requiresEveryTerm(provider) ? searchWithAnyTermFallback(searchOnce, search) : searchOnce(search) }, diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index 46711b27ae8..7591c26a748 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -714,17 +714,6 @@ describe('authorized live retrieval', () => { expect(mocks.search).toHaveBeenCalledTimes(2) }) - it('does not spend a second search on a Drive source whose verification walks folders', async () => { - mocks.requiresEveryTerm.mockReturnValue(true) - vi.mocked(livePolicyFor).mockReturnValue({ - ...defaultLiveSearchPolicy(), - fileTypes: ['application/pdf'], - }) - mocks.search.mockResolvedValue({ documents: [] }) - await searchLiveKnowledge.execute({ principal, input: { ...input, query: 'launch checklist' } }) - expect(mocks.search).toHaveBeenCalledTimes(1) - }) - it('checks the service source before returning member-visible candidates', async () => { const verify = vi.fn().mockResolvedValue(false) mocks.service.mockResolvedValue({ policy: defaultLiveSearchPolicy(), verify, partial: false }) diff --git a/apps/sim/lib/sim-search/live/google.ts b/apps/sim/lib/sim-search/live/google.ts index c0ae80fc9b9..9ec11532743 100644 --- a/apps/sim/lib/sim-search/live/google.ts +++ b/apps/sim/lib/sim-search/live/google.ts @@ -14,7 +14,6 @@ import { array, NativeSearchError, object, segment, string } from '@/lib/sim-sea import { interleaveByRank } from '@/lib/sim-search/live/pages' import { permitsResources } from '@/lib/sim-search/live/policy' import { providerText } from '@/lib/sim-search/live/text' -import { driveTextClause } from '@/lib/sim-search/live/text-match' import type { NativeClient, NativeDocument, @@ -25,6 +24,8 @@ import type { const DRIVE_FIELDS = 'id,name,mimeType,webViewLink,description,modifiedTime,owners(displayName,emailAddress)' +export const escapeDriveLiteral = (value: string) => + value.replace(/\\/g, '\\\\').replace(/'/g, "\\'") function driveDocument(row: Record): NativeDocument { return { @@ -47,7 +48,8 @@ export async function searchDrive( ): Promise { const text = nativeText(input) const query = - input.native?.query || (text ? driveTextClause(text, input.textMatch) : 'trashed = false') + input.native?.query || + (text ? `fullText contains '${escapeDriveLiteral(text)}'` : 'trashed = false') const dates = nativeDateBounds(input) const data = object( await client.json('/drive/v3/files', { diff --git a/apps/sim/lib/sim-search/live/providers.ts b/apps/sim/lib/sim-search/live/providers.ts index 59bb82570db..9f5e73eb3f2 100644 --- a/apps/sim/lib/sim-search/live/providers.ts +++ b/apps/sim/lib/sim-search/live/providers.ts @@ -90,7 +90,6 @@ export const LIVE_SEARCH_PROVIDERS = { avoid: 'bare words without a term and operator, which Drive rejects, and trashed or modifiedTime clauses, which the server adds from startDate/endDate. Drive search does not search comments or replies; find the file by title/content, then read it to retrieve its discussion. PDF and DOCX reads extract text within download and parsing limits; scanned PDFs need OCR and unsupported binaries provide metadata only. Saved Google Meet transcripts and generated notes are Google Docs: use Drive fullText search and read their content. Drive date filters use file modification time, not meeting time.', }, - requiresEveryTerm: true, previewText: previewDrive, search: searchDrive, read: (client, reference, options) => readDrive(client, reference.id, options.signal), diff --git a/apps/sim/lib/sim-search/live/scoped-search.ts b/apps/sim/lib/sim-search/live/scoped-search.ts index 2b2b6144d2b..9bda4dbe251 100644 --- a/apps/sim/lib/sim-search/live/scoped-search.ts +++ b/apps/sim/lib/sim-search/live/scoped-search.ts @@ -1,8 +1,8 @@ import type { LiveSearchProvider } from '@/lib/api/contracts/mothership-assistant-tools' import { parseCodaResourceUri } from '@/lib/sim-search/live/coda-uri' +import { escapeDriveLiteral } from '@/lib/sim-search/live/google' import { array, object, string } from '@/lib/sim-search/live/http' import { collectNativePages } from '@/lib/sim-search/live/pages' -import { driveTextClause, escapeDriveLiteral } from '@/lib/sim-search/live/text-match' import type { NativeClient, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types' /** Provider restrictions improve recall; the independent verifier remains authoritative. */ @@ -51,7 +51,9 @@ export async function searchWithinPolicy( provider, query: input.native?.query ?? - (input.query ? driveTextClause(input.query, input.textMatch) : 'trashed = false'), + (input.query + ? `fullText contains '${escapeDriveLiteral(input.query)}'` + : 'trashed = false'), ...input.native, project: id, }, @@ -107,7 +109,7 @@ export async function searchWithinPolicy( } const text = input.native?.query ?? - (input.query ? driveTextClause(input.query, input.textMatch) : 'trashed = false') + (input.query ? `fullText contains '${escapeDriveLiteral(input.query)}'` : 'trashed = false') const result = await search( withQuery( `(${text}) and (${[...folders].map((id) => `'${escapeDriveLiteral(id)}' in parents`).join(' or ')})` diff --git a/apps/sim/lib/sim-search/live/text-match.test.ts b/apps/sim/lib/sim-search/live/text-match.test.ts index b3cbe292d04..2b8e635a704 100644 --- a/apps/sim/lib/sim-search/live/text-match.test.ts +++ b/apps/sim/lib/sim-search/live/text-match.test.ts @@ -1,27 +1,17 @@ import { describe, expect, it } from 'vitest' -import { - confluenceTextClause, - driveTextClause, - jiraTextClause, -} from '@/lib/sim-search/live/text-match' +import { confluenceTextClause, jiraTextClause } from '@/lib/sim-search/live/text-match' describe('text match clauses', () => { it('matches either term of a two-term query, leaving out stopwords', () => { expect(jiraTextClause('what is the release process', 'any')).toBe( '(text ~ "release" OR text ~ "process")' ) - expect(driveTextClause('roadmap for planning', 'any')).toBe( - "(fullText contains 'planning' or fullText contains 'roadmap')" - ) }) it('requires two of the terms of a longer query anywhere in the full text', () => { expect(jiraTextClause('contractor offboarding runbook', 'any')).toBe( '((text ~ "offboarding" AND text ~ "contractor") OR (text ~ "offboarding" AND text ~ "runbook") OR (text ~ "contractor" AND text ~ "runbook"))' ) - expect(driveTextClause('contractor offboarding runbook', 'any')).toBe( - "((fullText contains 'offboarding' and fullText contains 'contractor') or (fullText contains 'offboarding' and fullText contains 'runbook') or (fullText contains 'contractor' and fullText contains 'runbook'))" - ) }) it('counts a term once whatever its case, so a pair always holds two different words', () => { @@ -42,9 +32,6 @@ describe('text match clauses', () => { expect(confluenceTextClause('ci/cd say"hi', 'any')).toBe( String.raw`siteSearch ~ "ci\\/cd say\\\"hi"` ) - expect(driveTextClause(String.raw`o'brien back\slash`, 'any')).toBe( - String.raw`(fullText contains 'back\\slash' or fullText contains 'o\'brien')` - ) }) it('falls back to the all-terms clause when fewer than two terms are distinctive', () => { diff --git a/apps/sim/lib/sim-search/live/text-match.ts b/apps/sim/lib/sim-search/live/text-match.ts index 492c9ebc4ed..2b47e730029 100644 --- a/apps/sim/lib/sim-search/live/text-match.ts +++ b/apps/sim/lib/sim-search/live/text-match.ts @@ -1,9 +1,9 @@ import { findTermMatches, queryTerms } from '@/lib/knowledge/search/snippet' /** - * How a plain query's words must match a document. Confluence, Jira and Drive full-text search - * require every word by default, so one word a page lacks hides it; `any` also matches pages that - * contain only some of the query's words. + * How a plain query's words must match a document. Confluence and Jira text search require every + * word by default, so one word a page lacks hides it; `any` also matches pages that contain only + * some of the query's words. */ export type TextMatch = 'all' | 'any' @@ -68,34 +68,25 @@ export function significantTerms(query: string): string[] { .slice(0, MAX_ANY_TERMS) } -/** - * An any-term clause: every pair of terms ORed together, so a match contains at least two of them - * somewhere in its full text, or either term of a two-term query. - */ -function anyTermClause( - terms: readonly string[], - clause: (term: string) => string, - and: string, - or: string -): string { - const groups = - terms.length === 2 - ? terms.map(clause) - : terms.flatMap((first, index) => - terms.slice(index + 1).map((second) => `(${clause(first)} ${and} ${clause(second)})`) - ) - return `(${groups.join(` ${or} `)})` -} - /** Escapes a value for a quoted JQL or CQL string, treating Lucene operators as literal text. */ const escapeSearchPhrase = (value: string) => JSON.stringify(value.replace(/[+\-&|!(){}[\]^"~*?:\\/]/g, '\\$&')) -/** The `text ~` clause for a plain query in Jira JQL. */ +/** + * The `text ~` clause for a plain query in Jira JQL. `any` ORs every pair of terms, so a match + * contains at least two of them somewhere in the issue, or either term of a two-term query. + */ export function jiraTextClause(text: string, match: TextMatch = 'all'): string { const terms = match === 'any' ? significantTerms(text) : [] if (terms.length < 2) return `text ~ ${escapeSearchPhrase(text)}` - return anyTermClause(terms, (term) => `text ~ ${escapeSearchPhrase(term)}`, 'AND', 'OR') + const clause = (term: string) => `text ~ ${escapeSearchPhrase(term)}` + const groups = + terms.length === 2 + ? terms.map(clause) + : terms.flatMap((first, index) => + terms.slice(index + 1).map((second) => `(${clause(first)} AND ${clause(second)})`) + ) + return `(${groups.join(' OR ')})` } /** @@ -107,21 +98,6 @@ export function confluenceTextClause(text: string, match: TextMatch = 'all'): st return `${match === 'any' ? 'siteSearch' : 'text'} ~ ${escapeSearchPhrase(text)}` } -export const escapeDriveLiteral = (value: string) => - value.replace(/\\/g, '\\\\').replace(/'/g, "\\'") - -/** The `fullText contains` clause for a plain query in Drive's q language. */ -export function driveTextClause(text: string, match: TextMatch = 'all'): string { - const terms = match === 'any' ? significantTerms(text) : [] - if (terms.length < 2) return `fullText contains '${escapeDriveLiteral(text)}'` - return anyTermClause( - terms, - (term) => `fullText contains '${escapeDriveLiteral(term)}'`, - 'and', - 'or' - ) -} - /** How many distinct terms occur as whole words in the text. */ export function termCoverage(text: string, terms: readonly string[]): number { return terms.filter((term) => findTermMatches(text, [term]).length > 0).length From 9e7f377b5c5a0374da7da5f9a6174028523f0c32 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 03:47:34 -0700 Subject: [PATCH 10/12] fix(search): say Drive previews carry text only for plain-text searches --- apps/sim/lib/sim-search/live/google.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/sim/lib/sim-search/live/google.ts b/apps/sim/lib/sim-search/live/google.ts index 9ec11532743..067cb658f2b 100644 --- a/apps/sim/lib/sim-search/live/google.ts +++ b/apps/sim/lib/sim-search/live/google.ts @@ -81,7 +81,7 @@ export async function searchDrive( nextCursor: string(data.nextPageToken) || undefined, partial: data.incompleteSearch === true, message: - 'Drive searches file names and full text, including Docs, Sheets and Slides. The top Docs and Slides results preview their text; other previews are file metadata, so read a result for its content.', + 'Drive searches file names and full text, including Docs, Sheets and Slides. For a plain-text search the top Docs and Slides results preview their text; other previews are file metadata, so read a result for its content.', } } From 17a0ced3308f241105c69179567118963e495d91 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 04:18:17 -0700 Subject: [PATCH 11/12] docs(search): document the provider term and preview helpers --- apps/sim/lib/sim-search/live/providers.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/sim/lib/sim-search/live/providers.ts b/apps/sim/lib/sim-search/live/providers.ts index 9f5e73eb3f2..7333304ae55 100644 --- a/apps/sim/lib/sim-search/live/providers.ts +++ b/apps/sim/lib/sim-search/live/providers.ts @@ -313,11 +313,16 @@ export function searchNativeProvider( return adapter.search(client, input) } +/** True when the provider's plain-text search matches only items containing every query word. */ export function requiresEveryTerm(provider: LiveSearchProviderId): boolean { const adapter = LIVE_SEARCH_PROVIDERS[provider] return 'requiresEveryTerm' in adapter && adapter.requiresEveryTerm === true } +/** + * Full text for a search result's preview, for providers whose search returns only an excerpt or + * metadata. Undefined when the provider or the document's kind has no single-request text form. + */ export async function previewNativeText( provider: LiveSearchProviderId, client: NativeClient, From 9470b5b80a9ec14537b59e0399eb485a5e4bd010 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 04:35:01 -0700 Subject: [PATCH 12/12] fix(search): keep provider queries in the search tool's guidance --- apps/sim/lib/knowledge/mcp/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/sim/lib/knowledge/mcp/server.ts b/apps/sim/lib/knowledge/mcp/server.ts index d0cf03d02af..dd92bca415e 100644 --- a/apps/sim/lib/knowledge/mcp/server.ts +++ b/apps/sim/lib/knowledge/mcp/server.ts @@ -124,7 +124,7 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe { title: 'Search', description: - 'Search this organization’s sources through their live APIs, within your access and the admin’s source settings. A plain query searches every source; use source and date filters to narrow results, and nativeQueries only for a provider’s own operators (exact phrases, fields such as a Jira project or Confluence space, OR across alternatives) or a continuation cursor. Results carry a passage around their match where the source provides text. Inspect live.accounts for provider status and continuation cursors, and live.guidance for query syntax. Results are candidates, not proof of complete coverage. When a passage does not answer the question, use read_document with the exact returned documentId, and cite citationUrl when available.', + 'Search this organization’s sources through their live APIs, within your access and the admin’s source settings. Use source and date filters to narrow results, or nativeQueries for provider queries and pagination. Results carry a passage around their match where the source provides text. Inspect live.accounts for provider status and continuation cursors, and live.guidance for query syntax. Results are candidates, not proof of complete coverage. When a passage does not answer the question, use read_document with the exact returned documentId, and cite citationUrl when available.', inputSchema: liveSearchMcpSchema, annotations: KNOWLEDGE_MCP_READ_ONLY, },