diff --git a/apps/sim/app/api/v2/files/uploads/[uploadId]/route.test.ts b/apps/sim/app/api/v2/files/uploads/[uploadId]/route.test.ts index 9638d18ff15..7a90a0f3ece 100644 --- a/apps/sim/app/api/v2/files/uploads/[uploadId]/route.test.ts +++ b/apps/sim/app/api/v2/files/uploads/[uploadId]/route.test.ts @@ -88,7 +88,8 @@ describe('GET /api/v2/files/uploads/[uploadId]', () => { expect(v2RouteMocks.operationRate).toHaveBeenCalledWith( expect.stringContaining('files.upload.read'), - expect.anything() + expect.anything(), + 1 ) expect(v2RouteMocks.operationRate).not.toHaveBeenCalledWith( expect.stringContaining('files.upload.cancel'), diff --git a/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.test.ts b/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.test.ts index 92cd87f0dbb..bef2b85433c 100644 --- a/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.test.ts +++ b/apps/sim/app/api/v2/workflows/[workflowId]/execute/route.test.ts @@ -500,11 +500,13 @@ describe('POST /api/v2/workflows/[workflowId]/execute', () => { expect(mockCheckOperationRate).toHaveBeenCalledTimes(2) expect(mockCheckOperationRate).toHaveBeenCalledWith( 'v2:workflows.execute:api-key:key-1', - expect.anything() + expect.anything(), + 1 ) expect(mockCheckOperationRate).toHaveBeenCalledWith( 'v2:workflows.execute:workspace:workspace-1', - expect.anything() + expect.anything(), + 1 ) expect(mockPreprocessExecution).toHaveBeenCalledWith( expect.objectContaining({ rateLimitCounter: 'sync' }) diff --git a/apps/sim/lib/api/contracts/knowledge/mcp.ts b/apps/sim/lib/api/contracts/knowledge/mcp.ts index 9a3d279959a..814a4e55409 100644 --- a/apps/sim/lib/api/contracts/knowledge/mcp.ts +++ b/apps/sim/lib/api/contracts/knowledge/mcp.ts @@ -17,7 +17,39 @@ export const organizationKnowledgeMcpContract = defineRouteContract({ }) export const liveSearchMcpSchema = searchWorkspaceInputSchema.strict() -export const readLiveDocumentMcpSchema = readDocumentInputSchema.strict() +/** Documents one read_document call may read; each counts as one read against the rate limit. */ +const MAX_MCP_READ_DOCUMENTS = 5 + +export const readLiveDocumentMcpSchema = readDocumentInputSchema + .extend({ + documentId: readDocumentInputSchema.shape.documentId.optional(), + documentIds: z + .array(readDocumentInputSchema.shape.documentId) + .min(1) + .max(MAX_MCP_READ_DOCUMENTS) + .optional() + .describe( + `Up to ${MAX_MCP_READ_DOCUMENTS} document IDs returned by search, read together from the beginning. Use instead of documentId.` + ), + }) + .strict() + .superRefine((input, context) => { + if (Boolean(input.documentId) === Boolean(input.documentIds)) + context.addIssue({ + code: 'custom', + path: ['documentId'], + message: 'Pass either documentId or documentIds.', + }) + if ( + input.documentIds && + (input.startChunkIndex !== undefined || input.startOffset !== undefined) + ) + context.addIssue({ + code: 'custom', + path: ['documentIds'], + message: 'Continue a document with documentId and its next offsets, not documentIds.', + }) + }) export const chatSearchMcpSchema = workspaceSearchFiltersSchema .extend({ diff --git a/apps/sim/lib/api/server/routes/v2-json-route.test.ts b/apps/sim/lib/api/server/routes/v2-json-route.test.ts index c287431dfc5..b58799290af 100644 --- a/apps/sim/lib/api/server/routes/v2-json-route.test.ts +++ b/apps/sim/lib/api/server/routes/v2-json-route.test.ts @@ -278,11 +278,13 @@ describe('defineV2JsonRoute', () => { expect(v2RouteMocks.operationRate).toHaveBeenCalledTimes(2) expect(v2RouteMocks.operationRate).toHaveBeenCalledWith( 'v2:widgets.update:api-key:key-1', - expect.objectContaining({ maxTokens: 100 }) + expect.objectContaining({ maxTokens: 100 }), + 1 ) expect(v2RouteMocks.operationRate).toHaveBeenCalledWith( 'v2:widgets.update:user:user-1', - expect.objectContaining({ maxTokens: 100 }) + expect.objectContaining({ maxTokens: 100 }), + 1 ) expect(response.headers.get('X-RateLimit-Limit')).toBe('100') expect(response.headers.get('X-RateLimit-Remaining')).toBe('12') diff --git a/apps/sim/lib/api/server/routes/v2-json-route.ts b/apps/sim/lib/api/server/routes/v2-json-route.ts index 69f219a6d37..ffab2e745f6 100644 --- a/apps/sim/lib/api/server/routes/v2-json-route.ts +++ b/apps/sim/lib/api/server/routes/v2-json-route.ts @@ -83,17 +83,19 @@ export const v2ApiKeyAuth = { export interface V2RateLimitPolicy { readonly kind: 'public_api' + /** `cost` charges a request that performs several operations, such as a batched read. */ enforce( request: NextRequest, auth: V2ApiKeyAuthContext, - operation: ApplicationOperation + operation: ApplicationOperation, + cost?: number ): Promise } export const v2RateLimits = { publicApi: { kind: 'public_api', - async enforce(request, auth, operation) { + async enforce(request, auth, operation, cost = 1) { const plan = (auth.rateLimitSubscription?.plan ?? 'free') as SubscriptionPlan const config = getRateLimit(plan, 'api-endpoint') const buckets = await Promise.all( @@ -101,7 +103,8 @@ export const v2RateLimits = { try { return await rateLimiter.checkRateLimitDirectOrThrow( `v2:${operation.id}:${subjectId}`, - config + config, + cost ) } catch (error) { throw new V2RouteInfrastructureError('rate_limit', error) diff --git a/apps/sim/lib/core/rate-limiter/rate-limiter.ts b/apps/sim/lib/core/rate-limiter/rate-limiter.ts index d2e132ecf1e..838ced01b68 100644 --- a/apps/sim/lib/core/rate-limiter/rate-limiter.ts +++ b/apps/sim/lib/core/rate-limiter/rate-limiter.ts @@ -227,15 +227,17 @@ export class RateLimiter { } /** - * Consume one token from an already-namespaced bucket and propagate storage - * failures. Declarative API adapters use this for credential/workspace - * buckets so an unavailable limiter is never mistaken for spare capacity. + * Consume tokens (one by default) from an already-namespaced bucket and + * propagate storage failures. Declarative API adapters use this for + * credential/workspace buckets so an unavailable limiter is never mistaken + * for spare capacity. */ async checkRateLimitDirectOrThrow( storageKey: string, - config: { maxTokens: number; refillRate: number; refillIntervalMs: number } + config: { maxTokens: number; refillRate: number; refillIntervalMs: number }, + tokens = 1 ): Promise { - const result = await this.storage.consumeTokens(storageKey, 1, config) + const result = await this.storage.consumeTokens(storageKey, tokens, config) if (!result.allowed) { logger.info('Rate limit exceeded', { storageKey, tokensRemaining: result.tokensRemaining }) } diff --git a/apps/sim/lib/knowledge/mcp/server.test.ts b/apps/sim/lib/knowledge/mcp/server.test.ts index 087fd8b2bd1..f7ec4449a40 100644 --- a/apps/sim/lib/knowledge/mcp/server.test.ts +++ b/apps/sim/lib/knowledge/mcp/server.test.ts @@ -163,6 +163,68 @@ describe('live organization Search MCP', () => { ) }) +describe('batched document reads', () => { + const readResult = (documentId: string) => ({ + documentId, + knowledgeBaseId: '', + knowledgeBaseName: 'Drive', + documentName: documentId, + sourceUrl: `https://example.com/${documentId}`, + connectorType: 'google_drive', + chunks: [], + hasMore: false, + next: null, + }) + const parsedBody = (result: CallToolResult) => { + const [part] = result.content + if (part?.type !== 'text') throw new Error('Expected a text result') + return JSON.parse(part.text) as { documents: Record[] } + } + + it('reports an unreadable document in its place and returns the others', async () => { + create() + mocks.liveRead.mockImplementation(async ({ input }: { input: { documentId: string } }) => { + if (input.documentId === 'gone') throw new OrchestrationError('not_found', 'Document moved') + if (input.documentId === 'broken') throw new Error('private backend detail') + return readResult(input.documentId) + }) + const result = await call('read_document', { documentIds: ['a', 'gone', 'broken', 'b'] }) + expect(result.isError).toBeUndefined() + expect(parsedBody(result).documents).toEqual([ + expect.objectContaining({ documentId: 'a', title: 'a' }), + { documentId: 'gone', error: 'Document moved' }, + { documentId: 'broken', error: 'Unable to read this document. Please try again.' }, + expect.objectContaining({ documentId: 'b', title: 'b' }), + ]) + }) + + it('charges the rate limit once per distinct document', async () => { + create() + mocks.liveRead.mockImplementation(async ({ input }: { input: { documentId: string } }) => + readResult(input.documentId) + ) + await call('read_document', { documentIds: ['a', 'b', 'a'] }) + expect(mocks.rateLimit).toHaveBeenCalledWith(request, auth, expect.anything(), 2) + expect(mocks.liveRead).toHaveBeenCalledTimes(2) + }) + + it('reads nothing when the batch exceeds the remaining rate limit', async () => { + create() + mocks.rateLimit.mockResolvedValueOnce(new Response(null, { status: 429 })) + expect((await call('read_document', { documentIds: ['a', 'b'] })).isError).toBe(true) + expect(mocks.liveRead).not.toHaveBeenCalled() + }) + + it.each([ + [{ documentId: 'a', documentIds: ['b'] }], + [{}], + [{ documentIds: ['a'], startOffset: 8000 }], + ])('rejects ambiguous read input %j', (input) => { + create() + expect(() => mocks.configs.get('read_document')?.inputSchema.parse(input)).toThrow() + }) +}) + describe('organization Search MCP tools', () => { it('does not return content denied by the canonical document ACL operation', async () => { create() diff --git a/apps/sim/lib/knowledge/mcp/server.ts b/apps/sim/lib/knowledge/mcp/server.ts index 449b1989d58..dd92bca415e 100644 --- a/apps/sim/lib/knowledge/mcp/server.ts +++ b/apps/sim/lib/knowledge/mcp/server.ts @@ -31,6 +31,29 @@ import { v2CaughtOrchestrationError } from '@/app/api/v2/lib/response' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const logger = createLogger('KnowledgeMcp') + +/** The message a caller may see for an expected failure; undefined for an internal error. */ +async function callerErrorMessage(error: unknown): Promise { + if (error instanceof LiveReadError) return error.message + const response = v2CaughtOrchestrationError(error) + if (!response) return undefined + const body: unknown = await response.json() + return isPlainRecord(body) && isPlainRecord(body.error) && typeof body.error.message === 'string' + ? body.error.message + : undefined +} + +/** One read_document result in the shape the tool returns it. */ +function projectReadResult(result: Awaited>) { + const { knowledgeBaseId: _knowledgeBaseId, knowledgeBaseName: _name, ...document } = result + return { + ...document, + title: result.documentName, + citationId: liveCitationId(result.documentId), + citationUrl: result.sourceUrl, + } +} + interface KnowledgeMcpContext { organizationId: string request: NextRequest @@ -43,13 +66,13 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe const principal = auth.principal const server = new McpServer({ name: 'Sim Search', version: '1.0.0' }) - const execute: KnowledgeMcpToolRunner = async (toolName, operation, toolSignal, run) => { + const execute: KnowledgeMcpToolRunner = async (toolName, operation, toolSignal, run, cost) => { const startedAt = performance.now() const signal = AbortSignal.any([request.signal, toolSignal]) let outcome: SearchMcpActivityInput['outcome'] = 'error' try { signal.throwIfAborted() - const limited = await v2RateLimits.publicApi.enforce(request, auth, operation) + const limited = await v2RateLimits.publicApi.enforce(request, auth, operation, cost) signal.throwIfAborted() if (limited) { outcome = 'rate_limited' @@ -68,18 +91,8 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe return result } catch (error) { if (signal.aborted) outcome = 'cancelled' - if (error instanceof LiveReadError) return toolError(error.message) - const response = v2CaughtOrchestrationError(error) - if (response) { - const body: unknown = await response.json() - if ( - isPlainRecord(body) && - isPlainRecord(body.error) && - typeof body.error.message === 'string' - ) { - return toolError(body.error.message) - } - } + const message = await callerErrorMessage(error) + if (message) return toolError(message) logger.error('Knowledge MCP operation failed', { operation: operation.id, error }) return toolError('Unable to complete this operation. Please try again.') } finally { @@ -111,7 +124,7 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe { title: 'Search', description: - 'Search this organization’s sources through their live APIs, within your access and the admin’s source settings. Use source and date filters to narrow results, or nativeQueries for provider queries and pagination. Inspect live.accounts for provider status and continuation cursors, and live.guidance for query syntax. Results are candidates, not proof of complete coverage. Use read_document with the exact returned documentId for context and cite citationUrl when available.', + 'Search this organization’s sources through their live APIs, within your access and the admin’s source settings. Use source and date filters to narrow results, or nativeQueries for provider queries and pagination. Results carry a passage around their match where the source provides text. Inspect live.accounts for provider status and continuation cursors, and live.guidance for query syntax. Results are candidates, not proof of complete coverage. When a passage does not answer the question, use read_document with the exact returned documentId, and cite citationUrl when available.', inputSchema: liveSearchMcpSchema, annotations: KNOWLEDGE_MCP_READ_ONLY, }, @@ -159,38 +172,59 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe { title: 'Read document', description: - 'Read a live document using the exact documentId returned by search. Access and the admin’s source settings are checked again on every read. When hasMore is true, pass next.startChunkIndex and next.startOffset with the same documentId to continue. Cite citationUrl when available.', + 'Read live documents using the exact document IDs returned by search: documentId for one, or documentIds to read several in one call rather than one call each. Access and the admin’s source settings are checked again on every read. When a document’s hasMore is true, pass its next.startChunkIndex and next.startOffset with the same documentId to continue. Cite citationUrl when available.', inputSchema: readLiveDocumentMcpSchema, annotations: KNOWLEDGE_MCP_READ_ONLY, }, - async (raw: unknown, extra: { signal: AbortSignal }) => - execute( + async (raw: unknown, extra: { signal: AbortSignal }) => { + const parsed = readLiveDocumentMcpSchema.safeParse(raw) + const batch = + parsed.success && parsed.data.documentIds ? [...new Set(parsed.data.documentIds)] : [] + return execute( 'read_document', knowledgeOperations.readDocument, extra.signal, async (registry, signal) => { - const input = readLiveDocumentMcpSchema.parse(raw) - const result = await readLiveDocument.execute({ - principal, - input: { ...input, organizationId, resultSecretRegistry: registry, signal }, - request, - }) - const { - knowledgeBaseId: _knowledgeBaseId, - knowledgeBaseName: _name, - ...document - } = result - return projectResult( - { - ...document, - title: result.documentName, - citationId: liveCitationId(result.documentId), - citationUrl: result.sourceUrl, - }, - registry + if (!parsed.success) throw parsed.error + const { documentId, documentIds: _documentIds, ...options } = parsed.data + const read = (id: string) => + readLiveDocument.execute({ + principal, + input: { + ...options, + documentId: id, + organizationId, + resultSecretRegistry: registry, + signal, + }, + request, + }) + if (documentId) return projectResult(projectReadResult(await read(documentId)), registry) + /** One unreadable document leaves the others; each failure is reported in its place. */ + const documents = await Promise.all( + batch.map(async (id) => { + try { + return projectReadResult(await read(id)) + } catch (error) { + signal.throwIfAborted() + const message = await callerErrorMessage(error) + if (!message) + logger.error('Knowledge MCP batched read failed', { + operation: knowledgeOperations.readDocument.id, + error, + }) + return { + documentId: id, + error: message ?? 'Unable to read this document. Please try again.', + } + } + }) ) - } + return projectResult({ documents }, registry) + }, + Math.max(1, batch.length) ) + } ) server.registerTool( @@ -198,7 +232,7 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe { title: 'Chat', description: - 'Ask the Sim Assistant to answer a question using your accessible organization sources. Returns an answer with citations and starts a new private conversation. Use source, modifiedAfter, or documentIds to narrow the evidence. No web search or source changes. Use search instead when you need raw passages.', + 'Ask the Sim Assistant to research a question in your accessible organization sources and write a cited answer. It runs its own multi-step search, so it usually takes 30 to 60 seconds and returns a synthesized answer rather than the sources. To answer a question yourself, use search and read_document, which return sources in seconds; use chat only when the user asks for the Sim Assistant’s answer. Starts a new private conversation. Use source, modifiedAfter, or documentIds to narrow the evidence. No web search or source changes.', inputSchema: chatSearchMcpSchema, annotations: { readOnlyHint: false, diff --git a/apps/sim/lib/knowledge/mcp/tool-runner.ts b/apps/sim/lib/knowledge/mcp/tool-runner.ts index 8b03011a7fa..a3828accab6 100644 --- a/apps/sim/lib/knowledge/mcp/tool-runner.ts +++ b/apps/sim/lib/knowledge/mcp/tool-runner.ts @@ -22,7 +22,9 @@ export type KnowledgeMcpToolRunner = ( toolName: SearchMcpActivityInput['toolName'], operation: ApplicationOperation, toolSignal: AbortSignal, - run: (registry: ResolvedSecretTraceRegistry, signal: AbortSignal) => Promise + run: (registry: ResolvedSecretTraceRegistry, signal: AbortSignal) => Promise, + /** Operations the call performs for the rate limit, such as each document a batched read reads. */ + cost?: number ) => Promise /** A tool result projected for its resolved secrets and bounded in size. */ diff --git a/apps/sim/lib/knowledge/search/diagnostics.ts b/apps/sim/lib/knowledge/search/diagnostics.ts index 2468262b182..81a13db8188 100644 --- a/apps/sim/lib/knowledge/search/diagnostics.ts +++ b/apps/sim/lib/knowledge/search/diagnostics.ts @@ -53,6 +53,7 @@ export type SearchStage = | 'live.session' | 'live.search' | 'live.verify' + | 'live.preview' | 'live.read' /** Fixed, content-free fields. Never pass queries, filters, document identities, SQL, or errors. */ diff --git a/apps/sim/lib/sim-search/live/account-session.ts b/apps/sim/lib/sim-search/live/account-session.ts index b230a4aca56..0092b4cff55 100644 --- a/apps/sim/lib/sim-search/live/account-session.ts +++ b/apps/sim/lib/sim-search/live/account-session.ts @@ -22,9 +22,15 @@ import type { LiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' import { livePolicyFor, loadLiveSearchPolicies } from '@/lib/sim-search/live/policy-store' import { LIVE_SEARCH_PROVIDER_CATALOG } from '@/lib/sim-search/live/provider-catalog' import { isSearchProviderEnabled } from '@/lib/sim-search/live/provider-rollout' -import { readNativeProvider, searchNativeProvider } from '@/lib/sim-search/live/providers' +import { + previewNativeText, + readNativeProvider, + requiresEveryTerm, + searchNativeProvider, +} from '@/lib/sim-search/live/providers' import { searchWithinPolicy } from '@/lib/sim-search/live/scoped-search' import { createLiveServiceSession } from '@/lib/sim-search/live/service-session' +import { searchWithAnyTermFallback } from '@/lib/sim-search/live/term-fallback' import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types' import { readZoomMcp, searchZoomMcp } from '@/lib/sim-search/live/zoom-mcp' @@ -45,6 +51,8 @@ export interface LiveAccountSession { /** True only when the document is inside the source boundary and the member may read it. */ verify(document: Reference): Promise read(reference: Reference, filters?: WorkspaceSearchFilters): Promise + /** Full text of a verified search result whose provider returned only an excerpt or metadata. */ + previewText(document: NativeDocument): Promise /** Checks a fetched document against the source settings saved now, not at session start. */ verifyCurrent(document: NativeDocument): Promise } @@ -189,12 +197,17 @@ export async function openLiveAccountSession( policy: boundary.policy, servicePartial: boundary.partial, async search(search) { - const scoped = boundary.scopeSearch ? boundary.scopeSearch(search) : search - if (scoped === null) return { documents: [] } - if (admin) return admin.search(scoped) - return searchWithinPolicy(provider, client, scoped, (request) => - client ? searchNativeProvider(provider, client, request) : searchMcp(request) - ) + const searchOnce = async (request: NativeSearchInput): Promise => { + const scoped = boundary.scopeSearch ? boundary.scopeSearch(request) : request + if (scoped === null) return { documents: [] } + if (admin) return admin.search(scoped) + return searchWithinPolicy(provider, client, scoped, (native) => + client ? searchNativeProvider(provider, client, native) : searchMcp(native) + ) + } + return client && requiresEveryTerm(provider) + ? searchWithAnyTermFallback(searchOnce, search) + : searchOnce(search) }, async verify(document) { if (!(await boundary.verify(document))) return false @@ -211,6 +224,10 @@ export async function openLiveAccountSession( }) return readMcp(reference) }, + async previewText(document) { + if (admin || !client) return undefined + return previewNativeText(provider, client, document) + }, async verifyCurrent(document) { if (!(await isSearchProviderEnabled(provider, resourceScopeFromOwner(owner)))) return false const current = await sourceBoundary( diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index d30e97119a7..7591c26a748 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -26,6 +26,8 @@ const mocks = vi.hoisted(() => ({ json: vi.fn(), service: vi.fn(), destroyPool: vi.fn(), + requiresEveryTerm: vi.fn(), + preview: vi.fn(), })) vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) vi.mock('@/lib/sim-search/live/service-session', () => ({ @@ -65,6 +67,8 @@ vi.mock('@/lib/sim-search/live/providers', () => ({ liveSearchGuidance: (providers: string[]) => `Live coverage: ${[...new Set(providers)]}`, searchNativeProvider: mocks.search, readNativeProvider: mocks.read, + requiresEveryTerm: mocks.requiresEveryTerm, + previewNativeText: mocks.preview, })) vi.mock('@/executor/utils/resolved-secret-content-projection', () => ({ projectResolvedSecretModelContent: (value: string) => ({ safe: true, value }), @@ -136,6 +140,8 @@ describe('authorized live retrieval', () => { documents: [{ ...document, id: 'src/a.ts', container: '42', kind: 'code' }], }) mocks.adminVerify.mockResolvedValue(true) + mocks.requiresEveryTerm.mockReset().mockReturnValue(false) + mocks.preview.mockReset().mockResolvedValue(undefined) }) describe('Zoom continuation integrity', () => { const connected = { @@ -611,6 +617,103 @@ describe('authorized live retrieval', () => { expect(result.live?.accounts.map((status) => status.queryIndex)).toEqual([0, 1]) expect(result.live?.guidance).toBe('Live coverage: slack') }) + describe('result previews', () => { + const excerpts = ['a', 'b', 'c', 'd'].map((id) => ({ + ...document, + id, + url: `https://docs.google.com/document/d/${id}/edit`, + content: `Excerpt ${id}`, + })) + const contents = (result: Awaited>) => + result.results.map((row) => row.content) + + it('replaces the excerpts of an account’s top results with their full text', async () => { + mocks.search.mockResolvedValue({ documents: excerpts }) + mocks.preview.mockImplementation(async (_provider, _client, { id }: { id: string }) => + id === 'a' ? undefined : `Full ${id} launch text` + ) + const result = await searchLiveKnowledge.execute({ principal, input }) + expect(contents(result)).toEqual([ + 'Excerpt a', + 'Full b launch text', + 'Full c launch text', + 'Excerpt d', + ]) + expect(mocks.preview).toHaveBeenCalledTimes(3) + }) + + it('previews the passage around a match deep inside a long document', async () => { + mocks.search.mockResolvedValue({ documents: excerpts.slice(0, 1) }) + mocks.preview.mockResolvedValue(`${'filler '.repeat(30_000)}the launch checklist`) + const [row] = (await searchLiveKnowledge.execute({ principal, input })).results + expect(row?.content).toContain('launch checklist') + }) + + it('keeps the provider excerpt when a preview read fails', async () => { + mocks.search.mockResolvedValue({ documents: excerpts.slice(0, 2) }) + mocks.preview + .mockRejectedValueOnce(new NativeSearchError('rate_limited', 'Slow down')) + .mockResolvedValueOnce('Full b launch text') + const result = await searchLiveKnowledge.execute({ principal, input }) + expect(contents(result)).toEqual(['Excerpt a', 'Full b launch text']) + expect(result.live?.accounts[0]?.status).toBe('ok') + }) + + it('keeps the provider excerpt when a preview read misses its deadline', async () => { + vi.useFakeTimers() + try { + mocks.search.mockResolvedValue({ documents: excerpts.slice(0, 1) }) + mocks.preview.mockReturnValue(new Promise(() => {})) + const pending = searchLiveKnowledge.execute({ principal, input }) + await vi.advanceTimersByTimeAsync(2500) + expect(contents(await pending)).toEqual(['Excerpt a']) + } finally { + vi.useRealTimers() + } + }) + + it('does not read previews for a native query, whose operators would anchor the passage', async () => { + mocks.search.mockResolvedValue({ documents: excerpts }) + await searchLiveKnowledge.execute({ + principal, + input: { + ...input, + query: '', + nativeQueries: [ + { provider: 'google_drive' as const, query: "fullText contains 'launch'" }, + ], + }, + }) + expect(mocks.preview).not.toHaveBeenCalled() + }) + + it('does not read previews for a listing without search terms', async () => { + mocks.search.mockResolvedValue({ documents: excerpts }) + await searchLiveKnowledge.execute({ + principal, + input: { ...input, query: '', filters: { sortBy: 'newest' } }, + }) + expect(mocks.preview).not.toHaveBeenCalled() + }) + }) + + it('retries an underfilled search with any term for providers that require every term', async () => { + mocks.requiresEveryTerm.mockReturnValue(true) + const loose = { ...document, id: 'loose', url: 'https://docs.google.com/document/d/loose/edit' } + mocks.search.mockImplementation(async (_provider, _client, search) => ({ + documents: search.textMatch === 'any' ? [loose, document] : [document], + })) + const result = await searchLiveKnowledge.execute({ + principal, + input: { ...input, query: 'launch checklist' }, + }) + expect(result.results.map((row) => decodeLiveReference(row.documentId).id)).toEqual([ + 'doc', + 'loose', + ]) + expect(mocks.search).toHaveBeenCalledTimes(2) + }) + it('checks the service source before returning member-visible candidates', async () => { const verify = vi.fn().mockResolvedValue(false) mocks.service.mockResolvedValue({ policy: defaultLiveSearchPolicy(), verify, partial: false }) diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 65ca860549c..6c92a86a95c 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -307,6 +307,42 @@ async function verifyCandidates(session: LiveAccountSession, candidates: LiveCan } } +/** Top results per account read for a passage when their provider returns only an excerpt. */ +const PREVIEW_READS = 3 +/** A slow preview read keeps the provider's excerpt rather than delaying the search. */ +const PREVIEW_DEADLINE_MS = 2500 +/** Text kept around the match from a preview read, so a long document stays bounded. */ +const PREVIEW_SOURCE_CHARACTERS = 100_000 + +/** + * Replaces the excerpt or metadata of an account's top results with their full text, so their + * preview is a passage around the match like other providers' results rather than a prompt to + * read. A read that fails or misses the deadline keeps the provider's own preview. + */ +async function withPreviewText( + session: LiveAccountSession, + candidates: LiveCandidate[], + query: string +): Promise { + const deadline = AbortSignal.timeout(PREVIEW_DEADLINE_MS) + const expired = new Promise((resolve) => + deadline.addEventListener('abort', () => resolve(undefined), { once: true }) + ) + const texts = await Promise.all( + candidates + .slice(0, PREVIEW_READS) + .map((candidate) => + Promise.race([session.previewText(candidate.document).catch(() => undefined), expired]) + ) + ) + return candidates.map((candidate, index) => { + const text = texts[index]?.trim() + if (!text) return candidate + const content = matchPassage(text, query, PREVIEW_SOURCE_CHARACTERS).content + return { ...candidate, document: { ...candidate.document, content } } + }) +} + /** Keeps the first of candidates a provider marked as the same item, such as a shared meeting. */ function firstOfEachDocument(candidates: LiveCandidate[]): LiveCandidate[] { const seen = new Set() @@ -507,6 +543,13 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ matchesLiveFilters(document, documentId, account.provider, queryFilters) ) ) + /** A native query's operators would anchor the passage, so only plain queries preview. */ + const previewed = + !native && input.query.trim() + ? await measureSearchStage('live.preview', () => + withPreviewText(session, matching, input.query) + ) + : matching const undatedExcluded = permitted.some(({ document }) => lacksFilterDate(document, account.provider, queryFilters) ) @@ -571,7 +614,7 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ }) : page.nextCursor, }, - results: matching.map((candidate, index) => { + results: previewed.map((candidate, index) => { const result = resultFor( candidate, account, diff --git a/apps/sim/lib/sim-search/live/atlassian.test.ts b/apps/sim/lib/sim-search/live/atlassian.test.ts index d6a9376c80b..d0d0816ddd5 100644 --- a/apps/sim/lib/sim-search/live/atlassian.test.ts +++ b/apps/sim/lib/sim-search/live/atlassian.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest' -import { readAtlassian, searchAtlassian } from '@/lib/sim-search/live/atlassian' +import { previewConfluence, readAtlassian, searchAtlassian } from '@/lib/sim-search/live/atlassian' import { NativeSearchError } from '@/lib/sim-search/live/http' import type { NativeClient } from '@/lib/sim-search/live/types' @@ -181,4 +181,19 @@ describe('Confluence live documents', () => { expect(page.message).toContain('cannot be read') expect(page.partial).toBe(true) }) + + it('keeps the search excerpt when a previewed page body has no text', async () => { + const page = (value: string) => + client({ [`${v2}/pages/7`]: { id: '7', title: 'Diagram', body: { view: { value } } } }) + const document = { + id: '7', + kind: 'page', + container: 'cloud', + title: 'Diagram', + url: '', + content: '', + } + expect(await previewConfluence(page(''), document)).toBeUndefined() + expect(await previewConfluence(page('

Rollout steps

'), document)).toBe('Rollout steps') + }) }) diff --git a/apps/sim/lib/sim-search/live/atlassian.ts b/apps/sim/lib/sim-search/live/atlassian.ts index 5c554666a04..b7e0cfc93e8 100644 --- a/apps/sim/lib/sim-search/live/atlassian.ts +++ b/apps/sim/lib/sim-search/live/atlassian.ts @@ -9,6 +9,7 @@ import { } from '@/lib/sim-search/live/http' import { interleaveByRank } from '@/lib/sim-search/live/pages' import { providerText } from '@/lib/sim-search/live/text' +import { confluenceTextClause, jiraTextClause } from '@/lib/sim-search/live/text-match' import type { NativeClient, NativeDocument, @@ -16,9 +17,6 @@ import type { NativeSearchInput, } from '@/lib/sim-search/live/types' -export const escapeSearchPhrase = (value: string) => - JSON.stringify(value.replace(/[+\-&|!(){}[\]^"~*?:\\/]/g, '\\$&')) - /** Place trusted scope before ORDER BY while respecting quoted JQL/CQL values. */ export function scopeAtlassianQuery(query: string, scope?: string, order?: string): string { if (!scope && !order) return query @@ -163,7 +161,7 @@ export async function searchAtlassian( await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, { body: { jql: scopeAtlassianQuery( - input.native?.query || (text ? `text ~ ${escapeSearchPhrase(text)}` : ''), + input.native?.query || (text ? jiraTextClause(text, input.textMatch) : ''), scope, order ), @@ -184,7 +182,7 @@ export async function searchAtlassian( query: { cql: scopeAtlassianQuery( input.native?.query || - `type IN (page, blogpost)${text ? ` AND text ~ ${escapeSearchPhrase(text)}` : ''}`, + `type IN (page, blogpost)${text ? ` AND ${confluenceTextClause(text, input.textMatch)}` : ''}`, scope, order ), @@ -246,6 +244,23 @@ export async function readAtlassian( return readConfluence(client, cloudId, string(site.url), id, kind) } +/** Page and blog post text for a search preview: one v2 read, reusing the search's site list. */ +export async function previewConfluence( + client: NativeClient, + document: NativeDocument +): Promise { + if (document.kind !== 'page' && document.kind !== 'blogpost') return undefined + const page = await readAtlassian( + client, + 'confluence', + document.id, + document.container, + document.kind + ) + /** A body without text reads as its title, which says less than the search excerpt. */ + return page.content === page.title ? undefined : page.content +} + /** * Reads through the v2 API, whose page, blog post, and space endpoints need only the granular * read scopes a Search connection grants; v1 content reads also need read:content-details. diff --git a/apps/sim/lib/sim-search/live/google.ts b/apps/sim/lib/sim-search/live/google.ts index 408ee7828bc..067cb658f2b 100644 --- a/apps/sim/lib/sim-search/live/google.ts +++ b/apps/sim/lib/sim-search/live/google.ts @@ -81,10 +81,25 @@ export async function searchDrive( nextCursor: string(data.nextPageToken) || undefined, partial: data.incompleteSearch === true, message: - 'Drive searches file names and full text, including Docs, Sheets and Slides. Previews are file metadata; read a result for its content.', + 'Drive searches file names and full text, including Docs, Sheets and Slides. For a plain-text search the top Docs and Slides results preview their text; other previews are file metadata, so read a result for its content.', } } +/** Docs and Slides export as plain text in one request; other kinds need a download and parse. */ +const PREVIEW_EXPORT_KINDS = new Set([ + 'application/vnd.google-apps.document', + 'application/vnd.google-apps.presentation', +]) + +/** Plain text of a Doc or Slides file for a search preview; Drive search returns only metadata. */ +export async function previewDrive( + client: NativeClient, + document: NativeDocument +): Promise { + if (!PREVIEW_EXPORT_KINDS.has(document.kind ?? '')) return undefined + return client.text(`/drive/v3/files/${segment(document.id)}/export`, { mimeType: 'text/plain' }) +} + export async function readDrive( client: NativeClient, id: string, diff --git a/apps/sim/lib/sim-search/live/providers.ts b/apps/sim/lib/sim-search/live/providers.ts index 1f74acef8f5..7333304ae55 100644 --- a/apps/sim/lib/sim-search/live/providers.ts +++ b/apps/sim/lib/sim-search/live/providers.ts @@ -1,9 +1,10 @@ import { MAX_NATIVE_QUERIES_PER_ACCOUNT } from '@/lib/api/contracts/mothership-assistant-tools' -import { readAtlassian, searchAtlassian } from '@/lib/sim-search/live/atlassian' +import { previewConfluence, readAtlassian, searchAtlassian } from '@/lib/sim-search/live/atlassian' import { readCoda, searchCoda } from '@/lib/sim-search/live/coda' import { readGitHub, searchGitHub } from '@/lib/sim-search/live/github' import { readGitLab, searchGitLab } from '@/lib/sim-search/live/gitlab' import { + previewDrive, readCalendar, readDrive, readGmail, @@ -45,6 +46,13 @@ interface NativeQueryGuide { interface NativeProvider { guide: NativeQueryGuide + /** Plain queries must contain every word, so an underfilled search retries with any term. */ + requiresEveryTerm?: true + /** + * Full text for a search preview, for providers whose search returns only an excerpt or + * metadata. At most one request; undefined for kinds without a cheap text form. + */ + previewText?(client: NativeClient, document: NativeDocument): Promise search(client: NativeClient, input: NativeSearchInput): Promise read( client: NativeClient, @@ -82,6 +90,7 @@ export const LIVE_SEARCH_PROVIDERS = { avoid: 'bare words without a term and operator, which Drive rejects, and trashed or modifiedTime clauses, which the server adds from startDate/endDate. Drive search does not search comments or replies; find the file by title/content, then read it to retrieve its discussion. PDF and DOCX reads extract text within download and parsing limits; scanned PDFs need OCR and unsupported binaries provide metadata only. Saved Google Meet transcripts and generated notes are Google Docs: use Drive fullText search and read their content. Drive date filters use file modification time, not meeting time.', }, + previewText: previewDrive, search: searchDrive, read: (client, reference, options) => readDrive(client, reference.id, options.signal), }, @@ -167,6 +176,7 @@ export const LIVE_SEARCH_PROVIDERS = { avoid: 'JQL with only an ORDER BY clause (Jira rejects unbounded queries), and identifying users other than currentUser() by display name or email; use their account ID.', }, + requiresEveryTerm: true, search: (client, input) => searchAtlassian(client, 'jira', input), read: (client, reference) => readAtlassian(client, 'jira', reference.id, reference.container), }, @@ -180,6 +190,8 @@ export const LIVE_SEARCH_PROVIDERS = { avoid: 'starting the query with a negative clause (NOT, !=, !~ or NOT IN), which CQL rejects; lead with a positive clause such as type IN (page, blogpost).', }, + requiresEveryTerm: true, + previewText: previewConfluence, search: (client, input) => searchAtlassian(client, 'confluence', input), read: (client, reference) => readAtlassian(client, 'confluence', reference.id, reference.container, reference.kind), @@ -301,6 +313,25 @@ export function searchNativeProvider( return adapter.search(client, input) } +/** True when the provider's plain-text search matches only items containing every query word. */ +export function requiresEveryTerm(provider: LiveSearchProviderId): boolean { + const adapter = LIVE_SEARCH_PROVIDERS[provider] + return 'requiresEveryTerm' in adapter && adapter.requiresEveryTerm === true +} + +/** + * Full text for a search result's preview, for providers whose search returns only an excerpt or + * metadata. Undefined when the provider or the document's kind has no single-request text form. + */ +export async function previewNativeText( + provider: LiveSearchProviderId, + client: NativeClient, + document: NativeDocument +): Promise { + const adapter = LIVE_SEARCH_PROVIDERS[provider] + return 'previewText' in adapter ? adapter.previewText(client, document) : undefined +} + export function readNativeProvider( provider: LiveSearchProviderId, client: NativeClient, diff --git a/apps/sim/lib/sim-search/live/service-session.ts b/apps/sim/lib/sim-search/live/service-session.ts index 9006ff86227..30c8601d79d 100644 --- a/apps/sim/lib/sim-search/live/service-session.ts +++ b/apps/sim/lib/sim-search/live/service-session.ts @@ -5,7 +5,7 @@ import { resolveConnectorAccessToken, resolveConnectorTokenUserId, } from '@/lib/knowledge/connectors/access-token' -import { escapeSearchPhrase, scopeAtlassianQuery } from '@/lib/sim-search/live/atlassian' +import { scopeAtlassianQuery } from '@/lib/sim-search/live/atlassian' import type { CodaMcpClient } from '@/lib/sim-search/live/coda-mcp' import { createCodaServiceVerifier } from '@/lib/sim-search/live/coda-service' import { createGitHubServiceVerifier } from '@/lib/sim-search/live/github-service' @@ -23,6 +23,7 @@ import type { LiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' import { LIVE_SEARCH_PROVIDER_CATALOG } from '@/lib/sim-search/live/provider-catalog' import { loadLiveGitHubSources, loadLiveServiceSource } from '@/lib/sim-search/live/service-sources' import { liveSourcePolicy } from '@/lib/sim-search/live/source-policy' +import { confluenceTextClause } from '@/lib/sim-search/live/text-match' import type { NativeClient, NativeDocument, NativeSearchInput } from '@/lib/sim-search/live/types' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' @@ -141,7 +142,8 @@ export async function createLiveServiceSession(input: { : []), ].join(' AND ') const query = - search.native?.query ?? (search.query ? `text ~ ${escapeSearchPhrase(search.query)}` : '') + search.native?.query ?? + (search.query ? confluenceTextClause(search.query, search.textMatch) : '') return { ...search, native: { provider, ...search.native, query: scopeAtlassianQuery(query, scope) }, diff --git a/apps/sim/lib/sim-search/live/term-fallback.test.ts b/apps/sim/lib/sim-search/live/term-fallback.test.ts new file mode 100644 index 00000000000..1284af22a1e --- /dev/null +++ b/apps/sim/lib/sim-search/live/term-fallback.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it, vi } from 'vitest' +import { NativeSearchError } from '@/lib/sim-search/live/http' +import { searchWithAnyTermFallback } from '@/lib/sim-search/live/term-fallback' +import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types' + +const input: NativeSearchInput = { query: 'contractor offboarding runbook', limit: 4, scopes: [] } + +function page(...documents: Partial[]): NativePage { + return { + documents: documents.map((document, index) => ({ + id: `doc-${index}`, + title: '', + url: '', + content: '', + ...document, + })), + } +} + +function provider(strict: NativePage, relaxed: NativePage | Error) { + return vi.fn(async (request: NativeSearchInput) => { + if (request.textMatch !== 'any') return strict + if (relaxed instanceof Error) throw relaxed + return relaxed + }) +} + +describe('searchWithAnyTermFallback', () => { + it('follows an exhausted all-terms page with partial matches, most terms first', async () => { + const search = provider( + page({ id: 'exact', title: 'Contractor offboarding runbook' }), + page( + { id: 'runbook', title: 'Runbook index' }, + { id: 'exact', title: 'Contractor offboarding runbook' }, + { id: 'in-preview', title: 'Offboarding', content: 'Steps for each contractor' }, + { id: 'in-title', title: 'Contractor offboarding form' }, + { id: 'contractors', title: 'Contractor directory' } + ) + ) + const result = await searchWithAnyTermFallback(search, input) + expect(result.documents.map((document) => document.id)).toEqual([ + 'exact', + 'in-title', + 'in-preview', + 'runbook', + ]) + expect(result.nextCursor).toBeUndefined() + expect(result.message).toContain('Few results contained every search term') + }) + + it('finds the best partial match beyond the provider’s loose top results', async () => { + const loose = Array.from({ length: 30 }, (_, index) => ({ + id: `loose-${index}`, + title: index === 20 ? 'Contractor offboarding form' : `Runbook ${index}`, + })) + const search = vi.fn(async (request: NativeSearchInput) => + request.textMatch === 'any' + ? { documents: page(...loose).documents.slice(0, request.limit) } + : page() + ) + const result = await searchWithAnyTermFallback(search, { ...input, limit: 2 }) + expect(result.documents.map((document) => document.title)).toEqual([ + 'Contractor offboarding form', + 'Runbook 0', + ]) + }) + + it('adds a few partial matches however large the page', async () => { + const partial = Array.from({ length: 12 }, () => ({ title: 'Contractor offboarding' })) + const search = provider(page(), page(...partial)) + const result = await searchWithAnyTermFallback(search, { ...input, limit: 20 }) + expect(result.documents).toHaveLength(5) + }) + + it('never relaxes a native query, which is the caller’s exact syntax', async () => { + const search = provider(page(), page({ id: 'loose' })) + const native = { provider: 'confluence' as const, query: 'text ~ "runbook"' } + expect((await searchWithAnyTermFallback(search, { ...input, native })).documents).toEqual([]) + expect(search).toHaveBeenCalledTimes(1) + }) + + it('keeps a continuable page so its cursor continues the query that produced it', async () => { + const strict = { ...page({ id: 'exact' }), nextCursor: 'page-2' } + const search = provider(strict, page({ id: 'loose' })) + expect(await searchWithAnyTermFallback(search, input)).toBe(strict) + expect(search).toHaveBeenCalledTimes(1) + }) + + it('does not relax a page that is already full', async () => { + const strict = page({}, {}, {}, {}) + const search = provider(strict, page({ id: 'loose' })) + expect(await searchWithAnyTermFallback(search, input)).toBe(strict) + expect(search).toHaveBeenCalledTimes(1) + }) + + it('does not relax a query with fewer than two distinctive terms', async () => { + const search = provider(page(), page({ id: 'loose' })) + await searchWithAnyTermFallback(search, { ...input, query: 'what is the runbook' }) + expect(search).toHaveBeenCalledTimes(1) + }) + + it('keeps the all-terms page as partial when the provider refuses the any-term search', async () => { + const search = provider( + page({ id: 'exact' }), + new NativeSearchError('rate_limited', 'Slow down') + ) + const result = await searchWithAnyTermFallback(search, input) + expect(result.documents.map((document) => document.id)).toEqual(['exact']) + expect(result).toMatchObject({ partial: true, message: 'Slow down' }) + }) + + it('propagates cancellation during the any-term search instead of returning a page', async () => { + const search = provider(page({ id: 'exact' }), new DOMException('Timed out', 'TimeoutError')) + await expect(searchWithAnyTermFallback(search, input)).rejects.toThrow('Timed out') + }) + + it('does not relax a date-sorted search, whose page is reordered by date', async () => { + const search = provider(page(), page({ id: 'loose' })) + await searchWithAnyTermFallback(search, { ...input, filters: { sortBy: 'newest' } }) + expect(search).toHaveBeenCalledTimes(1) + }) + + it('leaves the page untouched when the any-term search adds nothing new', async () => { + const strict = page({ id: 'exact' }) + const search = provider(strict, page({ id: 'exact' })) + expect(await searchWithAnyTermFallback(search, input)).toBe(strict) + }) +}) diff --git a/apps/sim/lib/sim-search/live/term-fallback.ts b/apps/sim/lib/sim-search/live/term-fallback.ts new file mode 100644 index 00000000000..0ceb1b0990b --- /dev/null +++ b/apps/sim/lib/sim-search/live/term-fallback.ts @@ -0,0 +1,85 @@ +import { dateSortDirection } from '@/lib/sim-search/live/dates' +import { NativeSearchError } from '@/lib/sim-search/live/http' +import { joinMessages } from '@/lib/sim-search/live/pages' +import { significantTerms, termCoverage } from '@/lib/sim-search/live/text-match' +import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types' + +/** + * Candidates the any-term search requests. Providers rank OR queries loosely, so the page is + * chosen from a wider pool by how many terms each candidate contains. + */ +const ANY_TERM_CANDIDATES = 25 + +/** Partial matches added to a page, so a loosely matching provider cannot crowd out other sources. */ +const ANY_TERM_RESULTS = 5 + +/** One provider item, however many queries returned it. */ +const documentKey = ({ container, kind, id }: NativeDocument) => + JSON.stringify([container ?? '', kind ?? '', id]) + +/** + * Searches with every query word required, then, when that exhausts the provider's matches + * without filling the page, with any of the query's significant terms. A natural-language query + * often contains one word the right page lacks, which an all-terms search turns into no result. + * + * All-terms matches keep their provider order ahead of the rest. At most a few matches are added, + * ordered by how many terms their title and preview contain, then how many their title contains, + * then provider order. The fallback runs only for a plain relevance-ordered query: native queries are + * the caller's exact syntax, and a date-sorted page is reordered by date. It never replaces a + * continuable page, so a cursor always continues the query that produced it. A provider refusal + * keeps the all-terms page, reported as partial with the provider's message; cancellation and + * transport failures propagate like the first search. + */ +export async function searchWithAnyTermFallback( + search: (input: NativeSearchInput) => Promise, + input: NativeSearchInput +): Promise { + const strict = await search(input) + const terms = significantTerms(input.query) + if ( + input.native || + dateSortDirection(input.filters) || + terms.length < 2 || + strict.nextCursor || + strict.hasMore || + strict.documents.length >= input.limit + ) + return strict + const relaxed = await search({ + ...input, + limit: Math.max(input.limit, ANY_TERM_CANDIDATES), + textMatch: 'any', + }).catch((error: unknown) => { + if (error instanceof NativeSearchError) return error + throw error + }) + if (relaxed instanceof NativeSearchError) + return { ...strict, partial: true, message: joinMessages([strict.message, relaxed.message]) } + const seen = new Set(strict.documents.map(documentKey)) + const additions = relaxed.documents + .filter((document) => !seen.has(documentKey(document))) + .map((document, rank) => ({ + document, + rank, + coverage: termCoverage(`${document.title}\n${document.content}`, terms), + titleCoverage: termCoverage(document.title, terms), + })) + .sort( + (left, right) => + right.coverage - left.coverage || + right.titleCoverage - left.titleCoverage || + left.rank - right.rank + ) + .slice(0, Math.min(ANY_TERM_RESULTS, input.limit - strict.documents.length)) + .map(({ document }) => document) + if (!additions.length) return strict + return { + ...strict, + documents: [...strict.documents, ...additions], + partial: strict.partial || relaxed.partial, + message: joinMessages([ + strict.message, + 'Few results contained every search term, so results containing some of the terms follow, ordered by how many they contain.', + ]), + } +} diff --git a/apps/sim/lib/sim-search/live/text-match.test.ts b/apps/sim/lib/sim-search/live/text-match.test.ts new file mode 100644 index 00000000000..2b8e635a704 --- /dev/null +++ b/apps/sim/lib/sim-search/live/text-match.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest' +import { confluenceTextClause, jiraTextClause } from '@/lib/sim-search/live/text-match' + +describe('text match clauses', () => { + it('matches either term of a two-term query, leaving out stopwords', () => { + expect(jiraTextClause('what is the release process', 'any')).toBe( + '(text ~ "release" OR text ~ "process")' + ) + }) + + it('requires two of the terms of a longer query anywhere in the full text', () => { + expect(jiraTextClause('contractor offboarding runbook', 'any')).toBe( + '((text ~ "offboarding" AND text ~ "contractor") OR (text ~ "offboarding" AND text ~ "runbook") OR (text ~ "contractor" AND text ~ "runbook"))' + ) + }) + + it('counts a term once whatever its case, so a pair always holds two different words', () => { + expect(jiraTextClause('API api rollout', 'any')).toBe('(text ~ "rollout" OR text ~ "API")') + }) + + it('relaxes a Confluence query to its relevance-ranked site search', () => { + expect(confluenceTextClause('release process', 'all')).toBe('text ~ "release process"') + expect(confluenceTextClause('what is the release process', 'any')).toBe( + 'siteSearch ~ "what is the release process"' + ) + }) + + it('escapes every term so a term cannot add query syntax', () => { + expect(jiraTextClause('ci/cd say"hi', 'any')).toBe( + String.raw`(text ~ "say\\\"hi" OR text ~ "ci\\/cd")` + ) + expect(confluenceTextClause('ci/cd say"hi', 'any')).toBe( + String.raw`siteSearch ~ "ci\\/cd say\\\"hi"` + ) + }) + + it('falls back to the all-terms clause when fewer than two terms are distinctive', () => { + expect(jiraTextClause('how does it work', 'any')).toBe('text ~ "how does it work"') + }) +}) diff --git a/apps/sim/lib/sim-search/live/text-match.ts b/apps/sim/lib/sim-search/live/text-match.ts new file mode 100644 index 00000000000..2b47e730029 --- /dev/null +++ b/apps/sim/lib/sim-search/live/text-match.ts @@ -0,0 +1,104 @@ +import { findTermMatches, queryTerms } from '@/lib/knowledge/search/snippet' + +/** + * How a plain query's words must match a document. Confluence and Jira text search require every + * word by default, so one word a page lacks hides it; `any` also matches pages that contain only + * some of the query's words. + */ +export type TextMatch = 'all' | 'any' + +/** Terms an any-term query sends; more would make the provider query long without adding recall. */ +const MAX_ANY_TERMS = 8 + +/** Words that match nearly every page, so they would make an any-term query return noise. */ +const STOPWORDS = new Set([ + 'about', + 'all', + 'and', + 'any', + 'are', + 'can', + 'could', + 'did', + 'does', + 'for', + 'from', + 'had', + 'has', + 'have', + 'how', + 'into', + 'its', + 'not', + 'our', + 'should', + 'that', + 'the', + 'their', + 'there', + 'this', + 'was', + 'were', + 'what', + 'when', + 'where', + 'which', + 'who', + 'why', + 'will', + 'with', + 'would', + 'you', + 'your', +]) + +/** + * The query's distinctive terms, longest first, each counted once whatever its case. Fewer than + * two means an any-term query would match the same pages as the all-terms one. + */ +export function significantTerms(query: string): string[] { + const seen = new Set() + return queryTerms(query) + .filter((term) => { + const key = term.toLowerCase() + if (STOPWORDS.has(key) || seen.has(key)) return false + seen.add(key) + return true + }) + .slice(0, MAX_ANY_TERMS) +} + +/** Escapes a value for a quoted JQL or CQL string, treating Lucene operators as literal text. */ +const escapeSearchPhrase = (value: string) => + JSON.stringify(value.replace(/[+\-&|!(){}[\]^"~*?:\\/]/g, '\\$&')) + +/** + * The `text ~` clause for a plain query in Jira JQL. `any` ORs every pair of terms, so a match + * contains at least two of them somewhere in the issue, or either term of a two-term query. + */ +export function jiraTextClause(text: string, match: TextMatch = 'all'): string { + const terms = match === 'any' ? significantTerms(text) : [] + if (terms.length < 2) return `text ~ ${escapeSearchPhrase(text)}` + const clause = (term: string) => `text ~ ${escapeSearchPhrase(term)}` + const groups = + terms.length === 2 + ? terms.map(clause) + : terms.flatMap((first, index) => + terms.slice(index + 1).map((second) => `(${clause(first)} AND ${clause(second)})`) + ) + return `(${groups.join(' OR ')})` +} + +/** + * The text clause for a plain query in Confluence CQL. `any` uses `siteSearch`, Confluence's own + * relevance search, which ranks a page with the query's rarer words above the many pages sharing + * one common word, where a `text ~` OR query ranks them the other way round. + */ +export function confluenceTextClause(text: string, match: TextMatch = 'all'): string { + return `${match === 'any' ? 'siteSearch' : 'text'} ~ ${escapeSearchPhrase(text)}` +} + +/** How many distinct terms occur as whole words in the text. */ +export function termCoverage(text: string, terms: readonly string[]): number { + return terms.filter((term) => findTermMatches(text, [term]).length > 0).length +} diff --git a/apps/sim/lib/sim-search/live/types.ts b/apps/sim/lib/sim-search/live/types.ts index 7f582aabfa5..24a06185e85 100644 --- a/apps/sim/lib/sim-search/live/types.ts +++ b/apps/sim/lib/sim-search/live/types.ts @@ -4,6 +4,7 @@ import type { NativeSearchQuery, } from '@/lib/api/contracts/mothership-assistant-tools' import type { LiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' +import type { TextMatch } from '@/lib/sim-search/live/text-match' export interface LiveAccount { id: string @@ -94,4 +95,6 @@ export interface NativeSearchInput { native?: NativeSearchQuery limit: number scopes: readonly string[] + /** How a plain query's words must match; providers without full-text search ignore it. */ + textMatch?: TextMatch }