diff --git a/apps/sim/lib/knowledge/search/snippet.ts b/apps/sim/lib/knowledge/search/snippet.ts index 2315eeb0f98..cb604e143ee 100644 --- a/apps/sim/lib/knowledge/search/snippet.ts +++ b/apps/sim/lib/knowledge/search/snippet.ts @@ -16,6 +16,12 @@ const HEADER_LINE = /^[A-Z][A-Za-z-]{1,15}: .*$/ */ const WORD_CHARACTER = /(?![\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Hangul}\p{sc=Thai}])[\p{L}\p{N}_]/u +const WORD_RUN = new RegExp(`(?:${WORD_CHARACTER.source})+`, 'gu') + +/** The words of a text, split where a run of `WORD_CHARACTER`s ends. */ +export function textWords(text: string): string[] { + return text.match(WORD_RUN) ?? [] +} /** * The document text without the header block some connectors prefix (the diff --git a/apps/sim/lib/sim-search/live/atlassian.test.ts b/apps/sim/lib/sim-search/live/atlassian.test.ts index d0d0816ddd5..4168a50fcfb 100644 --- a/apps/sim/lib/sim-search/live/atlassian.test.ts +++ b/apps/sim/lib/sim-search/live/atlassian.test.ts @@ -197,3 +197,51 @@ describe('Confluence live documents', () => { expect(await previewConfluence(page('
Rollout steps
'), document)).toBe('Rollout steps') }) }) + +describe('Jira any-term search', () => { + const issues = { + '/ex/jira/cloud/rest/api/3/search/jql': { + issues: [ + { key: 'ENG-1', fields: { summary: 'Token creator role request' } }, + { key: 'ENG-2', fields: { summary: 'Role request form' } }, + ], + }, + } + const query = { query: 'glean token creator role request', limit: 25, scopes: [] } + + it('keeps only issues holding at least half the query terms', async () => { + const page = await searchAtlassian(client(issues), 'jira', { ...query, textMatch: 'any' }) + expect(page.documents.map((document) => document.id)).toEqual(['ENG-1']) + }) + + it('counts only the summary and description, not the added status line', async () => { + const api = client({ + '/ex/jira/cloud/rest/api/3/search/jql': { + issues: [{ key: 'ENG-3', fields: { summary: 'Release update', status: { name: 'Open' } } }], + }, + }) + const page = await searchAtlassian(api, 'jira', { + query: 'release status update owner timeline', + limit: 25, + scopes: [], + textMatch: 'any', + }) + expect(page.documents).toEqual([]) + }) + + it('keeps every issue an all-terms search returns', async () => { + const page = await searchAtlassian(client(issues), 'jira', query) + expect(page.documents.map((document) => document.id)).toEqual(['ENG-1', 'ENG-2']) + }) + + it('leaves a Confluence any-term search to its relevance ranking', async () => { + const api = client({ + '/ex/confluence/cloud/wiki/rest/api/search': { + results: [{ content: { id: '5', type: 'page', title: 'Role request form' } }], + _links: {}, + }, + }) + const page = await searchAtlassian(api, 'confluence', { ...query, textMatch: 'any' }) + expect(page.documents.map((document) => document.id)).toEqual(['5']) + }) +}) diff --git a/apps/sim/lib/sim-search/live/atlassian.ts b/apps/sim/lib/sim-search/live/atlassian.ts index b80f4d87079..86721de66c2 100644 --- a/apps/sim/lib/sim-search/live/atlassian.ts +++ b/apps/sim/lib/sim-search/live/atlassian.ts @@ -9,7 +9,11 @@ import { } from '@/lib/sim-search/live/http' import { interleaveByRank } from '@/lib/sim-search/live/pages' import { providerText } from '@/lib/sim-search/live/text' -import { confluenceTextClause, jiraTextClause } from '@/lib/sim-search/live/text-match' +import { + confluenceTextClause, + holdsHalfTheTerms, + jiraTextClause, +} from '@/lib/sim-search/live/text-match' import type { NativeClient, NativeDocument, @@ -152,6 +156,9 @@ export async function searchAtlassian( const direction = dateSortDirection(input.filters) const order = direction ? `${field} ${direction.toUpperCase()}` : undefined const text = nativeText(input) + // Jira's any-term JQL matches issues sharing two common words; like minimum-should-match, + // keep only issues holding at least half the terms. + const relevant = input.textMatch === 'any' && text ? holdsHalfTheTerms(text) : undefined const pages = await Promise.all( selected.map(async (site) => { const cloudId = string(site.id) @@ -173,7 +180,15 @@ export async function searchAtlassian( }) ) return { - documents: array(data.issues).map((row) => issue(row, cloudId, origin)), + documents: array(data.issues) + .filter((row) => { + const fields = object(row.fields) + return ( + !relevant || + relevant(`${string(fields.summary)}\n${textContent(fields.description)}`) + ) + }) + .map((row) => issue(row, cloudId, origin)), excluded: false, next: string(data.nextPageToken) || undefined, } diff --git a/apps/sim/lib/sim-search/live/text-match.test.ts b/apps/sim/lib/sim-search/live/text-match.test.ts index acf083fadea..928d830d37d 100644 --- a/apps/sim/lib/sim-search/live/text-match.test.ts +++ b/apps/sim/lib/sim-search/live/text-match.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest' -import { confluenceTextClause, jiraTextClause } from '@/lib/sim-search/live/text-match' +import { + confluenceTextClause, + holdsHalfTheTerms, + jiraTextClause, +} from '@/lib/sim-search/live/text-match' describe('text match clauses', () => { it('matches either term of a two-term query, leaving out stopwords', () => { @@ -57,4 +61,30 @@ describe('text match clauses', () => { it('falls back to the all-terms clause when fewer than two terms are distinctive', () => { expect(jiraTextClause('how does it work', 'any')).toBe('text ~ "how does it work"') }) + + describe('holdsHalfTheTerms', () => { + it('needs at least half the distinct terms, rounding up', () => { + const holds = holdsHalfTheTerms('glean token creator role request') + expect(holds('Role request form')).toBe(false) + expect(holds('Token creator role')).toBe(true) + }) + + it('counts a word and its plural as one term in either direction', () => { + expect(holdsHalfTheTerms('glean token creator role request')('Glean tokens, creators')).toBe( + true + ) + expect( + holdsHalfTheTerms('database release checklist owner')('Databases and weekly releases') + ).toBe(true) + expect(holdsHalfTheTerms('databases releases checklist owner')('Database release')).toBe(true) + }) + + it('matches a term with punctuation inside it as written', () => { + expect(holdsHalfTheTerms('ci/cd pipeline failure owner')('CI/CD pipeline')).toBe(true) + }) + + it('accepts one matching term for a two-term query', () => { + expect(holdsHalfTheTerms('contractor offboarding')('Contractor directory')).toBe(true) + }) + }) }) diff --git a/apps/sim/lib/sim-search/live/text-match.ts b/apps/sim/lib/sim-search/live/text-match.ts index 7160078d6d3..a4a11419e24 100644 --- a/apps/sim/lib/sim-search/live/text-match.ts +++ b/apps/sim/lib/sim-search/live/text-match.ts @@ -1,4 +1,4 @@ -import { findTermMatches, queryTerms } from '@/lib/knowledge/search/snippet' +import { findTermMatches, queryTerms, textWords } from '@/lib/knowledge/search/snippet' /** * How a plain query's words must match a document. Confluence and Jira text search require every @@ -115,3 +115,21 @@ export function confluenceTextClause(text: string, match: TextMatch = 'all'): st export function termCoverage(text: string, terms: readonly string[]): number { return terms.filter((term) => findTermMatches(text, [term]).length > 0).length } + +/** + * A test for whether a text holds at least half the query's distinct terms. Query and text words + * are compared by their singular key, so a word and its regular plural count as one term as a + * stemmed search matches them; a term with punctuation inside it is matched as written. + */ +export function holdsHalfTheTerms(query: string): (text: string) => boolean { + const terms = distinctTerms(query, singularKey) + const keys = terms.map(singularKey) + const required = Math.ceil(terms.length / 2) + return (text) => { + const words = new Set(textWords(text).map(singularKey)) + const present = terms.filter( + (term, index) => words.has(keys[index]) || findTermMatches(text, [term]).length > 0 + ) + return present.length >= required + } +}