Skip to content

Commit 05f1545

Browse files
committed
refactor(search): apply the Jira half-the-terms floor in the Jira adapter
The floor tightens Jira's own any-term query, so it now sits beside that query in the Jira branch of searchAtlassian. The shared any-term fallback and the provider registry are back to their previous shape: no option, flag, or exported type. The term test is built once per search from the query, splits words with the shared WORD_CHARACTER definition, and is covered directly in text-match tests, with the adapter tests covering the wiring and Confluence staying untouched.
1 parent 62bda5e commit 05f1545

10 files changed

Lines changed: 113 additions & 142 deletions

File tree

‎apps/sim/lib/knowledge/search/snippet.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@ const HEADER_LINE = /^[A-Z][A-Za-z-]{1,15}: .*$/
1616
*/
1717
const WORD_CHARACTER =
1818
/(?![\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Hangul}\p{sc=Thai}])[\p{L}\p{N}_]/u
19+
const WORD_RUN = new RegExp(`(?:${WORD_CHARACTER.source})+`, 'gu')
20+
21+
/** The words of a text, split where a run of `WORD_CHARACTER`s ends. */
22+
export function textWords(text: string): string[] {
23+
return text.match(WORD_RUN) ?? []
24+
}
1925

2026
/**
2127
* The document text without the header block some connectors prefix (the

‎apps/sim/lib/sim-search/live/account-session.ts‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ import { livePolicyFor, loadLiveSearchPolicies } from '@/lib/sim-search/live/pol
2323
import { LIVE_SEARCH_PROVIDER_CATALOG } from '@/lib/sim-search/live/provider-catalog'
2424
import { isSearchProviderEnabled } from '@/lib/sim-search/live/provider-rollout'
2525
import {
26-
anyTermRetry,
2726
previewNativeText,
2827
readNativeProvider,
28+
requiresEveryTerm,
2929
searchNativeProvider,
3030
} from '@/lib/sim-search/live/providers'
3131
import { searchWithinPolicy } from '@/lib/sim-search/live/scoped-search'
@@ -205,9 +205,8 @@ export async function openLiveAccountSession(
205205
client ? searchNativeProvider(provider, client, native) : searchMcp(native)
206206
)
207207
}
208-
const retry = client ? anyTermRetry(provider) : undefined
209-
return retry
210-
? searchWithAnyTermFallback(searchOnce, search, signal, retry)
208+
return client && requiresEveryTerm(provider)
209+
? searchWithAnyTermFallback(searchOnce, search, signal)
211210
: searchOnce(search)
212211
},
213212
async verify(document) {

‎apps/sim/lib/sim-search/live/application.test.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ const mocks = vi.hoisted(() => ({
2626
json: vi.fn(),
2727
service: vi.fn(),
2828
destroyPool: vi.fn(),
29-
anyTermRetry: vi.fn(),
29+
requiresEveryTerm: vi.fn(),
3030
preview: vi.fn(),
3131
}))
3232
vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock)
@@ -67,7 +67,7 @@ vi.mock('@/lib/sim-search/live/providers', () => ({
6767
liveSearchGuidance: (providers: string[]) => `Live coverage: ${[...new Set(providers)]}`,
6868
searchNativeProvider: mocks.search,
6969
readNativeProvider: mocks.read,
70-
anyTermRetry: mocks.anyTermRetry,
70+
requiresEveryTerm: mocks.requiresEveryTerm,
7171
previewNativeText: mocks.preview,
7272
}))
7373
vi.mock('@/executor/utils/resolved-secret-content-projection', () => ({
@@ -140,7 +140,7 @@ describe('authorized live retrieval', () => {
140140
documents: [{ ...document, id: 'src/a.ts', container: '42', kind: 'code' }],
141141
})
142142
mocks.adminVerify.mockResolvedValue(true)
143-
mocks.anyTermRetry.mockReset().mockReturnValue(undefined)
143+
mocks.requiresEveryTerm.mockReset().mockReturnValue(false)
144144
mocks.preview.mockReset().mockResolvedValue(undefined)
145145
})
146146
describe('Zoom continuation integrity', () => {
@@ -698,7 +698,7 @@ describe('authorized live retrieval', () => {
698698
})
699699

700700
it('retries an underfilled search with any term for providers that require every term', async () => {
701-
mocks.anyTermRetry.mockReturnValue({})
701+
mocks.requiresEveryTerm.mockReturnValue(true)
702702
const loose = { ...document, id: 'loose', url: 'https://docs.google.com/document/d/loose/edit' }
703703
mocks.search.mockImplementation(async (_provider, _client, search) => ({
704704
documents: search.textMatch === 'any' ? [loose, document] : [document],

‎apps/sim/lib/sim-search/live/atlassian.test.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,3 +197,36 @@ describe('Confluence live documents', () => {
197197
expect(await previewConfluence(page('<p>Rollout steps</p>'), document)).toBe('Rollout steps')
198198
})
199199
})
200+
201+
describe('Jira any-term search', () => {
202+
const issues = {
203+
'/ex/jira/cloud/rest/api/3/search/jql': {
204+
issues: [
205+
{ key: 'ENG-1', fields: { summary: 'Token creator role request' } },
206+
{ key: 'ENG-2', fields: { summary: 'Role request form' } },
207+
],
208+
},
209+
}
210+
const query = { query: 'glean token creator role request', limit: 25, scopes: [] }
211+
212+
it('keeps only issues holding at least half the query terms', async () => {
213+
const page = await searchAtlassian(client(issues), 'jira', { ...query, textMatch: 'any' })
214+
expect(page.documents.map((document) => document.id)).toEqual(['ENG-1'])
215+
})
216+
217+
it('keeps every issue an all-terms search returns', async () => {
218+
const page = await searchAtlassian(client(issues), 'jira', query)
219+
expect(page.documents.map((document) => document.id)).toEqual(['ENG-1', 'ENG-2'])
220+
})
221+
222+
it('leaves a Confluence any-term search to its relevance ranking', async () => {
223+
const api = client({
224+
'/ex/confluence/cloud/wiki/rest/api/search': {
225+
results: [{ content: { id: '5', type: 'page', title: 'Role request form' } }],
226+
_links: {},
227+
},
228+
})
229+
const page = await searchAtlassian(api, 'confluence', { ...query, textMatch: 'any' })
230+
expect(page.documents.map((document) => document.id)).toEqual(['5'])
231+
})
232+
})

‎apps/sim/lib/sim-search/live/atlassian.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@ import {
99
} from '@/lib/sim-search/live/http'
1010
import { interleaveByRank } from '@/lib/sim-search/live/pages'
1111
import { providerText } from '@/lib/sim-search/live/text'
12-
import { confluenceTextClause, jiraTextClause } from '@/lib/sim-search/live/text-match'
12+
import {
13+
confluenceTextClause,
14+
holdsHalfTheTerms,
15+
jiraTextClause,
16+
} from '@/lib/sim-search/live/text-match'
1317
import type {
1418
NativeClient,
1519
NativeDocument,
@@ -152,6 +156,11 @@ export async function searchAtlassian(
152156
const direction = dateSortDirection(input.filters)
153157
const order = direction ? `${field} ${direction.toUpperCase()}` : undefined
154158
const text = nativeText(input)
159+
/**
160+
* Jira's any-term query ORs pairs of terms, which on a large site matches issues sharing two
161+
* common words; like a minimum-should-match, an issue must hold at least half the terms.
162+
*/
163+
const relevant = input.textMatch === 'any' && text ? holdsHalfTheTerms(text) : undefined
155164
const pages = await Promise.all(
156165
selected.map(async (site) => {
157166
const cloudId = string(site.id)
@@ -173,7 +182,9 @@ export async function searchAtlassian(
173182
})
174183
)
175184
return {
176-
documents: array(data.issues).map((row) => issue(row, cloudId, origin)),
185+
documents: array(data.issues)
186+
.map((row) => issue(row, cloudId, origin))
187+
.filter((document) => !relevant || relevant(`${document.title}\n${document.content}`)),
177188
excluded: false,
178189
next: string(data.nextPageToken) || undefined,
179190
}

‎apps/sim/lib/sim-search/live/providers.ts‎

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import {
2020
type LiveSearchProviderId,
2121
} from '@/lib/sim-search/live/provider-catalog'
2222
import { readSlack, searchSlack } from '@/lib/sim-search/live/slack'
23-
import type { AnyTermFallbackOptions } from '@/lib/sim-search/live/term-fallback'
2423
import type {
2524
NativeClient,
2625
NativeDocument,
@@ -47,12 +46,8 @@ interface NativeQueryGuide {
4746

4847
interface NativeProvider {
4948
guide: NativeQueryGuide
50-
/**
51-
* Plain queries must contain every word, so an underfilled search retries with any term.
52-
* `requireHalfTheTerms` is for a provider whose any-term query ranks items sharing just a couple
53-
* of common words alongside close matches.
54-
*/
55-
anyTermRetry?: AnyTermFallbackOptions
49+
/** Plain queries must contain every word, so an underfilled search retries with any term. */
50+
requiresEveryTerm?: true
5651
/**
5752
* Full text for a search preview, for providers whose search returns only an excerpt or
5853
* metadata. At most one request; undefined for kinds without a cheap text form.
@@ -181,7 +176,7 @@ export const LIVE_SEARCH_PROVIDERS = {
181176
avoid:
182177
'JQL with only an ORDER BY clause (Jira rejects unbounded queries), and identifying users other than currentUser() by display name or email; use their account ID.',
183178
},
184-
anyTermRetry: { requireHalfTheTerms: true },
179+
requiresEveryTerm: true,
185180
search: (client, input) => searchAtlassian(client, 'jira', input),
186181
read: (client, reference) => readAtlassian(client, 'jira', reference.id, reference.container),
187182
},
@@ -195,7 +190,7 @@ export const LIVE_SEARCH_PROVIDERS = {
195190
avoid:
196191
'starting the query with a negative clause (NOT, !=, !~ or NOT IN), which CQL rejects; lead with a positive clause such as type IN (page, blogpost).',
197192
},
198-
anyTermRetry: { requireHalfTheTerms: false },
193+
requiresEveryTerm: true,
199194
previewText: previewConfluence,
200195
search: (client, input) => searchAtlassian(client, 'confluence', input),
201196
read: (client, reference) =>
@@ -318,13 +313,10 @@ export function searchNativeProvider(
318313
return adapter.search(client, input)
319314
}
320315

321-
/**
322-
* How the provider retries an underfilled plain search with any term; undefined when its
323-
* plain-text search does not require every query word.
324-
*/
325-
export function anyTermRetry(provider: LiveSearchProviderId): AnyTermFallbackOptions | undefined {
316+
/** True when the provider's plain-text search matches only items containing every query word. */
317+
export function requiresEveryTerm(provider: LiveSearchProviderId): boolean {
326318
const adapter = LIVE_SEARCH_PROVIDERS[provider]
327-
return 'anyTermRetry' in adapter ? adapter.anyTermRetry : undefined
319+
return 'requiresEveryTerm' in adapter && adapter.requiresEveryTerm === true
328320
}
329321

330322
/**

‎apps/sim/lib/sim-search/live/term-fallback.test.ts‎

Lines changed: 0 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -217,91 +217,6 @@ describe('searchWithAnyTermFallback', () => {
217217
expect(search).toHaveBeenCalledTimes(1)
218218
})
219219

220-
describe('when partial matches must contain half the terms', () => {
221-
const long: NativeSearchInput = {
222-
query: 'glean token creator role request',
223-
limit: 10,
224-
scopes: [],
225-
}
226-
const halfTheTerms = { requireHalfTheTerms: true }
227-
228-
it('drops a partial match containing fewer than half the terms', async () => {
229-
const search = provider(
230-
page(),
231-
page(
232-
{ id: 'two-terms', title: 'Role request form' },
233-
{ id: 'three-terms', title: 'Token creator role' },
234-
{ id: 'in-description', title: 'Access', content: 'Glean token request for my team' }
235-
)
236-
)
237-
const result = await searchWithAnyTermFallback(search, long, live, halfTheTerms)
238-
expect(result.documents.map((document) => document.id)).toEqual([
239-
'three-terms',
240-
'in-description',
241-
])
242-
})
243-
244-
it('counts a word and its plural as one term, as the provider matches them', async () => {
245-
const search = provider(
246-
page(),
247-
page({ id: 'plurals', title: 'Glean tokens', content: 'Creators file requests' })
248-
)
249-
const result = await searchWithAnyTermFallback(search, long, live, halfTheTerms)
250-
expect(result.documents.map((document) => document.id)).toEqual(['plurals'])
251-
})
252-
253-
it('counts a singular word whose plural adds -es, in either form', async () => {
254-
const search = provider(
255-
page(),
256-
page(
257-
{ id: 'singular', title: 'Database release' },
258-
{ id: 'plural', title: 'Databases', content: 'Weekly releases' }
259-
)
260-
)
261-
const result = await searchWithAnyTermFallback(
262-
search,
263-
{ query: 'database release checklist owner', limit: 10, scopes: [] },
264-
live,
265-
halfTheTerms
266-
)
267-
expect(result.documents.map((document) => document.id).sort()).toEqual(['plural', 'singular'])
268-
})
269-
270-
it('counts a term with punctuation inside it, such as ci/cd', async () => {
271-
const search = provider(page(), page({ id: 'ci', title: 'CI/CD pipeline' }))
272-
const result = await searchWithAnyTermFallback(
273-
search,
274-
{ query: 'ci/cd pipeline failure owner', limit: 10, scopes: [] },
275-
live,
276-
halfTheTerms
277-
)
278-
expect(result.documents.map((document) => document.id)).toEqual(['ci'])
279-
})
280-
281-
it('returns the all-terms page unchanged when no partial match is close enough', async () => {
282-
const strict = page({ id: 'exact', title: 'Glean token creator role request' })
283-
const search = provider(strict, page({ id: 'loose', title: 'Role request' }))
284-
expect(await searchWithAnyTermFallback(search, long, live, halfTheTerms)).toBe(strict)
285-
})
286-
287-
it('keeps a single matching term enough for a two-term query', async () => {
288-
const search = provider(page(), page({ id: 'one', title: 'Contractor directory' }))
289-
const result = await searchWithAnyTermFallback(
290-
search,
291-
{ query: 'contractor offboarding', limit: 4, scopes: [] },
292-
live,
293-
halfTheTerms
294-
)
295-
expect(result.documents.map((document) => document.id)).toEqual(['one'])
296-
})
297-
298-
it('leaves providers without the floor adding loose partial matches', async () => {
299-
const search = provider(page(), page({ id: 'two-terms', title: 'Role request form' }))
300-
const result = await searchWithAnyTermFallback(search, long, live)
301-
expect(result.documents.map((document) => document.id)).toEqual(['two-terms'])
302-
})
303-
})
304-
305220
it('leaves the page untouched when the any-term search adds nothing new', async () => {
306221
const strict = page({ id: 'exact' })
307222
const search = provider(strict, page({ id: 'exact' }))

‎apps/sim/lib/sim-search/live/term-fallback.ts‎

Lines changed: 3 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors'
33
import { dateSortDirection } from '@/lib/sim-search/live/dates'
44
import { NativeSearchError } from '@/lib/sim-search/live/http'
55
import { joinMessages } from '@/lib/sim-search/live/pages'
6-
import {
7-
containsHalfTheTerms,
8-
significantTerms,
9-
termCoverage,
10-
} from '@/lib/sim-search/live/text-match'
6+
import { significantTerms, termCoverage } from '@/lib/sim-search/live/text-match'
117
import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types'
128

139
/**
@@ -46,20 +42,11 @@ const documentKey = ({ container, kind, id }: NativeDocument) =>
4642
* slow OR query is cancelled rather than awaited. Whenever it fails short of the caller's own
4743
* cancellation (a provider refusal, the budget running out, a transport error), the all-terms page
4844
* is kept, reported as partial. Cancellation through `signal` propagates like the first search.
49-
*
50-
* With `requireHalfTheTerms`, a partial match must contain at least half the query's terms in its
51-
* title or text, for providers whose OR query matches items sharing only a couple of common words.
5245
*/
53-
export interface AnyTermFallbackOptions {
54-
/** Keep only partial matches holding at least half the query's terms. */
55-
requireHalfTheTerms?: boolean
56-
}
57-
5846
export async function searchWithAnyTermFallback(
5947
search: (input: NativeSearchInput) => Promise<NativePage>,
6048
input: NativeSearchInput,
61-
signal: AbortSignal,
62-
options: AnyTermFallbackOptions = {}
49+
signal: AbortSignal
6350
): Promise<NativePage> {
6451
const strict = await search(input)
6552
const terms = significantTerms(input.query)
@@ -106,12 +93,7 @@ export async function searchWithAnyTermFallback(
10693
}
10794
const seen = new Set(strict.documents.map(documentKey))
10895
const additions = relaxed.documents
109-
.filter(
110-
(document) =>
111-
!seen.has(documentKey(document)) &&
112-
(!options.requireHalfTheTerms ||
113-
containsHalfTheTerms(`${document.title}\n${document.content}`, input.query))
114-
)
96+
.filter((document) => !seen.has(documentKey(document)))
11597
.map((document, rank) => ({
11698
document,
11799
rank,

‎apps/sim/lib/sim-search/live/text-match.test.ts‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
import { describe, expect, it } from 'vitest'
2-
import { confluenceTextClause, jiraTextClause } from '@/lib/sim-search/live/text-match'
2+
import {
3+
confluenceTextClause,
4+
holdsHalfTheTerms,
5+
jiraTextClause,
6+
} from '@/lib/sim-search/live/text-match'
37

48
describe('text match clauses', () => {
59
it('matches either term of a two-term query, leaving out stopwords', () => {
@@ -57,4 +61,29 @@ describe('text match clauses', () => {
5761
it('falls back to the all-terms clause when fewer than two terms are distinctive', () => {
5862
expect(jiraTextClause('how does it work', 'any')).toBe('text ~ "how does it work"')
5963
})
64+
65+
describe('holdsHalfTheTerms', () => {
66+
it('needs at least half the distinct terms, rounding up', () => {
67+
const holds = holdsHalfTheTerms('glean token creator role request')
68+
expect(holds('Role request form')).toBe(false)
69+
expect(holds('Token creator role')).toBe(true)
70+
})
71+
72+
it('counts a word and its plural as one term in either direction', () => {
73+
expect(holdsHalfTheTerms('glean token creator role request')('Glean tokens, creators')).toBe(
74+
true
75+
)
76+
const holds = holdsHalfTheTerms('database release checklist owner')
77+
expect(holds('Database release')).toBe(true)
78+
expect(holds('Databases and weekly releases')).toBe(true)
79+
})
80+
81+
it('matches a term with punctuation inside it as written', () => {
82+
expect(holdsHalfTheTerms('ci/cd pipeline failure owner')('CI/CD pipeline')).toBe(true)
83+
})
84+
85+
it('accepts one matching term for a two-term query', () => {
86+
expect(holdsHalfTheTerms('contractor offboarding')('Contractor directory')).toBe(true)
87+
})
88+
})
6089
})

0 commit comments

Comments
 (0)