fix(search): count Jira half-term summaries by stemmed term, from a wider pool, most terms first

Author: waleedlatif1

apps/sim/lib/sim-search/live/atlassian.test.ts

@@ -350,13 +350,21 @@ describe('Jira plain search ranking', () => {
           ? { key: row, fields: { summary: row } }
           : { key: row.key, fields: { summary: row.summary } }
       )
-    const json = vi.fn(async (path: string, options?: { body?: { jql?: string } }) => {
-      if (path === '/oauth/token/accessible-resources') return [SITE]
-      if (path !== search) throw new Error(`Unexpected request: ${path}`)
-      const jql = options?.body?.jql ?? ''
-      if (jql.includes('summary ~')) return { issues: issues(jql.includes(' OR ') ? half : titled) }
-      return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) }
-    })
+    const json = vi.fn(
+      async (path: string, options?: { body?: { jql?: string; maxResults?: number } }) => {
+        if (path === '/oauth/token/accessible-resources') return [SITE]
+        if (path !== search) throw new Error(`Unexpected request: ${path}`)
+        const jql = options?.body?.jql ?? ''
+        if (jql.includes('summary ~'))
+          return {
+            issues: issues(jql.includes(' OR ') ? half : titled).slice(
+              0,
+              options?.body?.maxResults
+            ),
+          }
+        return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) }
+      }
+    )
     return { json, text: vi.fn(), bytes: vi.fn() } satisfies NativeClient
   }
   const query = { query: 'merged results pipelines', limit: 3, scopes: [] }
@@ -430,8 +438,8 @@ describe('Jira plain search ranking', () => {
       overflowing,
       [],
       [
-        { key: 'HALF-1', summary: 'Merged results broke the pipelines' },
         { key: 'LOOSE-1', summary: 'Pipelines dashboard' },
+        { key: 'HALF-1', summary: 'Merged results broke the dashboard' },
       ]
     )
     const page = await searchAtlassian(api, 'jira', query)
@@ -454,4 +462,36 @@ describe('Jira plain search ranking', () => {
     expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1'])
     expect(page.nextCursor).toBe('page-2')
   })
+
+  it('counts a word and its plural as one term before allowing the half-the-terms fallback', async () => {
+    const api = jira(overflowing, [], [{ key: 'ONE-1', summary: 'Rotation schedule' }])
+    const page = await searchAtlassian(api, 'jira', { ...query, query: 'token tokens rotation' })
+    expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1'])
+  })
+
+  it('orders half-the-terms summaries by how many terms they hold', async () => {
+    const api = jira(
+      overflowing,
+      [],
+      [
+        { key: 'TWO-1', summary: 'Merged results dashboard' },
+        { key: 'THREE-1', summary: 'Merged results in release pipelines' },
+      ]
+    )
+    const page = await searchAtlassian(api, 'jira', {
+      ...query,
+      query: 'merged results release pipelines',
+    })
+    expect(page.documents.map((document) => document.id)).toEqual(['THREE-1', 'TWO-1', 'NEW-3'])
+  })
+
+  it('finds half-the-terms summaries beyond the first page of pair matches', async () => {
+    const pairs = Array.from({ length: 10 }, (_, index) => ({
+      key: `PAIR-${index}`,
+      summary: 'Pipelines dashboard',
+    }))
+    const api = jira(overflowing, [], [...pairs, { key: 'HALF-9', summary: 'Merged pipelines' }])
+    const page = await searchAtlassian(api, 'jira', query)
+    expect(page.documents.map((document) => document.id)[0]).toBe('HALF-9')
+  })
 })

apps/sim/lib/sim-search/live/atlassian.ts

@@ -13,7 +13,7 @@ import {
   confluenceTextClause,
   holdsHalfTheTerms,
   jiraTextClause,
-  significantTerms,
+  stemmedTerms,
 } from '@/lib/sim-search/live/text-match'
 import type {
   NativeClient,
@@ -117,6 +117,12 @@ async function sites(client: NativeClient) {
   return array(await client.json('/oauth/token/accessible-resources', { memo: true }))
 }
 
+/**
+ * Pair matches fetched for the half-the-terms summary fallback; the page is filtered to summaries
+ * holding enough terms, so it starts from a wider pool than the caller's limit.
+ */
+const HALF_TERM_CANDIDATES = 50
+
 /**
  * One Atlassian site's page; `ranking` marks a page whose order or completeness differs from the
  * all-terms search.
@@ -176,67 +182,75 @@ export async function searchAtlassian(
       const cloudId = string(site.id)
       const origin = string(site.url).replace(/\/$/, '')
       if (provider === 'jira') {
-        /** One JQL page, keeping only issues whose summary and description pass `keep`. */
-        const searchIssues = async (
-          jql: string,
-          cursor?: string,
-          keep?: (summary: string, description: string) => boolean
-        ): Promise<SitePage> => {
+        /** One JQL page, each issue with its summary and description. */
+        const searchIssues = async (jql: string, cursor?: string, maxResults = input.limit) => {
           const data = object(
             await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, {
               body: {
                 jql: scopeAtlassianQuery(jql, scope, order),
-                maxResults: input.limit,
+                maxResults,
                 fields: ['summary', 'description', 'updated', 'creator', 'status'],
                 ...(cursor ? { nextPageToken: cursor } : {}),
               },
               signal: input.signal,
             })
           )
           return {
-            documents: array(data.issues)
-              .filter((row) => {
-                const fields = object(row.fields)
-                return !keep || keep(string(fields.summary), textContent(fields.description))
-              })
-              .map((row) => issue(row, cloudId, origin)),
-            excluded: false,
+            issues: array(data.issues).map((row) => {
+              const fields = object(row.fields)
+              return {
+                document: issue(row, cloudId, origin),
+                summary: string(fields.summary),
+                description: textContent(fields.description),
+              }
+            }),
             next: string(data.nextPageToken) || undefined,
           }
         }
         const exact = await searchIssues(
           input.native?.query || (text ? jiraTextClause(text, input.textMatch) : ''),
-          input.native?.cursor && single ? input.native.cursor : undefined,
-          relevant && ((summary, description) => relevant(`${summary}\n${description}`))
+          input.native?.cursor && single ? input.native.cursor : undefined
         )
+        const exactPage: SitePage = {
+          documents: exact.issues
+            .filter(
+              ({ summary, description }) => !relevant || relevant(`${summary}\n${description}`)
+            )
+            .map(({ document }) => document),
+          excluded: false,
+          next: exact.next,
+        }
         // JQL has no relevance order, so an all-terms match larger than a page comes back newest
         // first; issues whose summary holds every term, or else at least half of three or more
-        // terms, lead it instead.
-        if (!text || input.native || order || input.textMatch === 'any' || !exact.next) return exact
+        // terms (most terms first), lead it instead.
+        if (!text || input.native || order || input.textMatch === 'any' || !exact.next)
+          return exactPage
+        const terms = stemmedTerms(text)
         const [every, half] = await Promise.allSettled([
           searchIssues(jiraTextClause(text, 'all', 'summary')),
-          significantTerms(text).length >= 3
-            ? searchIssues(
-                jiraTextClause(text, 'any', 'summary'),
-                undefined,
-                holdsHalfTheTerms(text)
-              )
+          terms.count >= 3
+            ? searchIssues(jiraTextClause(text, 'any', 'summary'), undefined, HALF_TERM_CANDIDATES)
             : undefined,
         ])
         // A cancelled search stops here; any other failure keeps the all-terms page.
         input.signal?.throwIfAborted()
+        const required = Math.ceil(terms.count / 2)
         const titled =
-          every.status === 'fulfilled' && every.value.documents.length
-            ? every.value.documents
-            : half.status === 'fulfilled'
-              ? (half.value?.documents ?? [])
+          every.status === 'fulfilled' && every.value.issues.length
+            ? every.value.issues.map(({ document }) => document)
+            : half.status === 'fulfilled' && half.value
+              ? half.value.issues
+                  .map(({ document, summary }) => ({ document, held: terms.held(summary) }))
+                  .filter(({ held }) => held >= required)
+                  .sort((left, right) => right.held - left.held)
+                  .map(({ document }) => document)
               : []
-        if (!titled.length) return exact
+        if (!titled.length) return exactPage
         const leading = new Set(titled.map((document) => document.id))
         return {
           documents: [
             ...titled,
-            ...exact.documents.filter((document) => !leading.has(document.id)),
+            ...exactPage.documents.filter((document) => !leading.has(document.id)),
           ].slice(0, input.limit),
           excluded: false,
           ranking: 'relevance',
@@ -304,7 +318,7 @@ export async function searchAtlassian(
         : '') +
       (pages.some((result) => result.ranking === 'relevance')
         ? provider === 'jira'
-          ? ' More issues contained every word than fit on one page, so issues whose summary holds the most query words come first; refine the query, or page with a native JQL query.'
+          ? ' More issues contained every word than fit on one page, so issues whose summary holds every word, or else most of them, come first; refine the query, or page with a native JQL query.'
           : ' More pages contained every word than fit on one page, so this page is ordered by Confluence relevance search; refine the query, or page with a native CQL query.'
         : '') +
       (pages.some((result) => result.ranking === 'unavailable')

apps/sim/lib/sim-search/live/text-match.ts

@@ -126,19 +126,27 @@ export function termCoverage(text: string, terms: readonly string[]): number {
 }
 
 /**
- * A test for whether a text holds at least half the query's distinct terms. Query and text words
- * are compared by their singular key, so a word and its regular plural count as one term as a
- * stemmed search matches them; a term with punctuation inside it is matched as written.
+ * The query's distinct terms as a stemmed search counts them, and how many of them a text holds.
+ * Query and text words are compared by their singular key, so a word and its regular plural count
+ * as one term; a term with punctuation inside it is matched as written.
  */
-export function holdsHalfTheTerms(query: string): (text: string) => boolean {
+export function stemmedTerms(query: string): { count: number; held: (text: string) => number } {
   const terms = distinctTerms(query, singularKey)
   const keys = terms.map(singularKey)
-  const required = Math.ceil(terms.length / 2)
-  return (text) => {
-    const words = new Set(textWords(text).map(singularKey))
-    const present = terms.filter(
-      (term, index) => words.has(keys[index]) || findTermMatches(text, [term]).length > 0
-    )
-    return present.length >= required
+  return {
+    count: terms.length,
+    held: (text) => {
+      const words = new Set(textWords(text).map(singularKey))
+      return terms.filter(
+        (term, index) => words.has(keys[index]) || findTermMatches(text, [term]).length > 0
+      ).length
+    },
   }
 }
+
+/** A test for whether a text holds at least half the query's distinct terms, as `stemmedTerms` counts them. */
+export function holdsHalfTheTerms(query: string): (text: string) => boolean {
+  const { count, held } = stemmedTerms(query)
+  const required = Math.ceil(count / 2)
+  return (text) => held(text) >= required
+}