improvement(search): lead an overflowing Jira all-terms match with issues whose summary holds every term

Author: waleedlatif1

apps/sim/lib/sim-search/live/atlassian.test.ts

@@ -331,3 +331,84 @@ describe('Confluence plain search ranking', () => {
     expect(own.nextCursor).toBe('page-2')
   })
 })
+
+describe('Jira plain search ranking', () => {
+  const search = '/ex/jira/cloud/rest/api/3/search/jql'
+  /** Answers the all-terms query with `exact` and the summary query with `titled`, or its failure. */
+  function jira(exact: { ids: string[]; more: boolean }, titled: string[] | (() => never)) {
+    const issues = (ids: string[]) => ids.map((key) => ({ key, fields: { summary: key } }))
+    const json = vi.fn(async (path: string, options?: { body?: { jql?: string } }) => {
+      if (path === '/oauth/token/accessible-resources') return [SITE]
+      if (path !== search) throw new Error(`Unexpected request: ${path}`)
+      if (options?.body?.jql?.includes('summary ~'))
+        return { issues: issues(typeof titled === 'function' ? titled() : titled) }
+      return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) }
+    })
+    return { json, text: vi.fn(), bytes: vi.fn() } satisfies NativeClient
+  }
+  const query = { query: 'merged results pipelines', limit: 3, scopes: [] }
+  const overflowing = { ids: ['NEW-3', 'NEW-2', 'NEW-1'], more: true }
+
+  it('leads an all-terms match too large for one page with issues whose summary holds every term', async () => {
+    const api = jira(overflowing, ['TOP-1', 'NEW-2'])
+    const page = await searchAtlassian(api, 'jira', query)
+    expect(page.documents.map((document) => document.id)).toEqual(['TOP-1', 'NEW-2', 'NEW-3'])
+    expect(page.hasMore).toBe(true)
+    expect(page.partial).toBe(true)
+    expect(page.nextCursor).toBeUndefined()
+  })
+
+  it('keeps the all-terms page and its cursor when no summary holds every term', async () => {
+    const page = await searchAtlassian(jira(overflowing, []), 'jira', query)
+    expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1'])
+    expect(page.nextCursor).toBe('page-2')
+    expect(page.partial).toBe(false)
+  })
+
+  it.each([
+    ['a provider refusal', new NativeSearchError('rate_limited', 'Rate limited.', 30)],
+    ['a request timeout', new Error('Request timed out after 10000ms')],
+  ])(
+    'keeps the all-terms page and its cursor when the summary search fails on %s',
+    async (_, failure) => {
+      const api = jira(overflowing, () => {
+        throw failure
+      })
+      const page = await searchAtlassian(api, 'jira', query)
+      expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1'])
+      expect(page.nextCursor).toBe('page-2')
+    }
+  )
+
+  it('propagates a cancellation during the summary search', async () => {
+    const caller = new AbortController()
+    const reason = new DOMException('Search cancelled', 'AbortError')
+    const api = jira(overflowing, () => {
+      caller.abort(reason)
+      throw reason
+    })
+    await expect(searchAtlassian(api, 'jira', { ...query, signal: caller.signal })).rejects.toBe(
+      reason
+    )
+  })
+
+  it('keeps an all-terms match that fits on one page without a summary search', async () => {
+    const api = jira({ ids: ['ONLY-1'], more: false }, ['TOP-1'])
+    const page = await searchAtlassian(api, 'jira', query)
+    expect(page.documents.map((document) => document.id)).toEqual(['ONLY-1'])
+    expect(api.json).toHaveBeenCalledTimes(2)
+  })
+
+  it('keeps date order, native queries and any-term matches on their own page', async () => {
+    for (const input of [
+      { ...query, filters: { sortBy: 'newest' as const } },
+      { ...query, native: { provider: 'jira' as const, query: 'text ~ "merged results"' } },
+      { ...query, textMatch: 'any' as const },
+    ]) {
+      const api = jira({ ids: ['NEW-3', 'NEW-2'], more: true }, ['TOP-1'])
+      const page = await searchAtlassian(api, 'jira', input)
+      expect(page.documents.map((document) => document.id)).not.toContain('TOP-1')
+      expect(api.json).toHaveBeenCalledTimes(2)
+    }
+  })
+})

apps/sim/lib/sim-search/live/atlassian.ts

@@ -12,6 +12,7 @@ import { providerText } from '@/lib/sim-search/live/text'
 import {
   confluenceTextClause,
   holdsHalfTheTerms,
+  jiraSummaryClause,
   jiraTextClause,
 } from '@/lib/sim-search/live/text-match'
 import type {
@@ -175,33 +176,55 @@ export async function searchAtlassian(
       const cloudId = string(site.id)
       const origin = string(site.url).replace(/\/$/, '')
       if (provider === 'jira') {
-        const data = object(
-          await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, {
-            body: {
-              jql: scopeAtlassianQuery(
-                input.native?.query || (text ? jiraTextClause(text, input.textMatch) : ''),
-                scope,
-                order
-              ),
-              maxResults: input.limit,
-              fields: ['summary', 'description', 'updated', 'creator', 'status'],
-              ...(input.native?.cursor && single ? { nextPageToken: input.native.cursor } : {}),
-            },
-            signal: input.signal,
-          })
-        )
-        return {
-          documents: array(data.issues)
-            .filter((row) => {
-              const fields = object(row.fields)
-              return (
-                !relevant ||
-                relevant(`${string(fields.summary)}\n${textContent(fields.description)}`)
-              )
+        const searchIssues = async (jql: string, cursor?: string): Promise<SitePage> => {
+          const data = object(
+            await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, {
+              body: {
+                jql: scopeAtlassianQuery(jql, scope, order),
+                maxResults: input.limit,
+                fields: ['summary', 'description', 'updated', 'creator', 'status'],
+                ...(cursor ? { nextPageToken: cursor } : {}),
+              },
+              signal: input.signal,
             })
-            .map((row) => issue(row, cloudId, origin)),
-          excluded: false,
-          next: string(data.nextPageToken) || undefined,
+          )
+          return {
+            documents: array(data.issues)
+              .filter((row) => {
+                const fields = object(row.fields)
+                return (
+                  !relevant ||
+                  relevant(`${string(fields.summary)}\n${textContent(fields.description)}`)
+                )
+              })
+              .map((row) => issue(row, cloudId, origin)),
+            excluded: false,
+            next: string(data.nextPageToken) || undefined,
+          }
+        }
+        const exact = await searchIssues(
+          input.native?.query || (text ? jiraTextClause(text, input.textMatch) : ''),
+          input.native?.cursor && single ? input.native.cursor : undefined
+        )
+        // JQL has no relevance order, so an all-terms match larger than a page comes back newest
+        // first; issues whose summary holds every term lead it instead.
+        if (!text || input.native || order || input.textMatch === 'any' || !exact.next) return exact
+        try {
+          const titled = await searchIssues(jiraSummaryClause(text))
+          if (!titled.documents.length) return exact
+          const leading = new Set(titled.documents.map((document) => document.id))
+          return {
+            documents: [
+              ...titled.documents,
+              ...exact.documents.filter((document) => !leading.has(document.id)),
+            ].slice(0, input.limit),
+            excluded: false,
+            ranking: 'relevance',
+          }
+        } catch {
+          // A cancelled search stops here; any other failure keeps the all-terms page.
+          input.signal?.throwIfAborted()
+          return exact
         }
       }
       const searchPages = async (clause: string, cursor?: string) => {
@@ -265,7 +288,9 @@ export async function searchAtlassian(
         ? ' Matches other than pages, blog posts, and spaces (attachments, comments, whiteboards, folders, databases) were excluded because they cannot be read.'
         : '') +
       (pages.some((result) => result.ranking === 'relevance')
-        ? ' More pages contained every word than fit on one page, so this page is ordered by Confluence relevance search; refine the query, or page with a native CQL query.'
+        ? provider === 'jira'
+          ? ' More issues contained every word than fit on one page, so issues whose summary holds every word come first; refine the query, or page with a native JQL query.'
+          : ' More pages contained every word than fit on one page, so this page is ordered by Confluence relevance search; refine the query, or page with a native CQL query.'
         : '') +
       (pages.some((result) => result.ranking === 'unavailable')
         ? ' Relevance ranking was unavailable, so this page keeps the order of the all-terms search.'

apps/sim/lib/sim-search/live/text-match.ts

@@ -106,6 +106,14 @@ export function jiraTextClause(text: string, match: TextMatch = 'all'): string {
   return `(${groups.join(' OR ')})`
 }
 
+/**
+ * The `summary ~` clause for a plain query in Jira JQL: issues whose summary holds every word.
+ * JQL has no relevance order, so this stands in for a title boost.
+ */
+export function jiraSummaryClause(text: string): string {
+  return `summary ~ ${escapeSearchPhrase(text)}`
+}
+
 /**
  * The text clause for a plain query in Confluence CQL. `any` uses `siteSearch`, Confluence's own
  * relevance search, which ranks a page with the query's rarer words above the many pages sharing