improvement(search): share the half-terms threshold and convert Jira descriptions only when filtered

Author: waleedlatif1

apps/sim/lib/sim-search/live/atlassian.test.ts

@@ -344,24 +344,19 @@ describe('Jira plain search ranking', () => {
     titled: Issue[] | (() => never),
     half: Issue[] | (() => never) = []
   ) {
-    const issues = (rows: Issue[] | (() => never)) =>
-      (typeof rows === 'function' ? rows() : rows).map((row) =>
-        typeof row === 'string'
-          ? { key: row, fields: { summary: row } }
-          : { key: row.key, fields: { summary: row.summary } }
-      )
+    const toRow = (row: Issue) => (typeof row === 'string' ? { key: row, summary: row } : row)
+    const issues = (rows: Issue[]) =>
+      rows.map(toRow).map(({ key, summary }) => ({ key, fields: { summary } }))
     const json = vi.fn(
       async (path: string, options?: { body?: { jql?: string; maxResults?: number } }) => {
         if (path === '/oauth/token/accessible-resources') return [SITE]
         if (path !== search) throw new Error(`Unexpected request: ${path}`)
         const jql = options?.body?.jql ?? ''
-        if (jql.includes('summary ~'))
-          return {
-            issues: issues(jql.includes(' OR ') ? half : titled).slice(
-              0,
-              options?.body?.maxResults
-            ),
-          }
+        if (jql.includes('summary ~')) {
+          const rows = jql.includes(' OR ') ? half : titled
+          const picked = typeof rows === 'function' ? rows() : rows
+          return { issues: issues(picked).slice(0, options?.body?.maxResults) }
+        }
         return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) }
       }
     )

apps/sim/lib/sim-search/live/atlassian.ts

@@ -118,8 +118,9 @@ async function sites(client: NativeClient) {
 }
 
 /**
- * Pair matches fetched for the half-the-terms summary fallback; the page is filtered to summaries
- * holding enough terms, so it starts from a wider pool than the caller's limit.
+ * Pair matches fetched for the half-the-terms summary fallback. Jira returns them newest first,
+ * not by relevance, and the page is then filtered to summaries holding enough terms, so it starts
+ * from a wider pool than the caller's limit or the any-term fallback's.
  */
 const HALF_TERM_CANDIDATES = 50
 
@@ -182,8 +183,11 @@ export async function searchAtlassian(
       const cloudId = string(site.id)
       const origin = string(site.url).replace(/\/$/, '')
       if (provider === 'jira') {
-        /** One JQL page, each issue with its summary and description. */
-        const searchIssues = async (jql: string, cursor?: string, maxResults = input.limit) => {
+        /** One JQL page, each issue with its summary and raw fields. */
+        const searchIssues = async (
+          jql: string,
+          { cursor, maxResults = input.limit }: { cursor?: string; maxResults?: number } = {}
+        ) => {
           const data = object(
             await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, {
               body: {
@@ -201,20 +205,21 @@ export async function searchAtlassian(
               return {
                 document: issue(row, cloudId, origin),
                 summary: string(fields.summary),
-                description: textContent(fields.description),
+                fields,
               }
             }),
             next: string(data.nextPageToken) || undefined,
           }
         }
         const exact = await searchIssues(
           input.native?.query || (text ? jiraTextClause(text, input.textMatch) : ''),
-          input.native?.cursor && single ? input.native.cursor : undefined
+          { cursor: input.native?.cursor && single ? input.native.cursor : undefined }
         )
         const exactPage: SitePage = {
           documents: exact.issues
             .filter(
-              ({ summary, description }) => !relevant || relevant(`${summary}\n${description}`)
+              ({ summary, fields }) =>
+                !relevant || relevant(`${summary}\n${textContent(fields.description)}`)
             )
             .map(({ document }) => document),
           excluded: false,
@@ -229,22 +234,22 @@ export async function searchAtlassian(
         const [every, half] = await Promise.allSettled([
           searchIssues(jiraTextClause(text, 'all', 'summary')),
           terms.count >= 3
-            ? searchIssues(jiraTextClause(text, 'any', 'summary'), undefined, HALF_TERM_CANDIDATES)
+            ? searchIssues(jiraTextClause(text, 'any', 'summary'), {
+                maxResults: HALF_TERM_CANDIDATES,
+              })
             : undefined,
         ])
         // A cancelled search stops here; any other failure keeps the all-terms page.
         input.signal?.throwIfAborted()
-        const required = Math.ceil(terms.count / 2)
-        const titled =
-          every.status === 'fulfilled' && every.value.issues.length
-            ? every.value.issues.map(({ document }) => document)
-            : half.status === 'fulfilled' && half.value
-              ? half.value.issues
-                  .map(({ document, summary }) => ({ document, held: terms.held(summary) }))
-                  .filter(({ held }) => held >= required)
-                  .sort((left, right) => right.held - left.held)
-                  .map(({ document }) => document)
-              : []
+        let titled =
+          every.status === 'fulfilled' ? every.value.issues.map(({ document }) => document) : []
+        if (!titled.length && half.status === 'fulfilled' && half.value) {
+          titled = half.value.issues
+            .map(({ document, summary }) => ({ document, held: terms.held(summary) }))
+            .filter(({ held }) => held >= terms.required)
+            .sort((left, right) => right.held - left.held)
+            .map(({ document }) => document)
+        }
         if (!titled.length) return exactPage
         const leading = new Set(titled.map((document) => document.id))
         return {

apps/sim/lib/sim-search/live/text-match.ts

@@ -130,11 +130,17 @@ export function termCoverage(text: string, terms: readonly string[]): number {
  * Query and text words are compared by their singular key, so a word and its regular plural count
  * as one term; a term with punctuation inside it is matched as written.
  */
-export function stemmedTerms(query: string): { count: number; held: (text: string) => number } {
+export function stemmedTerms(query: string): {
+  count: number
+  /** At least half the terms, the minimum a partial match must hold. */
+  required: number
+  held: (text: string) => number
+} {
   const terms = distinctTerms(query, singularKey)
   const keys = terms.map(singularKey)
   return {
     count: terms.length,
+    required: Math.ceil(terms.length / 2),
     held: (text) => {
       const words = new Set(textWords(text).map(singularKey))
       return terms.filter(
@@ -144,9 +150,11 @@ export function stemmedTerms(query: string): { count: number; held: (text: strin
   }
 }
 
-/** A test for whether a text holds at least half the query's distinct terms, as `stemmedTerms` counts them. */
+/**
+ * A test for whether a text holds at least half the query's distinct terms, as `stemmedTerms`
+ * counts them.
+ */
 export function holdsHalfTheTerms(query: string): (text: string) => boolean {
-  const { count, held } = stemmedTerms(query)
-  const required = Math.ceil(count / 2)
+  const { required, held } = stemmedTerms(query)
   return (text) => held(text) >= required
 }