improvement(search): require half the query's terms in Jira partial matches

apps/sim/lib/knowledge/search/snippet.ts

@@ -16,6 +16,12 @@ const HEADER_LINE = /^[A-Z][A-Za-z-]{1,15}: .*$/
  */
 const WORD_CHARACTER =
   /(?![\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Hangul}\p{sc=Thai}])[\p{L}\p{N}_]/u
+const WORD_RUN = new RegExp(`(?:${WORD_CHARACTER.source})+`, 'gu')
+
+/** The words of a text, split where a run of `WORD_CHARACTER`s ends. */
+export function textWords(text: string): string[] {
+  return text.match(WORD_RUN) ?? []
+}
 
 /**
  * The document text without the header block some connectors prefix (the

apps/sim/lib/sim-search/live/atlassian.test.ts

@@ -197,3 +197,51 @@ describe('Confluence live documents', () => {
     expect(await previewConfluence(page('<p>Rollout steps</p>'), document)).toBe('Rollout steps')
   })
 })
+
+describe('Jira any-term search', () => {
+  const issues = {
+    '/ex/jira/cloud/rest/api/3/search/jql': {
+      issues: [
+        { key: 'ENG-1', fields: { summary: 'Token creator role request' } },
+        { key: 'ENG-2', fields: { summary: 'Role request form' } },
+      ],
+    },
+  }
+  const query = { query: 'glean token creator role request', limit: 25, scopes: [] }
+
+  it('keeps only issues holding at least half the query terms', async () => {
+    const page = await searchAtlassian(client(issues), 'jira', { ...query, textMatch: 'any' })
+    expect(page.documents.map((document) => document.id)).toEqual(['ENG-1'])
+  })
+
+  it('counts only the summary and description, not the added status line', async () => {
+    const api = client({
+      '/ex/jira/cloud/rest/api/3/search/jql': {
+        issues: [{ key: 'ENG-3', fields: { summary: 'Release update', status: { name: 'Open' } } }],
+      },
+    })
+    const page = await searchAtlassian(api, 'jira', {
+      query: 'release status update owner timeline',
+      limit: 25,
+      scopes: [],
+      textMatch: 'any',
+    })
+    expect(page.documents).toEqual([])
+  })
+
+  it('keeps every issue an all-terms search returns', async () => {
+    const page = await searchAtlassian(client(issues), 'jira', query)
+    expect(page.documents.map((document) => document.id)).toEqual(['ENG-1', 'ENG-2'])
+  })
+
+  it('leaves a Confluence any-term search to its relevance ranking', async () => {
+    const api = client({
+      '/ex/confluence/cloud/wiki/rest/api/search': {
+        results: [{ content: { id: '5', type: 'page', title: 'Role request form' } }],
+        _links: {},
+      },
+    })
+    const page = await searchAtlassian(api, 'confluence', { ...query, textMatch: 'any' })
+    expect(page.documents.map((document) => document.id)).toEqual(['5'])
+  })
+})

apps/sim/lib/sim-search/live/atlassian.ts

@@ -9,7 +9,11 @@ import {
 } from '@/lib/sim-search/live/http'
 import { interleaveByRank } from '@/lib/sim-search/live/pages'
 import { providerText } from '@/lib/sim-search/live/text'
-import { confluenceTextClause, jiraTextClause } from '@/lib/sim-search/live/text-match'
+import {
+  confluenceTextClause,
+  holdsHalfTheTerms,
+  jiraTextClause,
+} from '@/lib/sim-search/live/text-match'
 import type {
   NativeClient,
   NativeDocument,
@@ -152,6 +156,9 @@ export async function searchAtlassian(
   const direction = dateSortDirection(input.filters)
   const order = direction ? `${field} ${direction.toUpperCase()}` : undefined
   const text = nativeText(input)
+  // Jira's any-term JQL matches issues sharing two common words; like minimum-should-match,
+  // keep only issues holding at least half the terms.
+  const relevant = input.textMatch === 'any' && text ? holdsHalfTheTerms(text) : undefined
   const pages = await Promise.all(
     selected.map(async (site) => {
       const cloudId = string(site.id)
@@ -173,7 +180,15 @@ export async function searchAtlassian(
           })
         )
         return {
-          documents: array(data.issues).map((row) => issue(row, cloudId, origin)),
+          documents: array(data.issues)
+            .filter((row) => {
+              const fields = object(row.fields)
+              return (
+                !relevant ||
+                relevant(`${string(fields.summary)}\n${textContent(fields.description)}`)
+              )
+            })
+            .map((row) => issue(row, cloudId, origin)),
           excluded: false,
           next: string(data.nextPageToken) || undefined,
         }

apps/sim/lib/sim-search/live/text-match.test.ts

@@ -1,5 +1,9 @@
 import { describe, expect, it } from 'vitest'
-import { confluenceTextClause, jiraTextClause } from '@/lib/sim-search/live/text-match'
+import {
+  confluenceTextClause,
+  holdsHalfTheTerms,
+  jiraTextClause,
+} from '@/lib/sim-search/live/text-match'
 
 describe('text match clauses', () => {
   it('matches either term of a two-term query, leaving out stopwords', () => {
@@ -57,4 +61,30 @@ describe('text match clauses', () => {
   it('falls back to the all-terms clause when fewer than two terms are distinctive', () => {
     expect(jiraTextClause('how does it work', 'any')).toBe('text ~ "how does it work"')
   })
+
+  describe('holdsHalfTheTerms', () => {
+    it('needs at least half the distinct terms, rounding up', () => {
+      const holds = holdsHalfTheTerms('glean token creator role request')
+      expect(holds('Role request form')).toBe(false)
+      expect(holds('Token creator role')).toBe(true)
+    })
+
+    it('counts a word and its plural as one term in either direction', () => {
+      expect(holdsHalfTheTerms('glean token creator role request')('Glean tokens, creators')).toBe(
+        true
+      )
+      expect(
+        holdsHalfTheTerms('database release checklist owner')('Databases and weekly releases')
+      ).toBe(true)
+      expect(holdsHalfTheTerms('databases releases checklist owner')('Database release')).toBe(true)
+    })
+
+    it('matches a term with punctuation inside it as written', () => {
+      expect(holdsHalfTheTerms('ci/cd pipeline failure owner')('CI/CD pipeline')).toBe(true)
+    })
+
+    it('accepts one matching term for a two-term query', () => {
+      expect(holdsHalfTheTerms('contractor offboarding')('Contractor directory')).toBe(true)
+    })
+  })
 })

apps/sim/lib/sim-search/live/text-match.ts

@@ -1,4 +1,4 @@
-import { findTermMatches, queryTerms } from '@/lib/knowledge/search/snippet'
+import { findTermMatches, queryTerms, textWords } from '@/lib/knowledge/search/snippet'
 
 /**
  * How a plain query's words must match a document. Confluence and Jira text search require every
@@ -115,3 +115,21 @@ export function confluenceTextClause(text: string, match: TextMatch = 'all'): st
 export function termCoverage(text: string, terms: readonly string[]): number {
   return terms.filter((term) => findTermMatches(text, [term]).length > 0).length
 }
+
+/**
+ * A test for whether a text holds at least half the query's distinct terms. Query and text words
+ * are compared by their singular key, so a word and its regular plural count as one term as a
+ * stemmed search matches them; a term with punctuation inside it is matched as written.
+ */
+export function holdsHalfTheTerms(query: string): (text: string) => boolean {
+  const terms = distinctTerms(query, singularKey)
+  const keys = terms.map(singularKey)
+  const required = Math.ceil(terms.length / 2)
+  return (text) => {
+    const words = new Set(textWords(text).map(singularKey))
+    const present = terms.filter(
+      (term, index) => words.has(keys[index]) || findTermMatches(text, [term]).length > 0
+    )
+    return present.length >= required
+  }
+}