improvement(knowledge): widen a search to 50 chunks only when the reader asks for more

Author: waleedlatif1

apps/sim/app/o/[organizationId]/search/search.test.tsx

@@ -135,7 +135,7 @@ async function editDraft(value: string) {
 function expectVisibleQuery(query: string) {
   expect(searchInput().value).toBe(query)
   expect(container.querySelector('a[data-source-link]')?.textContent).toBe(`${query} launch plan`)
-  expect(mocks.search).toHaveBeenLastCalledWith(scope, query, {})
+  expect(mocks.search).toHaveBeenLastCalledWith(scope, query, {}, 20)
   expect(document.activeElement).toBe(searchInput())
 }
 

apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx

@@ -167,12 +167,12 @@ describe('result paging and the custom window', () => {
     similarity: 0.5,
   })
 
-  it('shows ten documents at a time and reveals more on request', async () => {
+  it('offers more only after a full first page, and asks for the wider search on request', async () => {
     mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } })
-    mocks.search.mockReturnValue({
+    const page = (length: number) => ({
       data: {
         query: 'launch',
-        results: Array.from({ length: 25 }, (_, n) => result(n)),
+        results: Array.from({ length }, (_, n) => result(n)),
         retrieval: { status: 'complete', timedOutLegs: [] },
       },
       isPending: false,
@@ -181,20 +181,31 @@ describe('result paging and the custom window', () => {
       isError: false,
       refetch: mocks.retry,
     })
+    mocks.search.mockReturnValue(page(20))
+    await render()
+    expect(mocks.search.mock.calls.at(-1)![3]).toBe(20)
+    const more = () =>
+      [...container.querySelectorAll('button')].find((b) => b.textContent === 'Show more')
+    expect(more()).toBeDefined()
+    await act(async () => more()!.click())
+    /** The wider search is its own request; the first paint was never widened. */
+    expect(mocks.search.mock.calls.at(-1)![3]).toBe(50)
+    expect(more()).toBeUndefined()
+    mocks.search.mockReturnValue(page(7))
     await render()
-    expect(container.textContent).toContain('Document 9')
-    expect(container.textContent).not.toContain('Document 10')
-    const more = [...container.querySelectorAll('button')].find(
-      (b) => b.textContent === 'Show more'
-    )!
-    expect(more).toBeDefined()
-    await act(async () => more.click())
-    expect(container.textContent).toContain('Document 19')
-    expect(container.textContent).not.toContain('Document 20')
+    expect(more()).toBeUndefined()
   })
 
   it('searches a custom window as an inclusive range of days', async () => {
     mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } })
+    mocks.search.mockReturnValue({
+      data: { query: 'launch', results: [], retrieval: { status: 'complete', timedOutLegs: [] } },
+      isPending: false,
+      isFetching: false,
+      isPlaceholderData: false,
+      isError: false,
+      refetch: mocks.retry,
+    })
     await render(undefined, '?updated=custom&from=2026-09-01&to=2026-09-10')
     const filters = mocks.search.mock.calls.at(-1)![2]
     expect(filters.modifiedAfter).toBe('2026-09-01T00:00:00.000Z')

apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx

@@ -4,9 +4,10 @@ import { useState } from 'react'
 import { Chip, ChipDatePicker, ChipLink, cn } from '@sim/emcn'
 import { useQueryStates } from 'nuqs'
 import { ActivityStatus } from '@/components/ui/activity-status'
-import type {
-  WorkspaceKnowledgeSearchResult,
-  WorkspaceSearchFilters,
+import {
+  WORKSPACE_KNOWLEDGE_SEARCH_LIMITS,
+  type WorkspaceKnowledgeSearchResult,
+  type WorkspaceSearchFilters,
 } from '@/lib/api/contracts/knowledge'
 import { useSession } from '@/lib/auth/auth-client'
 import { type ResourceScope, resourceScopeKey } from '@/lib/core/resource-scope'
@@ -27,8 +28,6 @@ import { useSearchIndex, useSearchSourceOverview } from '@/hooks/queries/kb/conn
 import { useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge'
 
 const DAY_MS = 24 * 60 * 60 * 1000
-/** Cards shown before the reader asks for more; the search itself returns several pages' worth. */
-const RESULTS_PAGE_SIZE = 10
 /** Every result without a connector is an upload; the filter names them so. */
 const UPLOAD_SOURCE = 'upload'
 
@@ -129,7 +128,8 @@ interface SearchResultsProps {
 function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
   const [hasShownFilters, setHasShownFilters] = useState(false)
   const [searchedAt] = useState(Date.now)
-  const [shown, setShown] = useState(RESULTS_PAGE_SIZE)
+  /** More results are a second, wider search: the first paint stays as quick as it is. */
+  const [expanded, setExpanded] = useState(false)
   const {
     data: index,
     isPending: basesPending,
@@ -158,7 +158,17 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
     isPlaceholderData,
     isError: searchFailed,
     refetch: refetchSearch,
-  } = useWorkspaceKnowledgeSearch(scope, query, searchFilters)
+  } = useWorkspaceKnowledgeSearch(
+    scope,
+    query,
+    searchFilters,
+    expanded
+      ? WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.expanded
+      : WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.initial
+  )
+  /** A full first page may collapse to few cards, yet more documents may still match. */
+  const mayHaveMore =
+    !expanded && (search?.results.length ?? 0) >= WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.initial
   const { data: overview } = useSearchSourceOverview(scope)
   const indexing = (overview?.providers ?? [])
     .filter((provider) => provider.isSyncing)
@@ -292,7 +302,7 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
           className={cn('flex flex-col', isPlaceholderData && 'opacity-60')}
           onKeyDown={handleResultsKeyDown}
         >
-          {documents.slice(0, shown).map((result) => {
+          {documents.map((result) => {
             const source = toSource(result, query, scope)
             return (
               <SourceCard
@@ -311,9 +321,9 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
               />
             )
           })}
-          {documents.length > shown && (
+          {mayHaveMore && (
             <div className='flex px-2 py-2'>
-              <Chip variant='border' onClick={() => setShown((count) => count + RESULTS_PAGE_SIZE)}>
+              <Chip variant='border' disabled={isFetching} onClick={() => setExpanded(true)}>
                 Show more
               </Chip>
             </div>

apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx

@@ -212,7 +212,7 @@ describe('search refinement with the real query cache and URL state', () => {
       expect(requests.at(-1)?.body).toEqual({
         organizationId: 'organization',
         query: 'launch',
-        topK: 50,
+        topK: 20,
         filters: expectedFilters,
       })
       expect(container.querySelector('h1')).toBeNull()

apps/sim/hooks/queries/kb/knowledge.test.ts

@@ -214,9 +214,11 @@ describe('knowledge query placeholder scope', () => {
     const query = captureQuery(() =>
       useWorkspaceKnowledgeSearch('workspace-1', 'new query', { source: 'slack' })
     )
-    expect(query.queryKey).toEqual(
-      knowledgeKeys.search('workspace-1', 'new query', { source: 'slack' }, 'reader')
-    )
+    /** The limit is the key's last part, so the wider search never evicts the first paint. */
+    expect(query.queryKey).toEqual([
+      ...knowledgeKeys.search('workspace-1', 'new query', { source: 'slack' }, 'reader'),
+      20,
+    ])
     expect(knowledgeKeys.search('workspace-1', 'query', { source: 'slack' })).not.toEqual(
       knowledgeKeys.search('workspace-1', 'query', { source: 'gitlab' })
     )

apps/sim/hooks/queries/kb/knowledge.ts

@@ -51,8 +51,10 @@ import {
   updateKnowledgeChunkContract,
   updateKnowledgeDocumentContract,
   updateKnowledgeDocumentTagsContract,
+  WORKSPACE_KNOWLEDGE_SEARCH_LIMITS,
   type WorkspaceKnowledgeSearchBody,
   type WorkspaceKnowledgeSearchData,
+  type WorkspaceKnowledgeSearchLimit,
 } from '@/lib/api/contracts/knowledge'
 import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge/search'
 import { useSession } from '@/lib/auth/auth-client'
@@ -89,8 +91,6 @@ export const KNOWLEDGE_DOCUMENT_LIST_STALE_TIME = 60 * 1000
 export const KNOWLEDGE_CHUNK_LIST_STALE_TIME = 60 * 1000
 export const KNOWLEDGE_CHUNK_SEARCH_STALE_TIME = 60 * 1000
 export const WORKSPACE_KNOWLEDGE_SEARCH_STALE_TIME = 60 * 1000
-/** Chunks one search asks for: several pages of documents once collapsed to one card each. */
-export const WORKSPACE_KNOWLEDGE_SEARCH_RESULT_LIMIT = 50
 export const KNOWLEDGE_TAG_DEFINITION_LIST_STALE_TIME = 60 * 1000
 export const KNOWLEDGE_TAG_USAGE_STALE_TIME = 60 * 1000
 export const KNOWLEDGE_DOCUMENT_TAG_DEFINITION_LIST_STALE_TIME = 60 * 1000
@@ -1209,7 +1209,8 @@ async function searchWorkspaceKnowledge(
 export function useWorkspaceKnowledgeSearch(
   owner: string | ResourceScope | undefined,
   query: string,
-  filters?: WorkspaceSearchFilters
+  filters?: WorkspaceSearchFilters,
+  limit: WorkspaceKnowledgeSearchLimit = WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.initial
 ) {
   const { data: session } = useSession()
   const queryClient = useQueryClient()
@@ -1224,14 +1225,15 @@ export function useWorkspaceKnowledgeSearch(
   const scopeKey =
     scope?.kind === 'workspace' ? scope.workspaceId : scope ? resourceScopeKey(scope) : undefined
   return useQuery({
-    queryKey: knowledgeKeys.search(scopeKey, trimmed, filters, userId),
+    /** The limit is the key's last part, so asking for more never evicts the first paint. */
+    queryKey: [...knowledgeKeys.search(scopeKey, trimmed, filters, userId), limit],
     queryFn: ({ signal }) =>
       searchWorkspaceKnowledge(
         {
           ...(scope ? resourceScopeFields(scope) : {}),
           query: trimmed,
           filters,
-          topK: WORKSPACE_KNOWLEDGE_SEARCH_RESULT_LIMIT,
+          topK: limit,
         },
         signal
       ),

apps/sim/lib/api/contracts/knowledge/search.ts

@@ -194,6 +194,11 @@ export const workspaceSearchFiltersSchema = z
   })
 export type WorkspaceSearchFilters = z.output<typeof workspaceSearchFiltersSchema>
 
+/** Chunks a search asks for at first paint, and once the reader asks for more; both within `topK`'s bound. */
+export const WORKSPACE_KNOWLEDGE_SEARCH_LIMITS = { initial: 20, expanded: 50 } as const
+export type WorkspaceKnowledgeSearchLimit =
+  (typeof WORKSPACE_KNOWLEDGE_SEARCH_LIMITS)[keyof typeof WORKSPACE_KNOWLEDGE_SEARCH_LIMITS]
+
 export const workspaceKnowledgeSearchBodySchema = resourceOwnerSchema.safeExtend({
   filters: workspaceSearchFiltersSchema.optional(),
   query: z.string().trim().min(1, 'A search query is required').max(2000, 'Query is too long'),