improvement(knowledge): page search results, take a custom date window, and rerank a person's search (#8060)

* improvement(knowledge): page search results, take a custom date window, and rerank a person's search

- the Search page asks for 50 chunks, collapses them to one card per document, shows ten at a time and reveals more on request
- `updated` gains a custom range: `from`/`to` days in the URL, sent as an inclusive `modifiedAfter`/`modifiedBefore` window; `modifiedBefore` joins the filter contract, the filter intersection, the document conditions, the on-row date test, the filtered-set estimate and the bounded probe
- the dashboard search opts into the platform's cross-encoder reranker whenever a Cohere key is configured; reranking stays best-effort

* improvement(knowledge): widen a search to 50 chunks only when the reader asks for more

* improvement(knowledge): give a filter-first probe the budget its index-driven read can use

* improvement(knowledge): keep the default scan for a walk that asks the document per tuple

* fix(knowledge): check the window on the request, rerank with any key, and start a refined search at its first page

- the filters schema stays a plain object so the Assistant's search input can still extend it; the window's order is checked on the request body
- the dashboard asks for reranking outright, and the use case reranks only when the workspace or the platform holds a key
- a refinement of the filters starts over at the first page after the reader asked for more
- a custom window's days are the reader's local days

* fix(knowledge): judge a caller's reranker key by the resolver's policy, and let a custom window be cleared

- the credential check tries the platform key before a workspace lookup, and counts a caller's own key only off hosted Sim, as the resolver does
- a custom window with no days yet searches nothing and says so; another window drops the days; the picker's Clear reaches the URL through a forwarded `onClear`

* test(knowledge): assert only the dropped days, not the URL adapter's next flush

* chore(knowledge): narrow the filters once before estimating them

* fix(knowledge): wait for both days of a custom window, show the chosen range, and keep the default scan while rows are unfilled

- a custom window searches only once both days are chosen; the picker shows the chosen range instead of a fixed label
- while the projection still holds rows the backfill has not filled, an on-row walk keeps the default scan cap, since an unfilled row is decided through its document; the answer is read off the unfilled-rows index and remembered for a minute

* fix(knowledge): show the filters while a custom window waits for its days

Author: waleedlatif1

apps/sim/app/api/knowledge/search/route.test.ts

@@ -47,6 +47,9 @@ describe('workspace search route', () => {
     expect(call.input.signal).toBe(request.signal)
     expect(call.input.allowPartialResults).toBe(true)
     expect(call.input.vectorBudgetMs).toBe(3000)
+    /** A person's search asks for reranking; the use case reranks when a credential exists. */
+    expect(call.input.rerankerEnabled).toBe(true)
+    expect(call.input.rerankerModel).toBe('rerank-v4.0-fast')
     controller.abort()
     expect(call.input.signal.aborted).toBe(true)
     await expect(response.json()).resolves.toEqual({

apps/sim/app/api/knowledge/search/route.ts

@@ -7,6 +7,7 @@ import {
 import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies'
 import { knowledgeOperations } from '@/lib/knowledge/application/operations'
 import { searchScopedKnowledge } from '@/lib/knowledge/application/workspace-search'
+import { DEFAULT_RERANKER_MODEL } from '@/lib/knowledge/reranker-models'
 import { sourceAuthor } from '@/lib/knowledge/search/author'
 
 const DIRECT_SEARCH_VECTOR_BUDGET_MS = 3000
@@ -28,6 +29,13 @@ export const POST = defineInternalJsonRoute({
     topK: body.topK,
     allowPartialResults: true,
     vectorBudgetMs: DIRECT_SEARCH_VECTOR_BUDGET_MS,
+    /**
+     * A person's search is reranked by a cross-encoder whenever the workspace or the platform
+     * holds a key for one; the use case checks that before spending a call, and reranking stays
+     * best-effort, so a provider outage leaves the fused order in place.
+     */
+    rerankerEnabled: true,
+    rerankerModel: DEFAULT_RERANKER_MODEL,
     surface: 'dashboard' as const,
     signal: request.signal,
   }),

apps/sim/app/o/[organizationId]/search/search.test.tsx

@@ -135,7 +135,7 @@ async function editDraft(value: string) {
 function expectVisibleQuery(query: string) {
   expect(searchInput().value).toBe(query)
   expect(container.querySelector('a[data-source-link]')?.textContent).toBe(`${query} launch plan`)
-  expect(mocks.search).toHaveBeenLastCalledWith(scope, query, {})
+  expect(mocks.search).toHaveBeenLastCalledWith(scope, query, {}, 20)
   expect(document.activeElement).toBe(searchInput())
 }
 

apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx

@@ -48,10 +48,13 @@ afterEach(() => {
   act(() => root.unmount())
   vi.unstubAllGlobals()
 })
-async function render(scope: ResourceScope = { kind: 'workspace', workspaceId: 'workspace' }) {
+async function render(
+  scope: ResourceScope = { kind: 'workspace', workspaceId: 'workspace' },
+  searchParams = ''
+) {
   await act(async () =>
     root.render(
-      <NuqsTestingAdapter>
+      <NuqsTestingAdapter searchParams={searchParams}>
         <KnowledgeSearchResults scope={scope} query='launch' onSummarize={vi.fn()} />
       </NuqsTestingAdapter>
     )
@@ -148,3 +151,133 @@ describe('source setup navigation', () => {
     expect(container.querySelector('a')?.getAttribute('href')).toBe(href)
   })
 })
+
+describe('result paging and the custom window', () => {
+  const result = (n: number) => ({
+    documentId: `doc-${n}`,
+    knowledgeBaseId: 'kb',
+    knowledgeBaseName: 'Index',
+    documentName: `Document ${n}`,
+    sourceUrl: null,
+    connectorType: 'slack',
+    sourceModifiedAt: null,
+    author: null,
+    content: 'launch notes',
+    chunkIndex: 0,
+    similarity: 0.5,
+  })
+
+  it('offers more only after a full first page, and asks for the wider search on request', async () => {
+    mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } })
+    const page = (length: number) => ({
+      data: {
+        query: 'launch',
+        results: Array.from({ length }, (_, n) => result(n)),
+        retrieval: { status: 'complete', timedOutLegs: [] },
+      },
+      isPending: false,
+      isFetching: false,
+      isPlaceholderData: false,
+      isError: false,
+      refetch: mocks.retry,
+    })
+    mocks.search.mockReturnValue(page(20))
+    await render()
+    expect(mocks.search.mock.calls.at(-1)![3]).toBe(20)
+    const more = () =>
+      [...container.querySelectorAll('button')].find((b) => b.textContent === 'Show more')
+    expect(more()).toBeDefined()
+    await act(async () => more()!.click())
+    /** The wider search is its own request; the first paint was never widened. */
+    expect(mocks.search.mock.calls.at(-1)![3]).toBe(50)
+    expect(more()).toBeUndefined()
+    mocks.search.mockReturnValue(page(7))
+    await render()
+    expect(more()).toBeUndefined()
+  })
+
+  it('starts a refined search over at the first page after the reader asked for more', async () => {
+    mocks.overview.mockReturnValue({
+      data: {
+        providers: [{ connectorType: 'slack', isSyncing: false }],
+        hasSearchableDocuments: true,
+      },
+    })
+    mocks.search.mockReturnValue({
+      data: {
+        query: 'launch',
+        results: Array.from({ length: 20 }, (_, n) => result(n)),
+        retrieval: { status: 'complete', timedOutLegs: [] },
+      },
+      isPending: false,
+      isFetching: false,
+      isPlaceholderData: false,
+      isError: false,
+      refetch: mocks.retry,
+    })
+    await render()
+    const button = (label: string) =>
+      [...container.querySelectorAll('button')].find((b) => b.textContent === label)!
+    await act(async () => button('Show more').click())
+    expect(mocks.search.mock.calls.at(-1)![3]).toBe(50)
+    await act(async () => button('Slack').click())
+    expect(mocks.search.mock.calls.at(-1)![2]).toEqual({ source: 'slack' })
+    expect(mocks.search.mock.calls.at(-1)![3]).toBe(20)
+  })
+
+  it('drops the custom days when another window is chosen', async () => {
+    mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } })
+    mocks.search.mockReturnValue({
+      data: { query: 'launch', results: [], retrieval: { status: 'complete', timedOutLegs: [] } },
+      isPending: false,
+      isFetching: false,
+      isPlaceholderData: false,
+      isError: false,
+      refetch: mocks.retry,
+    })
+    await render(undefined, '?updated=custom&from=2026-09-01&to=2026-09-10')
+    expect(mocks.search.mock.calls.at(-1)![2]).toHaveProperty('modifiedBefore')
+    const anyTime = [...container.querySelectorAll('button')].find(
+      (b) => b.textContent === 'Any time'
+    )!
+    await act(async () => anyTime.click())
+    expect(mocks.search.mock.calls.at(-1)![2]).toEqual({})
+  })
+
+  it('searches nothing while a custom window has no days yet', async () => {
+    mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } })
+    mocks.search.mockReturnValue({
+      data: undefined,
+      isPending: true,
+      isFetching: false,
+      isPlaceholderData: false,
+      isError: false,
+      refetch: mocks.retry,
+    })
+    await render(undefined, '?updated=custom')
+    expect(mocks.search.mock.calls.at(-1)![1]).toBe('')
+    expect(container.textContent).toContain('Choose the days to search.')
+    /** The filters, and the picker among them, are shown so the days can be chosen. */
+    expect(container.textContent).toContain('Updated between')
+    /** One day alone is not a window either; a deep link with only `from` waits for `to`. */
+    await render(undefined, '?updated=custom&from=2026-09-01')
+    expect(mocks.search.mock.calls.at(-1)![1]).toBe('')
+  })
+
+  it('searches a custom window as an inclusive range of days', async () => {
+    mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } })
+    mocks.search.mockReturnValue({
+      data: { query: 'launch', results: [], retrieval: { status: 'complete', timedOutLegs: [] } },
+      isPending: false,
+      isFetching: false,
+      isPlaceholderData: false,
+      isError: false,
+      refetch: mocks.retry,
+    })
+    await render(undefined, '?updated=custom&from=2026-09-01&to=2026-09-10')
+    const filters = mocks.search.mock.calls.at(-1)![2]
+    /** The days are the reader's own: local midnight to the last millisecond of the local day. */
+    expect(filters.modifiedAfter).toBe(new Date(2026, 8, 1).toISOString())
+    expect(filters.modifiedBefore).toBe(new Date(2026, 8, 11, 0, 0, 0, -1).toISOString())
+  })
+})

apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx

@@ -1,12 +1,13 @@
 'use client'
 
 import { useState } from 'react'
-import { Chip, ChipLink, cn } from '@sim/emcn'
+import { Chip, ChipDatePicker, ChipLink, cn } from '@sim/emcn'
 import { useQueryStates } from 'nuqs'
 import { ActivityStatus } from '@/components/ui/activity-status'
-import type {
-  WorkspaceKnowledgeSearchResult,
-  WorkspaceSearchFilters,
+import {
+  WORKSPACE_KNOWLEDGE_SEARCH_LIMITS,
+  type WorkspaceKnowledgeSearchResult,
+  type WorkspaceSearchFilters,
 } from '@/lib/api/contracts/knowledge'
 import { useSession } from '@/lib/auth/auth-client'
 import { type ResourceScope, resourceScopeKey } from '@/lib/core/resource-scope'
@@ -27,6 +28,17 @@ import { useSearchIndex, useSearchSourceOverview } from '@/hooks/queries/kb/conn
 import { useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge'
 
 const DAY_MS = 24 * 60 * 60 * 1000
+
+/**
+ * The picker names calendar days; the URL keeps them as dates. A day's bounds are its local
+ * midnight and the last millisecond before the next, so "September 1" means the reader's own day.
+ */
+function startOfLocalDay(day: Date): Date {
+  return new Date(day.getUTCFullYear(), day.getUTCMonth(), day.getUTCDate())
+}
+function endOfLocalDay(day: Date): Date {
+  return new Date(day.getUTCFullYear(), day.getUTCMonth(), day.getUTCDate() + 1, 0, 0, 0, -1)
+}
 /** Every result without a connector is an upload; the filter names them so. */
 const UPLOAD_SOURCE = 'upload'
 
@@ -127,6 +139,11 @@ interface SearchResultsProps {
 function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
   const [hasShownFilters, setHasShownFilters] = useState(false)
   const [searchedAt] = useState(Date.now)
+  /**
+   * More results are a second, wider search: the first paint stays as quick as it is, and a
+   * refinement of the filters starts over at the first page.
+   */
+  const [expandedFor, setExpandedFor] = useState<string | null>(null)
   const {
     data: index,
     isPending: basesPending,
@@ -136,20 +153,42 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
   } = useSearchIndex(scope)
   const [filters, setFilters] = useQueryStates(searchFilterParsers, resourceUrlKeys)
   const window = UPDATED_WINDOWS.find((entry) => entry.id === filters.updated)
+  /** A custom window is inclusive of both days; `to` runs to the end of its day. */
+  const custom = filters.updated === 'custom'
   const searchFilters: WorkspaceSearchFilters = {
     ...(filters.source ? { source: filters.source } : {}),
     ...(window?.days
       ? { modifiedAfter: new Date(searchedAt - window.days * DAY_MS).toISOString() }
       : {}),
+    ...(custom && filters.from && filters.to
+      ? {
+          modifiedAfter: startOfLocalDay(filters.from).toISOString(),
+          modifiedBefore: endOfLocalDay(filters.to).toISOString(),
+        }
+      : {}),
   }
+  const filtersKey = JSON.stringify(searchFilters)
+  const expanded = expandedFor === filtersKey
+  /** A custom window is two-ended: until both days are chosen, nothing is searched. */
+  const awaitingRange = custom && !(filters.from && filters.to)
   const {
     data: search,
     isPending,
     isFetching,
     isPlaceholderData,
     isError: searchFailed,
     refetch: refetchSearch,
-  } = useWorkspaceKnowledgeSearch(scope, query, searchFilters)
+  } = useWorkspaceKnowledgeSearch(
+    scope,
+    awaitingRange ? '' : query,
+    searchFilters,
+    expanded
+      ? WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.expanded
+      : WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.initial
+  )
+  /** A full first page may collapse to few cards, yet more documents may still match. */
+  const mayHaveMore =
+    !expanded && (search?.results.length ?? 0) >= WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.initial
   const { data: overview } = useSearchSourceOverview(scope)
   const indexing = (overview?.providers ?? [])
     .filter((provider) => provider.isSyncing)
@@ -175,8 +214,12 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
       : null
 
   const showResults = !noSources && !failed && !basesPending && documents.length > 0
+  /** A custom window waiting for its days must show the filters, or the picker is unreachable. */
   const showFilters =
-    hasShownFilters || showResults || (!noSources && !pending && !failed && !!search && !partial)
+    hasShownFilters ||
+    showResults ||
+    awaitingRange ||
+    (!noSources && !pending && !failed && !!search && !partial)
   if (showFilters && !hasShownFilters) setHasShownFilters(true)
 
   return noSources ? (
@@ -196,7 +239,11 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
     <div className='flex flex-col'>
       <div className='flex items-center gap-2 px-2 py-2'>
         <div className='min-w-0 flex-1'>
-          {fetching || (pending && !failed) ? (
+          {awaitingRange ? (
+            <p role='status' className='text-[var(--text-muted)] text-caption'>
+              Choose the days to search.
+            </p>
+          ) : fetching || (pending && !failed) ? (
             <ActivityStatus label={pending ? 'Searching…' : 'Updating results…'} isActive />
           ) : (
             <p role='status' className='text-[var(--text-muted)] text-caption'>
@@ -257,11 +304,29 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
               shape='round'
               active={filters.updated === window.id}
               aria-pressed={filters.updated === window.id}
-              onClick={() => setFilters({ updated: window.id })}
+              onClick={() =>
+                setFilters(
+                  window.id === 'custom'
+                    ? { updated: window.id }
+                    : { updated: window.id, from: null, to: null }
+                )
+              }
             >
               {window.label}
             </Chip>
           ))}
+          {custom && (
+            <ChipDatePicker
+              mode='range'
+              placeholder='Updated between'
+              startDate={filters.from?.toISOString().slice(0, 10)}
+              endDate={filters.to?.toISOString().slice(0, 10)}
+              onRangeChange={(start, end) =>
+                void setFilters({ from: new Date(start), to: new Date(end) })
+              }
+              onClear={() => void setFilters({ from: null, to: null })}
+            />
+          )}
         </div>
       )}
       {showResults && (
@@ -291,6 +356,17 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
               />
             )
           })}
+          {mayHaveMore && (
+            <div className='flex px-2 py-2'>
+              <Chip
+                variant='border'
+                disabled={isFetching}
+                onClick={() => setExpandedFor(filtersKey)}
+              >
+                Show more
+              </Chip>
+            </div>
+          )}
         </div>
       )}
     </div>

apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx

@@ -212,6 +212,7 @@ describe('search refinement with the real query cache and URL state', () => {
       expect(requests.at(-1)?.body).toEqual({
         organizationId: 'organization',
         query: 'launch',
+        topK: 20,
         filters: expectedFilters,
       })
       expect(container.querySelector('h1')).toBeNull()

apps/sim/app/workspace/[workspaceId]/home/search-params.ts

@@ -1,4 +1,4 @@
-import { parseAsString, parseAsStringLiteral } from 'nuqs/server'
+import { parseAsIsoDate, parseAsString, parseAsStringLiteral } from 'nuqs/server'
 
 /**
  * Co-located, typed URL query-param definition for the home/Chat surface.
@@ -26,19 +26,23 @@ export const resourceUrlKeys = {
   clearOnDefault: true,
 } as const
 
-/** The recency windows a search can be narrowed to. */
+/** The recency windows a search can be narrowed to; `custom` reads its bounds from `from` and `to`. */
 export const UPDATED_WINDOWS = [
   { id: 'any', label: 'Any time', days: null },
   { id: '7d', label: 'Past week', days: 7 },
   { id: '30d', label: 'Past month', days: 30 },
+  { id: 'custom', label: 'Custom range', days: null },
 ] as const
 const UPDATED_WINDOW_IDS = UPDATED_WINDOWS.map((window) => window.id)
 
 /**
  * Shared result filters for organization search. `source` is a connector type
- * or `upload`, absent for every source.
+ * or `upload`, absent for every source; `from` and `to` are the days of a custom
+ * window, inclusive, and mean nothing unless `updated` is `custom`.
  */
 export const searchFilterParsers = {
   source: parseAsString,
   updated: parseAsStringLiteral(UPDATED_WINDOW_IDS).withDefault('any'),
+  from: parseAsIsoDate,
+  to: parseAsIsoDate,
 } as const