diff --git a/apps/sim/lib/sim-search/live/read-window.test.ts b/apps/sim/lib/sim-search/live/read-window.test.ts new file mode 100644 index 00000000000..e464a48d7ac --- /dev/null +++ b/apps/sim/lib/sim-search/live/read-window.test.ts @@ -0,0 +1,16 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { encodedWindowEnd } from '@/lib/sim-search/live/read-window' + +const encoded = (text: string) => JSON.stringify(text).length - 2 + +describe('encodedWindowEnd', () => { + it('returns the longest window it found to fit when escapes lead the text', () => { + const content = `${'"'.repeat(1_000)}${'x'.repeat(20_000)}` + const end = encodedWindowEnd(content, 0, 20_000, 8_000) + expect(end).toBe(19_000) + expect(encoded(content.slice(0, end))).toBeLessThanOrEqual(20_000) + }) +}) diff --git a/apps/sim/lib/sim-search/live/read-window.ts b/apps/sim/lib/sim-search/live/read-window.ts index 335f0d3c4df..eebaf6be4ba 100644 --- a/apps/sim/lib/sim-search/live/read-window.ts +++ b/apps/sim/lib/sim-search/live/read-window.ts @@ -1,6 +1,9 @@ /** Characters one standard read returns, the page budget indexed document reads use. */ export const READ_WINDOW_CHARACTERS = 8000 +/** Times a window is shortened to fit its budget before the minimum is used instead. */ +const MAX_SHRINKS = 3 + /** * Where a window of `content` from `start` ends so its text, as JSON encodes it, fits `budget` * characters. JSON escapes quotes, backslashes and line breaks, so a window is shortened until its @@ -13,9 +16,10 @@ export function encodedWindowEnd( minimumEnd: number ): number { let end = Math.min(content.length, start + budget) - for (let attempt = 0; attempt < 3 && end > minimumEnd; attempt++) { + for (let shrinks = 0; end > minimumEnd; shrinks++) { const encoded = JSON.stringify(content.slice(start, end)).length - 2 if (encoded <= budget) return end + if (shrinks === MAX_SHRINKS) break end = start + Math.floor(((end - start) * budget) / encoded) } return minimumEnd