fix(tables): fail fast at query batch guard

Author: TheodoreSpeaks

apps/sim/lib/table/__tests__/service-filter-threading.test.ts

@@ -219,6 +219,19 @@ describe('queryRows byte budget', () => {
     updatedAt: new Date('2024-01-01'),
   })
 
+  const mockRowsThroughBatchSafetyLimit = () => {
+    const largeRow = row(1, TABLE_LIMITS.MAX_ROW_SIZE_BYTES)
+    const smallRow = row(2, 0)
+    let drainBatch = 0
+    dbChainMockFns.limit.mockResolvedValueOnce([])
+    dbChainMockFns.limit.mockImplementation(async (ask: number) => {
+      drainBatch++
+      const rows = Array.from({ length: ask }, () => smallRow)
+      if (drainBatch === 1) rows[0] = largeRow
+      return rows
+    })
+  }
+
   it('returns an empty page with a null cursor', async () => {
     const result = await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1')
     expect(result.rows).toEqual([])
@@ -246,6 +259,26 @@ describe('queryRows byte budget', () => {
     expect(result.nextCursor).toBeNull()
   })
 
+  it('fails fast instead of silently truncating an unbounded query at the batch safety limit', async () => {
+    mockRowsThroughBatchSafetyLimit()
+
+    await expect(
+      queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1')
+    ).rejects.toThrow(/internal batches to complete safely/)
+  })
+
+  it('returns a continuation cursor when a bounded query reaches the batch safety limit', async () => {
+    mockRowsThroughBatchSafetyLimit()
+
+    const result = await queryRows(
+      TABLE,
+      { limit: 1_000_000, includeTotal: false, withExecutions: false },
+      'req-1'
+    )
+
+    expect(result.nextCursor).not.toBeNull()
+  })
+
   it('byte-cuts a BOUNDED page and returns a resume cursor instead of throwing', async () => {
     const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6)
     dbChainMockFns.limit.mockResolvedValueOnce([])

apps/sim/lib/table/rows/service.ts

@@ -1271,13 +1271,9 @@ interface BoundedFetchResult {
 }
 
 /**
- * Belt-and-braces bound on drain iterations.
- *
- * Unreachable only because every iteration either consumes at least one row or cuts, and a bounded
- * page's `limit` is capped at {@link TABLE_LIMITS.MAX_QUERY_LIMIT} — so the limit cut always fires
- * first. That makes the two constants exactly tight: raising `MAX_QUERY_LIMIT` above this bound
- * would let the loop exit with rows still unread and `hasMore: false`, which clients now trust as
- * end-of-table (they terminate on `nextCursor`, which this decides). Raise both together.
+ * Safety bound on drain iterations. Reaching it never proves source exhaustion: the drain performs
+ * one final witness read so bounded queries return a continuation cursor and unbounded queries fail
+ * fast instead of returning a partial result that looks complete.
  */
 const MAX_QUERY_BATCHES = 1000
 
@@ -1318,6 +1314,7 @@ async function fetchRowsBounded(params: BoundedFetchParams): Promise<BoundedFetc
   let anchor = params.seek
   let anchorOffset = params.startOffset
   let consumedSinceAnchor = 0
+  let sourceExhausted = false
 
   const nextBatchRows = (): number => {
     if (rows.length === 0) return Math.min(limit ?? firstBatchCap, firstBatchCap)
@@ -1369,7 +1366,10 @@ async function fetchRowsBounded(params: BoundedFetchParams): Promise<BoundedFetc
     const target = Math.min(nextBatchRows(), limitRemaining)
     const ask = target + 1 // +1 = witness row proving more data exists past a cut
     const batch = await runBatch(anchor, anchorOffset + consumedSinceAnchor, ask)
-    if (batch.length === 0) break
+    if (batch.length === 0) {
+      sourceExhausted = true
+      break
+    }
 
     let cut = false
     for (const row of batch) {
@@ -1408,7 +1408,23 @@ async function fetchRowsBounded(params: BoundedFetchParams): Promise<BoundedFetc
     }
     if (cut) break
     // Short batch = the source is exhausted; hasMore stays false.
-    if (batch.length < ask) break
+    if (batch.length < ask) {
+      sourceExhausted = true
+      break
+    }
+  }
+
+  if (!sourceExhausted && !hasMore) {
+    const witness = await runBatch(anchor, anchorOffset + consumedSinceAnchor, 1)
+    if (witness.length > 0) {
+      if (limit === undefined) {
+        throw new TableQueryValidationError(
+          `Query requires more than ${MAX_QUERY_BATCHES} internal batches to complete safely. Add a filter or a limit to narrow the result.`,
+          'TABLE_QUERY_RESULT_TOO_LARGE'
+        )
+      }
+      hasMore = true
+    }
   }
 
   return {