fix(db): unify ordered window semantics

.changeset/unify-ordered-window-semantics.md

@@ -0,0 +1,6 @@
+---
+'@tanstack/db': patch
+'@tanstack/db-ivm': patch
+---
+
+Use one total order and applied row provenance for lazy ordered windows so cursor pagination, live updates, and window replay preserve exact results.

packages/db-ivm/src/utils.ts

@@ -185,6 +185,14 @@ function range(start: number, end: number): Array<number> {
 export function compareKeys(a: string | number, b: string | number): number {
   // Same type: compare directly
   if (typeof a === typeof b) {
+    if (typeof a === `number` && typeof b === `number`) {
+      const aIsNaN = Number.isNaN(a)
+      const bIsNaN = Number.isNaN(b)
+      if (aIsNaN || bIsNaN) {
+        if (aIsNaN && bIsNaN) return 0
+        return aIsNaN ? 1 : -1
+      }
+    }
     if (a < b) return -1
     if (a > b) return 1
     return 0

packages/db-ivm/tests/utils.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from 'vitest'
 import { Temporal } from 'temporal-polyfill'
-import { DefaultMap, serializeValue } from '../src/utils.js'
+import { DefaultMap, compareKeys, serializeValue } from '../src/utils.js'
 import { hash } from '../src/hashing/index.js'
 
 describe(`DefaultMap`, () => {
@@ -30,6 +30,14 @@ describe(`DefaultMap`, () => {
   })
 })
 
+describe(`compareKeys`, () => {
+  it(`orders finite numeric keys before NaN`, () => {
+    expect(compareKeys(1, Number.NaN)).toBeLessThan(0)
+    expect(compareKeys(Number.NaN, 1)).toBeGreaterThan(0)
+    expect(compareKeys(Number.NaN, Number.NaN)).toBe(0)
+  })
+})
+
 describe(`serializeValue`, () => {
   it(`preserves the established JSON form for ordinary keys`, () => {
     expect(serializeValue(`user1`)).toBe(`"user1"`)

packages/db/package.json

@@ -21,7 +21,7 @@
     "dev": "vite build --watch",
     "lint": "eslint . --fix",
     "test": "vitest --run",
-    "test:oracles": "vitest --run tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts"
+    "test:oracles": "vitest --run tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts"
   },
   "type": "module",
   "main": "dist/cjs/index.cjs",

packages/db/src/collection/change-events.ts

@@ -11,8 +11,9 @@ import {
   optimizeExpressionWithIndexes,
 } from '../utils/index-optimization.js'
 import { ensureIndexForField } from '../indexes/auto-index.js'
-import { makeComparator } from '../utils/comparison.js'
+import { ReverseIndex } from '../indexes/reverse-index.js'
 import { buildCompareOptions } from '../query/compiler/order-by'
+import { TotalOrder } from '../query/total-order.js'
 import type {
   ChangeMessage,
   CollectionLike,
@@ -358,7 +359,30 @@ function getOrderedKeys<T extends object, TKey extends string | number>(
         // Take the keys that match the filter and limit
         // if no limit is provided `index.keyCount` is used,
         // i.e. we will take all keys that match the filter
-        return index.takeFromStart(limit ?? index.keyCount, filterFn)
+        if (!(index instanceof ReverseIndex)) {
+          return index.takeFromStart(limit ?? index.keyCount, filterFn)
+        }
+
+        // Reversing a value index also reverses keys inside an equal-value
+        // bucket, but query TotalOrder keeps its public-key tie-break ascending.
+        // Refine all matching indexed rows locally so a limit cannot cut the
+        // wrong side of a tied boundary.
+        const totalOrder = new TotalOrder(orderBy, collection)
+        const indexedEntries = index
+          .takeFromStart(index.keyCount, filterFn)
+          .flatMap((key) => {
+            const value = collection.get(key)
+            return value === undefined ? [] : [{ key, value }]
+          })
+        indexedEntries.sort((left, right) =>
+          totalOrder.compareEntries(
+            [left.key, left.value],
+            [right.key, right.value],
+          ),
+        )
+        return indexedEntries
+          .slice(0, limit ?? indexedEntries.length)
+          .map(({ key }) => key)
       }
     }
   }
@@ -375,24 +399,10 @@ function getOrderedKeys<T extends object, TKey extends string | number>(
     }
   }
 
-  // Sort using makeComparator
-  const compare = (a: { key: TKey; value: T }, b: { key: TKey; value: T }) => {
-    for (const clause of orderBy) {
-      const compareFn = makeComparator(clause.compareOptions)
-
-      // Extract values for comparison
-      const aValue = extractValueFromItem(a.value, clause.expression)
-      const bValue = extractValueFromItem(b.value, clause.expression)
-
-      const result = compareFn(aValue, bValue)
-      if (result !== 0) {
-        return result
-      }
-    }
-    return 0
-  }
-
-  allItems.sort(compare)
+  const totalOrder = new TotalOrder(orderBy, collection)
+  allItems.sort((left, right) =>
+    totalOrder.compareEntries([left.key, left.value], [right.key, right.value]),
+  )
   const sortedKeys = allItems.map((item) => item.key)
 
   // Apply limit if provided
@@ -403,23 +413,3 @@ function getOrderedKeys<T extends object, TKey extends string | number>(
   // if no limit is provided, we will return all keys
   return sortedKeys
 }
-
-/**
- * Helper function to extract a value from an item based on an expression
- */
-function extractValueFromItem(item: any, expression: BasicExpression): any {
-  if (expression.type === `ref`) {
-    const propRef = expression
-    let value = item
-    for (const pathPart of propRef.path) {
-      value = value?.[pathPart]
-    }
-    return value
-  } else if (expression.type === `val`) {
-    return expression.value
-  } else {
-    // It must be a function
-    const evaluator = compileSingleRowExpression(expression)
-    return evaluator(item as Record<string, unknown>)
-  }
-}

packages/db/src/collection/state.ts

@@ -3,6 +3,11 @@ import { SortedMap } from '../SortedMap'
 import { enrichRowWithVirtualProps } from '../virtual-props.js'
 import { SyncTransactionAbortedError } from '../errors.js'
 import { createDeferred } from '../deferred'
+import {
+  copySyncRequestProvenance,
+  getSyncRequestProvenance,
+  setSyncRequestProvenance,
+} from '../load-subset-request-provenance.js'
 import { DIRECT_TRANSACTION_METADATA_KEY } from './transaction-metadata.js'
 import type {
   VirtualOrigin,
@@ -42,6 +47,8 @@ interface PendingSyncedTransaction<
     deletes: Set<TKey>
   }
   preserveHydrationSeedKeys?: boolean
+  /** Exact subset request whose commit produced this transaction. */
+  requestSignal?: AbortSignal
   /**
    * When true, this transaction should be processed immediately even if there
    * are persisting user transactions. Used by manual write operations (writeInsert,
@@ -339,13 +346,15 @@ export class CollectionStateManager<
         : this.enrichWithVirtualProps(change.previousValue, change.key)
       : undefined
 
-    return {
+    const enriched = {
       key: change.key,
       type: change.type,
       value: enrichedValue,
       previousValue: enrichedPreviousValue,
       metadata: change.metadata,
     } as ChangeMessage<WithVirtualProps<TOutput, TKey>, TKey>
+    copySyncRequestProvenance(change, enriched)
+    return enriched
   }
 
   /**
@@ -940,13 +949,50 @@ export class CollectionStateManager<
       const changedKeys = new Set<TKey>()
       for (const transaction of committedSyncedTransactions) {
         for (const operation of transaction.operations) {
-          changedKeys.add(operation.key as TKey)
+          const key = operation.key as TKey
+          changedKeys.add(key)
         }
         for (const [key] of transaction.rowMetadataWrites) {
           changedKeys.add(key)
         }
       }
 
+      type AppliedRequestProvenance = {
+        version: {
+          value: TOutput | undefined
+          origin: VirtualOrigin | undefined
+        }
+        hasOrdinarySource: boolean
+        requestSignals: Set<AbortSignal>
+      }
+      const requestProvenanceByKey = new Map<TKey, AppliedRequestProvenance>()
+      const provenanceForSignal = (signal: AbortSignal | undefined) => ({
+        hasOrdinarySource: signal === undefined,
+        requestSignals:
+          signal === undefined
+            ? new Set<AbortSignal>()
+            : new Set<AbortSignal>([signal]),
+      })
+      const recordRequestProvenance = (
+        key: TKey,
+        signal: AbortSignal | undefined,
+      ) => {
+        const version = {
+          value: this.syncedData.get(key),
+          origin: this.rowOrigins.get(key),
+        }
+        const previous = requestProvenanceByKey.get(key)
+        if (previous !== undefined && deepEquals(previous.version, version)) {
+          if (signal === undefined) previous.hasOrdinarySource = true
+          else previous.requestSignals.add(signal)
+          return
+        }
+        requestProvenanceByKey.set(key, {
+          version,
+          ...provenanceForSignal(signal),
+        })
+      }
+
       const virtualSnapshotKeys = new Set(changedKeys)
       for (const key of this.pendingOptimisticDirectUpserts) {
         virtualSnapshotKeys.add(key)
@@ -1013,7 +1059,16 @@ export class CollectionStateManager<
               truncateOptimisticSnapshot?.upserts.get(key) ||
               this.syncedData.get(key)
             if (previousValue !== undefined) {
-              events.push({ type: `delete`, key, value: previousValue })
+              const event: ChangeMessage<TOutput, TKey> = {
+                type: `delete`,
+                key,
+                value: previousValue,
+              }
+              setSyncRequestProvenance(
+                event,
+                provenanceForSignal(transaction.requestSignal),
+              )
+              events.push(event)
             }
           }
 
@@ -1105,6 +1160,7 @@ export class CollectionStateManager<
               this.pendingOptimisticDirectDeletes.delete(key)
               break
           }
+          recordRequestProvenance(key, transaction.requestSignal)
           if (!transaction.preserveHydrationSeedKeys) {
             this.hydrationSeedKeys.delete(key)
             this.hydratedKeys.delete(key)
@@ -1114,9 +1170,9 @@ export class CollectionStateManager<
         for (const [key, metadataWrite] of transaction.rowMetadataWrites) {
           if (metadataWrite.type === `delete`) {
             this.syncedMetadata.delete(key)
-            continue
+          } else {
+            this.syncedMetadata.set(key, metadataWrite.value)
           }
-          this.syncedMetadata.set(key, metadataWrite.value)
         }
 
         for (const [
@@ -1226,6 +1282,7 @@ export class CollectionStateManager<
               this.isThisCollection(mutation.collection) &&
               mutation.optimistic
             ) {
+              requestProvenanceByKey.delete(mutation.key)
               switch (mutation.type) {
                 case `insert`:
                 case `update`:
@@ -1261,13 +1318,15 @@ export class CollectionStateManager<
           this.pendingOptimisticUpserts.delete(key)
           this.pendingLocalOrigins.delete(key)
         }
+        requestProvenanceByKey.delete(key)
       }
       for (const key of this.pendingOptimisticDirectDeletes) {
         if (!changedKeys.has(key)) {
           changedKeys.add(key)
         }
         this.pendingOptimisticDeletes.delete(key)
         this.pendingLocalOrigins.delete(key)
+        requestProvenanceByKey.delete(key)
       }
       this.pendingOptimisticDirectUpserts.clear()
       this.pendingOptimisticDirectDeletes.clear()
@@ -1379,6 +1438,17 @@ export class CollectionStateManager<
         }
       }
 
+      for (const event of events) {
+        if (getSyncRequestProvenance(event) !== undefined) continue
+        const provenance = requestProvenanceByKey.get(event.key)
+        if (provenance !== undefined) {
+          setSyncRequestProvenance(event, {
+            hasOrdinarySource: provenance.hasOrdinarySource,
+            requestSignals: new Set(provenance.requestSignals),
+          })
+        }
+      }
+
       // Update cached size after synced data changes
       this.size = this.calculateSize()