diff --git a/.changeset/calm-oracles-check-loss.md b/.changeset/calm-oracles-check-loss.md new file mode 100644 index 0000000000..3e22586590 --- /dev/null +++ b/.changeset/calm-oracles-check-loss.md @@ -0,0 +1,14 @@ +--- +'@tanstack/db': patch +'@tanstack/db-sqlite-persistence-core': patch +'@tanstack/electric-db-collection': patch +'@tanstack/query-db-collection': patch +--- + +Retain query-backed rows until their explicit owners release them. + +Harden Electric resume and lifecycle handling so partial updates cannot materialize unknown or moved-out rows, stale async work and waiters cannot cross cleanup or restart—including automatic garbage collection—and valid batches behave the same across callback partitions and persistence hydration. + +Preserve hydrated baseline rows during persistence reloads, accept complete-row updates from explicit full-replica resumes, retain committed match evidence until reset, and restart persisted resumes when hydration completion cannot be verified. + +Reduce live-update work to scale with the incoming batch instead of the full collection while preserving conservative reset recovery and committed mutation evidence. diff --git a/.changeset/lazy-runtime-reference-identities.md b/.changeset/lazy-runtime-reference-identities.md new file mode 100644 index 0000000000..807b0f5ce7 --- /dev/null +++ b/.changeset/lazy-runtime-reference-identities.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Lazily initialize runtime reference identities to avoid generating random values during Cloudflare Worker module evaluation. diff --git a/AGENTS.md b/AGENTS.md index 1d6fbf182b..29eadb5275 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -389,6 +389,29 @@ classifier, fixture, or assertion that let it pass. Use that analysis to suggest the smallest test or oracle improvement that would catch the same class of bug, not only the reported example. +### Keep Oracles Independent + +An oracle is useful only when its expected result comes from a source independent +of the implementation under test. Do not translate production branches, state +machines, classifiers, or helper functions into a second implementation and call +that an oracle. Both copies can encode the same wrong assumption. + +- Derive expected behavior from public contracts, documented prior behavior, + mathematical laws, or a separately specified reference model. +- Keep the reference model structurally different from production. Do not import + the production helper or reuse its classifications to compute expected results. +- Preserve existing contract tests unless a product or design decision explicitly + changes the contract. Rewriting a passing expectation to match new production + behavior is a design review, not routine test maintenance. +- When production work suggests an oracle change, compare the old and new + semantics with counterexamples before editing the oracle. +- Use hostile mutants to prove the oracle rejects plausible wrong designs, + including the mistake production currently makes. A green oracle without a + demonstrated kill is weak evidence. +- Use process grammar to explore lifecycle paths, and design grammar to challenge + the oracle's reference semantics. More generated traces cannot repair a wrong + reference model. + ### Name Tests After Behavior Test names should state the behavior they prove. Do not put issue or pull diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 14358dd3e9..f2ca644007 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -815,10 +815,12 @@ class PersistedCollectionRuntime< private started = false private startupMetadataPromise: Promise | null = null private startPromise: Promise | null = null + private resumeBaselinePromise: Promise | null = null + private lifecycleGeneration = 0 private internalApplyDepth = 0 private appliedReceiptSequence = 0 private readonly pendingAppliedReceipts = new Map>() - private isHydrating = false + private hydratingGeneration: number | null = null private coordinatorUnsubscribe: (() => void) | null = null private indexAddedUnsubscribe: (() => void) | null = null private indexRemovedUnsubscribe: (() => void) | null = null @@ -841,6 +843,8 @@ class PersistedCollectionRuntime< ) {} setSyncControls(syncControls: SyncControlFns): void { + this.advanceLifecycle() + const commit = syncControls.commit this.syncControls = { ...syncControls, @@ -880,7 +884,7 @@ class PersistedCollectionRuntime< } isHydratingNow(): boolean { - return this.isHydrating + return this.hydratingGeneration === this.lifecycleGeneration } isApplyingInternally(): boolean { @@ -910,20 +914,56 @@ class PersistedCollectionRuntime< return this.startPromise } - this.startPromise = this.startInternal() + const lifecycleGeneration = this.lifecycleGeneration + this.startPromise = this.startInternal(lifecycleGeneration) return this.startPromise } + ensureResumeBaselineHydrated(): Promise { + if (this.resumeBaselinePromise) { + return this.resumeBaselinePromise + } + + const lifecycleGeneration = this.lifecycleGeneration + this.resumeBaselinePromise = (async () => { + await this.ensureStarted() + if (lifecycleGeneration !== this.lifecycleGeneration) return + if (this.syncMode !== `on-demand`) return + + await this.hydrateBaseline(lifecycleGeneration) + })() + return this.resumeBaselinePromise + } + + private async hydrateBaseline(lifecycleGeneration: number): Promise { + if (lifecycleGeneration !== this.lifecycleGeneration) return + + const baseline = {} + this.activeSubsets.set(this.getSubsetKey(baseline), baseline) + const appliedCursor = this.appliedReceiptSequence + await this.applyMutex.run(async () => { + if (lifecycleGeneration !== this.lifecycleGeneration) return + await this.hydrateSubsetUnsafe(baseline, { + requestRemoteEnsure: false, + lifecycleGeneration, + }) + }) + if (lifecycleGeneration !== this.lifecycleGeneration) return + await this.waitForAppliedReceiptsAfter(appliedCursor) + } + async ensureStartupMetadataLoaded(): Promise { if (this.startupMetadataPromise) { return this.startupMetadataPromise } - this.startupMetadataPromise = this.loadStartupMetadataInternal() + const lifecycleGeneration = this.lifecycleGeneration + this.startupMetadataPromise = + this.loadStartupMetadataInternal(lifecycleGeneration) return this.startupMetadataPromise } - private async startInternal(): Promise { + private async startInternal(lifecycleGeneration: number): Promise { if (this.started) { return } @@ -931,28 +971,28 @@ class PersistedCollectionRuntime< this.started = true await this.ensureStartupMetadataLoaded() + if (lifecycleGeneration !== this.lifecycleGeneration) return const indexBootstrapSnapshot = this.collection?.getIndexMetadata() ?? [] this.attachIndexLifecycleListeners() await this.bootstrapPersistedIndexes(indexBootstrapSnapshot) + if (lifecycleGeneration !== this.lifecycleGeneration) return if (this.syncMode !== `on-demand`) { - this.activeSubsets.set(this.getSubsetKey({}), {}) - const appliedCursor = this.appliedReceiptSequence - await this.applyMutex.run(() => - this.hydrateSubsetUnsafe({}, { requestRemoteEnsure: false }), - ) - await this.waitForAppliedReceiptsAfter(appliedCursor) + await this.hydrateBaseline(lifecycleGeneration) } } - private async loadStartupMetadataInternal(): Promise { + private async loadStartupMetadataInternal( + lifecycleGeneration: number, + ): Promise { // Restore stream position from the database so that new mutations // don't collide with previously applied transactions. if (this.persistence.adapter.getStreamPosition) { const position = await this.persistence.adapter.getStreamPosition( this.collectionId, ) + if (lifecycleGeneration !== this.lifecycleGeneration) return this.observeStreamPosition( position.latestTerm, position.latestSeq, @@ -960,11 +1000,8 @@ class PersistedCollectionRuntime< ) } - await this.loadCollectionMetadataIntoCollection() - } - - private async loadCollectionMetadataIntoCollection(): Promise { const collectionMetadata = await this.loadCollectionMetadataSnapshot() + if (lifecycleGeneration !== this.lifecycleGeneration) return this.replaceCollectionMetadataSnapshot(collectionMetadata) } @@ -1017,14 +1054,17 @@ class PersistedCollectionRuntime< options: LoadSubsetOptions, upstreamLoadSubset?: (options: LoadSubsetOptions) => true | Promise, ): Promise { + const lifecycleGeneration = this.lifecycleGeneration this.activeSubsets.set(this.getSubsetKey(options), options) const appliedCursor = this.appliedReceiptSequence await this.applyMutex.run(() => this.hydrateSubsetUnsafe(options, { requestRemoteEnsure: this.mode === `sync-present`, + lifecycleGeneration, }), ) + if (lifecycleGeneration !== this.lifecycleGeneration) return await this.waitForAppliedReceiptsAfter(appliedCursor) if (upstreamLoadSubset) { @@ -1055,9 +1095,13 @@ class PersistedCollectionRuntime< } async forceReloadSubset(options: LoadSubsetOptions): Promise { + const lifecycleGeneration = this.lifecycleGeneration this.activeSubsets.set(this.getSubsetKey(options), options) await this.applyMutex.run(() => - this.hydrateSubsetUnsafe(options, { requestRemoteEnsure: false }), + this.hydrateSubsetUnsafe(options, { + requestRemoteEnsure: false, + lifecycleGeneration, + }), ) } @@ -1176,6 +1220,8 @@ class PersistedCollectionRuntime< } cleanup(): void { + this.advanceLifecycle() + this.coordinatorUnsubscribe?.() this.coordinatorUnsubscribe = null @@ -1198,6 +1244,15 @@ class PersistedCollectionRuntime< this.queuedHydrationTransactions.length = 0 this.queuedTxCommitted.length = 0 this.clearSyncControls() + this.collection = null + } + + private advanceLifecycle(): void { + this.lifecycleGeneration++ + this.started = false + this.startupMetadataPromise = null + this.startPromise = null + this.resumeBaselinePromise = null } private withInternalApply(task: () => TResult): TResult { @@ -1250,15 +1305,19 @@ class PersistedCollectionRuntime< options: LoadSubsetOptions, config: { requestRemoteEnsure: boolean + lifecycleGeneration: number }, ): Promise { - this.isHydrating = true + this.hydratingGeneration = config.lifecycleGeneration try { const rows = await this.loadSubsetRowsUnsafe(options) + if (config.lifecycleGeneration !== this.lifecycleGeneration) return this.applyRowsToCollection(rows) } finally { - this.isHydrating = false + if (this.hydratingGeneration === config.lifecycleGeneration) { + this.hydratingGeneration = null + } } await this.flushQueuedHydrationTransactionsUnsafe() @@ -1907,7 +1966,7 @@ class PersistedCollectionRuntime< // of both local and remote mutations. The seq dedup in // processCommittedTxUnsafe prevents double-processing of our own writes. if (isTxCommittedPayload(payload)) { - if (this.isHydrating) { + if (this.isHydratingNow()) { this.queuedTxCommitted.push(payload) return } @@ -2132,17 +2191,20 @@ class PersistedCollectionRuntime< } private async reloadActiveSubsetsUnsafe(): Promise { + const lifecycleGeneration = this.lifecycleGeneration const activeSubsetOptions = this.activeSubsets.size > 0 ? Array.from(this.activeSubsets.values()) : [{}] - this.isHydrating = true + this.hydratingGeneration = lifecycleGeneration try { const mergedRows = new Map() const collectionMetadata = await this.loadCollectionMetadataSnapshot() + if (lifecycleGeneration !== this.lifecycleGeneration) return for (const options of activeSubsetOptions) { const subsetRows = await this.loadSubsetRowsUnsafe(options) + if (lifecycleGeneration !== this.lifecycleGeneration) return for (const row of subsetRows) { mergedRows.set(row.key, { value: row.value, @@ -2160,7 +2222,9 @@ class PersistedCollectionRuntime< collectionMetadata, ) } finally { - this.isHydrating = false + if (this.hydratingGeneration === lifecycleGeneration) { + this.hydratingGeneration = null + } } await this.flushQueuedHydrationTransactionsUnsafe() @@ -2276,6 +2340,7 @@ function createWrappedSyncConfig< const getOpenTransaction = () => transactionStack[transactionStack.length - 1] let fullStartPromise: Promise | null = null + const startupState = { cleanedUp: false } const cancelledLoadKeys = new Set() const loadSubscriptionIds = new WeakMap() let nextLoadSubscriptionId = 0 @@ -2308,11 +2373,14 @@ function createWrappedSyncConfig< const wrappedParams = { ...params, markReady: () => { + if (startupState.cleanedUp) return void (fullStartPromise ?? runtime.ensureStarted()) .then(() => { + if (startupState.cleanedUp) return params.markReady() }) .catch((error) => { + if (startupState.cleanedUp) return console.warn( `Failed persisted sync startup before markReady:`, error, @@ -2321,6 +2389,7 @@ function createWrappedSyncConfig< }) }, begin: (options?: { immediate?: boolean }) => { + if (startupState.cleanedUp) return const transaction: OpenSyncTransaction = { operations: [], rowMetadataWrites: new Map(), @@ -2337,6 +2406,7 @@ function createWrappedSyncConfig< } }, write: (message: ChangeMessageOrDeleteKeyMessage) => { + if (startupState.cleanedUp) return const normalization = runtime.normalizeSyncWriteMessage(message) const openTransaction = getOpenTransaction() @@ -2382,7 +2452,12 @@ function createWrappedSyncConfig< metadata: params.metadata ? { row: { + whenHydrated: () => + startupState.cleanedUp + ? Promise.resolve() + : runtime.ensureResumeBaselineHydrated(), get: (key: TKey) => { + if (startupState.cleanedUp) return undefined const openTransaction = getOpenTransaction() const pendingWrite = openTransaction?.rowMetadataWrites.get(key) @@ -2397,8 +2472,11 @@ function createWrappedSyncConfig< return params.metadata!.row.get(key) }, scanPersisted: (options?: PersistedRowScanOptions) => - runtime.scanPersistedRows(options), + startupState.cleanedUp + ? Promise.resolve([]) + : runtime.scanPersistedRows(options), set: (key: TKey, value: unknown) => { + if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() if (!openTransaction) { throw new InvalidPersistedCollectionConfigError( @@ -2414,6 +2492,7 @@ function createWrappedSyncConfig< } }, delete: (key: TKey) => { + if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() if (!openTransaction) { throw new InvalidPersistedCollectionConfigError( @@ -2430,6 +2509,7 @@ function createWrappedSyncConfig< }, collection: { get: (key: string) => { + if (startupState.cleanedUp) return undefined const openTransaction = getOpenTransaction() const pendingWrite = openTransaction?.collectionMetadataWrites.get(key) @@ -2441,6 +2521,7 @@ function createWrappedSyncConfig< return params.metadata!.collection.get(key) }, set: (key: string, value: unknown) => { + if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() if (!openTransaction) { throw new InvalidPersistedCollectionConfigError( @@ -2456,6 +2537,7 @@ function createWrappedSyncConfig< } }, delete: (key: string) => { + if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() if (!openTransaction) { throw new InvalidPersistedCollectionConfigError( @@ -2470,6 +2552,7 @@ function createWrappedSyncConfig< } }, list: (prefix?: string) => { + if (startupState.cleanedUp) return [] const merged = new Map( params .metadata!.collection.list() @@ -2500,6 +2583,7 @@ function createWrappedSyncConfig< } : undefined, truncate: () => { + if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() if (!openTransaction) { params.truncate() @@ -2518,6 +2602,7 @@ function createWrappedSyncConfig< } }, commit: (signal?: AbortSignal) => { + if (startupState.cleanedUp) return true const openTransaction = transactionStack.pop() if (!openTransaction) { return params.commit(signal) @@ -2572,7 +2657,6 @@ function createWrappedSyncConfig< } let sourceResult: SyncConfigRes = {} - const startupState = { cleanedUp: false } fullStartPromise = runtime.ensureStarted() const sourceResultPromise = (async () => { await runtime.ensureStartupMetadataLoaded() @@ -2605,6 +2689,7 @@ function createWrappedSyncConfig< await runtime.loadSubset(options, resolvedSourceResult.loadSubset) }, unloadSubset: (options: LoadSubsetOptions) => { + if (startupState.cleanedUp) return cancelledLoadKeys.add(getLoadKey(options)) runtime.unloadSubset(options, sourceResult.unloadSubset) }, diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 606f0d75e7..91530c507b 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { BasicIndex, DbClient, @@ -1744,6 +1744,161 @@ describe(`persistedCollectionOptions`, () => { expect(collection.get(`2`)).toBeUndefined() }) + it(`does not let a stale invalidation reload overwrite a restarted lifecycle`, async () => { + const adapter = createRecordingAdapter([{ id: `1`, title: `Initial` }]) + const coordinator = createCoordinatorHarness() + const originalLoadSubset = adapter.loadSubset.bind(adapter) + let loadCalls = 0 + let releaseStaleReload!: () => void + let releaseFreshReload!: () => void + const staleReloadGate = new Promise((resolve) => { + releaseStaleReload = resolve + }) + const freshReloadGate = new Promise((resolve) => { + releaseFreshReload = resolve + }) + adapter.loadSubset = async (...args) => { + loadCalls++ + if (loadCalls === 2) { + await staleReloadGate + return [ + { + key: `1`, + value: { id: `1`, title: `Stale reload` }, + }, + ] + } + if (loadCalls === 3) await freshReloadGate + return originalLoadSubset(...args) + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + await collection.preload() + await flushAsyncWork() + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `tx-stale-reload`, + latestRowVersion: 1, + requiresFullReload: true, + }) + for (let attempt = 0; attempt < 20 && loadCalls < 2; attempt++) { + await flushAsyncWork() + } + expect(loadCalls).toBe(2) + + await collection.cleanup() + adapter.rows.set(`1`, { id: `1`, title: `Restarted` }) + collection.startSyncImmediate() + releaseStaleReload() + for (let attempt = 0; attempt < 20 && loadCalls < 3; attempt++) { + await flushAsyncWork() + } + expect(loadCalls).toBe(3) + expect(collection.get(`1`)?.title).not.toBe(`Stale reload`) + + releaseFreshReload() + for ( + let attempt = 0; + attempt < 20 && collection.get(`1`)?.title !== `Restarted`; + attempt++ + ) { + await flushAsyncWork() + } + expect(stripVirtualProps(collection.get(`1`))).toEqual({ + id: `1`, + title: `Restarted`, + }) + await collection.cleanup() + }) + + it(`does not let stale reload metadata start row loading after restart`, async () => { + const adapter = createRecordingAdapter([{ id: `1`, title: `Initial` }]) + const coordinator = createCoordinatorHarness() + const originalLoadCollectionMetadata = + adapter.loadCollectionMetadata!.bind(adapter) + const originalLoadSubset = adapter.loadSubset.bind(adapter) + let metadataCalls = 0 + let subsetCalls = 0 + let releaseStaleMetadata!: () => void + const staleMetadataGate = new Promise((resolve) => { + releaseStaleMetadata = resolve + }) + adapter.loadCollectionMetadata = async (...args) => { + metadataCalls++ + if (metadataCalls === 2) await staleMetadataGate + return originalLoadCollectionMetadata(...args) + } + adapter.loadSubset = async (...args) => { + subsetCalls++ + return originalLoadSubset(...args) + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + await collection.preload() + await flushAsyncWork() + expect(metadataCalls).toBe(1) + expect(subsetCalls).toBe(1) + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `tx-stale-metadata`, + latestRowVersion: 1, + requiresFullReload: true, + }) + for (let attempt = 0; attempt < 20 && metadataCalls < 2; attempt++) { + await flushAsyncWork() + } + expect(metadataCalls).toBe(2) + + await collection.cleanup() + adapter.rows.set(`1`, { id: `1`, title: `Restarted` }) + collection.startSyncImmediate() + releaseStaleMetadata() + for ( + let attempt = 0; + attempt < 20 && (metadataCalls < 3 || subsetCalls < 2); + attempt++ + ) { + await flushAsyncWork() + } + + expect(metadataCalls).toBe(3) + expect(subsetCalls).toBe(2) + expect(stripVirtualProps(collection.get(`1`))).toEqual({ + id: `1`, + title: `Restarted`, + }) + await collection.cleanup() + }) + it(`retries queued remote subset ensure after transient failures`, async () => { const adapter = createRecordingAdapter() let ensureCalls = 0 @@ -1997,6 +2152,89 @@ describe(`persistedCollectionOptions`, () => { title: `Updated`, }) }) + + it(`keeps a hydrated resume baseline across narrow full reloads`, async () => { + const adapter = createRecordingAdapter([ + { id: `1`, title: `Narrow` }, + { id: `2`, title: `Baseline only` }, + ]) + const loadSubset = adapter.loadSubset.bind(adapter) + adapter.loadSubset = async (...args) => { + const rows = await loadSubset(...args) + return args[1].where ? rows.filter((row) => row.key === `1`) : rows + } + const coordinator = createCoordinatorHarness() + let hydrateBaseline: (() => Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + syncMode: `on-demand`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady, metadata }) => { + hydrateBaseline = ( + metadata?.row as + | { whenHydrated?: () => Promise } + | undefined + )?.whenHydrated + markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + collection.startSyncImmediate() + await vi.waitFor(() => expect(hydrateBaseline).toBeTypeOf(`function`)) + await hydrateBaseline!() + expect(collection.has(`2`)).toBe(true) + + await collection._sync.loadSubset({ + where: new IR.Func(`eq`, [new IR.PropRef([`id`]), new IR.Value(`1`)]), + }) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `full-reload`, + latestRowVersion: 1, + requiresFullReload: true, + }) + await flushAsyncWork() + await flushAsyncWork() + + expect(collection.has(`2`)).toBe(true) + await collection.cleanup() + }) + + it(`ignores late wrapped sync writes after cleanup`, async () => { + let lateWrite!: (message: { type: `insert`; value: Todo }) => void + const collection = createCollection( + persistedCollectionOptions({ + id: `late-write-after-cleanup`, + getKey: (item) => item.id, + sync: { + sync: ({ write, markReady }) => { + lateWrite = (message) => write(message) + markReady() + }, + }, + persistence: { adapter: createNoopAdapter() }, + }), + ) + + await collection.preload() + await collection.cleanup() + + expect(() => + lateWrite({ + type: `insert`, + value: { id: `late`, title: `Late` }, + }), + ).not.toThrow() + expect(collection.has(`late`)).toBe(false) + }) }) describe(`persisted key and identifier helpers`, () => { diff --git a/packages/db/package.json b/packages/db/package.json index 1857935bba..806615adb5 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -21,7 +21,8 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", - "test:oracles": "vitest --run tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts" + "test:oracles": "vitest --run tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-space-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts", + "bench:nested-includes": "vitest bench tests/query/includes-performance.bench.ts --run" }, "type": "module", "main": "dist/cjs/index.cjs", diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index 7b61a13503..a4f06b4c81 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -47,6 +47,64 @@ import type { TransactionScope } from '../transactions.js' export type { CollectionIndexMetadata } from './events.js' +const collectionSyncConfigFactory: unique symbol = Symbol.for( + `@tanstack/db.collectionSyncConfig.factory`, +) as never +const collectionSyncConfigCleanup: unique symbol = Symbol.for( + `@tanstack/db.collectionSyncConfig.cleanup`, +) as never + +type CollectionSyncConfigWithFactory = TSync & { + readonly [collectionSyncConfigFactory]: ( + this: TSync, + utilities: object, + ) => TSync +} + +/** @internal Lets adapters bind a sync config to each collection instance. */ +export function withCollectionSyncConfigFactory( + sync: TSync, + factory: (source: TSync, utilities: object) => TSync, +): CollectionSyncConfigWithFactory { + Object.defineProperty(sync, collectionSyncConfigFactory, { + value(this: TSync, utilities: object) { + return factory(this, utilities) + }, + // Preserve the hook when callers wrap a sync config with object spread. + enumerable: true, + }) + return sync as CollectionSyncConfigWithFactory +} + +/** @internal Registers work owned before an adapter sync starts. */ +export function withCollectionSyncConfigCleanup( + sync: TSync, + cleanup: () => void, +): TSync { + Object.defineProperty(sync, collectionSyncConfigCleanup, { + value: cleanup, + enumerable: false, + }) + return sync +} + +function materializeCollectionSyncConfig( + sync: TSync, + utilities: object, +): TSync { + const factory = ( + sync as unknown as Partial> + )[collectionSyncConfigFactory] + return factory ? factory.call(sync, utilities) : sync +} + +function cleanupCollectionSyncConfig(sync: object): void { + const cleanup = ( + sync as unknown as { [collectionSyncConfigCleanup]?: () => void } + )[collectionSyncConfigCleanup] + cleanup?.() +} + /** * Enhanced Collection interface that includes both data type T and utilities TUtils * @template T - The type of items in the collection @@ -261,14 +319,6 @@ export function createCollection( const collection = new CollectionImpl( options, ) - - // Attach utils to collection - if (options.utils) { - collection.utils = options.utils - } else { - collection.utils = {} - } - return collection } @@ -336,10 +386,20 @@ export class CollectionImpl< } // Set default values for optional config properties + const collectionUtils = config.utils ?? {} + const collectionSync = materializeCollectionSyncConfig( + config.sync, + collectionUtils, + ) this.config = { ...config, + sync: collectionSync, autoIndex: config.autoIndex ?? `off`, + utils: collectionUtils, } + // Attach utilities before eager sync starts so adapters can bind helpers + // during sync setup. Preserve the adapter's object identity by default. + this.utils = collectionUtils if (this.config.autoIndex === `eager` && !config.defaultIndexType) { throw new CollectionConfigurationError( @@ -353,10 +413,12 @@ export class CollectionImpl< this._changes = new CollectionChangesManager() this._events = new CollectionEventsManager() this._indexes = new CollectionIndexesManager() - this._lifecycle = new CollectionLifecycleManager(config, this.id) - this._mutations = new CollectionMutationsManager(config, this.id) - this._state = new CollectionStateManager(config) - this._sync = new CollectionSyncManager(config, this.id) + this._lifecycle = new CollectionLifecycleManager(this.config, this.id, () => + cleanupCollectionSyncConfig(this.config.sync), + ) + this._mutations = new CollectionMutationsManager(this.config, this.id) + this._state = new CollectionStateManager(this.config) + this._sync = new CollectionSyncManager(this.config, this.id) this.comparisonOpts = buildCompareOptionsFromConfig(config) diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index 661e8410d0..c98400b14c 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -37,13 +37,19 @@ export class CollectionLifecycleManager< public onFirstReadyCallbacks: Array<() => void> = [] private idleCallbackId: number | null = null private syncError: unknown + private cleanupConfig: () => void /** * Creates a new CollectionLifecycleManager instance */ - constructor(config: CollectionConfig, id: string) { + constructor( + config: CollectionConfig, + id: string, + cleanupConfig: () => void = () => {}, + ) { this.config = config this.id = id + this.cleanupConfig = cleanupConfig } setDeps(deps: { @@ -249,6 +255,7 @@ export class CollectionLifecycleManager< if (hasTime) { // Perform all cleanup operations except events + this.cleanupConfig() this.sync.cleanup() this.state.cleanup() this.changes.cleanup() diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 794bfcc4a5..c1aebe7788 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -615,6 +615,7 @@ create recursive Collection machinery. | Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | | Collection facades, event coherence, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | | Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | +| Constructed and retained Collection facades | `packages/db/tests/query/includes-space-oracle.test.ts` | | Route-context discovery and transport across recursive and join boundaries | `packages/db/tests/query/includes-context-transport-oracle.test.ts` | | Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | | Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | @@ -633,6 +634,11 @@ reported seed and shrink path while reducing a failure. Replay a broad campaign with `TANSTACK_DB_ORACLE_SEED= pnpm test:oracles`, then add the smallest case as a deterministic regression trace. +Run the reported 20-by-2-by-5-by-10 nested-Collection benchmark with +`pnpm bench:nested-includes` from `packages/db`. The space oracle is the stable +CI contract; benchmark timings are diagnostic and must not become a fixed +wall-clock threshold. + The broad relationship history changes correlation keys rather than freezing them. Set `TANSTACK_DB_ORACLE_STATISTICS=1` to print its generated depth, relationship-change, optimistic, and delete distribution. Collection-valued, diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index f350e88850..cfe731ff44 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -45,6 +45,12 @@ export type FacadePublication = { rollback: () => void } +export type BucketFacadeMetrics = { + created: number + active: number + retired: number +} + /** * The only stateful boundary outside the materialization graph. It turns inert * bucket references into stable public Collection facades and applies the @@ -60,6 +66,7 @@ export class BucketFacadeAdapter { private readonly entries = new Map>() private readonly retiredEntries = new Map>() private resolvedValues = new WeakMap() + private createdEntries = 0 constructor( private readonly parentId: string, @@ -92,6 +99,20 @@ export class BucketFacadeAdapter { return this.pending.size > 0 || this.pendingActivity.size > 0 } + getMetrics(): BucketFacadeMetrics { + const count = (entries: Map>) => + [...entries.values()].reduce( + (total, byBucket) => total + byBucket.size, + 0, + ) + + return { + created: this.createdEntries, + active: count(this.entries), + retired: count(this.retiredEntries), + } + } + flush(): FacadePublication { const snapshot = this.snapshot() const deferredEntries = new Set() @@ -401,6 +422,7 @@ export class BucketFacadeAdapter { order, currentOrder: new Map(), } + this.createdEntries++ byBucket.set(bucketKey, entry) return entry } diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index be53fed522..95ecb0d404 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -161,6 +161,7 @@ export class CollectionConfigBuilder< private bucketFacadesCache: | ReturnType[`facades`] | undefined + private bucketFacadeAdapter: BucketFacadeAdapter | undefined // Map of opaque source ID to subscription readonly subscriptions: Record = {} @@ -276,6 +277,12 @@ export class CollectionConfigBuilder< getWindow: this.getWindow.bind(this), [LIVE_QUERY_INTERNAL]: { getBuilder: () => this, + getBucketFacadeMetrics: () => + this.bucketFacadeAdapter?.getMetrics() ?? { + created: 0, + active: 0, + retired: 0, + }, hasCustomGetKey: !!this.config.getKey, hasJoins: this.hasJoins(this.query), hasDistinct: !!this.query.distinct, @@ -891,7 +898,13 @@ export class CollectionConfigBuilder< syncState.messagesCount += count }, ) - syncState.unsubscribeCallbacks.add(() => bucketFacades.cleanup()) + this.bucketFacadeAdapter = bucketFacades + syncState.unsubscribeCallbacks.add(() => { + bucketFacades.cleanup() + if (this.bucketFacadeAdapter === bucketFacades) { + this.bucketFacadeAdapter = undefined + } + }) // Flush pending changes and reset the accumulator. // Called at the end of each graph run to commit all accumulated changes. diff --git a/packages/db/src/query/live/internal.ts b/packages/db/src/query/live/internal.ts index 3c6a706f40..a0ccde6165 100644 --- a/packages/db/src/query/live/internal.ts +++ b/packages/db/src/query/live/internal.ts @@ -1,4 +1,5 @@ import type { CollectionConfigBuilder } from './collection-config-builder.js' +import type { BucketFacadeMetrics } from './bucket-facade-adapter.js' /** * Symbol for accessing internal utilities that should not be part of the public API @@ -10,6 +11,7 @@ export const LIVE_QUERY_INTERNAL = Symbol(`liveQueryInternal`) */ export type LiveQueryInternalUtils = { getBuilder: () => CollectionConfigBuilder + getBucketFacadeMetrics: () => BucketFacadeMetrics hasCustomGetKey: boolean hasJoins: boolean hasDistinct: boolean diff --git a/packages/db/src/query/runtime-reference-identity.ts b/packages/db/src/query/runtime-reference-identity.ts index 15d7b82b6d..e41aeaf67a 100644 --- a/packages/db/src/query/runtime-reference-identity.ts +++ b/packages/db/src/query/runtime-reference-identity.ts @@ -21,8 +21,17 @@ export function createRuntimeReferenceIdentityFactory(): ( } } -export const getRuntimeReferenceIdentity = - createRuntimeReferenceIdentityFactory() +let runtimeReferenceIdentityFactory: + | ReturnType + | undefined + +export function getRuntimeReferenceIdentity( + value: object, +): RuntimeReferenceIdentity { + runtimeReferenceIdentityFactory ??= createRuntimeReferenceIdentityFactory() + + return runtimeReferenceIdentityFactory(value) +} function createRuntimeReferenceNamespace(): string { const randomValues = new Uint32Array(4) diff --git a/packages/db/tests/get-key-query-planning.test.ts b/packages/db/tests/get-key-query-planning.test.ts new file mode 100644 index 0000000000..728e427725 --- /dev/null +++ b/packages/db/tests/get-key-query-planning.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from 'vitest' +import { createCollection } from '../src/collection/index.js' +import { currentStateAsChanges } from '../src/collection/change-events.js' +import { Func, PropRef, Value } from '../src/query/ir.js' + +describe(`getKey query planning`, () => { + it(`does not treat arbitrary getKey code as query field metadata`, async () => { + type Row = { id: string; fallback: string } + let getKeyCalls = 0 + const collection = createCollection({ + id: `conditional-get-key-query-planning`, + getKey: (row) => { + getKeyCalls++ + return row.id === `special` ? row.fallback : row.id + }, + autoIndex: `off`, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ + type: `insert`, + value: { id: `special`, fallback: `actual-key` }, + }) + commit() + markReady() + }, + }, + }) + + try { + await collection.stateWhenReady() + const callsAfterSync = getKeyCalls + const where = new Func(`eq`, [new PropRef([`id`]), new Value(`special`)]) + + expect( + currentStateAsChanges(collection, { where, optimizedOnly: true }), + ).toBeUndefined() + expect(getKeyCalls).toBe(callsAfterSync) + expect( + currentStateAsChanges(collection, { where })?.map( + (change) => change.key, + ), + ).toEqual([`actual-key`]) + expect(getKeyCalls).toBe(callsAfterSync) + } finally { + await collection.cleanup() + } + }) +}) diff --git a/packages/db/tests/query/includes-performance.bench.ts b/packages/db/tests/query/includes-performance.bench.ts new file mode 100644 index 0000000000..251a9c2635 --- /dev/null +++ b/packages/db/tests/query/includes-performance.bench.ts @@ -0,0 +1,22 @@ +import { bench, describe } from 'vitest' +import { createNestedCollectionFixture } from './includes-space-oracle-fixture.js' + +describe(`nested Collection materialization`, () => { + bench( + `constructs and preloads the 20-by-2-by-5-by-10 tree`, + async () => { + const fixture = await createNestedCollectionFixture(20) + try { + await fixture.live.preload() + } finally { + await fixture.cleanup() + } + }, + { + iterations: 10, + time: 0, + warmupIterations: 2, + warmupTime: 0, + }, + ) +}) diff --git a/packages/db/tests/query/includes-space-oracle-fixture.ts b/packages/db/tests/query/includes-space-oracle-fixture.ts new file mode 100644 index 0000000000..cc52961b73 --- /dev/null +++ b/packages/db/tests/query/includes-space-oracle-fixture.ts @@ -0,0 +1,105 @@ +import { createCollection } from '../../src/collection/index.js' +import { BTreeIndex } from '../../src/indexes/btree-index.js' +import { localOnlyCollectionOptions } from '../../src/local-only.js' +import { createLiveQueryCollection, eq } from '../../src/query/index.js' + +type RootRow = { id: string } +type BranchRow = { id: string; rootId: string } +type TwigRow = { id: string; branchId: string } +type LeafRow = { id: string; twigId: string } + +let fixtureId = 0 + +function createRows(rootCount: number) { + const roots: Array = [] + const branches: Array = [] + const twigs: Array = [] + const leaves: Array = [] + + for (let rootIndex = 0; rootIndex < rootCount; rootIndex++) { + const rootId = `root-${rootIndex}` + roots.push({ id: rootId }) + for (let branchIndex = 0; branchIndex < 2; branchIndex++) { + const branchId = `branch-${rootIndex}-${branchIndex}` + branches.push({ id: branchId, rootId }) + for (let twigIndex = 0; twigIndex < 5; twigIndex++) { + const twigId = `twig-${rootIndex}-${branchIndex}-${twigIndex}` + twigs.push({ id: twigId, branchId }) + for (let leafIndex = 0; leafIndex < 10; leafIndex++) { + leaves.push({ + id: `leaf-${rootIndex}-${branchIndex}-${twigIndex}-${leafIndex}`, + twigId, + }) + } + } + } + } + + return { roots, branches, twigs, leaves } +} + +function source(name: string, rows: Array) { + return createCollection( + localOnlyCollectionOptions({ + id: `includes-space-${fixtureId}-${name}`, + getKey: (row: T) => row.id, + initialData: rows, + }), + ) +} + +export async function createNestedCollectionFixture(rootCount: number) { + fixtureId++ + const rows = createRows(rootCount) + const sources = { + roots: source(`roots`, rows.roots), + branches: source(`branches`, rows.branches), + twigs: source(`twigs`, rows.twigs), + leaves: source(`leaves`, rows.leaves), + } + + await Promise.all( + Object.values(sources).map((collection) => collection.preload()), + ) + sources.branches.createIndex((row) => row.rootId, { indexType: BTreeIndex }) + sources.twigs.createIndex((row) => row.branchId, { indexType: BTreeIndex }) + sources.leaves.createIndex((row) => row.twigId, { indexType: BTreeIndex }) + + const live = createLiveQueryCollection((q) => + q.from({ root: sources.roots }).select(({ root }) => ({ + id: root.id, + branches: q + .from({ branch: sources.branches }) + .where(({ branch }) => eq(branch.rootId, root.id)) + .select(({ branch }) => ({ + id: branch.id, + twigs: q + .from({ twig: sources.twigs }) + .where(({ twig }) => eq(twig.branchId, branch.id)) + .select(({ twig }) => ({ + id: twig.id, + leaves: q + .from({ leaf: sources.leaves }) + .where(({ leaf }) => eq(leaf.twigId, twig.id)) + .select(({ leaf }) => ({ id: leaf.id })), + })), + })), + })), + ) + + return { + live, + expectedFacadeCount: rootCount + rootCount * 2 + rootCount * 2 * 5, + cleanup: async () => { + const results = await Promise.allSettled([ + live.cleanup(), + ...Object.values(sources).map((collection) => collection.cleanup()), + ]) + const rejection = results.find( + (result): result is PromiseRejectedResult => + result.status === `rejected`, + ) + if (rejection) throw rejection.reason + }, + } +} diff --git a/packages/db/tests/query/includes-space-oracle.test.ts b/packages/db/tests/query/includes-space-oracle.test.ts new file mode 100644 index 0000000000..27a9aa71dd --- /dev/null +++ b/packages/db/tests/query/includes-space-oracle.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest' +import { LIVE_QUERY_INTERNAL } from '../../src/query/live/internal.js' +import { createNestedCollectionFixture } from './includes-space-oracle-fixture.js' + +describe(`nested Collection materialization space oracle`, () => { + it(`constructs exactly one facade per reachable bucket`, async () => { + const fixture = await createNestedCollectionFixture(20) + try { + await fixture.live.preload() + + expect( + fixture.live.utils[LIVE_QUERY_INTERNAL].getBucketFacadeMetrics(), + ).toEqual({ + created: fixture.expectedFacadeCount, + active: fixture.expectedFacadeCount, + retired: 0, + }) + } finally { + await fixture.cleanup() + } + }) +}) diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index ca745d9de5..6b730cbda3 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -309,6 +309,27 @@ describe(`semantic expression identity`, () => { }, ) + it(`does not initialize runtime reference identities during module evaluation`, async () => { + const getRandomValues = vi.fn((values: Uint32Array) => values) + vi.stubGlobal(`crypto`, { getRandomValues }) + vi.resetModules() + + try { + const { getRuntimeReferenceIdentity } = await import( + `../../src/query/runtime-reference-identity.js` + ) + + expect(getRandomValues).not.toHaveBeenCalled() + + getRuntimeReferenceIdentity({}) + getRuntimeReferenceIdentity({}) + + expect(getRandomValues).toHaveBeenCalledOnce() + } finally { + vi.unstubAllGlobals() + } + }) + it(`does not reuse reference identities across runtimes`, () => { const firstRuntime = createRuntimeReferenceIdentityFactory() const secondRuntime = createRuntimeReferenceIdentityFactory() diff --git a/packages/electric-db-collection/package.json b/packages/electric-db-collection/package.json index 0f5118b513..b49e296d19 100644 --- a/packages/electric-db-collection/package.json +++ b/packages/electric-db-collection/package.json @@ -21,6 +21,7 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest run", + "test:oracles": "vitest run tests/electric-oracle.property.test.ts", "test:e2e": "vitest run --config vitest.e2e.config.ts" }, "type": "module", @@ -49,10 +50,10 @@ "@electric-sql/client": "^1.5.15", "@standard-schema/spec": "^1.1.0", "@tanstack/db": "workspace:*", - "@tanstack/store": "^0.9.2", "debug": "^4.4.3" }, "devDependencies": { + "@tanstack/query-core": "^5.90.20", "@types/debug": "^4.1.12", "@types/pg": "^8.16.0", "@vitest/coverage-istanbul": "^3.2.4", diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 39e6540a6a..1ef3fb863f 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -4,12 +4,13 @@ import { isControlMessage, isVisibleInSnapshot, } from '@electric-sql/client' -import { Store } from '@tanstack/store' import DebugModule from 'debug' import { DeduplicatedLoadSubset, and, withCollectionConfigFactory, + withCollectionSyncConfigCleanup, + withCollectionSyncConfigFactory, } from '@tanstack/db' import { ExpectedNumberInAwaitTxIdError, @@ -49,6 +50,7 @@ import type { LoadSubsetOptions, SyncAppliedReceipt, SyncConfig, + SyncMetadataApi, SyncMode, UpdateMutationFnParams, UtilsRecord, @@ -64,6 +66,17 @@ import type { ShapeStreamOptions, } from '@electric-sql/client' +type ElectricSyncMetadataWithPersistedScan = SyncMetadataApi< + string | number +> & { + row: SyncMetadataApi[`row`] & { + whenHydrated?: () => Promise + scanPersisted?: (options?: { + metadataOnly?: boolean + }) => Promise> + } +} + // Re-export for user convenience in custom match functions export { isChangeMessage, isControlMessage } from '@electric-sql/client' @@ -111,6 +124,47 @@ type ElectricSyncMeta = { seenTxids: Array } +type ElectricLifecycleEvidence = { + seenTxids: Set + seenSnapshots: Array + hydratedResumeState?: ElectricResumeState +} + +type ElectricPendingMatch> = { + matchFn: (message: Message) => boolean + resolve: (value: boolean) => void + reject: (error: Error) => void + timeoutId: ReturnType + matched: boolean +} + +type ElectricMatchBuffer> = { + committedMessages: Array> + pendingMessages: Array> +} + +function exportElectricSyncMeta( + evidence: ElectricLifecycleEvidence, +): ElectricSyncMeta { + const resume = evidence.hydratedResumeState + return { + version: 1, + ...(resume ? { resume } : {}), + seenTxids: Array.from(evidence.seenTxids).sort((a, b) => a - b), + } +} + +function importElectricSyncMeta( + evidence: ElectricLifecycleEvidence, + meta: unknown, +): void { + const parsed = parseElectricSyncMeta(meta) + if (!parsed) return + + evidence.hydratedResumeState = parsed.resume + evidence.seenTxids = new Set(parsed.seenTxids) +} + function parseElectricResumeState( value: unknown, ): ElectricResumeState | undefined { @@ -215,7 +269,21 @@ function getNewestElectricResumeState( ): ElectricResumeState | undefined { if (!current) return incoming if (!incoming) return current - return incoming.updatedAt >= current.updatedAt ? incoming : current + if (incoming.updatedAt > current.updatedAt) return incoming + if (incoming.updatedAt < current.updatedAt) return current + + const isSameState = + current.kind === incoming.kind && + (current.kind === `reset` || + (incoming.kind === `resume` && + current.offset === incoming.offset && + current.handle === incoming.handle && + current.shapeId === incoming.shapeId)) + + // Equal timestamps have no causal ordering. Preserve an identical state, + // but collapse every conflict to reset so merge order cannot resurrect an + // offset that another source has already declared unsafe. + return isSameState ? current : { kind: `reset`, updatedAt: current.updatedAt } } /** @@ -439,6 +507,47 @@ function isMustRefetchMessage>( return isControlMessage(message) && message.headers.control === `must-refetch` } +function planBatchPresence>( + messages: ReadonlyArray>, + getKey: (row: T) => string | number, + hasKnownKey: (key: string | number) => boolean, + validatesResume: boolean, +): { + messageKeys: Map, string | number> + hasUnseenUpdate: boolean +} { + const presence = new Map() + const messageKeys = new Map, string | number>() + let usesKnownBaseline = true + + for (const message of messages) { + if (isMustRefetchMessage(message)) { + presence.clear() + usesKnownBaseline = false + validatesResume = false + continue + } + if (!isChangeMessage(message)) continue + + const rowId = getKey(message.value) + messageKeys.set(message, rowId) + const operation = message.headers.operation + if (operation === `delete`) { + presence.set(rowId, false) + continue + } + + const isKnown = + presence.get(rowId) ?? (usesKnownBaseline && hasKnownKey(rowId)) + if (validatesResume && operation === `update` && !isKnown) { + return { messageKeys, hasUnseenUpdate: true } + } + presence.set(rowId, true) + } + + return { messageKeys, hasUnseenUpdate: false } +} + function isSnapshotEndMessage>( message: Message, ): message is SnapshotEndMessage { @@ -529,6 +638,7 @@ function createLoadSubsetDedupe>({ commit, getCommitCursor, waitForCommitsAfter, + onLoadSubset, collectionId, encodeColumnName, signal, @@ -545,6 +655,7 @@ function createLoadSubsetDedupe>({ commit: (signal?: AbortSignal) => SyncAppliedReceipt getCommitCursor: () => number waitForCommitsAfter: (cursor: number) => Promise + onLoadSubset?: () => void collectionId?: string /** * Optional function to encode column names (e.g., camelCase to snake_case). @@ -578,6 +689,7 @@ function createLoadSubsetDedupe>({ } const loadSubset = async (opts: LoadSubsetOptions) => { + onLoadSubset?.() const commitCursor = getCommitCursor() if (opts.signal?.aborted) return @@ -726,6 +838,241 @@ export interface ElectricCollectionUtils< awaitMatch: AwaitMatchFn } +/** Owns evidence and pending work for one materialized Collection. */ +class ElectricLifecycle> { + private readonly evidence: ElectricLifecycleEvidence = { + seenTxids: new Set(), + seenSnapshots: [], + } + + private readonly pendingMatches = new Map>() + private readonly pendingTxidWaits = new Map< + number, + { + txId: Txid + resolve: (value: boolean) => void + reject: (error: Error) => void + timeoutId: ReturnType + } + >() + private matchBuffer: ElectricMatchBuffer = { + committedMessages: [], + pendingMessages: [], + } + private nextWaiterId = 0 + private epoch = 0 + private active = false + + constructor(private readonly collectionId?: string) {} + + readonly utils: ElectricCollectionUtils = { + awaitTxId: (txId, timeout) => this.awaitTxId(txId, timeout), + awaitMatch: (matchFn, timeout) => this.awaitMatch(matchFn, timeout), + } + + start(): number { + if (this.active) this.retire() + this.active = true + this.epoch++ + this.matchBuffer = { committedMessages: [], pendingMessages: [] } + return this.epoch + } + + isActive(epoch: number): boolean { + return this.active && this.epoch === epoch + } + + retire(epoch?: number): void { + if (epoch !== undefined && !this.isActive(epoch)) return + this.active = false + this.epoch++ + + for (const match of this.pendingMatches.values()) { + clearTimeout(match.timeoutId) + match.reject(new StreamAbortedError(this.collectionId)) + } + this.pendingMatches.clear() + + for (const waiter of this.pendingTxidWaits.values()) { + clearTimeout(waiter.timeoutId) + waiter.reject(new StreamAbortedError(this.collectionId)) + } + this.pendingTxidWaits.clear() + this.matchBuffer = { committedMessages: [], pendingMessages: [] } + this.evidence.hydratedResumeState = undefined + } + + exportMeta(): ElectricSyncMeta { + return exportElectricSyncMeta(this.evidence) + } + + importMeta(meta: unknown): void { + importElectricSyncMeta(this.evidence, meta) + this.resolveTxidWaiters() + } + + get resumeState(): ElectricResumeState | undefined { + return this.evidence.hydratedResumeState + } + + set resumeState(value: ElectricResumeState | undefined) { + this.evidence.hydratedResumeState = value + } + + publishEvidence( + txids: ReadonlySet, + snapshots: ReadonlyArray, + ): void { + txids.forEach((txid) => this.evidence.seenTxids.add(txid)) + this.evidence.seenSnapshots.push(...snapshots) + this.resolveTxidWaiters() + } + + private hasTxid(txId: Txid): boolean { + return ( + this.evidence.seenTxids.has(txId) || + this.evidence.seenSnapshots.some((snapshot) => + isVisibleInSnapshot(txId, snapshot), + ) + ) + } + + private resolveTxidWaiters(): void { + for (const [waitId, waiter] of this.pendingTxidWaits) { + if (!this.hasTxid(waiter.txId)) continue + clearTimeout(waiter.timeoutId) + this.pendingTxidWaits.delete(waitId) + waiter.resolve(true) + } + } + + private async awaitTxId( + txId: Txid, + timeout: number = 5000, + ): Promise { + debug( + `${this.collectionId ? `[${this.collectionId}] ` : ``}awaitTxId called with txid %d`, + txId, + ) + if (typeof txId !== `number`) { + throw new ExpectedNumberInAwaitTxIdError(typeof txId, this.collectionId) + } + if (this.hasTxid(txId)) return true + + return new Promise((resolve, reject) => { + const waitId = this.nextWaiterId++ + const timeoutId = setTimeout(() => { + this.pendingTxidWaits.delete(waitId) + reject(new TimeoutWaitingForTxIdError(txId, this.collectionId)) + }, timeout) + this.pendingTxidWaits.set(waitId, { + txId, + resolve, + reject, + timeoutId, + }) + }) + } + + private async awaitMatch( + matchFn: MatchFunction, + timeout: number = 3000, + ): Promise { + debug( + `${this.collectionId ? `[${this.collectionId}] ` : ``}awaitMatch called with custom function`, + ) + + for (const message of this.matchBuffer.committedMessages) { + if (!matchFn(message)) continue + return true + } + for (const message of this.matchBuffer.pendingMessages) { + if (matchFn(message)) return this.registerMatch(matchFn, timeout, true) + } + return this.registerMatch(matchFn, timeout, false) + } + + private registerMatch( + matchFn: MatchFunction, + timeout: number, + matched: boolean, + ): Promise { + return new Promise((resolve, reject) => { + const matchId = this.nextWaiterId++ + const timeoutId = setTimeout(() => { + this.pendingMatches.delete(matchId) + reject(new TimeoutWaitingForMatchError(this.collectionId)) + }, timeout) + this.pendingMatches.set(matchId, { + matchFn, + resolve, + reject, + timeoutId, + matched, + }) + }) + } + + beginMatchGeneration(messages: ReadonlyArray>): void { + if (!messages.some(isMustRefetchMessage)) return + this.matchBuffer = { committedMessages: [], pendingMessages: [] } + for (const match of this.pendingMatches.values()) match.matched = false + } + + observeMatchMessage(message: Message): void { + if ( + isChangeMessage(message) || + isMoveOutMessage(message) || + isMoveInMessage(message) + ) { + this.matchBuffer.pendingMessages.push(message) + let overflow = + this.matchBuffer.committedMessages.length + + this.matchBuffer.pendingMessages.length - + 1000 + if (overflow > 0) { + const committedOverflow = Math.min( + overflow, + this.matchBuffer.committedMessages.length, + ) + this.matchBuffer.committedMessages.splice(0, committedOverflow) + overflow -= committedOverflow + if (overflow > 0) { + this.matchBuffer.pendingMessages.splice(0, overflow) + } + } + } + + for (const [matchId, match] of this.pendingMatches) { + if (match.matched) continue + try { + match.matched = match.matchFn(message) + } catch (error) { + clearTimeout(match.timeoutId) + this.pendingMatches.delete(matchId) + match.reject(error instanceof Error ? error : new Error(String(error))) + } + } + } + + commitMatches(): void { + this.matchBuffer.committedMessages.push(...this.matchBuffer.pendingMessages) + this.matchBuffer.pendingMessages = [] + if (this.matchBuffer.committedMessages.length > 1000) { + this.matchBuffer.committedMessages.splice( + 0, + this.matchBuffer.committedMessages.length - 1000, + ) + } + for (const [matchId, match] of this.pendingMatches) { + if (!match.matched) continue + clearTimeout(match.timeoutId) + this.pendingMatches.delete(matchId) + match.resolve(true) + } + } +} + /** * Creates Electric collection options for use with a standard Collection * @@ -779,267 +1126,55 @@ export function electricCollectionOptions>( utils: ElectricCollectionUtils schema?: any } { - const seenTxids = new Store>(new Set([])) - const seenSnapshots = new Store>([]) - const hydratedResumeState = new Store( - undefined, - ) + let descriptorLifecycle = new ElectricLifecycle(config.id) + let utilityLifecycle = descriptorLifecycle const internalSyncMode = config.syncMode ?? `eager` const finalSyncMode = internalSyncMode === `progressive` ? `on-demand` : internalSyncMode - const pendingMatches = new Store< - Map< - string, - { - matchFn: (message: Message) => boolean - resolve: (value: boolean) => void - reject: (error: Error) => void - timeoutId: ReturnType - matched: boolean - } - > - >(new Map()) - - // Buffer messages since last up-to-date to handle race conditions - const currentBatchMessages = new Store>>([]) - - // Track whether the current batch has been committed (up-to-date received) - // This allows awaitMatch to resolve immediately for messages from committed batches - const batchCommitted = new Store(false) - - /** - * Helper function to remove multiple matches from the pendingMatches store - */ - const removePendingMatches = (matchIds: Array) => { - if (matchIds.length > 0) { - pendingMatches.setState((current) => { - const newMatches = new Map(current) - matchIds.forEach((id) => newMatches.delete(id)) - return newMatches - }) - } - } - - /** - * Helper function to resolve and cleanup matched pending matches - */ - const resolveMatchedPendingMatches = () => { - const matchesToResolve: Array = [] - pendingMatches.state.forEach((match, matchId) => { - if (match.matched) { - clearTimeout(match.timeoutId) - match.resolve(true) - matchesToResolve.push(matchId) - debug( - `${config.id ? `[${config.id}] ` : ``}awaitMatch resolved on up-to-date for match %s`, - matchId, - ) - } + const boundLifecycles = new WeakMap>() + const awaitTxId: AwaitTxIdFn = (txId, timeout) => + utilityLifecycle.utils.awaitTxId(txId, timeout) + + const awaitMatch: AwaitMatchFn = (matchFn, timeout) => + utilityLifecycle.utils.awaitMatch(matchFn, timeout) + + const createSync = () => + createElectricSync(config.shapeOptions, { + getLifecycle: (collection) => + boundLifecycles.get(collection) ?? descriptorLifecycle, + syncMode: internalSyncMode, + collectionId: config.id, + testHooks: config[ELECTRIC_TEST_HOOKS], }) - removePendingMatches(matchesToResolve) - } - const sync = createElectricSync(config.shapeOptions, { - seenTxids, - seenSnapshots, - hydratedResumeState, - syncMode: internalSyncMode, - pendingMatches, - currentBatchMessages, - batchCommitted, - removePendingMatches, - resolveMatchedPendingMatches, - collectionId: config.id, - testHooks: config[ELECTRIC_TEST_HOOKS], - }) - - /** - * Wait for a specific transaction ID to be synced - * @param txId The transaction ID to wait for as a number - * @param timeout Optional timeout in milliseconds (defaults to 5000ms) - * @returns Promise that resolves when the txId is synced - */ - const awaitTxId: AwaitTxIdFn = async ( - txId: Txid, - timeout: number = 5000, - ): Promise => { - debug( - `${config.id ? `[${config.id}] ` : ``}awaitTxId called with txid %d`, - txId, - ) - if (typeof txId !== `number`) { - throw new ExpectedNumberInAwaitTxIdError(typeof txId, config.id) - } - - // First check if the txid is in the seenTxids store - const hasTxid = seenTxids.state.has(txId) - if (hasTxid) return true - - // Then check if the txid is in any of the seen snapshots - const hasSnapshot = seenSnapshots.state.some((snapshot) => - isVisibleInSnapshot(txId, snapshot), - ) - if (hasSnapshot) return true - - return new Promise((resolve, reject) => { - const cleanup = () => { - clearTimeout(timeoutId) - subSeenTxids.unsubscribe() - subSeenSnapshots.unsubscribe() - } - - const timeoutId = setTimeout(() => { - cleanup() - reject(new TimeoutWaitingForTxIdError(txId, config.id)) - }, timeout) - - const subSeenTxids = seenTxids.subscribe(() => { - if (seenTxids.state.has(txId)) { - debug( - `${config.id ? `[${config.id}] ` : ``}awaitTxId found match for txid %o`, - txId, - ) - cleanup() - resolve(true) - } - }) - - const subSeenSnapshots = seenSnapshots.subscribe(() => { - const visibleSnapshot = seenSnapshots.state.find((snapshot) => - isVisibleInSnapshot(txId, snapshot), - ) - if (visibleSnapshot) { - debug( - `${config.id ? `[${config.id}] ` : ``}awaitTxId found match for txid %o in snapshot %o`, - txId, - visibleSnapshot, - ) - cleanup() - resolve(true) - } - }) - }) - } - - /** - * Wait for a custom match function to find a matching message - * @param matchFn Function that returns true when a message matches - * @param timeout Optional timeout in milliseconds (defaults to 5000ms) - * @returns Promise that resolves when a matching message is found - */ - const awaitMatch: AwaitMatchFn = async ( - matchFn: MatchFunction, - timeout: number = 3000, - ): Promise => { - debug( - `${config.id ? `[${config.id}] ` : ``}awaitMatch called with custom function`, - ) - - return new Promise((resolve, reject) => { - const matchId = Math.random().toString(36) - - const cleanupMatch = () => { - pendingMatches.setState((current) => { - const newMatches = new Map(current) - newMatches.delete(matchId) - return newMatches - }) - } - - const onTimeout = () => { - cleanupMatch() - reject(new TimeoutWaitingForMatchError(config.id)) - } - - const timeoutId = setTimeout(onTimeout, timeout) - - // We need access to the stream messages to check against the match function - // This will be handled by the sync configuration - const checkMatch = (message: Message) => { - if (matchFn(message)) { - debug( - `${config.id ? `[${config.id}] ` : ``}awaitMatch found matching message, waiting for up-to-date`, - ) - // Mark as matched but don't resolve yet - wait for up-to-date - pendingMatches.setState((current) => { - const newMatches = new Map(current) - const existing = newMatches.get(matchId) - if (existing) { - newMatches.set(matchId, { ...existing, matched: true }) - } - return newMatches - }) - return true - } - return false - } - - // Check against current batch messages first to handle race conditions - for (const message of currentBatchMessages.state) { - if (matchFn(message)) { - // If batch is committed (up-to-date already received), resolve immediately - // just like awaitTxId does when it finds a txid in seenTxids - if (batchCommitted.state) { - debug( - `${config.id ? `[${config.id}] ` : ``}awaitMatch found immediate match in committed batch, resolving immediately`, - ) - clearTimeout(timeoutId) - resolve(true) - return - } - - // If batch is not yet committed, register match and wait for up-to-date - debug( - `${config.id ? `[${config.id}] ` : ``}awaitMatch found immediate match in current batch, waiting for up-to-date`, - ) - pendingMatches.setState((current) => { - const newMatches = new Map(current) - newMatches.set(matchId, { - matchFn: checkMatch, - resolve, - reject, - timeoutId, - matched: true, // Already matched, will resolve on up-to-date - }) - return newMatches - }) - return - } - } - - // Store the match function for the sync process to use - // We'll add this to a pending matches store - pendingMatches.setState((current) => { - const newMatches = new Map(current) - newMatches.set(matchId, { - matchFn: checkMatch, - resolve, - reject, - timeoutId, - matched: false, - }) - return newMatches - }) - }) - } + const sync = createSync() /** * Process matching strategy and wait for synchronization */ const processMatchingStrategy = async ( result: MatchingStrategy, + waitForTxId: AwaitTxIdFn, ): Promise => { // Only wait if result contains txid if (result && `txid` in result) { const timeout = result.timeout // Handle both single txid and array of txids if (Array.isArray(result.txid)) { - await Promise.all(result.txid.map((txid) => awaitTxId(txid, timeout))) + await Promise.all(result.txid.map((txid) => waitForTxId(txid, timeout))) } else { - await awaitTxId(result.txid, timeout) + await waitForTxId(result.txid, timeout) } } // If result is void/undefined, don't wait - mutation completes immediately } + const getMutationAwaitTxId = (params: unknown): AwaitTxIdFn => { + const collection = ( + params as { + collection?: { utils?: { awaitTxId?: AwaitTxIdFn } } + } + ).collection + return collection?.utils?.awaitTxId ?? awaitTxId + } // Create wrapper handlers for direct persistence operations that handle different matching strategies const wrappedOnInsert = config.onInsert @@ -1051,7 +1186,10 @@ export function electricCollectionOptions>( >, ) => { const handlerResult = await config.onInsert!(params) - await processMatchingStrategy(handlerResult) + await processMatchingStrategy( + handlerResult, + getMutationAwaitTxId(params), + ) return handlerResult } : undefined @@ -1065,7 +1203,10 @@ export function electricCollectionOptions>( >, ) => { const handlerResult = await config.onUpdate!(params) - await processMatchingStrategy(handlerResult) + await processMatchingStrategy( + handlerResult, + getMutationAwaitTxId(params), + ) return handlerResult } : undefined @@ -1079,7 +1220,10 @@ export function electricCollectionOptions>( >, ) => { const handlerResult = await config.onDelete!(params) - await processMatchingStrategy(handlerResult) + await processMatchingStrategy( + handlerResult, + getMutationAwaitTxId(params), + ) return handlerResult } : undefined @@ -1093,37 +1237,62 @@ export function electricCollectionOptions>( ...restConfig } = config - const options = { - ...restConfig, - syncMode: finalSyncMode, - sync: { - ...sync, - exportSyncMeta: (): ElectricSyncMeta => ({ - version: 1, - ...(hydratedResumeState.state - ? { resume: hydratedResumeState.state } - : {}), - seenTxids: Array.from(seenTxids.state).sort((a, b) => a - b), - }), - importSyncMeta: (meta: unknown): void => { - const parsed = parseElectricSyncMeta(meta) - if (!parsed) { - return - } - - hydratedResumeState.setState(() => parsed.resume) - seenTxids.setState(() => new Set(parsed.seenTxids)) + const utilityTemplate: ElectricCollectionUtils = { + awaitTxId, + awaitMatch, + } + const consumeDescriptorLifecycle = (): ElectricLifecycle => { + const lifecycle = descriptorLifecycle + descriptorLifecycle = new ElectricLifecycle(config.id) + utilityLifecycle = lifecycle + return lifecycle + } + const createBoundSync = ( + source: SyncConfig, + utilities: object, + ): SyncConfig => { + const lifecycle = consumeDescriptorLifecycle() + let collectionKey: object | undefined + Object.assign(utilities, lifecycle.utils) + + const boundSync: SyncConfig = { + ...source, + sync: (params) => { + collectionKey = params.collection + boundLifecycles.set(params.collection, lifecycle) + return source.sync(params) }, + exportSyncMeta: () => lifecycle.exportMeta(), + importSyncMeta: (meta) => lifecycle.importMeta(meta), + mergeSyncMeta: mergeElectricSyncMeta, + } + return withCollectionSyncConfigCleanup(boundSync, () => { + lifecycle.retire() + if (collectionKey) boundLifecycles.delete(collectionKey) + }) + } + const syncTemplate = withCollectionSyncConfigFactory( + { + ...sync, + exportSyncMeta: () => descriptorLifecycle.exportMeta(), + importSyncMeta: (meta) => descriptorLifecycle.importMeta(meta), mergeSyncMeta: mergeElectricSyncMeta, }, + createBoundSync, + ) + const options = { + ...restConfig, + syncMode: finalSyncMode, + sync: syncTemplate, onInsert: wrappedOnInsert, onUpdate: wrappedOnUpdate, onDelete: wrappedOnDelete, - utils: { - awaitTxId, - awaitMatch, - }, + utils: utilityTemplate, } + Object.defineProperty(options, `utils`, { + enumerable: true, + get: () => ({ ...utilityTemplate }), + }) return withCollectionConfigFactory(options, () => ( @@ -1141,46 +1310,14 @@ function createElectricSync>( shapeOptions: ShapeStreamOptions>, options: { syncMode: ElectricSyncMode - seenTxids: Store> - seenSnapshots: Store> - hydratedResumeState: Store - pendingMatches: Store< - Map< - string, - { - matchFn: (message: Message) => boolean - resolve: (value: boolean) => void - reject: (error: Error) => void - timeoutId: ReturnType - matched: boolean - } - > - > - currentBatchMessages: Store>> - batchCommitted: Store - removePendingMatches: (matchIds: Array) => void - resolveMatchedPendingMatches: () => void + getLifecycle: (collection: object) => ElectricLifecycle collectionId?: string testHooks?: ElectricTestHooks }, ): SyncConfig { - const { - seenTxids, - seenSnapshots, - hydratedResumeState, - syncMode, - pendingMatches, - currentBatchMessages, - batchCommitted, - removePendingMatches, - resolveMatchedPendingMatches, - collectionId, - testHooks, - } = options - const MAX_BATCH_MESSAGES = 1000 // Safety limit for message buffer - - // Store for the relation schema information - const relationSchema = new Store(undefined) + const { getLifecycle, syncMode, collectionId, testHooks } = options + + let relationSchema: string | undefined const tagCache = new Map() @@ -1414,6 +1551,7 @@ function createElectricSync>( begin: () => void, write: (message: ChangeMessageOrDeleteKeyMessage) => void, transactionStarted: boolean, + onDelete: (rowId: RowId) => void, ): boolean => { if (tagLength === undefined) { debug( @@ -1441,6 +1579,7 @@ function createElectricSync>( type: `delete`, key: rowId, }) + onDelete(rowId) } } } @@ -1478,7 +1617,7 @@ function createElectricSync>( */ const getSyncMetadata = (): Record => { // Use the stored schema if available, otherwise default to 'public' - const schema = relationSchema.state || `public` + const schema = relationSchema || `public` return { relation: shapeOptions.params?.table @@ -1487,10 +1626,13 @@ function createElectricSync>( } } - let unsubscribeStream: () => void - return { sync: (params: Parameters[`sync`]>[0]) => { + const lifecycle = getLifecycle(params.collection) + const lifecycleEpoch = lifecycle.start() + const isActiveLifecycle = () => lifecycle.isActive(lifecycleEpoch) + Object.assign(params.collection.utils, lifecycle.utils) + const { begin, write, @@ -1526,9 +1668,15 @@ function createElectricSync>( return parseElectricResumeState(persistedResumeState) } + const persistedMetadata = metadata as + | ElectricSyncMetadataWithPersistedScan + | undefined + const scanPersisted = persistedMetadata?.row.scanPersisted + const whenHydrated = persistedMetadata?.row.whenHydrated + const persistedResumeState = getNewestElectricResumeState( readPersistedResumeState(), - hydratedResumeState.state, + lifecycle.resumeState, ) const shapeIdentity = getStableShapeIdentity({ url: shapeOptions.url, @@ -1537,11 +1685,27 @@ function createElectricSync>( const hasIncompatiblePersistedResume = persistedResumeState?.kind === `resume` && persistedResumeState.shapeId !== shapeIdentity + const hasUnverifiablePersistedResume = + shapeOptions.offset === undefined && + shapeOptions.handle === undefined && + persistedResumeState?.kind === `resume` && + scanPersisted !== undefined && + whenHydrated === undefined const canUsePersistedResume = shapeOptions.offset === undefined && shapeOptions.handle === undefined && persistedResumeState?.kind === `resume` && - !hasIncompatiblePersistedResume + !hasIncompatiblePersistedResume && + !hasUnverifiablePersistedResume + const hasExplicitResumeOffset = + shapeOptions.offset !== undefined && shapeOptions.offset !== `-1` + const receivesCompleteRows = shapeOptions.params?.replica === `full` + // Eager and progressive streams that start after the initial offset can + // only apply partial updates when the local materialization is complete. + const requiresCompleteResume = + syncMode !== `on-demand` && + (canUsePersistedResume || + (hasExplicitResumeOffset && !receivesCompleteRows)) // Wrap markReady to wait for test hook in progressive mode let progressiveReadyGate: Promise | null = null @@ -1589,15 +1753,8 @@ function createElectricSync>( } } - // Cleanup pending matches on abort abortController.signal.addEventListener(`abort`, () => { - pendingMatches.setState((current) => { - current.forEach((match) => { - clearTimeout(match.timeoutId) - match.reject(new StreamAbortedError()) - }) - return new Map() // Clear all pending matches - }) + lifecycle.retire(lifecycleEpoch) }) const stream = new ShapeStream({ @@ -1650,12 +1807,19 @@ function createElectricSync>( // resume starts from an already-committed stream offset, so the next // up-to-date message must not run the initial atomic swap again. let hasReceivedUpToDate = - syncMode === `progressive` && canUsePersistedResume + syncMode === `progressive` && requiresCompleteResume + // A must-refetch starts a new snapshot generation. Until its up-to-date + // commit is applied, old Collection keys cannot make an update valid and + // the durable resume marker must remain reset. + let isResettingSnapshot = false + let resetGeneration = 0 // Progressive mode state // Helper to determine if we're buffering the initial sync const isBufferingInitialSync = () => - syncMode === `progressive` && !hasReceivedUpToDate + syncMode === `progressive` && + !hasReceivedUpToDate && + !isResettingSnapshot const bufferedMessages: Array> = [] // Buffer change messages during initial sync // Track keys that have been synced to handle overlapping subset queries. @@ -1663,8 +1827,18 @@ function createElectricSync>( // for each response. We convert subsequent inserts to updates to avoid // duplicate key errors when the row's data has changed between requests. const syncedKeys = new Set() + // This is the logical key set for the current stream generation. Unlike + // collection.keys(), it includes uncommitted changes from earlier + // callbacks, so accepting an update cannot depend on batch partitioning. + const knownKeys = new Set( + collection._state.syncedData.keys(), + ) + let resumeInvalid = false const stageResumeMetadata = () => { + if (!isActiveLifecycle() || resumeInvalid) { + return + } const shapeHandle = stream.shapeHandle const lastOffset = stream.lastOffset if (!shapeHandle || lastOffset === `-1`) { @@ -1678,7 +1852,7 @@ function createElectricSync>( shapeId: shapeIdentity, updatedAt: Date.now(), } - hydratedResumeState.setState(() => resumeState) + lifecycle.resumeState = resumeState metadata?.collection.set(`electric:resume`, resumeState) } @@ -1687,7 +1861,7 @@ function createElectricSync>( kind: `reset`, updatedAt: Date.now(), } - hydratedResumeState.setState(() => resetState) + lifecycle.resumeState = resetState if (metadata) { begin({ immediate: true }) @@ -1696,7 +1870,7 @@ function createElectricSync>( } } - if (hasIncompatiblePersistedResume) { + if (hasIncompatiblePersistedResume || hasUnverifiablePersistedResume) { commitResetResumeMetadataImmediately() } @@ -1768,6 +1942,11 @@ function createElectricSync>( commit, getCommitCursor: () => commitSequence, waitForCommitsAfter, + onLoadSubset: () => { + for (const rowId of collection._state.syncedData.keys()) { + knownKeys.add(rowId) + } + }, collectionId, // Pass the columnMapper's encode function to transform column names // (e.g., camelCase to snake_case) when compiling SQL for subset queries @@ -1776,32 +1955,64 @@ function createElectricSync>( signal: abortController.signal, }) - unsubscribeStream = stream.subscribe((messages: Array>) => { + const resumeKeysPromise = !requiresCompleteResume + ? undefined + : whenHydrated + ? whenHydrated().then(() => [] as Array<{ key: string | number }>) + : undefined + let areResumeKeysReady = !requiresCompleteResume || !resumeKeysPromise + const pendingResumeBatches: Array>> = [] + let unsubscribeStream: () => void = () => {} + + const processMessages = (messages: Array>): void => { + if (!isActiveLifecycle() || resumeInvalid) { + return + } + + // Plan against a sparse callback overlay. This keeps one-row live + // updates O(batch size), regardless of the materialized row count. + const { messageKeys, hasUnseenUpdate } = planBatchPresence( + messages, + (row) => collection.getKeyFromItem(row), + (rowId) => knownKeys.has(rowId), + requiresCompleteResume && !isResettingSnapshot, + ) + + // A resumed eager/progressive stream assumes its persisted rows form a + // complete materialization at the saved offset. Electric updates only + // carry changed columns, so applying one without a prior row would + // create a durable partial row. Reject the whole batch and persist a + // reset marker so the next sync starts from a full snapshot. + if (requiresCompleteResume && hasUnseenUpdate) { + resumeInvalid = true + if (transactionStarted) { + const cancellation = new AbortController() + cancellation.abort() + commit(cancellation.signal) + transactionStarted = false + } + syncedKeys.clear() + newTxids.clear() + newSnapshots.length = 0 + commitResetResumeMetadataImmediately() + streamErrorVersion++ + unsubscribeStream() + abortController.abort() + markError( + new Error( + `Electric resume state referenced an unseen row; a full snapshot is required`, + ), + ) + return + } + // Track commit point type - up-to-date takes precedence as it also triggers progressive mode atomic swap let commitPoint: `up-to-date` | `subset-end` | null = null - // Don't clear the buffer between batches - this preserves messages for awaitMatch - // to find even if multiple batches arrive before awaitMatch is called. - // The buffer is naturally limited by MAX_BATCH_MESSAGES (oldest messages are dropped). - // Reset batchCommitted since we're starting a new batch - batchCommitted.setState(() => false) + lifecycle.beginMatchGeneration(messages) for (const message of messages) { - // Add message to current batch buffer (for race condition handling) - if ( - isChangeMessage(message) || - isMoveOutMessage(message) || - isMoveInMessage(message) - ) { - currentBatchMessages.setState((currentBuffer) => { - const newBuffer = [...currentBuffer, message] - // Limit buffer size for safety - if (newBuffer.length > MAX_BATCH_MESSAGES) { - newBuffer.splice(0, newBuffer.length - MAX_BATCH_MESSAGES) - } - return newBuffer - }) - } + lifecycle.observeMatchMessage(message) // Check for txids in the message and add them to our store // Skip during buffered initial sync in progressive mode (txids will be extracted during atomic swap) @@ -1814,34 +2025,29 @@ function createElectricSync>( message.headers.txids?.forEach((txid) => newTxids.add(txid)) } - // Check pending matches against this message - // Note: matchFn will mark matches internally, we don't resolve here - const matchesToRemove: Array = [] - pendingMatches.state.forEach((match, matchId) => { - if (!match.matched) { - try { - match.matchFn(message) - } catch (err) { - // If matchFn throws, clean up and reject the promise - clearTimeout(match.timeoutId) - match.reject( - err instanceof Error ? err : new Error(String(err)), - ) - matchesToRemove.push(matchId) - debug(`matchFn error: %o`, err) - } + if (isChangeMessage(message)) { + const rowId = messageKeys.get(message)! + const operation = message.headers.operation + if ( + operation === `update` && + !receivesCompleteRows && + !knownKeys.has(rowId) + ) { + continue } - }) - - // Remove matches that errored - removePendingMatches(matchesToRemove) + if (operation === `delete`) { + knownKeys.delete(rowId) + } else { + knownKeys.add(rowId) + } + } if (isChangeMessage(message)) { // Check if the message contains schema information const schema = message.headers.schema if (schema && typeof schema === `string`) { // Store the schema for future use if it's a valid string - relationSchema.setState(() => schema) + relationSchema = schema } // In buffered initial sync of progressive mode, buffer messages instead of writing @@ -1887,6 +2093,10 @@ function createElectricSync>( begin, write, transactionStarted, + (rowId) => { + knownKeys.delete(rowId) + syncedKeys.delete(rowId) + }, ) } } else if (isMoveInMessage(message)) { @@ -1917,6 +2127,9 @@ function createElectricSync>( // Clear synced keys tracking since we're starting fresh syncedKeys.clear() + knownKeys.clear() + isResettingSnapshot = true + resetGeneration++ // Reset the loadSubset deduplication state since we're starting fresh // This ensures that previously loaded predicates don't prevent refetching after truncate @@ -1932,6 +2145,9 @@ function createElectricSync>( if (commitPoint !== null) { let applied: SyncAppliedReceipt = true const wasBufferingInitialSync = isBufferingInitialSync() + const finishesReset = + isResettingSnapshot && commitPoint === `up-to-date` + const finishingResetGeneration = resetGeneration // PROGRESSIVE MODE: Atomic swap on first up-to-date (not subset-end) // EXCEPTION: Skip atomic swap if a transaction is already started (e.g., from must-refetch). // In that case, do a normal commit to properly close the existing transaction. @@ -1977,6 +2193,10 @@ function createElectricSync>( begin, write, transactionStarted, + (rowId) => { + knownKeys.delete(rowId) + syncedKeys.delete(rowId) + }, ) } else if (isMoveInMessage(bufferedMsg)) { // Process buffered move-in messages during atomic swap @@ -1999,7 +2219,9 @@ function createElectricSync>( // Normal mode or on-demand: commit transaction if one was started // Both up-to-date and subset-end trigger a commit if (transactionStarted) { - stageResumeMetadata() + if (!isResettingSnapshot || finishesReset) { + stageResumeMetadata() + } applied = commit() transactionStarted = false } else if (commitPoint === `up-to-date` && metadata) { @@ -2019,47 +2241,82 @@ function createElectricSync>( ) } + if (finishesReset) { + const finishReset = () => { + if (resetGeneration === finishingResetGeneration) { + isResettingSnapshot = false + } + } + if (applied === true) { + finishReset() + } else { + void applied.then(finishReset, () => undefined) + } + } + // Track that we've received the first up-to-date for progressive mode if (commitPoint === `up-to-date`) { hasReceivedUpToDate = true } - // Always commit txids when we receive up-to-date, regardless of transaction state - seenTxids.setState((currentTxids) => { - const clonedSeen = new Set(currentTxids) - if (newTxids.size > 0) { - debug( - `${collectionId ? `[${collectionId}] ` : ``}new txids synced from pg %O`, - Array.from(newTxids), - ) - } - newTxids.forEach((txid) => clonedSeen.add(txid)) - newTxids.clear() - return clonedSeen - }) - - // Always commit snapshots when we receive up-to-date, regardless of transaction state - seenSnapshots.setState((currentSnapshots) => { - const seen = [...currentSnapshots, ...newSnapshots] - newSnapshots.forEach((snapshot) => - debug( - `${collectionId ? `[${collectionId}] ` : ``}new snapshot synced from pg %o`, - snapshot, - ), + // Stream evidence is the acknowledgement boundary used by mutation + // handlers. It must publish before a parked applied receipt or the + // optimistic transaction and its acknowledgement can deadlock. + if (newTxids.size > 0) { + debug( + `${collectionId ? `[${collectionId}] ` : ``}new txids synced from pg %O`, + Array.from(newTxids), ) - newSnapshots.length = 0 - return seen - }) - - // Resolve all matched pending matches on up-to-date or subset-end - // Set batchCommitted BEFORE resolving to avoid timing window where late awaitMatch - // calls could register as "matched" after resolver pass already ran - batchCommitted.setState(() => true) + } + newSnapshots.forEach((snapshot) => + debug( + `${collectionId ? `[${collectionId}] ` : ``}new snapshot synced from pg %o`, + snapshot, + ), + ) + lifecycle.publishEvidence(newTxids, newSnapshots) + newTxids.clear() + newSnapshots.length = 0 + lifecycle.commitMatches() + } + } - resolveMatchedPendingMatches() + unsubscribeStream = stream.subscribe((messages: Array>) => { + if (!areResumeKeysReady) { + pendingResumeBatches.push([...messages]) + return } + processMessages(messages) }) + if (!areResumeKeysReady && resumeKeysPromise) { + void resumeKeysPromise.then( + (rows) => { + if (abortController.signal.aborted) return + + rows.forEach((row) => knownKeys.add(row.key)) + for (const rowId of collection._state.syncedData.keys()) { + knownKeys.add(rowId) + } + areResumeKeysReady = true + + const queuedBatches = pendingResumeBatches.splice(0) + queuedBatches.forEach(processMessages) + }, + (error: unknown) => { + if (abortController.signal.aborted) return + + pendingResumeBatches.length = 0 + resumeInvalid = true + commitResetResumeMetadataImmediately() + streamErrorVersion++ + unsubscribeStream() + abortController.abort() + markError(error) + }, + ) + } + // Return the deduplicated loadSubset if available (on-demand or progressive mode) // The loadSubset method is auto-bound, so it can be safely returned directly return { @@ -2069,9 +2326,10 @@ function createElectricSync>( unsubscribeStream() // Abort the abort controller to stop the stream abortController.abort() + pendingResumeBatches.length = 0 // Reset deduplication tracking so collection can load fresh data if restarted loadSubsetDedupe?.reset() - hydratedResumeState.setState(() => undefined) + lifecycle.retire(lifecycleEpoch) }, } }, diff --git a/packages/electric-db-collection/tests/ORACLE_MUTATIONS.md b/packages/electric-db-collection/tests/ORACLE_MUTATIONS.md new file mode 100644 index 0000000000..2248ecce0c --- /dev/null +++ b/packages/electric-db-collection/tests/ORACLE_MUTATIONS.md @@ -0,0 +1,85 @@ +# Electric oracle mutation ledger + +Run each mutation alone from `packages/electric-db-collection`, confirm the +named test fails, then restore the source before trying the next mutation. +The baseline command is: + +```sh +pnpm test:oracles +``` + +1. Collection-local evidence + + In `src/electric.ts`, replace `consumeDescriptorEvidence()` with + `descriptorEvidence` when a bound sync is created. + + Killed by: `generated process grammar preserves lifecycle and +concurrent-collection isolation`. + +2. Stale callback isolation + + In `src/electric.ts`, remove the active-lifecycle term from either + `processMessages` lifecycle guard. + + Killed by: `settles every startup, hydration, snapshot availability, +commit, and cleanup permutation` and `keeps stream cleanup and stale +callbacks scoped to their lifecycle`. + +3. Acknowledgement liveness + + In `src/electric.ts`, delay `seenTxids`, `seenSnapshots`, and matched-message + publication until a pending applied receipt resolves. + + Killed by: `txid tracking > should simulate the complete flow` and the + direct-persistence-handler flow tests. Those handlers must receive stream + acknowledgement before the parked optimistic transaction can finish. + +4. Durable convergence + + In `runPersistedTrace` in `electric-oracle.property.test.ts`, make the + wrapped `applyCommittedTx` resolve without calling the saved adapter method. + + Killed by: `denotational reference, Electric, persisted Electric, and query +adapters converge across controls and publication epochs`. + +5. Late match evidence + + Clear committed match messages when the next change-bearing batch starts. + + Killed by: `keeps committed match evidence across newer writer batches`. + The paired `clears committed match evidence when the stream must refetch` + test prevents the opposite error of retaining evidence across a reset. + +6. Persisted on-demand presence + + Remove the `onLoadSubset` presence refresh before an Electric subset starts. + + Killed by: `applies on-demand catch-up updates to hydrated persisted rows`. + The expected row comes from the persisted adapter and the public merge + contract, not Electric's `knownKeys` state machine. + +7. Resume capability fencing + + Accept a persisted offset when `scanPersisted` exists but `whenHydrated` + does not. + + Killed by: `restarts a persisted resume when hydration completion is +unavailable`. + +8. Complete-row discrimination + + Treat every resumed `update` as a partial row, including updates from a + `replica: 'full'` stream. + + Killed by: `accepts complete replica updates from an explicit eager resume`. + The paired `rejects an unseen partial update from an explicit eager resume` + test proves that the exception does not admit partial rows. + +Use this focused form while iterating: + +```sh +pnpm exec vitest run tests/electric-oracle.property.test.ts -t '' +``` + +These mutants test the named laws. They do not claim exhaustive mutation +coverage of the package. diff --git a/packages/electric-db-collection/tests/electric-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-oracle.property.test.ts new file mode 100644 index 0000000000..963de96a2f --- /dev/null +++ b/packages/electric-db-collection/tests/electric-oracle.property.test.ts @@ -0,0 +1,3285 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { createCollection, createTransaction } from '@tanstack/db' +import { ShapeStream } from '@electric-sql/client' +import { QueryClient } from '@tanstack/query-core' +import { persistedCollectionOptions } from '../../db-sqlite-persistence-core/src' +import { queryCollectionOptions } from '../../query-db-collection/src/query' +import { electricCollectionOptions } from '../src/electric' +import type { Collection, SyncMetadataApi } from '@tanstack/db' +import type { ChangeMessage, Message, Offset, Row } from '@electric-sql/client' +import type { + PersistedTx, + PersistenceAdapter, +} from '../../db-sqlite-persistence-core/src' +import type { ElectricCollectionUtils, ElectricSyncMode } from '../src/electric' + +type OracleRow = Row & { + id: number + name: string + stable: string +} + +const mockSubscribe = vi.fn() +const shapeId = `{"params":{"table":"test_table"},"url":"http://test-url"}` +const mockStream = { + subscribe: mockSubscribe, + requestSnapshot: vi.fn().mockResolvedValue(undefined), + fetchSnapshot: vi.fn().mockResolvedValue({ metadata: {}, data: [] }), + forceDisconnectAndRefresh: vi.fn().mockResolvedValue(undefined), + isUpToDate: false, + shapeHandle: undefined as string | undefined, + lastOffset: `-1` as string, +} + +function createDeferred(): { + promise: Promise + resolve: (value: T | PromiseLike) => void +} { + let resolve!: (value: T | PromiseLike) => void + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise + }) + return { promise, resolve } +} + +vi.mock(`@electric-sql/client`, async () => { + const actual = await vi.importActual(`@electric-sql/client`) + return { + ...actual, + ShapeStream: vi.fn(() => mockStream), + } +}) + +function everyContiguousPartition(values: Array): Array>> { + if (values.length === 0) return [[]] + const partitions: Array>> = [] + const boundaryCount = values.length - 1 + for (let mask = 0; mask < 1 << boundaryCount; mask++) { + const batches: Array> = [[values[0]!]] + for (let index = 1; index < values.length; index++) { + if ((mask & (1 << (index - 1))) !== 0) { + batches.push([values[index]!]) + } else { + batches.at(-1)!.push(values[index]!) + } + } + partitions.push(batches) + } + return partitions +} + +function isLegalElectricPartition( + batches: Array>>, +): boolean { + return batches.every((batch) => { + const resetIndex = batch.findIndex( + (message) => + (message.headers as Record).control === `must-refetch`, + ) + if (resetIndex < 0) return true + return !batch.slice(0, resetIndex).some((message) => { + const control = (message.headers as Record).control + return control === `up-to-date` || control === `subset-end` + }) + }) +} + +function createMetadata(seed: ReadonlyMap): { + api: SyncMetadataApi + state: Map +} { + const state = new Map(seed) + return { + state, + api: { + row: { + get: () => undefined, + set: () => {}, + delete: () => {}, + }, + collection: { + get: (key) => state.get(key), + set: (key, value) => { + state.set(key, value) + }, + delete: (key) => { + state.delete(key) + }, + list: (prefix) => + Array.from(state, ([key, value]) => ({ key, value })).filter( + ({ key }) => !prefix || key.startsWith(prefix), + ), + }, + }, + } +} + +function resumeState(): ReadonlyMap { + return new Map([ + [ + `electric:resume`, + { + kind: `resume`, + offset: `10_0`, + handle: `shape-1`, + shapeId, + updatedAt: 1, + }, + ], + ]) +} + +function createPersistedAdapter( + collectionMetadata: Map, + rows: Map, + loadGate: Promise = Promise.resolve(), +): PersistenceAdapter { + return { + loadSubset: () => + loadGate.then(() => Array.from(rows, ([key, value]) => ({ key, value }))), + loadCollectionMetadata: () => + Promise.resolve( + Array.from(collectionMetadata, ([key, value]) => ({ key, value })), + ), + applyCommittedTx: (_collectionId: string, tx: PersistedTx) => { + for (const mutation of tx.collectionMetadataMutations ?? []) { + if (mutation.type === `delete`) { + collectionMetadata.delete(mutation.key) + } else { + collectionMetadata.set(mutation.key, mutation.value) + } + } + if (tx.truncate) rows.clear() + for (const mutation of tx.mutations) { + if (mutation.type === `delete`) { + rows.delete(mutation.key) + } else if (mutation.type === `update`) { + rows.set(mutation.key, { + ...rows.get(mutation.key), + ...mutation.value, + } as OracleRow) + } else { + rows.set(mutation.key, mutation.value as OracleRow) + } + } + return Promise.resolve() + }, + ensureIndex: () => Promise.resolve(), + } +} + +function createOracleCollection( + id: string, + syncMode: ElectricSyncMode, + metadata: SyncMetadataApi, + shapeResumeOptions: { offset?: Offset; handle?: string } = {}, +) { + let subscriber!: (messages: Array>) => void + const unsubscribe = vi.fn() + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return unsubscribe + }) + const options = electricCollectionOptions({ + id, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + ...shapeResumeOptions, + }, + syncMode, + getKey: (row) => row.id, + startSync: true, + }) + const originalSync = options.sync + return { + collection: createCollection({ + ...options, + sync: { + sync: (params: Parameters[0]) => + originalSync.sync({ ...params, metadata }), + }, + }), + subscriber, + unsubscribe, + } +} + +type TraceResult = { + rows: Array<[string | number, string, string]> + snapshots: Array> + status: string + resume: unknown +} + +type PersistedTraceResult = TraceResult & { + durableRows: Array<[string | number, string, string]> + durableResume: unknown + persistenceCommits: number +} + +type ReferenceState = { + committed: Map + pending: Map +} + +function rowsFromCollection( + collection: Collection, +): Array<[string | number, string, string]> { + return Array.from( + collection, + ([key, row]): [string | number, string, string] => [ + key, + row.name, + row.stable, + ], + ).sort(([left], [right]) => String(left).localeCompare(String(right))) +} + +function rowsFromMap( + rows: ReadonlyMap, +): Array<[string | number, string, string]> { + return Array.from(rows, ([key, row]): [string | number, string, string] => [ + key, + row.name, + row.stable, + ]).sort(([left], [right]) => String(left).localeCompare(String(right))) +} + +function applyReferenceBatch( + state: ReferenceState, + batch: ReadonlyArray>, +): boolean { + let commits = false + for (const message of batch) { + const headers = message.headers as Record + const control = headers.control + if (typeof control === `string`) { + if (control === `must-refetch`) { + state.pending.clear() + } + if (control === `up-to-date` || control === `subset-end`) commits = true + continue + } + if (!(`value` in message)) continue + const value = message.value + const id = value.id + if (headers.operation === `delete`) { + state.pending.delete(id) + } else if (headers.operation === `insert`) { + state.pending.set(id, value) + } else { + const current = state.pending.get(id) + if (current) { + state.pending.set(id, { ...current, ...value } as OracleRow) + } + } + } + if (commits) state.committed = new Map(state.pending) + return commits +} + +function expectedSnapshots( + prefix: Array>>, + batches: Array>>, +): Array> { + const state: ReferenceState = { + committed: new Map(), + pending: new Map(), + } + const observed: Array> = [] + for (const batch of prefix) applyReferenceBatch(state, batch) + for (const batch of batches) { + applyReferenceBatch(state, batch) + observed.push(rowsFromMap(state.committed)) + } + return observed +} + +function recomputeCommittedRows( + batches: ReadonlyArray>>, + unknownUpdate: `ignore` | `promote-complete` = `ignore`, +): Array<[string | number, string, string]> { + let commitBatchIndex = -1 + for (let index = 0; index < batches.length; index++) { + const commits = batches[index]!.some((message) => { + const control = (message.headers as Record).control + return control === `up-to-date` || control === `subset-end` + }) + if (commits) commitBatchIndex = index + } + if (commitBatchIndex < 0) return [] + + // A control message commits its entire callback, including messages that + // happen to follow the control inside that atomic delivery. + const committedPrefix = batches.slice(0, commitBatchIndex + 1).flat() + let resetIndex = -1 + for (let index = 0; index < committedPrefix.length; index++) { + if ( + (committedPrefix[index]!.headers as Record).control === + `must-refetch` + ) { + resetIndex = index + } + } + + const rows = new Map() + for (const message of committedPrefix.slice(resetIndex + 1)) { + if (!(`value` in message)) continue + const headers = message.headers as Record + const value = message.value + if (headers.operation === `delete`) { + rows.delete(value.id) + } else if (headers.operation === `insert`) { + rows.set(value.id, value) + } else { + const current = rows.get(value.id) + if (current) { + rows.set(value.id, { ...current, ...value } as OracleRow) + } else if ( + unknownUpdate === `promote-complete` && + typeof value.name === `string` && + typeof value.stable === `string` + ) { + rows.set(value.id, value) + } + } + } + return rowsFromMap(rows) +} + +function recomputedSnapshots( + prefix: Array>>, + batches: Array>>, +): Array> { + const history = [...prefix] + return batches.map((batch) => { + history.push(batch) + return recomputeCommittedRows(history) + }) +} + +function observableResume(value: unknown): unknown { + if (value === null || typeof value !== `object`) return value + const state = value as Record + return { + kind: state.kind, + offset: state.offset, + handle: state.handle, + shapeId: state.shapeId, + } +} + +async function runTrace( + id: string, + syncMode: ElectricSyncMode, + prefix: Array>>, + batches: Array>>, + seed: ReadonlyMap = new Map(), +): Promise { + const metadata = createMetadata(seed) + const { collection, subscriber } = createOracleCollection( + id, + syncMode, + metadata.api, + ) + for (const batch of prefix) subscriber(batch) + const snapshots: Array> = [] + for (const batch of batches) { + subscriber(batch) + snapshots.push(rowsFromCollection(collection)) + } + const rows = rowsFromCollection(collection) + const result = { + rows, + snapshots, + status: collection.status, + resume: observableResume(metadata.state.get(`electric:resume`)), + } + await collection.cleanup() + return result +} + +async function runPersistedTrace( + id: string, + syncMode: ElectricSyncMode, + batches: Array>>, +): Promise { + let subscriber!: (messages: Array>) => void + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const persistedRows = new Map() + const persistedMetadata = new Map() + const adapter = createPersistedAdapter(persistedMetadata, persistedRows) + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + let persistenceCommits = 0 + adapter.applyCommittedTx = (...args) => { + persistenceCommits++ + return applyCommittedTx(...args) + } + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode, + getKey: (row) => row.id, + startSync: true, + }), + persistence: { + adapter, + }, + }), + ) + collection.startSyncImmediate() + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`), { + interval: 1, + timeout: 250, + }) + + const snapshots: Array> = [] + const history: Array>> = [] + for (const batch of batches) { + subscriber(batch) + history.push(batch) + const expected = recomputeCommittedRows(history) + await vi.waitFor( + () => expect(rowsFromCollection(collection)).toEqual(expected), + { interval: 1, timeout: 250 }, + ) + snapshots.push(rowsFromCollection(collection)) + } + await vi.waitFor(() => expect(collection.status).toBe(`ready`), { + interval: 1, + timeout: 250, + }) + const exported = collection.config.sync.exportSyncMeta?.() as + | { resume?: unknown } + | undefined + await vi.waitFor( + () => { + expect(persistenceCommits).toBeGreaterThan(0) + expect(rowsFromMap(persistedRows)).toEqual(rowsFromCollection(collection)) + }, + { interval: 1, timeout: 250 }, + ) + const result = { + rows: rowsFromCollection(collection), + snapshots, + status: collection.status, + resume: observableResume(exported?.resume), + durableRows: rowsFromMap(persistedRows), + durableResume: observableResume(persistedMetadata.get(`electric:resume`)), + persistenceCommits, + } + await collection.cleanup() + return result +} + +async function runQueryTrace( + id: string, + batches: Array>>, +): Promise { + const queryClient = new QueryClient({ + defaultOptions: { + queries: { + gcTime: Number.POSITIVE_INFINITY, + retry: false, + staleTime: Number.POSITIVE_INFINITY, + }, + }, + }) + const queryKey = [id] as const + let queryRows: Array = [] + const collection = createCollection( + queryCollectionOptions({ + id, + queryClient, + queryKey, + queryFn: () => Promise.resolve(queryRows), + getKey: (row) => row.id, + startSync: true, + }), + ) + await collection.preload() + + const snapshots: TraceResult[`snapshots`] = [] + const history: Array>> = [] + for (const batch of batches) { + history.push(batch) + const commits = batch.some((message) => { + const control = (message.headers as Record).control + return control === `up-to-date` || control === `subset-end` + }) + if (commits) { + queryRows = recomputeCommittedRows(history).map( + ([rowId, name, stable]) => ({ + id: Number(rowId), + name, + stable, + }), + ) + queryClient.setQueryData(queryKey, queryRows) + await vi.waitFor( + () => { + expect(rowsFromCollection(collection)).toEqual( + queryRows.map((row): [number, string, string] => [ + row.id, + row.name, + row.stable, + ]), + ) + }, + { interval: 1, timeout: 250 }, + ) + } + snapshots.push(rowsFromCollection(collection)) + } + + const result: TraceResult = { + rows: rowsFromCollection(collection), + snapshots, + status: collection.status, + resume: undefined, + } + await collection.cleanup() + queryClient.clear() + return result +} + +function change( + operation: `insert` | `update` | `delete`, + id: number, + name: string, +): Message { + const value = + operation === `insert` + ? { id, name, stable: `stable-${id}` } + : operation === `update` + ? { id, name } + : { id } + return { + key: String(id), + value: value as OracleRow, + headers: { operation }, + } +} + +const upToDate: Message = { + headers: { control: `up-to-date` }, +} +const subsetEnd: Message = { + headers: { control: `subset-end` }, +} +const mustRefetch: Message = { + headers: { control: `must-refetch` }, +} + +type PartitionScenario = { + name: string + prefix: Array>> + messages: Array> + expectedRows: Array<[number, string, string]> + resume: boolean +} + +type HistoryToken = { + operation: `insert` | `update` | `delete` + id: number + name: string +} + +type DesignToken = + | HistoryToken + | { operation: `reset` | `commit` | `subset` | `neutral` } + +type ProcessSlot = `a` | `b` + +type ProcessCommand = + | { kind: `create`; slot: ProcessSlot } + | { kind: `import`; slot: ProcessSlot; txid: number; resume: boolean } + | { kind: `preload`; slot: ProcessSlot } + | { + kind: `batch` + slot: ProcessSlot + operation: HistoryToken[`operation`] + id: number + name: string + txid: number + } + | { kind: `reset`; slot: ProcessSlot } + | { kind: `snapshot`; slot: ProcessSlot; id: number; name: string } + | { kind: `cleanup`; slot: ProcessSlot } + | { kind: `restart`; slot: ProcessSlot } + +type ProcessRuntime = { + collection: Collection + subscriber?: (messages: Array>) => void + reference: ReferenceState + seenTxids: Set + resumeAvailable: boolean + requiresCompleteResume: boolean + resettingSnapshot: boolean + terminalError: boolean + active: boolean + retired: boolean + preloadPromises: Array> +} + +const processCommandArb: fc.Arbitrary = fc.oneof( + fc.record({ + kind: fc.constant(`create` as const), + slot: fc.constantFrom(`a`, `b`), + }), + fc.record({ + kind: fc.constant(`import` as const), + slot: fc.constantFrom(`a`, `b`), + txid: fc.integer({ min: 1, max: 50 }), + resume: fc.boolean(), + }), + fc.record({ + kind: fc.constant(`preload` as const), + slot: fc.constantFrom(`a`, `b`), + }), + fc.record({ + kind: fc.constant(`batch` as const), + slot: fc.constantFrom(`a`, `b`), + operation: fc.constantFrom( + `insert`, + `update`, + `delete`, + ), + id: fc.integer({ min: 1, max: 3 }), + name: fc.string({ maxLength: 8 }), + txid: fc.integer({ min: 51, max: 100 }), + }), + fc.record({ + kind: fc.constant(`reset` as const), + slot: fc.constantFrom(`a`, `b`), + }), + fc.record({ + kind: fc.constant(`snapshot` as const), + slot: fc.constantFrom(`a`, `b`), + id: fc.integer({ min: 1, max: 3 }), + name: fc.string({ maxLength: 8 }), + }), + fc.record({ + kind: fc.constant(`cleanup` as const), + slot: fc.constantFrom(`a`, `b`), + }), + fc.record({ + kind: fc.constant(`restart` as const), + slot: fc.constantFrom(`a`, `b`), + }), +) + +const designTokenArb: fc.Arbitrary = fc.oneof( + fc.record({ + operation: fc.constantFrom( + `insert`, + `update`, + `delete`, + ), + id: fc.integer({ min: 1, max: 3 }), + name: fc.string({ maxLength: 8 }), + }), + fc.record({ + operation: fc.constantFrom<`reset` | `commit` | `subset` | `neutral`>( + `reset`, + `commit`, + `subset`, + `neutral`, + ), + }), +) + +type SchedulerEvent = + | `startup-promise` + | `hydration` + | `snapshot-available` + | `commit` + | `cleanup` + +function permutations(values: ReadonlyArray): Array> { + if (values.length <= 1) return [[...values]] + const result: Array> = [] + values.forEach((value, index) => { + const rest = [...values.slice(0, index), ...values.slice(index + 1)] + for (const suffix of permutations(rest)) result.push([value, ...suffix]) + }) + return result +} + +async function drainScheduler(): Promise { + for (let turn = 0; turn < 12; turn++) await Promise.resolve() +} + +function buildValidHistory(tokens: Array): { + messages: Array> + expectedRows: Array<[number, string, string]> +} { + const state = new Map() + const messages: Array> = [] + + for (const token of tokens) { + if (token.operation === `delete`) { + if (!state.has(token.id)) continue + messages.push(change(`delete`, token.id, token.name)) + state.delete(token.id) + continue + } + + const operation = + token.operation === `update` && !state.has(token.id) + ? `insert` + : token.operation + messages.push(change(operation, token.id, token.name)) + state.set(token.id, { + name: token.name, + stable: `stable-${token.id}`, + }) + } + + return { + messages: [...messages, upToDate], + expectedRows: Array.from(state, ([id, row]): [number, string, string] => [ + id, + row.name, + row.stable, + ]).sort(([left], [right]) => left - right), + } +} + +function designMessage(token: DesignToken): Message { + if (token.operation === `reset`) return mustRefetch + if (token.operation === `commit`) return upToDate + if (token.operation === `subset`) return subsetEnd + if (token.operation === `neutral`) { + return { + headers: { + control: `snapshot-end`, + xmin: `100`, + xmax: `150`, + xip_list: [], + }, + } + } + if (!(`id` in token)) throw new Error(`Unknown design token`) + return change(token.operation, token.id, token.name) +} + +function buildDifferentialHistory( + tokens: Array, +): Array> { + const known = new Set([99]) + const messages: Array> = [ + change(`insert`, 99, `baseline`), + upToDate, + ] + + for (const token of tokens) { + if (token.operation === `reset`) { + known.clear() + messages.push(mustRefetch) + continue + } + if ( + token.operation === `commit` || + token.operation === `subset` || + token.operation === `neutral` + ) { + messages.push(designMessage(token)) + continue + } + if (!(`id` in token)) throw new Error(`Unknown differential token`) + if (token.operation === `delete`) { + if (!known.delete(token.id)) continue + messages.push(change(`delete`, token.id, token.name)) + continue + } + const operation = + token.operation === `update` && !known.has(token.id) + ? `insert` + : token.operation + known.add(token.id) + messages.push(change(operation, token.id, token.name)) + } + + messages.push(subsetEnd) + return messages +} + +async function runProcessGrammar( + idPrefix: string, + generated: Array, +): Promise { + const subscribers: Array<(messages: Array>) => void> = [] + const runtimes = new Map() + const allPreloads: Array> = [] + let generation = 0 + mockSubscribe.mockReset() + mockSubscribe.mockImplementation((callback) => { + subscribers.push(callback) + return vi.fn() + }) + + const createRuntime = async (slot: ProcessSlot) => { + const previous = runtimes.get(slot) + if (previous) await previous.collection.cleanup() + generation++ + const collection = createCollection( + electricCollectionOptions({ + id: `${idPrefix}-${slot}-${generation}`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (row) => row.id, + startSync: false, + }), + ) + runtimes.set(slot, { + collection, + reference: { committed: new Map(), pending: new Map() }, + seenTxids: new Set(), + resumeAvailable: false, + requiresCompleteResume: false, + resettingSnapshot: false, + terminalError: false, + active: false, + retired: false, + preloadPromises: [], + }) + } + + const startRuntime = (runtime: ProcessRuntime, preload: boolean) => { + if (runtime.active) return + const subscriberIndex = subscribers.length + if (preload) { + const promise = runtime.collection.preload() + runtime.preloadPromises.push(promise) + allPreloads.push(promise) + } else { + runtime.collection.startSyncImmediate() + } + const subscriber = subscribers[subscriberIndex] + if (!subscriber) throw new Error(`Electric stream did not subscribe`) + runtime.subscriber = subscriber + runtime.requiresCompleteResume = runtime.resumeAvailable + runtime.resettingSnapshot = false + runtime.terminalError = false + runtime.active = true + runtime.retired = false + } + + const assertAllSlots = () => { + runtimes.forEach((runtime) => { + const expected = runtime.active + ? rowsFromMap(runtime.reference.committed) + : [] + expect(rowsFromCollection(runtime.collection)).toEqual(expected) + if (runtime.terminalError) { + expect(runtime.collection.status).toBe(`error`) + } + if (!runtime.active && runtime.retired) { + expect(runtime.collection.status).toBe(`cleaned-up`) + } + }) + } + + const execute = async (command: ProcessCommand) => { + if (command.kind === `create`) { + await createRuntime(command.slot) + return + } + const runtime = runtimes.get(command.slot) + if (!runtime) return + + if (command.kind === `import`) { + if (runtime.active) return + runtime.collection.config.sync.importSyncMeta?.({ + version: 1, + seenTxids: [command.txid], + ...(command.resume + ? { + resume: { + kind: `resume`, + offset: `10_0`, + handle: `shape-1`, + shapeId, + updatedAt: command.txid, + }, + } + : {}), + }) + await expect( + runtime.collection.utils.awaitTxId(command.txid, 20), + ).resolves.toBe(true) + runtime.resumeAvailable ||= command.resume + runtime.seenTxids.add(command.txid) + for (const [otherSlot, other] of runtimes) { + if (otherSlot === command.slot || other.seenTxids.has(command.txid)) { + continue + } + await expect( + other.collection.utils.awaitTxId(command.txid, 2), + ).rejects.toThrow() + } + return + } + + if (command.kind === `preload`) { + startRuntime(runtime, true) + return + } + if (command.kind === `restart`) { + startRuntime(runtime, false) + return + } + if (command.kind === `cleanup`) { + await runtime.collection.cleanup() + runtime.active = false + runtime.retired = true + runtime.subscriber = undefined + runtime.reference = { + committed: new Map(), + pending: new Map(), + } + runtime.resumeAvailable = false + runtime.requiresCompleteResume = false + runtime.resettingSnapshot = false + runtime.terminalError = false + return + } + if (!runtime.active || !runtime.subscriber || runtime.terminalError) return + + if (command.kind === `reset`) { + runtime.subscriber([mustRefetch]) + applyReferenceBatch(runtime.reference, [mustRefetch]) + runtime.resettingSnapshot = true + return + } + + if (command.kind === `snapshot`) { + const messages = [change(`insert`, command.id, command.name), upToDate] + runtime.subscriber(messages) + applyReferenceBatch(runtime.reference, messages) + runtime.resettingSnapshot = false + runtime.resumeAvailable = true + return + } + + const evidenceChange = change( + command.operation, + command.id, + command.name, + ) as ChangeMessage + const messages: Array> = [ + { + ...evidenceChange, + headers: { + operation: command.operation, + txids: [command.txid], + }, + }, + upToDate, + ] + const observesTxid = + command.operation !== `update` || + runtime.reference.pending.has(command.id) + const invalidResume = + runtime.requiresCompleteResume && + !runtime.resettingSnapshot && + command.operation === `update` && + !runtime.reference.pending.has(command.id) + const txidOutcome = observesTxid + ? runtime.collection.utils.awaitTxId(command.txid, 100) + : undefined + runtime.subscriber(messages) + if (invalidResume) { + runtime.terminalError = true + runtime.resumeAvailable = false + return + } + applyReferenceBatch(runtime.reference, messages) + runtime.resettingSnapshot = false + runtime.resumeAvailable = true + if (txidOutcome) { + await expect(txidOutcome).resolves.toBe(true) + runtime.seenTxids.add(command.txid) + } + } + + const requiredPrefix: Array = [ + { kind: `create`, slot: `a` }, + { kind: `create`, slot: `b` }, + { kind: `import`, slot: `a`, txid: 1, resume: true }, + { kind: `import`, slot: `b`, txid: 2, resume: false }, + { kind: `preload`, slot: `a` }, + { kind: `preload`, slot: `b` }, + { + kind: `batch`, + slot: `a`, + operation: `insert`, + id: 1, + name: `a-initial`, + txid: 51, + }, + { + kind: `batch`, + slot: `b`, + operation: `insert`, + id: 1, + name: `b-initial`, + txid: 52, + }, + { kind: `reset`, slot: `a` }, + { kind: `snapshot`, slot: `a`, id: 2, name: `a-snapshot` }, + ] + const requiredSuffix: Array = [ + { kind: `cleanup`, slot: `a` }, + { kind: `restart`, slot: `a` }, + { + kind: `batch`, + slot: `a`, + operation: `insert`, + id: 3, + name: `a-restarted`, + txid: 53, + }, + { kind: `cleanup`, slot: `b` }, + { kind: `restart`, slot: `b` }, + { + kind: `batch`, + slot: `b`, + operation: `insert`, + id: 3, + name: `b-restarted`, + txid: 54, + }, + ] + + try { + for (const command of [ + ...requiredPrefix, + ...generated, + ...requiredSuffix, + ]) { + await execute(command) + assertAllSlots() + } + } finally { + await Promise.all( + Array.from(runtimes.values(), ({ collection }) => collection.cleanup()), + ) + await Promise.allSettled(allPreloads) + mockSubscribe.mockReset() + } +} + +async function runSchedulerPermutation( + id: string, + order: Array, +): Promise<{ + outcome: `resolved` | `aborted` + acknowledgedBeforeDurable: boolean +}> { + const startup = createDeferred() + const hydration = createDeferred() + const commit = createDeferred() + const persistedMetadata = new Map(resumeState()) + const persistedRows = new Map([ + [1, { id: 1, name: `persisted`, stable: `stable-1` }], + ]) + const adapter = createPersistedAdapter( + persistedMetadata, + persistedRows, + hydration.promise, + ) + const loadMetadata = adapter.loadCollectionMetadata!.bind(adapter) + adapter.loadCollectionMetadata = async (...args) => { + await startup.promise + return loadMetadata(...args) + } + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + adapter.applyCommittedTx = async (...args) => { + await commit.promise + return applyCommittedTx(...args) + } + + let subscriber: ((messages: Array>) => void) | undefined + let snapshotRequested = false + let snapshotDelivered = false + const deliveryPhases = new Set<`before-cleanup` | `after-cleanup`>() + let cleanupCompleted = false + let acknowledgedBeforeDurable = false + const schedulerStream = { + ...mockStream, + subscribe: (callback: (messages: Array>) => void) => { + subscriber = callback + return vi.fn() + }, + } + vi.mocked(ShapeStream).mockReset() + vi.mocked(ShapeStream).mockImplementation(() => schedulerStream as never) + + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (row) => row.id, + startSync: false, + }), + persistence: { adapter }, + }), + ) + let matchSettlement: `resolved` | `aborted` | `timed-out` | undefined + let txidSettlement: `resolved` | `aborted` | `timed-out` | undefined + const matchOutcome = collection.utils + .awaitMatch((message) => `value` in message && message.value.id === 1, 250) + .then( + () => `resolved` as const, + (error: unknown) => + /aborted/i.test(String(error)) + ? (`aborted` as const) + : (`timed-out` as const), + ) + .then((outcome) => (matchSettlement = outcome)) + const txidOutcome = collection.utils + .awaitTxId(91, 250) + .then( + () => `resolved` as const, + (error: unknown) => + /aborted/i.test(String(error)) + ? (`aborted` as const) + : (`timed-out` as const), + ) + .then((outcome) => (txidSettlement = outcome)) + collection.startSyncImmediate() + + const deliverSnapshotIfPossible = () => { + if (!snapshotRequested || snapshotDelivered || !subscriber) return + snapshotDelivered = true + deliveryPhases.add(cleanupCompleted ? `after-cleanup` : `before-cleanup`) + const update = change(`update`, 1, `scheduled`) as ChangeMessage + subscriber([ + { ...update, headers: { operation: `update`, txids: [91] } }, + upToDate, + ]) + } + + const assertSchedulerCheckpoint = () => { + expect(matchSettlement === undefined).toBe(txidSettlement === undefined) + if (matchSettlement === `resolved`) { + expect(txidSettlement).toBe(`resolved`) + expect(snapshotDelivered).toBe(true) + acknowledgedBeforeDurable ||= persistedRows.get(1)?.name === `persisted` + } + if (matchSettlement === `aborted`) { + expect(txidSettlement).toBe(`aborted`) + expect(collection.status).toBe(`cleaned-up`) + } + expect(matchSettlement).not.toBe(`timed-out`) + expect(txidSettlement).not.toBe(`timed-out`) + } + + try { + for (const event of order) { + if (event === `startup-promise`) startup.resolve() + if (event === `hydration`) hydration.resolve() + if (event === `snapshot-available`) snapshotRequested = true + if (event === `commit`) commit.resolve() + if (event === `cleanup`) { + await collection.cleanup() + cleanupCompleted = true + } + await drainScheduler() + deliverSnapshotIfPossible() + await drainScheduler() + assertSchedulerCheckpoint() + } + + startup.resolve() + hydration.resolve() + commit.resolve() + snapshotRequested = true + await drainScheduler() + deliverSnapshotIfPossible() + await drainScheduler() + assertSchedulerCheckpoint() + + const [match, txid] = await Promise.all([matchOutcome, txidOutcome]) + if (match === `timed-out` || txid === `timed-out`) { + throw new Error(`scheduler waiter timed out`) + } + expect(match).toBe(txid) + expect(collection.status).toBe(`cleaned-up`) + expect(rowsFromCollection(collection)).toEqual([]) + if (deliveryPhases.has(`after-cleanup`)) { + expect(persistedRows.get(1)?.name).toBe(`persisted`) + } + + if (order[0] === `cleanup`) { + expect(match).toBe(`aborted`) + expect(subscriber).toBeUndefined() + expect(persistedRows.get(1)?.name).toBe(`persisted`) + } + if (match === `resolved`) { + expect(persistedRows.get(1)).toEqual({ + id: 1, + name: `scheduled`, + stable: `stable-1`, + }) + } + return { outcome: match, acknowledgedBeforeDurable } + } finally { + await collection.cleanup() + vi.mocked(ShapeStream).mockReset() + vi.mocked(ShapeStream).mockImplementation(() => mockStream as never) + } +} + +describe(`Electric adapter laws`, () => { + let processGrammarRun = 0 + + beforeEach(() => { + vi.clearAllMocks() + mockStream.isUpToDate = false + mockStream.shapeHandle = `shape-current` + mockStream.lastOffset = `20_0` + }) + + fcTest.prop([fc.array(processCommandArb, { maxLength: 20 })], { + numRuns: 20, + })( + `generated process grammar preserves lifecycle and concurrent-collection isolation`, + async (commands) => { + processGrammarRun++ + await runProcessGrammar(`process-grammar-${processGrammarRun}`, commands) + }, + ) + + it(`stops lifecycle replay after an invalid resumed update`, async () => { + await runProcessGrammar(`process-grammar-terminal-error`, [ + { + kind: `batch`, + slot: `a`, + operation: `update`, + id: 1, + name: `unseen`, + txid: 55, + }, + { kind: `snapshot`, slot: `a`, id: 1, name: `stale callback` }, + ]) + }) + + fcTest.prop( + [fc.array(designTokenArb, { minLength: 1, maxLength: 7 }), fc.nat()], + { numRuns: 24 }, + )( + `operational and denotational reference designs agree before checking production`, + async (tokens, partitionSeed) => { + const prefix = [[upToDate]] + const messages = [...tokens.map(designMessage), upToDate] + const partitions = everyContiguousPartition(messages) + const selected = [ + [messages], + messages.map((message) => [message]), + partitions[partitionSeed % partitions.length]!, + ].filter(isLegalElectricPartition) + + for (const [partitionId, partition] of selected.entries()) { + const operational = expectedSnapshots(prefix, partition) + const denotational = recomputedSnapshots(prefix, partition) + expect(operational).toEqual(denotational) + + for (const syncMode of [`eager`, `on-demand`, `progressive`] as const) { + const actual = await runTrace( + `design-grammar-${syncMode}-${partitionId}`, + syncMode, + prefix, + partition, + ) + expect(actual.snapshots).toEqual(denotational) + } + } + }, + ) + + it(`distinguishes plausible unseen-update and reset-visibility semantics`, async () => { + const completeUnseenUpdate: ChangeMessage = { + key: `1`, + value: { id: 1, name: `complete`, stable: `stable-1` }, + headers: { operation: `update` }, + } + const updateHistory = [[completeUnseenUpdate, upToDate]] + expect(recomputeCommittedRows(updateHistory, `ignore`)).toEqual([]) + expect(recomputeCommittedRows(updateHistory, `promote-complete`)).toEqual([ + [1, `complete`, `stable-1`], + ]) + const updateActual = await runTrace( + `design-unseen-update`, + `eager`, + [], + updateHistory, + ) + expect(updateActual.rows).toEqual( + recomputeCommittedRows(updateHistory, `ignore`), + ) + + const resetHistory = [ + [change(`insert`, 1, `committed`), upToDate], + [mustRefetch], + ] + const atomicReset = recomputedSnapshots([], resetHistory) + const immediateReset: typeof atomicReset = [atomicReset[0]!, []] + expect(atomicReset).not.toEqual(immediateReset) + const resetActual = await runTrace( + `design-reset-visibility`, + `eager`, + [], + resetHistory, + ) + expect(resetActual.snapshots).toEqual(atomicReset) + }) + + it(`distinguishes callback-atomic and subset publication semantics`, async () => { + const callbackHistory = [ + [ + change(`insert`, 1, `before-control`), + upToDate, + change(`update`, 1, `after-control`), + ], + ] + const callbackAtomic = recomputeCommittedRows(callbackHistory) + const freezeAtControl: typeof callbackAtomic = [ + [1, `before-control`, `stable-1`], + ] + expect(callbackAtomic).toEqual([[1, `after-control`, `stable-1`]]) + expect(callbackAtomic).not.toEqual(freezeAtControl) + expect(isLegalElectricPartition([[upToDate, mustRefetch]])).toBe(false) + + const readyPrefix = [[upToDate]] + const subsetHistory = [ + [change(`insert`, 1, `subset-publication`)], + [subsetEnd], + ] + const subsetPublishes = recomputedSnapshots(readyPrefix, subsetHistory) + const upToDateOnly: typeof subsetPublishes = [[], []] + expect(subsetPublishes).not.toEqual(upToDateOnly) + + for (const syncMode of [`eager`, `on-demand`, `progressive`] as const) { + const callbackActual = await runTrace( + `design-callback-atomic-${syncMode}`, + syncMode, + [], + callbackHistory, + ) + expect(callbackActual.rows).toEqual(callbackAtomic) + + const subsetActual = await runTrace( + `design-subset-publication-${syncMode}`, + syncMode, + readyPrefix, + subsetHistory, + ) + expect(subsetActual.snapshots).toEqual(subsetPublishes) + } + }) + + it(`settles every startup, hydration, snapshot availability, commit, and cleanup permutation`, async () => { + const events: Array = [ + `startup-promise`, + `hydration`, + `snapshot-available`, + `commit`, + `cleanup`, + ] + let permutationIndex = 0 + const outcomes = new Set<`resolved` | `aborted`>() + let sawAcknowledgementBeforeDurability = false + for (const order of permutations(events)) { + const currentIndex = permutationIndex++ + try { + const result = await runSchedulerPermutation( + `scheduler-permutation-${currentIndex}`, + order, + ) + outcomes.add(result.outcome) + sawAcknowledgementBeforeDurability ||= result.acknowledgedBeforeDurable + } catch (error) { + throw new Error( + `scheduler permutation ${currentIndex} failed: ${order.join(` → `)}`, + { cause: error }, + ) + } + } + expect(outcomes).toEqual(new Set([`resolved`, `aborted`])) + expect(sawAcknowledgementBeforeDurability).toBe(true) + }, 30_000) + + fcTest.prop( + [ + fc.tuple( + fc.string({ maxLength: 8 }), + fc.string({ maxLength: 8 }), + fc.string({ maxLength: 8 }), + fc.string({ maxLength: 8 }), + ), + ], + { numRuns: 10 }, + )( + `batch partition is invariant across Electric modes and phases`, + async (names) => { + const [first, second, updated, reinserted] = names + const readyPrefix = [ + [change(`insert`, 1, first), change(`insert`, 2, second), upToDate], + ] + const scenarios: Array = [ + { + name: `bootstrap`, + prefix: [], + messages: [ + change(`insert`, 1, first), + change(`insert`, 2, second), + change(`update`, 1, updated), + change(`delete`, 2, second), + change(`insert`, 2, reinserted), + upToDate, + ], + expectedRows: [ + [1, updated, `stable-1`], + [2, reinserted, `stable-2`], + ], + resume: false, + }, + { + name: `steady`, + prefix: readyPrefix, + messages: [ + change(`update`, 1, updated), + change(`delete`, 2, second), + change(`insert`, 2, reinserted), + upToDate, + ], + expectedRows: [ + [1, updated, `stable-1`], + [2, reinserted, `stable-2`], + ], + resume: false, + }, + { + name: `split-delete-update`, + prefix: [[change(`insert`, 1, first), upToDate]], + messages: [ + change(`delete`, 1, first), + change(`update`, 1, updated), + upToDate, + ], + expectedRows: [], + resume: false, + }, + { + name: `subset`, + prefix: readyPrefix, + messages: [ + change(`update`, 1, updated), + change(`delete`, 2, second), + subsetEnd, + ], + expectedRows: [[1, updated, `stable-1`]], + resume: false, + }, + { + name: `must-refetch`, + prefix: readyPrefix, + messages: [ + mustRefetch, + change(`insert`, 1, first), + change(`update`, 1, updated), + change(`insert`, 2, reinserted), + upToDate, + ], + expectedRows: [ + [1, updated, `stable-1`], + [2, reinserted, `stable-2`], + ], + resume: false, + }, + { + name: `must-refetch-unseen-update`, + prefix: readyPrefix, + messages: [mustRefetch, change(`update`, 1, updated), upToDate], + expectedRows: [], + resume: false, + }, + { + name: `must-refetch-subset`, + prefix: readyPrefix, + messages: [ + mustRefetch, + change(`insert`, 1, first), + subsetEnd, + change(`update`, 1, updated), + upToDate, + ], + expectedRows: [[1, updated, `stable-1`]], + resume: false, + }, + { + name: `resume`, + prefix: [], + messages: [ + change(`insert`, 1, first), + change(`update`, 1, updated), + upToDate, + ], + expectedRows: [[1, updated, `stable-1`]], + resume: true, + }, + ] + + for (const syncMode of [`eager`, `on-demand`, `progressive`] as const) { + for (const scenario of scenarios) { + const seed = scenario.resume ? resumeState() : new Map() + const atomic = await runTrace( + `${syncMode}-${scenario.name}-atomic`, + syncMode, + scenario.prefix, + [scenario.messages], + seed, + ) + expect(atomic.rows).toEqual(scenario.expectedRows) + expect(atomic.status).toBe(`ready`) + expect(atomic.resume).toEqual( + expect.objectContaining({ kind: `resume`, offset: `20_0` }), + ) + + let partitionId = 0 + for (const partition of everyContiguousPartition(scenario.messages)) { + const currentPartition = partitionId++ + const split = await runTrace( + `${syncMode}-${scenario.name}-${currentPartition}`, + syncMode, + scenario.prefix, + partition, + seed, + ) + expect( + { + rows: split.rows, + status: split.status, + resume: split.resume, + }, + `${syncMode}/${scenario.name}/partition-${currentPartition}: ${JSON.stringify({ partition, atomic, split })}`, + ).toEqual({ + rows: atomic.rows, + status: atomic.status, + resume: atomic.resume, + }) + expect(split.snapshots).toEqual( + expectedSnapshots(scenario.prefix, partition), + ) + } + } + } + }, + 30_000, + ) + + fcTest.prop( + [ + fc.array( + fc.record({ + operation: fc.constantFrom( + `insert`, + `update`, + `delete`, + ), + id: fc.integer({ min: 1, max: 3 }), + name: fc.string({ maxLength: 8 }), + }), + { minLength: 1, maxLength: 5 }, + ), + ], + { numRuns: 20 }, + )( + `generated valid histories are invariant under every batch partition`, + async (tokens) => { + const history = buildValidHistory(tokens) + for (const syncMode of [`eager`, `on-demand`, `progressive`] as const) { + let partitionId = 0 + for (const partition of everyContiguousPartition(history.messages)) { + const result = await runTrace( + `generated-${syncMode}-${partitionId++}`, + syncMode, + [], + partition, + ) + expect(result.rows).toEqual(history.expectedRows) + expect(result.snapshots).toEqual(expectedSnapshots([], partition)) + expect(result.status).toBe(`ready`) + } + } + }, + ) + + fcTest.prop( + [fc.array(designTokenArb, { minLength: 1, maxLength: 7 }), fc.nat()], + { numRuns: 20 }, + )( + `denotational reference, Electric, persisted Electric, and query adapters converge across controls and publication epochs`, + async (tokens, partitionSeed) => { + const messages = buildDifferentialHistory(tokens) + const partitions = everyContiguousPartition(messages).filter( + isLegalElectricPartition, + ) + const partition = partitions[partitionSeed % partitions.length]! + const referenceSnapshots = recomputedSnapshots([], partition) + + for (const syncMode of [`eager`, `on-demand`, `progressive`] as const) { + const direct = await runTrace( + `direct-differential-${syncMode}`, + syncMode, + [], + partition, + ) + const persisted = await runPersistedTrace( + `persisted-differential-${syncMode}`, + syncMode, + partition, + ) + const query = await runQueryTrace( + `query-differential-${syncMode}`, + partition, + ) + + expect(direct.snapshots).toEqual(referenceSnapshots) + expect(persisted.snapshots).toEqual(referenceSnapshots) + expect(query.snapshots).toEqual(referenceSnapshots) + expect(persisted.rows).toEqual(direct.rows) + expect(query.rows).toEqual(direct.rows) + expect(persisted.status).toBe(direct.status) + expect(query.status).toBe(`ready`) + expect(persisted.resume).toEqual(direct.resume) + expect(persisted.durableRows).toEqual(direct.rows) + expect(persisted.durableResume).toEqual(direct.resume) + expect(persisted.persistenceCommits).toBeGreaterThan(0) + } + }, + 30_000, + ) + + fcTest.prop( + [ + fc.integer({ min: 1, max: 20 }), + fc.string({ maxLength: 8 }), + fc.string({ maxLength: 8 }), + ], + { numRuns: 20 }, + )( + `generated invalid resume transitions fail under every batch partition`, + async (id, completeName, partialName) => { + const messages = [ + change(`delete`, id, completeName), + change(`update`, id, partialName), + upToDate, + ] + + for (const syncMode of [`eager`, `progressive`] as const) { + for (const [partitionId, partition] of everyContiguousPartition( + messages, + ).entries()) { + const metadata = createMetadata(resumeState()) + const trace = createOracleCollection( + `generated-invalid-${syncMode}-${id}-${partitionId}`, + syncMode, + metadata.api, + ) + trace.subscriber([change(`insert`, id, completeName), upToDate]) + for (const batch of partition) trace.subscriber(batch) + + expect(trace.collection.status).toBe(`error`) + expect(trace.collection.get(id)).toEqual( + expect.objectContaining({ stable: `stable-${id}` }), + ) + expect(metadata.state.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `reset` }), + ) + await trace.collection.cleanup() + } + } + }, + ) + + for (const syncMode of [`eager`, `progressive`] as const) { + it(`rejects unseen resumed updates and recovers on the next ${syncMode} lifecycle`, async () => { + const metadata = createMetadata(resumeState()) + const old = createOracleCollection( + `invalid-${syncMode}-resume`, + syncMode, + metadata.api, + ) + + old.subscriber([change(`update`, 1, `partial`)]) + expect(old.collection.status).toBe(`error`) + expect(old.collection.has(1)).toBe(false) + expect(old.unsubscribe).toHaveBeenCalledOnce() + expect(metadata.state.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `reset` }), + ) + + old.subscriber([change(`insert`, 1, `old generation`), upToDate]) + expect(old.collection.has(1)).toBe(false) + expect(metadata.state.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `reset` }), + ) + await old.collection.cleanup() + + const fresh = createOracleCollection( + `fresh-${syncMode}-snapshot`, + syncMode, + metadata.api, + ) + fresh.subscriber([change(`insert`, 1, `complete snapshot`), upToDate]) + + expect(fresh.collection.status).toBe(`ready`) + expect(fresh.collection.get(1)?.name).toBe(`complete snapshot`) + expect(metadata.state.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `resume`, offset: `20_0` }), + ) + await fresh.collection.cleanup() + }) + + it(`treats delete then update as an invalid ${syncMode} resume`, async () => { + const metadata = createMetadata(resumeState()) + const trace = createOracleCollection( + `delete-update-${syncMode}-resume`, + syncMode, + metadata.api, + ) + + trace.subscriber([ + change(`insert`, 1, `seen`), + change(`delete`, 1, `seen`), + change(`update`, 1, `partial`), + ]) + + expect(trace.collection.status).toBe(`error`) + expect(trace.collection.has(1)).toBe(false) + expect(metadata.state.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `reset` }), + ) + await trace.collection.cleanup() + }) + + it(`rejects delete then update across every ${syncMode} resume partition`, async () => { + const messages = [ + change(`delete`, 1, `complete`), + change(`update`, 1, `partial`), + upToDate, + ] + + for (const [partitionId, partition] of everyContiguousPartition( + messages, + ).entries()) { + const metadata = createMetadata(resumeState()) + const trace = createOracleCollection( + `invalid-partition-${syncMode}-${partitionId}`, + syncMode, + metadata.api, + ) + trace.subscriber([change(`insert`, 1, `complete`), upToDate]) + for (const batch of partition) trace.subscriber(batch) + + expect(trace.collection.status).toBe(`error`) + expect(trace.collection.get(1)).toEqual( + expect.objectContaining({ stable: `stable-1` }), + ) + expect(metadata.state.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `reset` }), + ) + await trace.collection.cleanup() + } + }) + } + + it(`keeps stream cleanup and stale callbacks scoped to their lifecycle`, async () => { + const subscribers: Array<(messages: Array>) => void> = [] + const unsubscribes = [vi.fn(), vi.fn()] + mockSubscribe.mockImplementation((callback) => { + const index = subscribers.length + subscribers.push(callback) + return unsubscribes[index]! + }) + const options = electricCollectionOptions({ + id: `reused-sync-config`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (row) => row.id, + startSync: true, + }) + const createLifecycle = (id: string) => { + const metadata = createMetadata(resumeState()) + const lifecycleSync = options.sync + return createCollection({ + ...options, + id, + sync: { + ...lifecycleSync, + sync: (params: Parameters[0]) => + lifecycleSync.sync({ ...params, metadata: metadata.api }), + }, + }) + } + + const oldCollection = createLifecycle(`old-sync-lifecycle`) + const currentCollection = createLifecycle(`current-sync-lifecycle`) + + subscribers[0]!([change(`insert`, 1, `first row`), upToDate]) + subscribers[1]!([change(`insert`, 2, `second row`), upToDate]) + expect(oldCollection.get(1)?.name).toBe(`first row`) + expect(currentCollection.get(2)?.name).toBe(`second row`) + subscribers[0]!([change(`update`, 1, `first row updated`), upToDate]) + const currentMatch = currentCollection.utils.awaitMatch( + (message) => `value` in message && message.value.id === 3, + 100, + ) + const currentTxid = currentCollection.utils.awaitTxId(42, 100) + const currentSnapshotTxid = currentCollection.utils.awaitTxId(50, 100) + let currentMatchResolved = false + let currentTxidResolved = false + let currentSnapshotTxidResolved = false + void currentMatch.then(() => { + currentMatchResolved = true + }) + void currentTxid.then(() => { + currentTxidResolved = true + }) + void currentSnapshotTxid.then(() => { + currentSnapshotTxidResolved = true + }) + subscribers[0]!([ + change(`insert`, 3, `wrong lifecycle`), + { + headers: { + control: `snapshot-end`, + xmin: `100`, + xmax: `150`, + xip_list: [], + }, + }, + { headers: { control: `up-to-date`, txids: [42] } }, + ]) + await Promise.resolve() + expect(currentMatchResolved).toBe(false) + expect(currentTxidResolved).toBe(false) + expect(currentSnapshotTxidResolved).toBe(false) + + await oldCollection.cleanup() + + expect(unsubscribes[0]).toHaveBeenCalledOnce() + expect(unsubscribes[1]).not.toHaveBeenCalled() + + subscribers[0]!([change(`update`, 1, `stale`)]) + expect(unsubscribes[1]).not.toHaveBeenCalled() + subscribers[1]!([ + change(`insert`, 3, `still live`), + { + headers: { + control: `snapshot-end`, + xmin: `100`, + xmax: `150`, + xip_list: [], + }, + }, + { headers: { control: `up-to-date`, txids: [42] } }, + ]) + await expect(currentMatch).resolves.toBe(true) + await expect(currentTxid).resolves.toBe(true) + await expect(currentSnapshotTxid).resolves.toBe(true) + expect(currentCollection.get(3)?.name).toBe(`still live`) + await currentCollection.cleanup() + expect(unsubscribes[1]).toHaveBeenCalledOnce() + }) + + it(`binds sync metadata import and export to the receiving collection`, async () => { + const subscribers: Array<(messages: Array>) => void> = [] + mockSubscribe.mockImplementation((callback) => { + subscribers.push(callback) + return vi.fn() + }) + const options = electricCollectionOptions({ + id: `shared-metadata-config`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (row) => row.id, + startSync: true, + }) + const first = createCollection({ ...options, id: `metadata-first` }) + const second = createCollection({ ...options, id: `metadata-second` }) + + subscribers[0]!([{ headers: { control: `up-to-date`, txids: [11] } }]) + subscribers[1]!([{ headers: { control: `up-to-date`, txids: [22] } }]) + + expect(first.config.sync.exportSyncMeta?.()).toMatchObject({ + version: 1, + seenTxids: [11], + }) + expect(second.config.sync.exportSyncMeta?.()).toMatchObject({ + version: 1, + seenTxids: [22], + }) + + const importedResume = { + kind: `resume` as const, + offset: `7_0` as const, + handle: `shape-7`, + shapeId, + updatedAt: 7, + } + first.config.sync.importSyncMeta?.({ + version: 1, + resume: importedResume, + seenTxids: [99], + }) + + await expect(first.utils.awaitTxId(99, 50)).resolves.toBe(true) + await expect(second.utils.awaitTxId(99, 10)).rejects.toThrow() + expect(first.config.sync.exportSyncMeta?.()).toEqual({ + version: 1, + resume: importedResume, + seenTxids: [99], + }) + expect(second.config.sync.exportSyncMeta?.()).toMatchObject({ + version: 1, + seenTxids: [22], + }) + expect(second.config.sync.exportSyncMeta?.()).not.toMatchObject({ + resume: importedResume, + }) + + const third = createCollection({ ...options, id: `metadata-third` }) + await expect(third.utils.awaitTxId(99, 10)).rejects.toThrow() + expect(third.config.sync.exportSyncMeta?.()).not.toMatchObject({ + resume: importedResume, + }) + + await first.cleanup() + await second.cleanup() + await third.cleanup() + }) + + it(`consumes raw sync metadata when the next collection is materialized`, async () => { + mockSubscribe.mockImplementation(() => vi.fn()) + const options = electricCollectionOptions({ + id: `seeded-metadata-config`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (row) => row.id, + startSync: true, + }) + const importedResume = { + kind: `resume` as const, + offset: `8_0` as const, + handle: `shape-8`, + shapeId, + updatedAt: 8, + } + options.sync.importSyncMeta?.({ + version: 1, + resume: importedResume, + seenTxids: [55], + }) + + const seeded = createCollection({ ...options, id: `metadata-seeded` }) + await expect(seeded.utils.awaitTxId(55, 50)).resolves.toBe(true) + expect(seeded.config.sync.exportSyncMeta?.()).toMatchObject({ + resume: importedResume, + seenTxids: [55], + }) + + const unseeded = createCollection({ ...options, id: `metadata-unseeded` }) + await expect(unseeded.utils.awaitTxId(55, 10)).rejects.toThrow() + expect(unseeded.config.sync.exportSyncMeta?.()).toEqual({ + version: 1, + seenTxids: [], + }) + + await seeded.cleanup() + await unseeded.cleanup() + }) + + it(`binds imported evidence and pending matches before lazy sync starts`, async () => { + let subscriber!: (messages: Array>) => void + mockSubscribe.mockImplementation((callback) => { + subscriber = callback + return vi.fn() + }) + const options = electricCollectionOptions({ + id: `lazy-bound-evidence`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (row) => row.id, + startSync: false, + }) + const collection = createCollection(options) + const importedResume = { + kind: `resume` as const, + offset: `9_0` as const, + handle: `shape-9`, + shapeId, + updatedAt: 9, + } + collection.config.sync.importSyncMeta?.({ + version: 1, + resume: importedResume, + seenTxids: [33], + }) + + await expect(collection.utils.awaitTxId(33, 20)).resolves.toBe(true) + const pendingMatch = collection.utils.awaitMatch( + (message) => `value` in message && message.value.id === 5, + 100, + ) + const pendingTxid = collection.utils.awaitTxId(34, 100) + const preload = collection.preload() + expect(vi.mocked(ShapeStream).mock.calls.at(-1)?.[0]).toMatchObject({ + offset: `9_0`, + handle: `shape-9`, + }) + const evidenceChange = change( + `insert`, + 5, + `matched after start`, + ) as ChangeMessage + subscriber([ + { + ...evidenceChange, + headers: { operation: `insert`, txids: [34] }, + }, + upToDate, + ]) + + await expect(pendingMatch).resolves.toBe(true) + await expect(pendingTxid).resolves.toBe(true) + await preload + expect(collection.get(5)?.name).toBe(`matched after start`) + await collection.cleanup() + }) + + it(`keeps descriptor utilities captured before collection startup live`, async () => { + let subscriber!: (messages: Array>) => void + mockSubscribe.mockImplementation((callback) => { + subscriber = callback + return vi.fn() + }) + const options = electricCollectionOptions({ + id: `captured-descriptor-utilities`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (row) => row.id, + startSync: false, + }) + const { awaitMatch, awaitTxId } = options.utils + const collection = createCollection(options) + const pendingMatch = awaitMatch( + (message) => `value` in message && message.value.id === 77, + 100, + ) + const pendingTxid = awaitTxId(77, 100) + + const preload = collection.preload() + const evidenceChange = change( + `insert`, + 77, + `captured utilities`, + ) as ChangeMessage + subscriber([ + { + ...evidenceChange, + headers: { operation: `insert`, txids: [77] }, + }, + upToDate, + ]) + + await expect(pendingMatch).resolves.toBe(true) + await expect(pendingTxid).resolves.toBe(true) + await preload + await collection.cleanup() + }) + + it(`retires a pending pre-start match when a lazy collection is cleaned up`, async () => { + mockSubscribe.mockImplementation(() => vi.fn()) + const collection = createCollection( + electricCollectionOptions({ + id: `lazy-pre-start-cleanup`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (row) => row.id, + startSync: false, + }), + ) + const pendingMatch = collection.utils.awaitMatch(() => false, 100) + + await collection.cleanup() + + await expect(pendingMatch).rejects.toThrow(/aborted/i) + expect(mockSubscribe).not.toHaveBeenCalled() + }) + + it(`retires pre-start waiters when persisted metadata startup is interrupted`, async () => { + const metadataStarted = createDeferred() + const metadataGate = createDeferred() + const adapter = createPersistedAdapter(new Map(), new Map()) + adapter.loadCollectionMetadata = async () => { + metadataStarted.resolve() + await metadataGate.promise + return [] + } + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id: `persisted-startup-waiter-cleanup`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (row) => row.id, + startSync: false, + }), + persistence: { adapter }, + }), + ) + const pendingMatch = collection.utils.awaitMatch(() => false, 100) + const pendingTxid = collection.utils.awaitTxId(702, 100) + const preload = collection.preload() + await metadataStarted.promise + const matchOutcome = expect(pendingMatch).rejects.toThrow(/aborted/i) + const txidOutcome = expect(pendingTxid).rejects.toThrow(/aborted/i) + + await collection.cleanup() + + await matchOutcome + await txidOutcome + expect(mockSubscribe).not.toHaveBeenCalled() + metadataGate.resolve() + await preload + }) + + it(`retires pre-start waiters through automatic collection GC`, async () => { + const metadataGate = createDeferred() + const adapter = createPersistedAdapter(new Map(), new Map()) + adapter.loadCollectionMetadata = vi.fn(async () => { + await metadataGate.promise + return [] + }) + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id: `persisted-gc-waiter-cleanup`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (row) => row.id, + startSync: false, + gcTime: 10, + }), + persistence: { adapter }, + }), + ) + const pendingMatch = collection.utils.awaitMatch(() => false, 5000) + const pendingTxid = collection.utils.awaitTxId(703, 5000) + const preload = collection.preload() + await vi.waitFor( + () => expect(adapter.loadCollectionMetadata).toHaveBeenCalledOnce(), + { interval: 1, timeout: 250 }, + ) + const subscription = collection.subscribeChanges(() => {}) + const matchOutcome = pendingMatch.catch((error: unknown) => error) + const txidOutcome = pendingTxid.catch((error: unknown) => error) + + subscription.unsubscribe() + await vi.waitFor(() => expect(collection.status).toBe(`cleaned-up`), { + interval: 10, + timeout: 1500, + }) + const pendingSentinel = Symbol(`pending`) + const matchResult = await Promise.race([ + matchOutcome, + new Promise((resolve) => setTimeout(() => resolve(pendingSentinel), 50)), + ]) + const txidResult = await Promise.race([ + txidOutcome, + new Promise((resolve) => setTimeout(() => resolve(pendingSentinel), 50)), + ]) + + expect(matchResult).toEqual( + expect.objectContaining({ + message: expect.stringMatching(/aborted/i), + }), + ) + expect(txidResult).toEqual( + expect.objectContaining({ + message: expect.stringMatching(/aborted/i), + }), + ) + expect(mockSubscribe).not.toHaveBeenCalled() + metadataGate.resolve() + await preload + }) + + it(`retires every pending waiter when its collection lifecycle is cleaned up`, async () => { + mockSubscribe.mockImplementation(() => vi.fn()) + const lazy = createCollection( + electricCollectionOptions({ + id: `lazy-waiter-cleanup`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (row) => row.id, + startSync: false, + }), + ) + const lazyTxid = lazy.utils.awaitTxId(700, 100) + await lazy.cleanup() + await expect(lazyTxid).rejects.toThrow(/aborted/i) + + const active = createOracleCollection( + `active-waiter-cleanup`, + `eager`, + createMetadata(new Map()).api, + ) + const activeMatch = active.collection.utils.awaitMatch(() => false, 100) + const activeTxid = active.collection.utils.awaitTxId(701, 100) + await active.collection.cleanup() + + await expect(activeMatch).rejects.toThrow(/aborted/i) + await expect(activeTxid).rejects.toThrow(/aborted/i) + }) + + it(`settles waiters according to whether evidence or cleanup wins`, async () => { + for (const cleanupWins of [true, false]) { + const trace = createOracleCollection( + `waiter-race-${cleanupWins}`, + `eager`, + createMetadata(new Map()).api, + ) + const pendingMatch = trace.collection.utils.awaitMatch( + (message) => `value` in message && message.value.id === 91, + 100, + ) + const pendingTxid = trace.collection.utils.awaitTxId(91, 100) + const evidenceChange = change( + `insert`, + 91, + `race winner`, + ) as ChangeMessage + const evidence: Array> = [ + { + ...evidenceChange, + headers: { operation: `insert`, txids: [91] }, + }, + upToDate, + ] + + if (cleanupWins) { + await trace.collection.cleanup() + trace.subscriber(evidence) + await expect(pendingMatch).rejects.toThrow(/aborted/i) + await expect(pendingTxid).rejects.toThrow(/aborted/i) + expect(trace.collection.get(91)).toBeUndefined() + } else { + trace.subscriber(evidence) + await trace.collection.cleanup() + await expect(pendingMatch).resolves.toBe(true) + await expect(pendingTxid).resolves.toBe(true) + } + } + }) + + it(`keeps committed match evidence across newer writer batches`, async () => { + const metadata = createMetadata(new Map()) + const trace = createOracleCollection( + `await-match-batch-generation`, + `eager`, + metadata.api, + ) + + trace.subscriber([change(`insert`, 1, `old batch`), upToDate]) + trace.subscriber([change(`insert`, 2, `current batch`), upToDate]) + + await expect( + trace.collection.utils.awaitMatch( + (message) => `value` in message && message.value.name === `old batch`, + 20, + ), + ).resolves.toBe(true) + await trace.collection.cleanup() + }) + + it(`keeps committed match evidence across callbacks with no new data`, async () => { + const trace = createOracleCollection( + `await-match-neutral-callback`, + `eager`, + createMetadata(new Map()).api, + ) + + trace.subscriber([change(`insert`, 1, `committed`), upToDate]) + trace.subscriber([ + { + headers: { + control: `snapshot-end`, + xmin: `100`, + xmax: `150`, + xip_list: [], + }, + }, + ]) + trace.subscriber([]) + + await expect( + trace.collection.utils.awaitMatch( + (message) => `value` in message && message.value.id === 1, + 20, + ), + ).resolves.toBe(true) + await trace.collection.cleanup() + }) + + it(`clears committed match evidence when the stream must refetch`, async () => { + const trace = createOracleCollection( + `await-match-reset-generation`, + `eager`, + createMetadata(new Map()).api, + ) + + trace.subscriber([change(`insert`, 1, `old generation`), upToDate]) + trace.subscriber([mustRefetch, upToDate]) + + await expect( + trace.collection.utils.awaitMatch( + (message) => + `value` in message && message.value.name === `old generation`, + 20, + ), + ).rejects.toThrow(/Timeout waiting for custom match function/) + await trace.collection.cleanup() + }) + + it(`does not let a committed message satisfy awaitMatch after restart`, async () => { + const subscribers: Array<(messages: Array>) => void> = [] + mockSubscribe.mockImplementation((callback) => { + subscribers.push(callback) + return vi.fn() + }) + const collection = createCollection( + electricCollectionOptions({ + id: `await-match-lifecycle-generation`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (row) => row.id, + startSync: true, + }), + ) + + subscribers[0]!([change(`insert`, 1, `old lifecycle`), upToDate]) + await collection.cleanup() + collection.startSyncImmediate() + await vi.waitFor(() => expect(subscribers).toHaveLength(2)) + + await expect( + collection.utils.awaitMatch( + (message) => + `value` in message && message.value.name === `old lifecycle`, + 20, + ), + ).rejects.toThrow(/Timeout waiting for custom match function/) + await collection.cleanup() + }) + + for (const syncMode of [`eager`, `on-demand`, `progressive`] as const) { + it(`does not apply an unseen ${syncMode} update after must-refetch`, async () => { + const metadata = createMetadata(new Map()) + const trace = createOracleCollection( + `must-refetch-unseen-${syncMode}`, + syncMode, + metadata.api, + ) + trace.subscriber([change(`insert`, 1, `complete`), upToDate]) + + trace.subscriber([mustRefetch, change(`update`, 1, `partial`), upToDate]) + + expect(trace.collection.status).toBe(`ready`) + expect(trace.collection.has(1)).toBe(false) + await trace.collection.cleanup() + }) + + it(`keeps the ${syncMode} reset marker until the replacement snapshot is complete`, async () => { + const metadata = createMetadata(resumeState()) + const trace = createOracleCollection( + `durable-reset-${syncMode}`, + syncMode, + metadata.api, + ) + + trace.subscriber([mustRefetch]) + expect(metadata.state.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `reset` }), + ) + + trace.subscriber([change(`insert`, 1, `partial snapshot`), subsetEnd]) + expect(metadata.state.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `reset` }), + ) + + trace.subscriber([ + change(`update`, 2, `unseen`), + change(`update`, 1, `complete snapshot`), + upToDate, + ]) + expect(trace.collection.status).toBe(`ready`) + expect(trace.collection.get(1)).toEqual( + expect.objectContaining({ + id: 1, + name: `complete snapshot`, + stable: `stable-1`, + }), + ) + expect(trace.collection.has(2)).toBe(false) + expect(metadata.state.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `resume`, offset: `20_0` }), + ) + await trace.collection.cleanup() + }) + } + + it(`does not let an older applied receipt finish a newer reset`, async () => { + const metadata = createMetadata(resumeState()) + const trace = createOracleCollection( + `overlapping-reset-generations`, + `eager`, + metadata.api, + ) + const persistence = createDeferred() + const transaction = createTransaction({ + mutationFn: () => persistence.promise, + }) + + trace.subscriber([ + mustRefetch, + change(`insert`, 1, `first replacement`), + subsetEnd, + ]) + transaction.mutate(() => + trace.collection.insert({ + id: 99, + name: `optimistic`, + stable: `stable-99`, + }), + ) + trace.subscriber([upToDate]) + trace.subscriber([mustRefetch]) + await Promise.resolve() + + trace.subscriber([change(`update`, 2, `unseen`), upToDate]) + + expect(trace.collection.status).not.toBe(`error`) + expect(trace.collection.has(2)).toBe(false) + + persistence.resolve() + await transaction.isPersisted.promise + await trace.collection.cleanup() + }) + + it(`waits for a deferred applied receipt before publishing readiness`, async () => { + const metadata = createMetadata(new Map()) + const trace = createOracleCollection( + `deferred-applied-receipt`, + `eager`, + metadata.api, + ) + const persistence = createDeferred() + const transaction = createTransaction({ + mutationFn: () => persistence.promise, + }) + transaction.mutate(() => + trace.collection.insert({ + id: 99, + name: `optimistic`, + stable: `stable-99`, + }), + ) + + trace.subscriber([change(`insert`, 1, `synced`), upToDate]) + await Promise.resolve() + + expect(trace.collection.status).toBe(`loading`) + expect(trace.collection.has(1)).toBe(false) + + persistence.resolve() + await transaction.isPersisted.promise + await trace.collection.stateWhenReady() + + expect(trace.collection.status).toBe(`ready`) + expect(trace.collection.get(1)).toEqual( + expect.objectContaining({ stable: `stable-1` }), + ) + await trace.collection.cleanup() + }) + + it(`does not publish readiness after a parked receipt is rejected by cleanup`, async () => { + const metadata = createMetadata(new Map()) + const trace = createOracleCollection( + `rejected-applied-receipt`, + `eager`, + metadata.api, + ) + const persistence = createDeferred() + const transaction = createTransaction({ + mutationFn: () => persistence.promise, + }) + transaction.mutate(() => + trace.collection.insert({ + id: 99, + name: `optimistic`, + stable: `stable-99`, + }), + ) + trace.subscriber([change(`insert`, 1, `synced`), upToDate]) + await Promise.resolve() + expect(trace.collection.status).toBe(`loading`) + + await trace.collection.cleanup() + persistence.resolve() + await transaction.isPersisted.promise + await Promise.resolve() + + expect(trace.collection.status).toBe(`cleaned-up`) + expect(trace.collection.has(1)).toBe(false) + }) + + it(`accepts partial resumed updates for rows restored by persistence`, async () => { + let subscriber!: (messages: Array>) => void + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const collectionMetadata = new Map(resumeState()) + const persistedRows = new Map([ + [ + 1, + { + id: 1, + name: `persisted`, + stable: `stable-1`, + }, + ], + ]) + const electricOptions = electricCollectionOptions({ + id: `persisted-resume-oracle`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (row) => row.id, + startSync: true, + }) + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricOptions, + persistence: { + adapter: createPersistedAdapter(collectionMetadata, persistedRows), + }, + }), + ) + + collection.startSyncImmediate() + await collection._sync.loadSubset({ limit: 10 }) + expect(collection.get(1)).toEqual( + expect.objectContaining({ name: `persisted`, stable: `stable-1` }), + ) + expect(vi.mocked(ShapeStream).mock.calls.at(-1)?.[0]).toMatchObject({ + offset: `10_0`, + handle: `shape-1`, + }) + + subscriber([change(`update`, 1, `resumed update`), upToDate]) + await vi.waitFor(() => expect(collection.status).toBe(`ready`)) + + expect(collection.get(1)).toEqual( + expect.objectContaining({ + name: `resumed update`, + stable: `stable-1`, + }), + ) + await vi.waitFor(() => { + expect(persistedRows.get(1)).toEqual( + expect.objectContaining({ + name: `resumed update`, + stable: `stable-1`, + }), + ) + }) + await collection.cleanup() + }) + + it(`buffers resumed updates that arrive while persisted rows are hydrating`, async () => { + let subscriber: ((messages: Array>) => void) | undefined + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const hydration = createDeferred() + const collectionMetadata = new Map(resumeState()) + const persistedRows = new Map([ + [ + 1, + { + id: 1, + name: `persisted`, + stable: `stable-1`, + }, + ], + ]) + const electricOptions = electricCollectionOptions({ + id: `concurrent-persisted-resume-oracle`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (row) => row.id, + startSync: true, + }) + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricOptions, + persistence: { + adapter: createPersistedAdapter( + collectionMetadata, + persistedRows, + hydration.promise, + ), + }, + }), + ) + + collection.startSyncImmediate() + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`)) + expect(collection.status).not.toBe(`error`) + subscriber!([change(`update`, 1, `concurrent update`), upToDate]) + + expect(collection.status).not.toBe(`error`) + hydration.resolve() + await vi.waitFor(() => expect(collection.status).toBe(`ready`)) + expect(collection.get(1)).toEqual( + expect.objectContaining({ + name: `concurrent update`, + stable: `stable-1`, + }), + ) + await collection.cleanup() + }) + + it(`rehydrates persisted rows and resume metadata after cleanup and restart`, async () => { + const subscribers: Array<(messages: Array>) => void> = [] + mockSubscribe.mockImplementation((callback) => { + subscribers.push(callback) + return vi.fn() + }) + const collectionMetadata = new Map(resumeState()) + const persistedRows = new Map([ + [1, { id: 1, name: `persisted`, stable: `stable-1` }], + ]) + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id: `persisted-restart-oracle`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (row) => row.id, + startSync: true, + }), + persistence: { + adapter: createPersistedAdapter(collectionMetadata, persistedRows), + }, + }), + ) + + collection.startSyncImmediate() + await vi.waitFor(() => expect(collection.get(1)?.name).toBe(`persisted`)) + await vi.waitFor(() => expect(subscribers).toHaveLength(1)) + subscribers[0]!([upToDate]) + await collection.stateWhenReady() + + await collection.cleanup() + collection.startSyncImmediate() + await vi.waitFor(() => expect(subscribers).toHaveLength(2)) + await vi.waitFor(() => expect(collection.get(1)?.name).toBe(`persisted`)) + expect(collectionMetadata.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `resume` }), + ) + await collection.cleanup() + }) + + it(`does not let hydration from a cleaned-up lifecycle poison restart`, async () => { + const subscribers: Array<(messages: Array>) => void> = [] + mockSubscribe.mockImplementation((callback) => { + subscribers.push(callback) + return vi.fn() + }) + const firstHydration = createDeferred() + const collectionMetadata = new Map(resumeState()) + const persistedRows = new Map([ + [1, { id: 1, name: `current`, stable: `stable-1` }], + ]) + const adapter = createPersistedAdapter(collectionMetadata, persistedRows) + let hydrationCall = 0 + adapter.loadSubset = vi.fn(async () => { + hydrationCall++ + if (hydrationCall === 1) { + await firstHydration.promise + return [ + { + key: 1, + value: { id: 1, name: `stale`, stable: `stable-1` }, + }, + ] + } + return Array.from(persistedRows, ([key, value]) => ({ key, value })) + }) + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id: `persisted-stale-hydration-oracle`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (row) => row.id, + startSync: true, + }), + persistence: { adapter }, + }), + ) + + collection.startSyncImmediate() + await vi.waitFor(() => expect(adapter.loadSubset).toHaveBeenCalledTimes(1)) + await collection.cleanup() + collection.startSyncImmediate() + await vi.waitFor(() => expect(subscribers).toHaveLength(2)) + + firstHydration.resolve() + await vi.waitFor(() => expect(adapter.loadSubset).toHaveBeenCalledTimes(2)) + await vi.waitFor(() => expect(collection.get(1)?.name).toBe(`current`)) + subscribers[1]!([upToDate]) + await collection.stateWhenReady() + await collection.cleanup() + }) + + it(`does not hydrate persisted on-demand rows before subset demand`, async () => { + let subscriber: ((messages: Array>) => void) | undefined + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const collectionMetadata = new Map() + const persistedRows = new Map([ + [1, { id: 1, name: `persisted`, stable: `stable-1` }], + ]) + const adapter = createPersistedAdapter(collectionMetadata, persistedRows) + const loadSubset = vi.fn(adapter.loadSubset) + adapter.loadSubset = loadSubset + const electricOptions = electricCollectionOptions({ + id: `persisted-on-demand-oracle`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `on-demand`, + getKey: (row) => row.id, + startSync: true, + }) + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricOptions, + persistence: { adapter }, + }), + ) + + collection.startSyncImmediate() + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`)) + subscriber!([upToDate]) + await vi.waitFor(() => expect(collection.status).toBe(`ready`)) + + expect(loadSubset).not.toHaveBeenCalled() + expect(collection.has(1)).toBe(false) + await collection.cleanup() + }) + + it(`applies on-demand catch-up updates to hydrated persisted rows`, async () => { + let subscriber!: (messages: Array>) => void + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const collectionMetadata = new Map(resumeState()) + const persistedRows = new Map([ + [1, { id: 1, name: `persisted`, stable: `stable-1` }], + ]) + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id: `on-demand-persisted-catch-up`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `on-demand`, + getKey: (row) => row.id, + startSync: true, + }), + persistence: { + adapter: createPersistedAdapter(collectionMetadata, persistedRows), + }, + }), + ) + + collection.startSyncImmediate() + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`)) + await collection._sync.loadSubset({}) + subscriber([change(`update`, 1, `caught up`), upToDate]) + await vi.waitFor(() => expect(collection.status).toBe(`ready`)) + + expect(collection.get(1)).toEqual( + expect.objectContaining({ name: `caught up`, stable: `stable-1` }), + ) + expect(collectionMetadata.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `resume`, offset: `20_0` }), + ) + await collection.cleanup() + }) + + it.each([ + { + name: `malformed`, + seed: new Map([ + [ + `electric:resume`, + { + kind: `resume`, + offset: 10, + handle: `shape-1`, + shapeId, + updatedAt: 1, + }, + ], + ]), + }, + { + name: `reset`, + seed: new Map([ + [`electric:resume`, { kind: `reset`, updatedAt: 1 }], + ]), + }, + { + name: `incompatible`, + seed: new Map([ + [ + `electric:resume`, + { + kind: `resume`, + offset: `10_0`, + handle: `shape-1`, + shapeId: `different-shape`, + updatedAt: 1, + }, + ], + ]), + }, + ])(`starts a full snapshot for $name resume metadata`, async ({ seed }) => { + const metadata = createMetadata(seed) + const trace = createOracleCollection( + `non-resumable-metadata`, + `eager`, + metadata.api, + ) + + expect(vi.mocked(ShapeStream).mock.calls.at(-1)?.[0]).toMatchObject({ + offset: undefined, + handle: undefined, + }) + trace.subscriber([change(`insert`, 1, `full snapshot`), upToDate]) + + expect(trace.collection.status).toBe(`ready`) + expect(trace.collection.get(1)).toEqual( + expect.objectContaining({ stable: `stable-1` }), + ) + expect(metadata.state.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `resume`, offset: `20_0` }), + ) + await trace.collection.cleanup() + }) + + it(`does not mix an explicit resume option with persisted metadata`, async () => { + const metadata = createMetadata(resumeState()) + const trace = createOracleCollection( + `explicit-resume`, + `on-demand`, + metadata.api, + { handle: `explicit-handle` }, + ) + + expect(vi.mocked(ShapeStream).mock.calls.at(-1)?.[0]).toMatchObject({ + offset: `now`, + handle: `explicit-handle`, + }) + trace.subscriber([upToDate]) + expect(trace.collection.status).toBe(`ready`) + await trace.collection.cleanup() + }) + + it(`lets an equal-timestamp reset dominate a stale hydrated resume`, async () => { + let subscriber!: (messages: Array>) => void + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const options = electricCollectionOptions({ + id: `equal-timestamp-reset`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (row) => row.id, + startSync: true, + }) + options.sync.importSyncMeta?.({ + version: 1, + resume: { + kind: `resume`, + offset: `10_0`, + handle: `stale-handle`, + shapeId, + updatedAt: 1, + }, + seenTxids: [], + }) + const originalSync = options.sync + const metadata = createMetadata( + new Map([[`electric:resume`, { kind: `reset`, updatedAt: 1 }]]), + ) + const collection = createCollection({ + ...options, + sync: { + ...originalSync, + sync: (params: Parameters[0]) => + originalSync.sync({ ...params, metadata: metadata.api }), + }, + }) + + expect(vi.mocked(ShapeStream).mock.calls.at(-1)?.[0]).toMatchObject({ + offset: undefined, + handle: undefined, + }) + subscriber([change(`insert`, 1, `full snapshot`), upToDate]) + expect(collection.status).toBe(`ready`) + await collection.cleanup() + }) + + fcTest.prop( + [ + fc.integer({ min: 0, max: 3 }), + fc.integer({ min: 0, max: 3 }), + fc.integer({ min: 0, max: 3 }), + fc.string({ minLength: 1, maxLength: 8 }), + fc.string({ minLength: 1, maxLength: 8 }), + fc.string({ minLength: 1, maxLength: 8 }), + fc.boolean(), + ], + { numRuns: 50 }, + )( + `resume metadata merge is commutative and reset-safe on timestamp ties`, + ( + leftTimestamp, + rightTimestamp, + thirdTimestamp, + leftHandle, + rightHandle, + thirdHandle, + thirdIsReset, + ) => { + const options = electricCollectionOptions({ + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (row) => row.id, + }) + const merge = options.sync.mergeSyncMeta! + const left = { + version: 1, + resume: { + kind: `resume`, + offset: `10_0`, + handle: leftHandle, + shapeId, + updatedAt: leftTimestamp, + }, + seenTxids: [], + } + const right = { + version: 1, + resume: + leftTimestamp === rightTimestamp && leftHandle !== rightHandle + ? { kind: `reset`, updatedAt: rightTimestamp } + : { + kind: `resume`, + offset: `20_0`, + handle: rightHandle, + shapeId, + updatedAt: rightTimestamp, + }, + seenTxids: [], + } + const third = { + version: 1, + resume: thirdIsReset + ? { kind: `reset`, updatedAt: thirdTimestamp } + : { + kind: `resume`, + offset: `30_0`, + handle: thirdHandle, + shapeId, + updatedAt: thirdTimestamp, + }, + seenTxids: [], + } + + const leftThenRight = merge(left, right) + const rightThenLeft = merge(right, left) + expect(leftThenRight).toEqual(rightThenLeft) + expect(merge(left, left)).toEqual(left) + expect(merge(merge(left, right), third)).toEqual( + merge(left, merge(right, third)), + ) + if (leftTimestamp === rightTimestamp) { + expect(leftThenRight).toEqual( + expect.objectContaining({ + resume: expect.objectContaining({ kind: `reset` }), + }), + ) + } + }, + ) + + it(`rejects an unseen partial update from an explicit eager resume`, async () => { + const metadata = createMetadata(resumeState()) + const trace = createOracleCollection( + `explicit-eager-resume`, + `eager`, + metadata.api, + { offset: `10_0` as Offset, handle: `explicit-handle` }, + ) + + trace.subscriber([change(`update`, 1, `partial`), upToDate]) + + expect(trace.collection.status).toBe(`error`) + expect(trace.collection.has(1)).toBe(false) + expect(metadata.state.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `reset` }), + ) + await trace.collection.cleanup() + }) + + it(`accepts complete replica updates from an explicit eager resume`, async () => { + let subscriber!: (messages: Array>) => void + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const collection = createCollection( + electricCollectionOptions({ + id: `explicit-full-replica-resume`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table`, replica: `full` }, + offset: `10_0` as Offset, + handle: `explicit-handle`, + }, + syncMode: `eager`, + getKey: (row) => row.id, + startSync: true, + }), + ) + + subscriber([ + { + key: `1`, + value: { id: 1, name: `complete update`, stable: `stable-1` }, + headers: { operation: `update` }, + }, + upToDate, + ]) + + expect(collection.status).toBe(`ready`) + expect(collection.get(1)).toEqual( + expect.objectContaining({ + name: `complete update`, + stable: `stable-1`, + }), + ) + await collection.cleanup() + }) + + it(`restarts a persisted resume when hydration completion is unavailable`, async () => { + const metadata = createMetadata(resumeState()) + Object.assign(metadata.api.row, { + scanPersisted: () => Promise.resolve([{ key: 1 }]), + }) + const trace = createOracleCollection( + `unverifiable-persisted-resume`, + `eager`, + metadata.api, + ) + + expect(vi.mocked(ShapeStream).mock.calls.at(-1)?.[0]).toMatchObject({ + offset: undefined, + handle: undefined, + }) + trace.subscriber([change(`insert`, 1, `full snapshot`), upToDate]) + + expect(trace.collection.status).toBe(`ready`) + expect(trace.collection.get(1)).toEqual( + expect.objectContaining({ stable: `stable-1` }), + ) + await trace.collection.cleanup() + }) + + it(`ignores an unseen on-demand update without blocking readiness`, async () => { + const metadata = createMetadata(resumeState()) + const trace = createOracleCollection( + `on-demand-unseen-update`, + `on-demand`, + metadata.api, + ) + + trace.subscriber([change(`update`, 1, `partial`), upToDate]) + + expect(trace.collection.status).toBe(`ready`) + expect(trace.collection.has(1)).toBe(false) + expect(metadata.state.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `resume`, offset: `20_0` }), + ) + await trace.collection.cleanup() + }) + + it(`records match and txid evidence for an ignored on-demand update`, async () => { + const trace = createOracleCollection( + `on-demand-ignored-update-evidence`, + `on-demand`, + createMetadata(resumeState()).api, + ) + trace.subscriber([upToDate]) + + const pendingMatch = trace.collection.utils.awaitMatch( + (message) => `value` in message && message.value.id === 808, + 100, + ) + const pendingTxid = trace.collection.utils.awaitTxId(808, 100) + const update = change(`update`, 808, `ignored`) as ChangeMessage + update.headers.txids = [808] + + trace.subscriber([update, upToDate]) + + await Promise.all([ + expect(pendingMatch).resolves.toBe(true), + expect(pendingTxid).resolves.toBe(true), + ]) + expect(trace.collection.has(808)).toBe(false) + await trace.collection.cleanup() + }) +}) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index c913f8d973..1e1cc201be 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -376,6 +376,76 @@ describe(`Electric Integration`, () => { ) }) + it(`ignores an update for a key that has never been materialized`, () => { + subscriber([ + { + key: `2`, + value: { id: 2, name: `Only changed columns` }, + headers: { operation: `update` }, + }, + { headers: { control: `up-to-date` } }, + ]) + + expect(collection.has(2)).toBe(false) + }) + + it(`accepts an update after an insert for the same key in one batch`, () => { + subscriber([ + { + key: `2`, + value: { id: 2, name: `Initial value` }, + headers: { operation: `insert` }, + }, + { + key: `2`, + value: { id: 2, name: `Updated value` }, + headers: { operation: `update` }, + }, + { headers: { control: `up-to-date` } }, + ]) + + expect(collection.get(2)?.name).toBe(`Updated value`) + }) + + it(`accepts a progressive update after its insert in an earlier callback`, () => { + let testSubscriber!: (messages: Array>) => void + mockSubscribe.mockImplementation((callback) => { + testSubscriber = callback + return () => {} + }) + + const testCollection = createCollection( + electricCollectionOptions({ + id: `progressive-split-insert-update-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + + testSubscriber([ + { + key: `2`, + value: { id: 2, name: `Initial value` }, + headers: { operation: `insert` }, + }, + ]) + testSubscriber([ + { + key: `2`, + value: { id: 2, name: `Updated value` }, + headers: { operation: `update` }, + }, + ]) + testSubscriber([{ headers: { control: `up-to-date` } }]) + + expect(testCollection.get(2)?.name).toBe(`Updated value`) + }) + it(`should handle delete operations`, () => { // Insert and commit subscriber([ @@ -4163,6 +4233,120 @@ describe(`Electric Integration`, () => { }), ) }) + + it(`refuses an update for an unseen key and invalidates persisted resume state`, () => { + const metadataHarness = createInMemorySyncMetadataApi( + new Map([ + [ + `electric:resume`, + { + kind: `resume`, + offset: `10_0`, + handle: `shape-1`, + shapeId: `{"params":{"table":"test_table"},"url":"http://test-url"}`, + updatedAt: 1, + }, + ], + ]), + ) + mockStream.shapeHandle = `shape-1` + mockStream.lastOffset = `11_0` + + const baseOptions = electricCollectionOptions({ + id: `unseen-update-resume-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (item) => item.id as number, + startSync: true, + }) + const originalSync = baseOptions.sync + const testCollection = createCollection({ + ...baseOptions, + sync: { + sync: (params: Parameters[0]) => + originalSync.sync({ ...params, metadata: metadataHarness.api }), + }, + }) + + subscriber([ + { + key: `2`, + value: { id: 2, name: `Changed without immutable fields` }, + headers: { operation: `update` }, + }, + { headers: { control: `up-to-date` } }, + ]) + + expect(testCollection.has(2)).toBe(false) + expect(metadataHarness.collectionMetadata.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `reset` }), + ) + expect(testCollection.status).toBe(`error`) + }) + + it(`rejects a resumed batch that updates a key after deleting it`, () => { + const metadataHarness = createInMemorySyncMetadataApi( + new Map([ + [ + `electric:resume`, + { + kind: `resume`, + offset: `10_0`, + handle: `shape-1`, + shapeId: `{"params":{"table":"test_table"},"url":"http://test-url"}`, + updatedAt: 1, + }, + ], + ]), + ) + const baseOptions = electricCollectionOptions({ + id: `delete-then-update-resume-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + getKey: (item) => item.id as number, + startSync: true, + }) + const originalSync = baseOptions.sync + const testCollection = createCollection({ + ...baseOptions, + sync: { + sync: (params: Parameters[0]) => + originalSync.sync({ ...params, metadata: metadataHarness.api }), + }, + }) + + subscriber([ + { + key: `2`, + value: { id: 2, name: `Complete row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `up-to-date` } }, + ]) + expect(testCollection.has(2)).toBe(true) + + subscriber([ + { + key: `2`, + value: { id: 2 }, + headers: { operation: `delete` }, + }, + { + key: `2`, + value: { id: 2, name: `Partial replacement` }, + headers: { operation: `update` }, + }, + ]) + + expect(testCollection.get(2)?.name).toBe(`Complete row`) + expect(metadataHarness.collectionMetadata.get(`electric:resume`)).toEqual( + expect.objectContaining({ kind: `reset` }), + ) + }) }) // Tests for overlapping subset queries with duplicate keys diff --git a/packages/electric-db-collection/tests/tags.test.ts b/packages/electric-db-collection/tests/tags.test.ts index 2aa765ef2d..48af9c6b15 100644 --- a/packages/electric-db-collection/tests/tags.test.ts +++ b/packages/electric-db-collection/tests/tags.test.ts @@ -688,6 +688,42 @@ describe(`Electric Tag Tracking and GC`, () => { expect(collection.state.get(3)).toEqual({ id: 3, name: `User 3` }) }) + it(`does not accept a partial update after move-out deletes a row`, () => { + subscriber([ + { + key: `1`, + value: { id: 1, name: `complete`, stable: `preserved` }, + headers: { + operation: `insert`, + tags: [`hash1/hash2/hash3`], + }, + }, + { headers: { control: `up-to-date` } }, + ]) + + subscriber([ + { + headers: { + event: `move-out`, + patterns: [{ pos: 0, value: `hash1` }], + }, + }, + { headers: { control: `up-to-date` } }, + ]) + expect(collection.has(1)).toBe(false) + + subscriber([ + { + key: `1`, + value: { id: 1, name: `partial` }, + headers: { operation: `update` }, + }, + { headers: { control: `up-to-date` } }, + ]) + + expect(collection.has(1)).toBe(false) + }) + it(`should remove shared tags from all rows when move-out pattern matches`, () => { // Create tags where some are shared between rows const sharedTag1 = `hash1/hash2/hash3` // Shared by rows 1 and 2 diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index e6d95e3f57..6497de6f04 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -1570,14 +1570,20 @@ export function queryCollectionOptions( newItemsMap.forEach((newItem, key) => { const owners = getPersistedOwners(key) - if (!owners.has(hashedQueryKey)) { + const addsOwner = !owners.has(hashedQueryKey) + const insertsRow = !currentSyncedItems.has(key) + if (addsOwner) { owners.add(hashedQueryKey) - setPersistedOwners(key, owners) } addRowOwner(key, hashedQueryKey) - if (!currentSyncedItems.has(key)) { + if (insertsRow) { write({ type: `insert`, value: newItem }) } + if (addsOwner || insertsRow) { + // An insert clears stale metadata for its key. Stage ownership + // afterward so rows and ownership commit as one state change. + setPersistedOwners(key, owners) + } }) const applied = commit(signal) @@ -1937,6 +1943,12 @@ export function queryCollectionOptions( const hasListeners = observer?.hasListeners() ?? false + // Refcounts are explicit ownership tokens. A cache event can remove the + // observer while an eager or active acquisition still owns this query. + if (refcount > 0) { + return + } + if (hasListeners) { // During invalidateQueries, TanStack Query keeps internal listeners alive. // Leave refcount at 0 but keep observer so it can resubscribe. @@ -1944,16 +1956,6 @@ export function queryCollectionOptions( return } - // No listeners means the query is truly idle. - // Even if refcount > 0, we treat hasListeners as authoritative to prevent leaks. - // This can happen if subscriptions are GC'd without calling unloadSubset. - if (refcount > 0) { - console.warn( - `[cleanupQueryIfIdle] Invariant violation: refcount=${refcount} but no listeners. Cleaning up to prevent leak.`, - { hashedQueryKey }, - ) - } - if ( effectivePersistedGcTime !== undefined && metadata && diff --git a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts index ef9baa5da9..cf153b2475 100644 --- a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts +++ b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts @@ -1,7 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { QueryClient } from '@tanstack/query-core' import { createCollection, eq } from '@tanstack/db' -import { expectAssertionFailure } from '../../db/tests/expected-failure.js' import { TraceAssertionError } from '../../db/tests/trace-runner.js' import { queryCollectionOptions } from '../src/query.js' import type { Collection, SyncMetadataApi } from '@tanstack/db' @@ -140,102 +139,6 @@ function assertCheckpoint( } } -function asRecords({ - actual, - expected, -}: { - actual: unknown - expected: unknown -}): - | { - observed: Record - wanted: Record - } - | undefined { - if ( - !actual || - typeof actual !== `object` || - !expected || - typeof expected !== `object` - ) { - return undefined - } - - return { - observed: actual as Record, - wanted: expected as Record, - } -} - -function classifyEagerOwnerLoss(difference: { - actual: unknown - expected: unknown -}): boolean { - const records = asRecords(difference) - if (!records) return false - const { observed, wanted } = records - return ( - observed.status === `ready` && - Array.isArray(observed.rows) && - observed.rows.length === 0 && - observed.owners === 0 && - wanted.status === `ready` && - Array.isArray(wanted.rows) && - wanted.rows.length === 1 && - wanted.rows[0] === shared.id && - wanted.owners === 1 - ) -} - -function classifyInsertedOwnerMetadataLoss(difference: { - actual: unknown - expected: unknown -}): boolean { - const records = asRecords(difference) - if (!records) return false - const { observed, wanted } = records - return ( - Array.isArray(observed.persistedOwners) && - observed.persistedOwners.length === 0 && - Array.isArray(observed.metadataSetKeys) && - observed.metadataSetKeys.length === 1 && - observed.metadataSetKeys[0] === shared.id && - Array.isArray(wanted.persistedOwners) && - wanted.persistedOwners.length === 1 && - typeof wanted.persistedOwners[0] === `string` && - Array.isArray(wanted.metadataSetKeys) && - wanted.metadataSetKeys.length === 1 && - wanted.metadataSetKeys[0] === shared.id - ) -} - -function sameArray(actual: unknown, expected: unknown): boolean { - return ( - Array.isArray(actual) && - Array.isArray(expected) && - actual.length === expected.length && - actual.every((value, index) => value === expected[index]) - ) -} - -function classifyPersistedBaselineLoss(difference: { - actual: unknown - expected: unknown -}): boolean { - const records = asRecords(difference) - if (!records) return false - const { observed, wanted } = records - return ( - sameArray(observed.liveOwners, wanted.liveOwners) && - sameArray(observed.persistedOwners, wanted.insertedOwners) && - Array.isArray(observed.insertedOwners) && - observed.insertedOwners.length === 0 && - Array.isArray(wanted.persistedOwners) && - wanted.persistedOwners.length === 2 && - sameArray(observed.metadataSetKeys, wanted.metadataSetKeys) - ) -} - function recordMetadataWrites( metadata: SyncMetadataApi, recorder: MetadataRecorder, @@ -559,8 +462,8 @@ describe(`query collection ownership lifecycle oracle`, () => { ) }) - it(`#1631 keeps the eager owner when its last collection listener departs`, async () => { - const id = `ownership-eager-listener-1631` + it(`keeps the eager owner when its last collection listener departs`, async () => { + const id = `ownership-eager-listener` const { collection, maps, queryClient } = createOwnershipFixture({ id, syncMode: `eager`, @@ -589,40 +492,31 @@ describe(`query collection ownership lifecycle oracle`, () => { // without making the defect boundary depend on a timer. queryClient.removeQueries({ queryKey: [id], exact: true }) - const assertOwnerSurvives = expectAssertionFailure( - () => - Promise.resolve().then(() => { - assertCheckpoint( - 2, - { - status: collection.status, - rows: collectionRows(collection), - owners: ownersOf(maps, shared.id).length, - }, - { status: `ready`, rows: [shared.id], owners: 1 }, - ) - }), + assertCheckpoint( + 2, { - checkpoint: 2, - classify: classifyEagerOwnerLoss, + status: collection.status, + rows: collectionRows(collection), + owners: ownersOf(maps, shared.id).length, + ownedRows: rowsOwnedBy(maps, queryHash), + }, + { + status: `ready`, + rows: [shared.id], + owners: 1, + ownedRows: [shared.id], }, ) - - await assertOwnerSurvives() - expect(warning).toHaveBeenCalledOnce() - expect(warning).toHaveBeenCalledWith( - expect.stringContaining(`[cleanupQueryIfIdle]`), - { hashedQueryKey: queryHash }, - ) + expect(warning).not.toHaveBeenCalled() } finally { warning.mockRestore() } }) - it(`#1656 keeps the first persisted owner when a second query inserts another row`, async () => { + it(`keeps every persisted owner when overlapping queries insert rows`, async () => { const metadataRecorder: MetadataRecorder = { rowWrites: [] } const { collection, maps } = createOwnershipFixture({ - id: `ownership-persisted-baseline-1656`, + id: `ownership-persisted-baseline`, results: [[shared], [shared, listOnly]], metadataRecorder, }) @@ -631,59 +525,41 @@ describe(`query collection ownership lifecycle oracle`, () => { await collection._sync.loadSubset(detailSubset) const detailHash = onlyOwner(maps, shared.id) - // The production metadata API records the owner write, but the insert's - // commit currently loses it. Accept only that exact #1656 boundary. - const assertInsertedOwnerPersists = expectAssertionFailure( - () => - Promise.resolve().then(() => { - assertCheckpoint( - 0, - { - persistedOwners: persistedOwners( - collection._state.syncedMetadata, - shared.id, - ), - metadataSetKeys: setMetadataKeys(metadataRecorder), - }, - { persistedOwners: [detailHash], metadataSetKeys: [shared.id] }, - ) - }), - { checkpoint: 0, classify: classifyInsertedOwnerMetadataLoss }, + assertCheckpoint( + 0, + { + persistedOwners: persistedOwners( + collection._state.syncedMetadata, + shared.id, + ), + metadataSetKeys: setMetadataKeys(metadataRecorder), + }, + { persistedOwners: [detailHash], metadataSetKeys: [shared.id] }, ) - await assertInsertedOwnerPersists() await collection._sync.loadSubset(listSubset) const listHash = otherOwner(maps, shared.id, detailHash) - // A second insert loses its own owner and rebuilds the persisted baseline - // with only the later query, while the in-memory ownership remains sound. - const assertPersistedBaselineSurvives = expectAssertionFailure( - () => - Promise.resolve().then(() => { - assertCheckpoint( - 1, - { - liveOwners: ownersOf(maps, shared.id), - persistedOwners: persistedOwners( - collection._state.syncedMetadata, - shared.id, - ), - insertedOwners: persistedOwners( - collection._state.syncedMetadata, - listOnly.id, - ), - metadataSetKeys: setMetadataKeys(metadataRecorder), - }, - { - liveOwners: sorted([detailHash, listHash]), - persistedOwners: sorted([detailHash, listHash]), - insertedOwners: [listHash], - metadataSetKeys: [listOnly.id, shared.id], - }, - ) - }), - { checkpoint: 1, classify: classifyPersistedBaselineLoss }, + assertCheckpoint( + 1, + { + liveOwners: ownersOf(maps, shared.id), + persistedOwners: persistedOwners( + collection._state.syncedMetadata, + shared.id, + ), + insertedOwners: persistedOwners( + collection._state.syncedMetadata, + listOnly.id, + ), + metadataSetKeys: setMetadataKeys(metadataRecorder), + }, + { + liveOwners: sorted([detailHash, listHash]), + persistedOwners: sorted([detailHash, listHash]), + insertedOwners: [listHash], + metadataSetKeys: [listOnly.id, shared.id], + }, ) - await assertPersistedBaselineSurvives() collection._sync.unloadSubset(listSubset) assertCheckpoint( diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index b6f8266813..9f4c38f70a 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -7401,11 +7401,7 @@ describe(`QueryCollection`, () => { } }) - it(`should reset refcount after query GC and reload (stale refcount bug)`, async () => { - // This test catches Bug 2: stale refcounts after GC/remove - // When TanStack Query GCs a query, the refcount should be cleaned up - // Otherwise, reloading the same subset will start with a stale count - + it(`should reload a released subset without retaining a stale refcount`, async () => { const baseQueryKey = [`stale-refcount-test`] const items: Array = [ { id: `1`, name: `Item 1`, category: `A` }, @@ -7443,13 +7439,17 @@ describe(`QueryCollection`, () => { expect(collection.size).toBe(2) }) - // Force GC by calling removeQueries (simulates gcTime expiry) + // Release the first acquisition before its cache entry is removed. + // Cache events do not revoke active collection ownership. + await query1.cleanup() + await vi.waitFor(() => { + expect(collection.size).toBe(0) + }) + + // Force GC by calling removeQueries (simulates gcTime expiry). queryClient.removeQueries({ queryKey: baseQueryKey }) await flushPromises() - // BUG: queryRefCounts still has stale count, wasn't cleaned up by cleanupQuery - // When we load again, the refcount will be wrong (starts at 1 instead of 0, or accumulates) - // Reload the same query const query2 = createLiveQueryCollection({ query: (q) => @@ -7466,14 +7466,11 @@ describe(`QueryCollection`, () => { expect(collection.size).toBe(2) }) - // Cleanup - this should properly decrement from 1 to 0 and clean up + // Cleanup should decrement the new acquisition from one to zero. await query2.cleanup() await vi.waitFor(() => { - expect(collection.size).toBe(0) // Should be cleaned up + expect(collection.size).toBe(0) }) - - // BUG SYMPTOM: If refcount was stale (e.g. was 2, decremented to 1), - // the observer won't be destroyed and data won't be cleaned up }) it(`should handle mount/unmount/remount without breaking cache (destroyed observer bug)`, async () => { diff --git a/packages/react-db/tests/useLiveQuery.test.tsx b/packages/react-db/tests/useLiveQuery.test.tsx index 39396bdbfa..49f7207c25 100644 --- a/packages/react-db/tests/useLiveQuery.test.tsx +++ b/packages/react-db/tests/useLiveQuery.test.tsx @@ -13,6 +13,7 @@ import { gt, lte, sum, + toArray, } from '@tanstack/db' import { useEffect } from 'react' import { useLiveQuery } from '../src/useLiveQuery' @@ -2787,6 +2788,94 @@ describe(`Query Collections`, () => { expect(alphaRenderCount).toBe(settledAlphaRenders + 1) expect(betaRenderCount).toBe(settledBetaRenders) }) + + it(`keeps nested array includes on the render after a parent update`, async () => { + type Document = { + id: string + name: string + schemaId: string + } + type Schema = { + id: string + name: string + } + type Field = { + id: string + schemaId: string + name: string + } + + const documents = createCollection( + mockSyncCollectionOptions({ + id: `includes-react-documents`, + getKey: (document) => document.id, + initialData: [{ id: `d1`, name: `Before`, schemaId: `s1` }], + }), + ) + const schemas = createCollection( + mockSyncCollectionOptions({ + id: `includes-react-schemas`, + getKey: (schema) => schema.id, + initialData: [{ id: `s1`, name: `Schema` }], + }), + ) + const fields = createCollection( + mockSyncCollectionOptions({ + id: `includes-react-fields`, + getKey: (field) => field.id, + initialData: [{ id: `f1`, schemaId: `s1`, name: `Title` }], + }), + ) + + const { result } = renderHook(() => + useLiveQuery((q) => + q.from({ document: documents }).select(({ document }) => ({ + id: document.id, + name: document.name, + schema: toArray( + q + .from({ schema: schemas }) + .where(({ schema }) => eq(schema.id, document.schemaId)) + .select(({ schema }) => ({ + id: schema.id, + fields: toArray( + q + .from({ field: fields }) + .where(({ field }) => eq(field.schemaId, schema.id)) + .select(({ field }) => ({ + id: field.id, + name: field.name, + })), + ), + })), + ), + })), + ), + ) + + await waitFor(() => { + expect(result.current.data[0]).toMatchObject({ + name: `Before`, + schema: [{ id: `s1`, fields: [{ id: `f1`, name: `Title` }] }], + }) + }) + + act(() => { + documents.utils.begin() + documents.utils.write({ + type: `update`, + value: { id: `d1`, name: `After`, schemaId: `s1` }, + }) + documents.utils.commit() + }) + + await waitFor(() => { + expect(result.current.data[0]).toMatchObject({ + name: `After`, + schema: [{ id: `s1`, fields: [{ id: `f1`, name: `Title` }] }], + }) + }) + }) }) describe(`SSR hydration`, () => { diff --git a/packages/solid-db/tests/useLiveQuery.test.tsx b/packages/solid-db/tests/useLiveQuery.test.tsx index debfe95a51..85f9144fc1 100644 --- a/packages/solid-db/tests/useLiveQuery.test.tsx +++ b/packages/solid-db/tests/useLiveQuery.test.tsx @@ -8,6 +8,7 @@ import { createOptimisticAction, eq, gt, + toArray, } from '@tanstack/db' import { For, @@ -2738,3 +2739,152 @@ describe(`Query Collections`, () => { }) }) }) + +describe(`includes subqueries`, () => { + type Project = { + id: number + name: string + } + + type ProjectIssue = { + id: number + projectId: number + title: string + } + + function includedIssues(value: unknown): Array { + if (Array.isArray(value)) { + return value as Array + } + if ( + value !== null && + typeof value === `object` && + `toArray` in value && + Array.isArray(value.toArray) + ) { + return value.toArray as Array + } + return [] + } + + it(`updates a rendered array include after a child insert`, async () => { + const projects = createCollection( + mockSyncCollectionOptions({ + id: `includes-solid-array-projects`, + getKey: (project) => project.id, + initialData: [ + { id: 1, name: `Alpha` }, + { id: 2, name: `Beta` }, + ], + }), + ) + const issues = createCollection( + mockSyncCollectionOptions({ + id: `includes-solid-array-issues`, + getKey: (issue) => issue.id, + initialData: [ + { id: 10, projectId: 1, title: `Bug in Alpha` }, + { id: 20, projectId: 2, title: `Bug in Beta` }, + ], + }), + ) + + function TestComponent() { + const query = useLiveQuery((q) => + q.from({ project: projects }).select(({ project }) => ({ + id: project.id, + issueTitles: toArray( + q + .from({ issue: issues }) + .where(({ issue }) => eq(issue.projectId, project.id)) + .select(({ issue }) => ({ + id: issue.id, + title: issue.title, + })), + ), + })), + ) + + return ( + + {(project) => ( +

+ {project.issueTitles.map((issue) => issue.title).join(`|`)} +

+ )} + + ) + } + + const rendered = render(() => ) + await waitFor(() => { + expect(rendered.getByTestId(`project-1`).textContent).toBe(`Bug in Alpha`) + }) + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 11, projectId: 1, title: `Feature for Alpha` }, + }) + issues.utils.commit() + + await waitFor(() => { + expect(rendered.getByTestId(`project-1`).textContent).toBe( + `Bug in Alpha|Feature for Alpha`, + ) + }) + }) + + it(`populates an initially empty collection include after its first child insert`, async () => { + const projects = createCollection( + mockSyncCollectionOptions({ + id: `includes-solid-empty-projects`, + getKey: (project) => project.id, + initialData: [{ id: 1, name: `Alpha` }], + }), + ) + const issues = createCollection( + mockSyncCollectionOptions({ + id: `includes-solid-empty-issues`, + getKey: (issue) => issue.id, + initialData: [], + }), + ) + + const rendered = renderHook(() => + useLiveQuery((q) => + q.from({ project: projects }).select(({ project }) => ({ + id: project.id, + issues: q + .from({ issue: issues }) + .where(({ issue }) => eq(issue.projectId, project.id)) + .select(({ issue }) => ({ + id: issue.id, + projectId: issue.projectId, + title: issue.title, + })), + })), + ), + ) + + await waitFor(() => { + expect(rendered.result.isReady).toBe(true) + expect(includedIssues(rendered.result()[0]?.issues)).toEqual([]) + }) + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 10, projectId: 1, title: `Bug in Alpha` }, + }) + issues.utils.commit() + + await waitFor(() => { + expect( + includedIssues(rendered.result()[0]?.issues).map( + (issue) => issue.title, + ), + ).toEqual([`Bug in Alpha`]) + }) + }) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 943e48209a..19b94ea939 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1276,13 +1276,13 @@ importers: '@tanstack/db': specifier: workspace:* version: link:../db - '@tanstack/store': - specifier: ^0.9.2 - version: 0.9.2 debug: specifier: ^4.4.3 version: 4.4.3 devDependencies: + '@tanstack/query-core': + specifier: ^5.90.20 + version: 5.90.20 '@types/debug': specifier: ^4.1.12 version: 4.1.12