From 036e9fb95216dc19517ff89941a92d2131f1ae7e Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 29 Oct 2026 11:48:03 -0600 Subject: [PATCH 01/64] test(conformance): add the eager live-source adapter conformance oracle Add the private @tanstack/db-adapter-conformance package with a shared eager live-source oracle (#2118): an independent row-Map model, a capability-gated history grammar, a production driver contract, and a refinement check at settled checkpoints plus every publication. Drivers: a reference push adapter with 20 hostile mutants, Neon Realtime 0.1.1 (vendored, expected RED, `test:neon`), and TrailBase eager mode. - Law: READY-2/3/4, HEALTH-1/2/5, PUB-1/2/3/7, FID-4, LIFE-2/5/10, EVID-1/2/3, RETRY-1, RECON-1, RENEW-1 from the adapter law set v18. Authority: the sync callback contract in packages/db/src/types.ts and the status and health terms in docs/contributing/glossary.md. - Reach: fixed campaign (seed 2118, 400 histories) reaches all 14 declared states, fewest 22 (transient-failure-recovered); lowest count across 20 other seeds is 11. Minimum per campaign: 5. - RED: Neon fails READY-4 (status loading where error is owed) and HEALTH-1 (health.error null) because it calls the removed markError. A probe with markError mapped to fail then fails LIFE-10. - GREEN: reference oracle 46/46; TrailBase suite 151/151. - Mutant: each of 20 reference-adapter mutants is rejected with its law, including reverses-batch (PUB-1, per-change) and keeps-failed-source (LIFE-10). ORC-004 reconstruction, exclusion, and ablation controls pass. - Limits: no on-demand, mutations, or reload; publications observed only through subscribeChanges; EVID-3 does not detect a timed wait that rejects before its timeout; Neon and TrailBase claims cover controlled providers, not real clients. Co-authored-by: Isaac --- .prettierignore | 1 + docs/contributing/oracle-coverage.md | 34 + eslint.config.js | 2 + packages/db-adapter-conformance/MIGRATION.md | 74 + packages/db-adapter-conformance/README.md | 161 +++ packages/db-adapter-conformance/package.json | 59 + .../src/eager-live-source-oracle.ts | 1229 +++++++++++++++++ packages/db-adapter-conformance/src/index.ts | 24 + .../neon-realtime-oracle.property.test.ts | 79 ++ .../tests/neon/neon-realtime-driver.ts | 282 ++++ .../tests/neon/vendor/LICENSE | 202 +++ .../tests/neon/vendor/collection-options.ts | 335 +++++ .../tests/neon/vendor/no-infer.ts | 6 + .../reference-adapter-oracle.property.test.ts | 385 ++++++ .../tests/reference-adapter.ts | 387 ++++++ packages/db-adapter-conformance/tsconfig.json | 22 + .../db-adapter-conformance/vite.config.ts | 24 + .../vitest.neon.config.ts | 34 + packages/db/tests/oracle-config.ts | 3 + .../trailbase-db-collection/tests/ORACLE.md | 11 + .../tests/conformance-driver.ts | 162 +++ ...source-conformance-oracle.property.test.ts | 71 + 22 files changed, 3587 insertions(+) create mode 100644 packages/db-adapter-conformance/MIGRATION.md create mode 100644 packages/db-adapter-conformance/README.md create mode 100644 packages/db-adapter-conformance/package.json create mode 100644 packages/db-adapter-conformance/src/eager-live-source-oracle.ts create mode 100644 packages/db-adapter-conformance/src/index.ts create mode 100644 packages/db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts create mode 100644 packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts create mode 100644 packages/db-adapter-conformance/tests/neon/vendor/LICENSE create mode 100644 packages/db-adapter-conformance/tests/neon/vendor/collection-options.ts create mode 100644 packages/db-adapter-conformance/tests/neon/vendor/no-infer.ts create mode 100644 packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts create mode 100644 packages/db-adapter-conformance/tests/reference-adapter.ts create mode 100644 packages/db-adapter-conformance/tsconfig.json create mode 100644 packages/db-adapter-conformance/vite.config.ts create mode 100644 packages/db-adapter-conformance/vitest.neon.config.ts create mode 100644 packages/trailbase-db-collection/tests/conformance-driver.ts create mode 100644 packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts diff --git a/.prettierignore b/.prettierignore index 20eda14e54..bb7a4df2e7 100644 --- a/.prettierignore +++ b/.prettierignore @@ -9,3 +9,4 @@ pnpm-lock.yaml **/routeTree.gen.ts packages/convex-db-collection/e2e/backend/.convex/ packages/convex-db-collection/e2e/backend/convex/_generated/ +packages/db-adapter-conformance/tests/neon/vendor/ diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index a6616dd4df..756a603966 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -5965,3 +5965,37 @@ controls are separate files. The [chunk 26 record](oracle-reviews/2026-10-28-pr- retains exact source/evidence hashes and all ORC outcomes. Actual production bundle erasure, Typedoc generation, tooling GREEN and restored-old-tooling mutants remain required receiving checks. Generated docs remain CI-owned. + +## Adapter conformance: eager live sources (#2118) + +`packages/db-adapter-conformance/src/eager-live-source-oracle.ts` owns the +shared eager live-source laws from the adapter conformance law set (v18): +READY-2/3/4, HEALTH-1, PUB-1/2/3/7, FID-4, LIFE-2/5/10, HEALTH-5, EVID-1/2/3 +(`await-evidence`), HEALTH-2 and RETRY-1 for retried failures +(`retry-policy`), RECON-1 (`transport-reconnect`), and RENEW-1 +(`capability-renewal`). Its model is an independent row Map and sync-run +phase; drivers supply only a controlled provider and declare whether they +commit per batch or per change. PUB-1 checks every change publication during +a step, not only the settled checkpoint. The reference push adapter +(`tests/reference-adapter-oracle.property.test.ts`, +`adapter-conformance.eager-live-source.reference`) runs fixed and random +campaigns of 400 histories with declared reach, a per-change variant, and 20 +law-named hostile mutants, with ORC-004 reconstruction, exclusion, and +ablation controls. The Neon Realtime receiving driver +(`adapter-conformance.eager-live-source.neon-realtime`, `test:neon`) runs +`@neon/realtime-tanstack` 0.1.1 vendored at neon-pkgs `7ac55e3e` and is RED on +READY-4 and HEALTH-1 until that adapter moves to `fail()`; probes show LIFE-10 +(subscription kept open after an error state) behind them. Limits: no +on-demand loading, mutation handlers, or reload; publications are observed +through `subscribeChanges` only; EVID-3 does not distinguish a timed wait +that rejects before its timeout; the Neon and TrailBase claims cover +controlled providers, not real clients. Owner: +`packages/db-adapter-conformance`. Evidence: the package README. +Migration goal (maintainer, 2026-10-29): every collection adapter gets a +driver for each conformance family it declares; the plan and per-adapter +status are in `packages/db-adapter-conformance/MIGRATION.md`. Existing +adapter oracles keep their laws until a shared family checks the same law, +history, path, and observation. TrailBase (eager mode, per-change commits, +`adapter-conformance.eager-live-source.trailbase`) is GREEN; its lifecycle +oracle keeps its laws, so eager startup failure and stream-end health have +two owners until this entry moves them. diff --git a/eslint.config.js b/eslint.config.js index 8a93d67d2f..c1ad9142cd 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -12,6 +12,8 @@ export default [ `**/traildepot/**`, `examples/angular/**`, `packages/db-collection-e2e/vite.config.ts`, + // Third-party adapter source vendored unchanged as a conformance fixture. + `packages/db-adapter-conformance/tests/neon/vendor/**`, `packages/capacitor-db-sqlite-persistence/e2e/app/android/**`, `packages/capacitor-db-sqlite-persistence/e2e/app/ios/**`, // Expo expects Metro config in CommonJS format. diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md new file mode 100644 index 0000000000..d0bd048b52 --- /dev/null +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -0,0 +1,74 @@ +# Migrating every collection adapter onto the conformance oracles + +Goal (maintainer, 2026-10-29): every collection adapter runs the shared +conformance oracles through its own production driver, not only Neon. An +adapter is migrated when it has a driver for every oracle family whose +capabilities it declares in the law set's section A, and each driver is GREEN +or has a recorded RED with an owner. + +Migration does not delete an adapter's existing oracles. An existing owner +keeps its laws until a shared family checks the same law, history, path, and +observation. Then the coverage map moves the law to the shared owner, and the +old test either stays as supplemental coverage or is removed in its own +reviewed change. + +## Oracle families + +| Family | Status | Laws | +| ------------------------------------- | ----------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| Eager live source | Built (`src/eager-live-source-oracle.ts`) | READY-2/3/4, HEALTH-1/2/5, PUB-1/2/3/7, FID-4, LIFE-2/5/10, EVID-1/2/3, RETRY-1 (eager), RECON-1, RENEW-1 | +| Mutations and confirmation | Needed | MUT-1/2/4/5/7/10/13, FID-5 | +| On-demand loading | Needed | DEM-1 to DEM-23, MAP-1 to MAP-4 | +| Reload and retry | Needed | RELOAD-1 to RELOAD-5; RETRY-1 and HEALTH-2 beyond the eager source's retried failures | +| Client cache and refetch | Needed | CACHE-1 to CACHE-8, REF-1 to REF-7, DIR-1 to DIR-5 | +| Source shapes | Needed | RESET-1, RESUME-1 to RESUME-3, PROG-1, MEMB-1/2, PAGE-1 to PAGE-3, OWN-1/2, META-1 | +| Local and shared stores | Needed | SELF-1 to SELF-3, STORE-1 to STORE-4, PEER-1 | +| Persistence wrapper and storage hosts | Needed | P-_, S-_ | + +## Adapters + +| Adapter | Families that apply (section A) | Driver status | +| ----------------- | ------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Neon Realtime | eager live source, mutations | Eager: RED (READY-4, HEALTH-1, and LIFE-10 behind them); adapter update pending | +| Electric | eager live source, mutations, on-demand, source shapes, reload and retry, persistence | Not started | +| PowerSync | eager live source, mutations, on-demand | Not started | +| RxDB | eager live source, mutations | Not started | +| TrailBase | eager live source, mutations, on-demand | Eager: GREEN (2026-10-29), per-change commits. Driver `trailbase-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.trailbase`; declares no optional capability. Mutations, on-demand: not started | +| Convex | eager live source, mutations, on-demand, source shapes | Not started | +| query-db | client cache and refetch, on-demand, mutations, reload and retry, persistence | Not started; not a push source, so the eager family does not apply | +| local-only | local and shared stores, mutations | Not started | +| localStorage | local and shared stores, mutations | Not started | +| Persisted wrapper | persistence wrapper | Not started | +| Storage hosts (8) | storage hosts | Not started | + +## Suggested order + +1. **Push adapters on the existing family:** RxDB, TrailBase (done), PowerSync, Electric (eager mode), and Convex. Each needs only a controlled-provider driver. A RED result is a finding against that adapter. +2. **The Neon update, run against its driver.** This shows an adapter moving from RED to GREEN. +3. **The mutations family.** It applies to every adapter except the storage hosts. +4. **The other families,** one at a time. Each family's new driver work goes to the adapters that declare it. + +## Migration log + +- **2026-10-29, TrailBase, eager live source.** Both campaigns are GREEN. + - **Reach:** every declared state reached by at least five histories. + - **Driver evidence:** a temporary adapter edit, `fail()` only on a closed stream, was rejected with HEALTH-1 by both campaigns, then reverted. + - **Oracle changes this migration exposed:** + - PUB-7 replacement is now gated on `complete-snapshot`, because TrailBase lists once per sync run. + - The check now decides whether a driver supports a step before it moves the model. + - **Ownership:** `lifecycle-oracle.property.test.ts` keeps its laws. It also checks eager startup failure and stream-end health, so those laws now have two owners until the coverage map moves them. +- **2026-10-29, eager live source, closed limits.** The family now also + checks PUB-1 publications during each step, LIFE-10 release at failure, + HEALTH-5 detached failures and console diagnostics, EVID-3 timeouts, and + retried failures (`retry-policy`: HEALTH-2, RETRY-1). The evidence is in + `README.md`. + - **TrailBase** stays GREEN on the stronger check, with `per-change` + commits. It found no TrailBase defect. It found one checker defect: the + PUB-1 match took the first equal state, so a batch that returned to an + earlier state was flagged. The match now searches forward from the last + matched commit. + - **Neon** stays RED with READY-4 and HEALTH-1. Two probes, not kept, + showed what is behind them. With `markError` mapped to `fail`, Neon fails + LIFE-10: it keeps the raw subscription open after an error state until + cleanup. With that subscription also released once at the error state, + 300 histories pass. These two changes are the Neon update's scope. diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md new file mode 100644 index 0000000000..719ae519cb --- /dev/null +++ b/packages/db-adapter-conformance/README.md @@ -0,0 +1,161 @@ +# @tanstack/db-adapter-conformance + +Conformance oracles for TanStack DB sync adapters (issue #2118). An adapter +author supplies a production driver; the package supplies the laws, an +independent model, a history grammar, and the check that compares the +adapter's public behavior with the model at each checkpoint. + +The package is private while its first oracle settles. + +## Eager live-source oracle + +`src/eager-live-source-oracle.ts` checks an eager Collection whose provider +pushes a complete result and then committed change batches. Its opening +comment states the laws and their limits. In short: + +| Law | Checked behavior | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| READY-2, READY-3 | `loading` until the first complete delivery; then `ready` with exactly its rows | +| READY-4 | A failure before ready: status `error`, `preload()` rejects with the value, `health.error.cause` is the value | +| HEALTH-1 | A failure after ready: status stays `ready`, rows kept, `health.error.cause` is the value | +| PUB-1 | Every change publication during a step is one whole committed state, in commit order; per batch or per change, as the driver declares | +| PUB-2, PUB-3, PUB-7, FID-4 | Change events fold to the rows; batches fold onto rows; a complete delivery replaces (PUB-7 with `complete-snapshot`) | +| LIFE-2, LIFE-5 | Cleanup releases the provider subscription; a restart is a fresh, empty sync run | +| LIFE-10 | A failed provider's subscription is released at the failure, and no subscription is released twice | +| HEALTH-5 | No unhandled rejection and no undeclared `console.error` or `console.warn` | +| EVID-1, EVID-2, EVID-3 | `awaitTxId` fulfills after its batch applies, stays pending before, rejects after its timeout, and rejects at retirement | +| HEALTH-2, RETRY-1 | A retried failure stays hidden before ready, joins `failureReason` after it, and clears on the next delivery (`retry-policy`) | +| RECON-1 | Disconnect and reconnect keep status and rows | +| RENEW-1 | Capability renewal reuses the provider subscription and changes nothing visible | + +The law IDs come from the adapter conformance law set, version 18. + +### Writing a driver + +Implement `EagerLiveSourceDriver`: + +- **`create()`:** returns the Collection built by your adapter's options factory. +- **A `ControlledProvider`:** delivers exactly what the oracle asks for. It delivers snapshots and batches, fails, reports retried failures, disconnects, and completes renewals. `releases()` returns, for each subscription the adapter opened, how many times the adapter released it. +- **`awaitTxId`:** your adapter's utility, if you declare `await-evidence`. It receives the oracle's timeout. +- **`changeCommits`:** `per-batch` if your adapter commits a provider batch as one transaction, `per-change` if it commits each change alone. +- **`declaredDiagnostic`:** optional; accepts console diagnostics your adapter documents, so HEALTH-5 does not reject them. + +Declare only the capabilities your adapter supports; the grammar generates +only the steps they allow. Then run `checkEagerLiveSourceHistory` over +`eagerLiveSourceHistoryArb(capabilities)` in a fast-check property. Pass a +reach counter, so the run fails when histories miss a state the laws need. + +## Drivers + +- **Reference adapter** (`tests/reference-adapter.ts`): a small push adapter + written to conform. `tests/reference-adapter-oracle.property.test.ts` runs a + fixed-seed and a random campaign with declared reach, a per-change variant, + and 20 hostile mutants of the adapter, each of which the check must reject + with its law. + It runs in the default `pnpm test`. +- **Neon Realtime** (`tests/neon/`): `@neon/realtime-tanstack` 0.1.1, vendored + unchanged from `neondatabase/neon-pkgs` at + `7ac55e3e7b9e491cc53fa6225bab29d700b729e2` (Apache-2.0, see + `tests/neon/vendor/LICENSE`), except a `@ts-nocheck` header because it + targets the TanStack DB 0.9 sync API. It runs on this workspace's TanStack + DB with a controlled raw live-query subscription. It is expected to fail + until the adapter is updated, so it has its own config: + `pnpm --filter @tanstack/db-adapter-conformance test:neon`. + +- **TrailBase** (`packages/trailbase-db-collection/tests/conformance-driver.ts`): + the published adapter in eager mode over a controlled `RecordApi`, with + per-change commits and no optional capability. It runs in that package's + default `pnpm test`. + +## Evidence (2026-10-29) + +- **Law:** the table above. Authority: the sync callback contract in + `packages/db/src/types.ts`, the status and health terms in + `docs/contributing/glossary.md`, and the adapter law set v18. +- **Reach:** in the fixed campaign (seed 2118, 400 histories) on the reference + adapter, the histories that reached each declared state: + + | Declared state | Histories | + | ------------------------------ | --------- | + | first-ready | 184 | + | replacing-snapshot | 58 | + | change-batch | 77 | + | multi-change-batch | 28 | + | failure-before-ready | 83 | + | failure-after-ready | 24 | + | restart | 73 | + | wait-before-evidence-fulfilled | 23 | + | wait-timed-out | 60 | + | wait-rejected-at-retirement | 149 | + | transient-failure-reported | 48 | + | transient-failure-recovered | 22 | + | reconnect | 34 | + | renewal | 94 | + + The campaign executed 1,353 steps and skipped 1,101 as illegal at their + position, for example a batch before the first ready. Each campaign fails + when a state is reached fewer than five times. Across 20 other seeds, the + lowest count for any state was 11. + +- **GREEN:** `pnpm exec vitest run tests/reference-adapter-oracle.property.test.ts`: + 46 tests pass (both campaigns, the per-change pair, 19 calibrations, and the + grammar controls). TrailBase: its full suite, 151 tests, passes. +- **Mutants:** the fixed campaign rejects each with the law it breaks: + + | Mutant | Law | + | ----------------------------- | -------- | + | `ready-before-delivery` | READY-2 | + | `wraps-loading-failure` | READY-4 | + | `failure-clears-rows` | HEALTH-1 | + | `transient-is-fatal` | HEALTH-1 | + | `never-recovers` | HEALTH-2 | + | `transient-fails-loading` | RETRY-1 | + | `splits-batch` | PUB-1 | + | `reverses-batch` (per-change) | PUB-1 | + | `drops-deletes` | PUB-3 | + | `snapshot-merges` | PUB-7 | + | `leaks-subscription` | LIFE-2 | + | `restart-replays-rows` | LIFE-5 | + | `keeps-failed-source` | LIFE-10 | + | `double-release` | LIFE-10 | + | `detached-rejection` | HEALTH-5 | + | `logs-failure` | HEALTH-5 | + | `wait-off-by-one` | EVID-1 | + | `wait-ignores-timeout` | EVID-3 | + | `disconnect-fails` | RECON-1 | + | `renew-resubscribes` | RENEW-1 | + + `reverses-batch` reverses only batches of distinct keys, so its final rows + are right and only the intermediate publications are wrong. An earlier + mutant, which fulfilled a wait once any later txid was seen, was equivalent + within this grammar, because txids are sequential. It was replaced. + +- **Shrinking keeps the first violation (ORC-010):** `eagerLiveSourceProperty` treats a smaller history that fails a different law as passing. A shrunk counterexample therefore violates the law the campaign found. A cleanup error after a violation is kept with it in an `AggregateError`. +- **Grammar controls (ORC-004):** these run in the reference oracle file. + - **Reconstruction:** nine witnesses from the Neon and TrailBase suites and the retry contract are rebuilt as histories. Each is accepted with no skipped step and passes. + - **Exclusion:** five nearby invalid histories are each skipped by exactly one step. + - **Ablation:** removing each step kind loses the declared state that only it reaches. +- **Replay:** during a path replay, fixed campaigns and calibrations stand aside (`oracleReplayTest`). +- **RED (Neon):** `pnpm test:neon`. Both campaigns fail. The fixed campaign + shrinks to `[fail, wait]`: READY-4, status `loading` where `error` is owed. + Random seeds find READY-4 or, after ready, HEALTH-1: `health.error` is + `null`. + The adapter calls `markError`, which the current sync API does not provide, + so the failure never reaches core. + - A probe without failure steps passed 300 histories: deliveries, restart, + timed and untimed waits, reconnect, renewal, publications, and HEALTH-5. + - A probe with `markError` mapped to `fail` then fails LIFE-10: Neon keeps + its raw subscription open after the error state until cleanup. + - A probe that also releases the subscription once at the error state + passed 300 histories. Those two changes are the scope of the Neon update. +- **Limits:** + - No on-demand loading, mutation handlers, or reload. These belong to + other conformance families. + - Publications are recorded only through `subscribeChanges`. A framework + adapter's scheduling is not observed. + - EVID-3 passes a 5 ms timeout and checks for rejection 30 ms later. A + timed wait settles within its own step, so no evidence arrives during + it. An adapter that rejects a timed wait at once, before its timeout, + is not distinguished from one that waits. + - The Neon and TrailBase claims are limited to the controlled providers. A + claim about a real client needs a receiving witness against it (ORC-014). diff --git a/packages/db-adapter-conformance/package.json b/packages/db-adapter-conformance/package.json new file mode 100644 index 0000000000..1a6df7cc80 --- /dev/null +++ b/packages/db-adapter-conformance/package.json @@ -0,0 +1,59 @@ +{ + "name": "@tanstack/db-adapter-conformance", + "version": "0.0.0", + "private": true, + "description": "Conformance oracles that check a TanStack DB sync adapter against the adapter law set", + "author": "Kyle Mathews", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/TanStack/db.git", + "directory": "packages/db-adapter-conformance" + }, + "homepage": "https://tanstack.com/db", + "keywords": [ + "tanstack-db", + "conformance", + "oracle", + "sync-adapter" + ], + "scripts": { + "build": "vite build", + "lint": "eslint . --fix", + "test": "vitest --run", + "test:oracles": "vitest run tests/reference-adapter-oracle.property.test.ts", + "test:neon": "vitest run --config vitest.neon.config.ts" + }, + "type": "module", + "main": "dist/cjs/index.cjs", + "module": "dist/esm/index.js", + "types": "dist/esm/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "types": "./dist/cjs/index.d.cts", + "default": "./dist/cjs/index.cjs" + } + }, + "./package.json": "./package.json" + }, + "sideEffects": false, + "files": [ + "dist", + "src" + ], + "dependencies": { + "@tanstack/db": "workspace:*", + "fast-check": "^3.23.0" + }, + "devDependencies": { + "@fast-check/vitest": "^0.4.1", + "@neon/realtime": "0.1.2", + "@standard-schema/spec": "^1.1.0", + "@vitest/coverage-istanbul": "^4.1.12" + } +} diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts new file mode 100644 index 0000000000..21aca4b23c --- /dev/null +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -0,0 +1,1229 @@ +/** + * # Does an eager live-source sync adapter keep the core sync contract? + * + * A sync adapter connects one Collection to a provider that pushes a complete + * result first and committed change batches after it. This oracle checks the + * laws that every such adapter owes, whatever its provider: + * + * - **READY-2, READY-3.** After `preload()`, Collection status is `loading` + * until the first complete delivery applies. Then it is `ready`, `preload()` + * fulfills, and the rows equal that delivery. + * - **READY-4.** A provider failure before the first ready makes status + * `error`, rejects `preload()` with the failure value by identity, and + * reports that value as the health node's `error` cause. + * - **HEALTH-1.** A provider failure after the first ready keeps status + * `ready` and the rows, and reports the value as the health node's `error` + * cause with no `failureReason`. Before any failure, the node shows no + * problem. + * - **PUB-1.** Each change publication equals one whole committed state: the + * rows after one provider commit, never part of a commit, never an earlier + * state than one already published. A commit with no changes publishes + * nothing. The one empty callback core sends when a sync run first becomes + * ready (`emitEmptyReadyEvent` in `packages/db/src/collection/changes.ts`) + * is a readiness notification, not a commit publication. + * - **PUB-2.** Folding the Collection's change events gives the current rows. + * - **PUB-3, FID-4.** After a committed change batch applies, the rows equal + * the batch folded onto the previous rows. + * - **PUB-7** (capability `complete-snapshot`). A later complete delivery + * replaces the rows: keys it omits are gone. + * - **LIFE-2, LIFE-5.** Cleanup releases the provider subscription. A restart + * is a fresh sync run: one new provider subscription, status `loading`, and + * no rows until its own first delivery. + * - **LIFE-10.** A provider that fails ends its source: the adapter releases + * that subscription at the failure, and releases each subscription at most + * once across that end and the later cleanup. + * - **HEALTH-5.** No history causes an unhandled rejection, or an error- or + * warning-level console diagnostic that the driver does not declare. + * - **EVID-1, EVID-2, EVID-3** (capability `await-evidence`). + * `awaitTxId(txid)` fulfills once a committed batch carrying that id has + * applied, also when the batch came first, and stays pending before. A + * wait with a timeout and no evidence rejects after the timeout, and later + * evidence does not change that. A wait still pending when its sync run + * retires rejects. + * - **HEALTH-1, HEALTH-2, RETRY-1** (capability `retry-policy`). A transient + * failure that the adapter keeps retrying shows nothing before the first + * ready. After it, the failure joins the node's `failureReason`, oldest + * first, with `cause` the newest and no `error`. The next successful + * delivery clears it. + * - **RECON-1** (capability `transport-reconnect`). A transport disconnect and + * reconnect keep status and rows, and later deliveries still apply. + * - **RENEW-1** (capability `capability-renewal`). While the source's + * capability awaits renewal, status and rows do not change. Renewal reuses + * the existing provider subscription: no new subscription, no reset. + * + * Authority: the sync callback contract in `packages/db/src/types.ts` + * (`begin`, `commit`, `markReady`, `fail`, `reportError`, `reportHealthy`), + * the Collection status and health terms in `docs/contributing/glossary.md`, + * and the adapter conformance law set (issue #2118, version 18), whose law + * IDs are used here. + * + * Boundary. The oracle checks an eager Collection whose provider pushes one + * complete result at a time. On-demand loading, mutation handlers, and + * reload belong to other conformance families. A driver supplies a controlled + * provider, so a claim about a real provider client needs a driver for that + * client too (ORC-014). Rows are compared at a settled checkpoint, three + * event-loop turns after each step; every publication during the step is + * recorded and checked against PUB-1, so a torn intermediate state is seen. + * + * Layout. The production driver contract comes first, then the history + * grammar, the independent model, and the check that runs a history through + * a driver and compares each checkpoint with the model. + */ +import * as fc from 'fast-check' +import type { ChangeMessage, HealthNode } from '@tanstack/db' + +// --------------------------------------------------------------------------- +// Production driver contract +// --------------------------------------------------------------------------- + +/** The row every conformance history uses. A driver maps it to its provider. */ +export interface ConformanceRow { + readonly id: number + readonly title: string +} + +/** One change in a provider's committed batch. */ +export type ProviderChange = + | { readonly type: `upsert`; readonly row: ConformanceRow } + | { readonly type: `delete`; readonly id: number } + +/** Optional capabilities from the law set that this oracle can check. */ +export type EagerLiveSourceCapability = + | `complete-snapshot` + | `await-evidence` + | `transport-reconnect` + | `capability-renewal` + | `retry-policy` + +/** + * The public Collection surface the oracle reads. A driver passes the + * Collection that its adapter's options factory produced; the oracle starts + * it, restarts it, and cleans it up. + */ +export interface ObservedCollection { + readonly status: string + readonly health: HealthNode + values: () => IterableIterator + preload: () => Promise + cleanup: () => Promise + subscribeChanges: ( + callback: (changes: Array>) => void, + options?: { includeInitialState?: boolean }, + ) => { unsubscribe: () => void } +} + +/** + * The provider a driver controls. Each delivery acts on the provider + * subscription that the adapter opened most recently. A driver must deliver + * exactly what it is given; it must not repair or reorder deliveries. + */ +export interface ControlledProvider { + /** + * For each provider subscription the adapter opened, in order, how many + * times the adapter released it (unsubscribe, cancel, or reader release). + */ + releases: () => ReadonlyArray + /** Delivers a complete result: every row the source holds. */ + deliverSnapshot: (rows: ReadonlyArray) => void + /** Delivers one committed change batch that carries `txid`. */ + deliverChanges: (changes: ReadonlyArray, txid: string) => void + /** The provider stops with `error` and delivers nothing more. */ + fail: (error: Error) => void + /** A failure the adapter keeps retrying (capability `retry-policy`). */ + transientFailure?: (error: Error) => void + /** Drops the transport (capability `transport-reconnect`). */ + disconnect?: () => void + /** Restores the transport (capability `transport-reconnect`). */ + reconnect?: () => void + /** + * Supplies the replacement capability that the adapter requested and + * resolves after the adapter renewed its provider subscription with it + * (capability `capability-renewal`). Rejects when the adapter never asked. + */ + completeRenewal?: () => Promise + /** Renewals the adapter completed on its latest subscription. */ + renewals?: () => number +} + +/** One adapter under test. */ +export interface EagerLiveSourceFixture { + readonly collection: ObservedCollection + readonly provider: ControlledProvider + /** The adapter's `awaitTxId` utility (capability `await-evidence`). */ + awaitTxId?: (txid: string, timeout?: number) => Promise + /** Releases fixture resources after the history's final cleanup. */ + dispose?: () => void | Promise +} + +/** A production driver: it builds a fresh fixture for each history. */ +export interface EagerLiveSourceDriver { + readonly adapter: string + readonly capabilities: ReadonlySet + /** + * How the adapter commits one provider change batch (PUB-1): as one sync + * transaction, or as one per change because the provider commits each + * change on its own. + */ + readonly changeCommits: `per-batch` | `per-change` + /** + * Console diagnostics the adapter declares (HEALTH-5). Any other + * `console.error` or `console.warn` call is a violation. + */ + readonly declaredDiagnostic?: ( + level: `error` | `warn`, + args: ReadonlyArray, + ) => boolean + create: () => EagerLiveSourceFixture +} + +// --------------------------------------------------------------------------- +// History grammar +// --------------------------------------------------------------------------- + +/** + * A history starts one sync run with `preload()` and applies these steps. + * The model decides which steps are legal now; an illegal step is skipped and + * counted, so a generated list stays meaningful after shrinking. + * + * - `snapshot`: a complete delivery. Legal while the sync run is active and + * connected; after the first ready only with `complete-snapshot`. + * - `changes`: one committed batch, legal after the first ready. Its txid is + * the next in the sync run. A delete of a key the source does not hold is + * dropped, because a provider never sends one. + * - `wait`: `awaitTxId` for the last txid plus `ahead`, so a wait can target + * an applied batch (`ahead` 0) or a future one. With `timeout`, the wait + * carries a short timeout. + * - `fail`: the provider stops. + * - `transient-failure`: a failure the adapter keeps retrying. + * - `disconnect`, `reconnect`: transport loss and return. + * - `renew`: the adapter's renewal of an expiring capability completes. Legal + * once per sync run. + * - `restart`: cleanup, then `preload()` starts a new sync run. + * + * Keys range over 1..5 and titles over three values, so batches revisit + * keys, deletes hit live rows, and complete deliveries omit held keys. + */ +export type EagerLiveSourceStep = + | { readonly kind: `snapshot`; readonly rows: ReadonlyArray } + | { + readonly kind: `changes` + readonly changes: ReadonlyArray + } + | { + readonly kind: `wait` + readonly ahead: number + readonly timeout?: boolean + } + | { readonly kind: `fail` } + | { readonly kind: `transient-failure` } + | { readonly kind: `disconnect` } + | { readonly kind: `reconnect` } + | { readonly kind: `renew` } + | { readonly kind: `restart` } + +export type EagerLiveSourceHistory = ReadonlyArray + +const rowArb: fc.Arbitrary = fc.record({ + id: fc.integer({ min: 1, max: 5 }), + title: fc.constantFrom(`a`, `b`, `c`), +}) + +const changeArb: fc.Arbitrary = fc.oneof( + rowArb.map((row) => ({ type: `upsert` as const, row })), + fc.integer({ min: 1, max: 5 }).map((id) => ({ type: `delete` as const, id })), +) + +/** Generates histories whose steps use only `capabilities`. */ +export function eagerLiveSourceHistoryArb( + capabilities: ReadonlySet, +): fc.Arbitrary { + const steps: Array<{ + weight: number + arbitrary: fc.Arbitrary + }> = [ + { + weight: 4, + arbitrary: fc + .uniqueArray(rowArb, { selector: (row) => row.id, maxLength: 4 }) + .map((rows) => ({ kind: `snapshot` as const, rows })), + }, + { + weight: 6, + arbitrary: fc + .array(changeArb, { maxLength: 3 }) + .map((changes) => ({ kind: `changes` as const, changes })), + }, + { weight: 2, arbitrary: fc.constant({ kind: `fail` as const }) }, + { weight: 1, arbitrary: fc.constant({ kind: `restart` as const }) }, + ] + if (capabilities.has(`await-evidence`)) { + // One wait in four is timed. A timed wait settles at its own step, so + // only untimed waits can be fulfilled by a later batch. + steps.push({ + weight: 4, + arbitrary: fc + .record({ + ahead: fc.integer({ min: 0, max: 2 }), + timeout: fc.integer({ min: 0, max: 3 }).map((n) => n === 0), + }) + .map(({ ahead, timeout }) => ({ + kind: `wait` as const, + ahead, + timeout, + })), + }) + } + if (capabilities.has(`retry-policy`)) { + steps.push({ + weight: 3, + arbitrary: fc.constant({ kind: `transient-failure` as const }), + }) + } + if (capabilities.has(`transport-reconnect`)) { + steps.push( + // A reconnect needs a ready run and a disconnect before it, so both + // weigh enough that random seeds keep the reach minimum. + { weight: 2, arbitrary: fc.constant({ kind: `disconnect` as const }) }, + { weight: 3, arbitrary: fc.constant({ kind: `reconnect` as const }) }, + ) + } + if (capabilities.has(`capability-renewal`)) { + steps.push({ + weight: 2, + arbitrary: fc.constant({ kind: `renew` as const }), + }) + } + return fc.array(fc.oneof(...steps), { minLength: 1, maxLength: 12 }) +} + +// --------------------------------------------------------------------------- +// Independent model +// --------------------------------------------------------------------------- + +/** + * The model holds only what a legal next step can observe: + * + * - `phase` is the sync run's place in the initial gate. `loading` before the + * first complete delivery, `ready` after it, `loading-error` after a + * failure before ready, and `stopped` after a failure after ready. A + * failure ends deliveries and releases the source, so it is a phase and not + * a flag. + * - `rows` is the source's result as the provider delivered it: an + * independent Map, folded from steps, never read from the adapter. + * - `lastTxid` numbers the sync run's batches. A wait targets it. + * - `transient` lists the open transient failures, oldest first. Only a + * failure after the first ready is listed (RETRY-1). + * - `waits` records every `awaitTxId` call with the settlement the model + * expects, across sync runs, because a retired wait must reject. + * + * Model-only terms: `phase` combines Collection status with whether the + * provider can still deliver. `stopped` maps to status `ready` plus a node + * `error`; it is not a Collection status. + */ +type Phase = `loading` | `ready` | `loading-error` | `stopped` + +/** A wait's expected settlement. `unchecked` after its sync run failed. */ +type WaitExpectation = `pending` | `fulfilled` | `rejected` | `unchecked` + +type RowState = ReadonlyArray<[number, string]> + +interface ModelState { + run: number + phase: Phase + rows: Map + connected: boolean + lastTxid: number + committed: Set + renewed: boolean + failure: number | undefined + transient: Array + waits: Array<{ run: number; txid: number; expected: WaitExpectation }> +} + +/** What the model predicts at a checkpoint. */ +export interface ExpectedObservation { + status: `loading` | `ready` | `error` + rows: RowState + /** Index of the failure the health node's `error` must carry, or none. */ + healthError: number | null + /** Indices of the failures `failureReason.errors` must list, in order. */ + failureReason: ReadonlyArray + preload: `pending` | `fulfilled` | { rejectedWith: number } + waits: ReadonlyArray + /** Unreleased provider subscriptions: one while the provider runs. */ + openSubscriptions: number + /** Subscriptions opened in total: one per sync run, renewal included. */ + openedSubscriptions: number + renewalsThisRun: number +} + +function initialModel(): ModelState { + return { + run: 1, + phase: `loading`, + rows: new Map(), + connected: true, + lastTxid: 0, + committed: new Set(), + renewed: false, + failure: undefined, + transient: [], + waits: [], + } +} + +/** A delivery needs a running sync run whose provider is still connected. */ +function canDeliver(state: ModelState): boolean { + return ( + (state.phase === `loading` || state.phase === `ready`) && state.connected + ) +} + +/** Retiring a sync run rejects its pending waits (EVID-2). */ +function retireWaits(state: ModelState): void { + for (const wait of state.waits) { + if (wait.run === state.run && wait.expected === `pending`) { + wait.expected = `rejected` + } + } +} + +function rowState(rows: ReadonlyMap): RowState { + return [...rows].sort(([a], [b]) => a - b) +} + +/** A step the model accepts, with what it actually delivers. */ +type AcceptedStep = + | { kind: `snapshot`; rows: ReadonlyArray } + | { kind: `changes`; changes: Array; txid: number } + | { kind: `wait`; txid: number; timeout: boolean } + | { kind: `fail`; failure: number } + | { kind: `transient-failure`; failure: number } + | { kind: `disconnect` } + | { kind: `reconnect` } + | { kind: `renew` } + | { kind: `restart` } + +/** + * Applies one step to the model. Returns the accepted step and the row + * states that a publication during the step may show (PUB-1), or + * `undefined` when the step is illegal now. + */ +function stepModel( + state: ModelState, + step: EagerLiveSourceStep, + failures: { next: number }, + changeCommits: EagerLiveSourceDriver[`changeCommits`], +): { done: AcceptedStep; publishable: ReadonlyArray } | undefined { + switch (step.kind) { + case `snapshot`: { + if (!canDeliver(state)) return undefined + // PUB-7: a complete delivery replaces the rows. READY-3: the first one + // opens the initial gate. HEALTH-2: a successful delivery clears the + // open transient failures. + state.rows = new Map(step.rows.map((row) => [row.id, row.title])) + state.phase = `ready` + state.transient = [] + return { + done: { kind: `snapshot`, rows: step.rows }, + publishable: [rowState(state.rows)], + } + } + case `changes`: { + if (state.phase !== `ready` || !state.connected) return undefined + // PUB-3: fold the batch in order. A provider never deletes a key it + // does not hold, so such a delete is not delivered. PUB-1: each prefix + // is a committed state when the provider commits each change alone. + const delivered: Array = [] + const prefixes: Array = [] + for (const change of step.changes) { + if (change.type === `upsert`) { + state.rows.set(change.row.id, change.row.title) + delivered.push(change) + } else if (state.rows.has(change.id)) { + state.rows.delete(change.id) + delivered.push(change) + } else { + continue + } + prefixes.push(rowState(state.rows)) + } + state.lastTxid += 1 + state.committed.add(state.lastTxid) + state.transient = [] + // EVID-1: a wait for this txid fulfills once the batch applied. + for (const wait of state.waits) { + if ( + wait.run === state.run && + wait.txid === state.lastTxid && + wait.expected === `pending` + ) { + wait.expected = `fulfilled` + } + } + return { + done: { kind: `changes`, changes: delivered, txid: state.lastTxid }, + publishable: + changeCommits === `per-change` ? prefixes : [rowState(state.rows)], + } + } + case `wait`: { + if (state.phase !== `loading` && state.phase !== `ready`) return undefined + const txid = Math.max(1, state.lastTxid + step.ahead) + // EVID-1: evidence already applied fulfills the wait. EVID-3: a wait + // with a timeout and no evidence rejects once the timeout passes, which + // is before the next checkpoint. + const timeout = step.timeout === true + state.waits.push({ + run: state.run, + txid, + expected: state.committed.has(txid) + ? `fulfilled` + : timeout + ? `rejected` + : `pending`, + }) + return { done: { kind: `wait`, txid, timeout }, publishable: [] } + } + case `fail`: { + if (!canDeliver(state)) return undefined + const failure = failures.next++ + state.failure = failure + // READY-4 before the first ready; HEALTH-1 after it. LIFE-10: the + // source ends here. + state.phase = state.phase === `loading` ? `loading-error` : `stopped` + state.transient = [] + // No law fixes what a pending txid wait does after its source failed. + for (const wait of state.waits) { + if (wait.run === state.run && wait.expected === `pending`) { + wait.expected = `unchecked` + } + } + return { done: { kind: `fail`, failure }, publishable: [] } + } + case `transient-failure`: { + if (!canDeliver(state)) return undefined + const failure = failures.next++ + // RETRY-1: before the first ready a retried failure shows nothing. + // HEALTH-1: after it, the failure joins `failureReason`. + if (state.phase === `ready`) state.transient.push(failure) + return { done: { kind: `transient-failure`, failure }, publishable: [] } + } + case `disconnect`: { + if (!canDeliver(state)) return undefined + state.connected = false + return { done: { kind: `disconnect` }, publishable: [] } + } + case `reconnect`: { + if (state.connected) return undefined + if (state.phase !== `loading` && state.phase !== `ready`) return undefined + state.connected = true + return { done: { kind: `reconnect` }, publishable: [] } + } + case `renew`: { + if (!canDeliver(state) || state.renewed) return undefined + // RENEW-1: renewal changes neither rows nor status. + state.renewed = true + return { done: { kind: `renew` }, publishable: [] } + } + case `restart`: { + // LIFE-5: cleanup retires the sync run; the next one starts empty. + retireWaits(state) + state.run += 1 + state.phase = `loading` + state.rows = new Map() + state.connected = true + state.lastTxid = 0 + state.committed = new Set() + state.renewed = false + state.failure = undefined + state.transient = [] + return { done: { kind: `restart` }, publishable: [] } + } + } +} + +/** What the model predicts after an accepted step. */ +function expectedObservation(state: ModelState): ExpectedObservation { + const running = state.phase === `loading` || state.phase === `ready` + return { + status: + state.phase === `loading` + ? `loading` + : state.phase === `loading-error` + ? `error` + : `ready`, + rows: rowState(state.rows), + healthError: running ? null : state.failure!, + failureReason: state.transient, + preload: + state.phase === `loading` + ? `pending` + : state.phase === `loading-error` + ? { rejectedWith: state.failure! } + : `fulfilled`, + waits: state.waits.map((wait) => wait.expected), + // LIFE-10: a failed provider's source is released at its end. + openSubscriptions: running ? 1 : 0, + openedSubscriptions: state.run, + renewalsThisRun: state.renewed ? 1 : 0, + } +} + +// --------------------------------------------------------------------------- +// Refinement check +// --------------------------------------------------------------------------- + +/** Declared states whose laws need generated histories to reach them. */ +export const eagerLiveSourceReachStates = [ + `first-ready`, + `replacing-snapshot`, + `change-batch`, + `multi-change-batch`, + `failure-before-ready`, + `failure-after-ready`, + `restart`, + `wait-before-evidence-fulfilled`, + `wait-timed-out`, + `wait-rejected-at-retirement`, + `transient-failure-reported`, + `transient-failure-recovered`, + `reconnect`, + `renewal`, +] as const + +export type EagerLiveSourceReachState = + (typeof eagerLiveSourceReachStates)[number] + +const capabilityOfState: Partial< + Record +> = { + 'replacing-snapshot': `complete-snapshot`, + 'wait-before-evidence-fulfilled': `await-evidence`, + 'wait-timed-out': `await-evidence`, + 'wait-rejected-at-retirement': `await-evidence`, + 'transient-failure-reported': `retry-policy`, + 'transient-failure-recovered': `retry-policy`, + reconnect: `transport-reconnect`, + renewal: `capability-renewal`, +} + +/** The declared states a driver's capabilities let histories reach. */ +export function reachStatesFor( + capabilities: ReadonlySet, +): Array { + return eagerLiveSourceReachStates.filter((state) => { + const capability = capabilityOfState[state] + return capability === undefined || capabilities.has(capability) + }) +} + +/** A law violation at one checkpoint. */ +export class ConformanceViolation extends Error { + constructor( + readonly law: string, + readonly step: number, + readonly expected: unknown, + readonly actual: unknown, + readonly history: EagerLiveSourceHistory, + ) { + super( + `${law} violated at step ${step} of ${JSON.stringify(history)}: ` + + `expected ${JSON.stringify(expected)}, observed ${JSON.stringify(actual)}`, + ) + this.name = `ConformanceViolation` + } +} + +type Settlement = `pending` | `fulfilled` | { rejected: unknown } + +function recordSettlement(promise: Promise): { current: Settlement } { + const record: { current: Settlement } = { current: `pending` } + promise.then( + () => { + record.current = `fulfilled` + }, + (error: unknown) => { + record.current = { rejected: error } + }, + ) + return record +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** + * The checkpoint premise: the adapters in scope apply a delivery within a few + * event-loop turns. Three macrotask turns cover microtask receipts and one + * timer hop. A slower adapter shows as a mismatch at this cut, not as a pass. + */ +async function settle(): Promise { + for (let turn = 0; turn < 3; turn++) await sleep(0) +} + +/** The timeout a `wait` step with `timeout` passes, and how long it waits. */ +const waitTimeoutMs = 5 +const timeoutSettleMs = 30 + +/** The laws an observation answers to after each kind of step. */ +function lawFor(done: AcceptedStep, wasReady: boolean): string { + switch (done.kind) { + case `snapshot`: + return wasReady ? `PUB-7` : `READY-3` + case `changes`: + return `PUB-3/FID-4` + case `wait`: + return done.timeout ? `EVID-3` : `EVID-1` + case `fail`: + return wasReady ? `HEALTH-1` : `READY-4` + case `transient-failure`: + return wasReady ? `HEALTH-1` : `RETRY-1` + case `disconnect`: + case `reconnect`: + return `RECON-1` + case `renew`: + return `RENEW-1` + case `restart`: + return `LIFE-2/LIFE-5` + } +} + +export interface CheckOptions { + /** Called when the history enters a declared state (ORC-015). */ + reach?: (state: EagerLiveSourceReachState) => void +} + +/** + * Records detached failures and console diagnostics for one history + * (HEALTH-5). Each history installs it and removes it when it ends. + */ +function watchDetachedFailures(driver: EagerLiveSourceDriver) { + const found: Array = [] + const onRejection = (reason: unknown) => { + found.push(`unhandled rejection: ${String(reason)}`) + } + const hasProcess = typeof process !== `undefined` + if (hasProcess) process.on(`unhandledRejection`, onRejection) + const original = { error: console.error, warn: console.warn } + for (const level of [`error`, `warn`] as const) { + console[level] = (...args: Array) => { + if (!driver.declaredDiagnostic?.(level, args)) { + found.push(`console.${level}: ${args.map(String).join(` `)}`) + } + } + } + return { + found: () => found, + stop: () => { + if (hasProcess) process.off(`unhandledRejection`, onRejection) + console.error = original.error + console.warn = original.warn + }, + } +} + +/** + * Runs one history through a driver and compares every checkpoint with the + * model. Throws a `ConformanceViolation` naming the first violated law. The + * fixture is always cleaned up; a cleanup failure after a violation is kept + * as a secondary error and does not replace it (ORC-010). + */ +export async function checkEagerLiveSourceHistory( + driver: EagerLiveSourceDriver, + history: EagerLiveSourceHistory, + options: CheckOptions = {}, +): Promise<{ accepted: number; skipped: number }> { + const reach = options.reach ?? (() => {}) + const detached = watchDetachedFailures(driver) + const fixture = driver.create() + const { collection, provider } = fixture + const state = initialModel() + const failures: Array = [] + const failureCounter = { next: 0 } + const waits: Array<{ current: Settlement }> = [] + let preload = recordSettlement(collection.preload()) + let events = foldChanges(collection) + let accepted = 0 + let skipped = 0 + let primary: unknown + + const failure = (index: number) => failures[index]! + const violation = ( + law: string, + stepIndex: number, + want: unknown, + got: unknown, + ) => new ConformanceViolation(law, stepIndex, want, got, history) + + /** + * PUB-1: every publication during the step showed one whole committed + * state, in commit order, and none was empty. + */ + const checkPublications = ( + stepIndex: number, + publishable: ReadonlyArray, + becameReady: boolean, + ) => { + let last = -1 + let readiness = becameReady ? 1 : 0 + for (const publication of events.take()) { + if (publication === `empty`) { + if (readiness-- > 0) continue + throw violation(`PUB-1 (empty publication)`, stepIndex, `none`, `empty`) + } + // A publication shows a commit after the last one shown. Prefix states + // can repeat within a batch, so the search starts after that commit. + const index = publishable.findIndex( + (rows, commit) => commit > last && sameRows(rows, publication), + ) + if (index === -1) { + throw violation( + `PUB-1 (publication)`, + stepIndex, + publishable, + publication, + ) + } + last = index + } + } + + // `law` names the step's law. Observations whose law does not depend on + // the step name their own: HEALTH-2 when a delivery should have cleared a + // retried failure, EVID-1 for a wait that fulfilled without evidence, + // EVID-3 for a timed wait that did not reject, and LIFE-10 for a failed + // provider's subscription that stays open or is released twice. + const compare = async ( + stepIndex: number, + law: string, + delivery: boolean = false, + ) => { + await settle() + const expected = expectedObservation(state) + const rows = rowState( + new Map([...collection.values()].map((row) => [row.id, row.title])), + ) + const check = (name: string, ok: boolean, want: unknown, got: unknown) => { + if (!ok) throw violation(`${law} (${name})`, stepIndex, want, got) + } + const detachedNow = detached.found() + if (detachedNow.length > 0) { + throw violation(`HEALTH-5`, stepIndex, `no detached failure`, detachedNow) + } + check( + `status`, + collection.status === expected.status, + expected.status, + collection.status, + ) + check(`rows`, sameRows(rows, expected.rows), expected.rows, rows) + check(`change events`, sameRows(events.rows(), rows), rows, events.rows()) + const health = collection.health + if (expected.healthError === null) { + check(`health error`, health.error === null, null, health.error?.message) + } else { + check( + `health error cause`, + health.error?.cause === failure(expected.healthError), + failure(expected.healthError).message, + health.error?.message ?? null, + ) + } + const reasons = + health.failureReason?.errors.map((entry) => entry.cause) ?? [] + const wantReasons = expected.failureReason.map(failure) + const reasonLaw = delivery ? `HEALTH-2` : law + const checkReason = (ok: boolean, want: unknown, got: unknown) => { + if (!ok) + throw violation( + `${reasonLaw} (health failureReason)`, + stepIndex, + want, + got, + ) + } + checkReason( + reasons.length === wantReasons.length && + reasons.every((cause, index) => cause === wantReasons[index]) && + (wantReasons.length === 0 || + health.failureReason?.cause === wantReasons.at(-1)), + wantReasons.map((error) => error.message), + reasons.map(String), + ) + const want = expected.preload + const got = preload.current + check( + `preload`, + typeof want === `string` + ? got === want + : typeof got === `object` && + got.rejected === failure(want.rejectedWith), + want, + typeof got === `object` ? { rejected: String(got.rejected) } : got, + ) + expected.waits.forEach((wantWait, index) => { + if (wantWait === `unchecked`) return + const gotWait = waits[index]!.current + const gotKind = typeof gotWait === `object` ? `rejected` : gotWait + if (gotKind === wantWait) return + const waitLaw = + gotKind === `fulfilled` + ? `EVID-1` + : wantWait === `rejected` && state.waits[index]!.run === state.run + ? `EVID-3` + : law + throw violation( + `${waitLaw} (awaitTxId ${index})`, + stepIndex, + wantWait, + gotKind, + ) + }) + const releases = provider.releases() + const open = releases.filter((count) => count === 0).length + const failed = state.phase === `loading-error` || state.phase === `stopped` + if (open !== expected.openSubscriptions) { + throw violation( + `${failed ? `LIFE-10` : law} (open provider subscriptions)`, + stepIndex, + expected.openSubscriptions, + open, + ) + } + check( + `opened provider subscriptions`, + releases.length === expected.openedSubscriptions, + expected.openedSubscriptions, + releases.length, + ) + if (releases.some((count) => count > 1)) { + throw violation( + `LIFE-10 (releases per subscription)`, + stepIndex, + `at most 1`, + releases, + ) + } + if (provider.renewals) { + const renewals = provider.renewals() + check( + `renewals`, + renewals === expected.renewalsThisRun, + expected.renewalsThisRun, + renewals, + ) + } + } + + try { + await compare(-1, `READY-2`) + for (const [index, step] of history.entries()) { + const wasReady = state.phase === `ready` + const transientBefore = state.transient.length + // Support is decided before the model moves, so a skipped step leaves + // the model unchanged. + const result = supports(driver, fixture, step, wasReady) + ? stepModel(state, step, failureCounter, driver.changeCommits) + : undefined + if (result === undefined) { + skipped++ + continue + } + accepted++ + const { done, publishable } = result + switch (done.kind) { + case `snapshot`: + provider.deliverSnapshot(done.rows) + reach(wasReady ? `replacing-snapshot` : `first-ready`) + if (transientBefore > 0) reach(`transient-failure-recovered`) + break + case `changes`: + provider.deliverChanges(done.changes, String(done.txid)) + reach(`change-batch`) + if (done.changes.length > 1) reach(`multi-change-batch`) + if (transientBefore > 0) reach(`transient-failure-recovered`) + if ( + state.waits.some( + (wait, waitIndex) => + wait.run === state.run && + wait.txid === done.txid && + waits[waitIndex]?.current === `pending`, + ) + ) { + reach(`wait-before-evidence-fulfilled`) + } + break + case `wait`: { + const pending = fixture.awaitTxId!( + String(done.txid), + done.timeout ? waitTimeoutMs : undefined, + ) + waits.push(recordSettlement(pending)) + if (done.timeout) { + await sleep(timeoutSettleMs) + if (state.waits.at(-1)!.expected === `rejected`) + reach(`wait-timed-out`) + } + break + } + case `fail`: { + const error = new Error( + `conformance provider failure ${done.failure}`, + ) + failures[done.failure] = error + provider.fail(error) + reach(wasReady ? `failure-after-ready` : `failure-before-ready`) + break + } + case `transient-failure`: { + const error = new Error( + `conformance transient failure ${done.failure}`, + ) + failures[done.failure] = error + provider.transientFailure!(error) + if (wasReady) reach(`transient-failure-reported`) + break + } + case `disconnect`: + provider.disconnect!() + break + case `reconnect`: + provider.reconnect!() + reach(`reconnect`) + break + case `renew`: + await provider.completeRenewal!() + reach(`renewal`) + break + case `restart`: { + const pendingBefore = waits.some((wait) => wait.current === `pending`) + events.stop() + await collection.cleanup() + preload = recordSettlement(collection.preload()) + events = foldChanges(collection) + reach(`restart`) + if (pendingBefore) reach(`wait-rejected-at-retirement`) + break + } + } + await compare( + index, + lawFor(done, wasReady), + done.kind === `snapshot` || done.kind === `changes`, + ) + if (done.kind !== `restart`) { + checkPublications( + index, + publishable, + !wasReady && state.phase === `ready`, + ) + } else { + events.take() + } + } + } catch (error) { + primary = error + } + + // Final cleanup retires the last sync run: pending waits reject (EVID-2), + // every provider subscription is released exactly once (LIFE-2, LIFE-10), + // and nothing detached fails during teardown (HEALTH-5). + const cleanupErrors: Array = [] + try { + if ( + primary === undefined && + waits.some((wait) => wait.current === `pending`) + ) { + reach(`wait-rejected-at-retirement`) + } + events.stop() + await collection.cleanup() + await settle() + if (primary === undefined) { + retireWaits(state) + primary = retiredWaitViolation(state, waits, history) + } + const releases = provider.releases() + if (primary === undefined && !releases.every((count) => count === 1)) { + primary = violation( + `LIFE-2/LIFE-10 (releases per subscription)`, + history.length, + `exactly 1 each`, + releases, + ) + } + if (primary === undefined && detached.found().length > 0) { + primary = violation( + `HEALTH-5`, + history.length, + `no detached failure`, + detached.found(), + ) + } + } catch (error) { + cleanupErrors.push(error) + } finally { + detached.stop() + } + try { + await fixture.dispose?.() + } catch (error) { + cleanupErrors.push(error) + } + + if (primary !== undefined) { + if (cleanupErrors.length === 0) throw primary + throw new AggregateError(cleanupErrors, `conformance violation`, { + cause: primary, + }) + } + if (cleanupErrors.length > 0) { + throw new AggregateError(cleanupErrors, `fixture cleanup failed`) + } + return { accepted, skipped } +} + +/** EVID-2: after final cleanup, every wait still owed must have rejected. */ +function retiredWaitViolation( + state: ModelState, + waits: ReadonlyArray<{ current: Settlement }>, + history: EagerLiveSourceHistory, +): ConformanceViolation | undefined { + for (const [index, wait] of state.waits.entries()) { + if (wait.expected !== `rejected`) continue + const got = waits[index]!.current + if (typeof got !== `object`) { + return new ConformanceViolation( + `EVID-2 (awaitTxId ${index})`, + history.length, + `rejected`, + got, + history, + ) + } + } + return undefined +} + +/** A step the driver cannot express is skipped, as an illegal one is. */ +function supports( + driver: EagerLiveSourceDriver, + fixture: EagerLiveSourceFixture, + step: EagerLiveSourceStep, + ready: boolean, +): boolean { + switch (step.kind) { + case `snapshot`: + return !ready || driver.capabilities.has(`complete-snapshot`) + case `wait`: + return driver.capabilities.has(`await-evidence`) && !!fixture.awaitTxId + case `transient-failure`: + return ( + driver.capabilities.has(`retry-policy`) && + !!fixture.provider.transientFailure + ) + case `disconnect`: + case `reconnect`: + return driver.capabilities.has(`transport-reconnect`) + case `renew`: + return driver.capabilities.has(`capability-renewal`) + default: + return true + } +} + +function sameRows(a: RowState, b: RowState): boolean { + return ( + a.length === b.length && + a.every( + ([id, title], index) => b[index]![0] === id && b[index]![1] === title, + ) + ) +} + +/** + * Folds the Collection's change events into rows (PUB-2) and records the row + * state after each publication (PUB-1). The fold keeps every event, so a + * duplicate insert or a delete of an absent key shows as a mismatch rather + * than disappearing. It subscribes right after `preload()`, while the sync + * run holds no rows, so it needs no initial state. + */ +function foldChanges(collection: ObservedCollection) { + const rows = new Map() + let inconsistent = false + let publications: Array = [] + const subscription = collection.subscribeChanges((changes) => { + if (changes.length === 0) { + publications.push(`empty`) + return + } + for (const change of changes) { + if (change.type === `delete`) { + if (!rows.delete(change.key)) inconsistent = true + } else { + if (change.type === `insert` && rows.has(change.key)) + inconsistent = true + rows.set(change.key, change.value.title) + } + } + publications.push(rowState(rows)) + }) + return { + rows: (): RowState => + inconsistent ? [[-1, `inconsistent change events`]] : rowState(rows), + /** The publications since the last call. */ + take: () => { + const taken = publications + publications = [] + return taken + }, + stop: () => subscription.unsubscribe(), + } +} + +/** A reach counter, such as the one `createOracleReach` returns. */ +export interface ReachCounter { + beginHistory: () => void + reach: (state: EagerLiveSourceReachState) => void +} + +/** + * The generated property every driver runs. It checks each history with + * `checkEagerLiveSourceHistory` and counts reach. + * + * Shrinking keeps the first violation (ORC-010). After a history first fails, + * a smaller history that fails a different law, or fails outside the + * comparison, counts as passing, so the reported counterexample violates the + * law and observation that the campaign found. + */ +export function eagerLiveSourceProperty( + driver: EagerLiveSourceDriver, + reach?: ReachCounter, +) { + let target: string | undefined + return fc.asyncProperty( + eagerLiveSourceHistoryArb(driver.capabilities), + async (history) => { + reach?.beginHistory() + try { + await checkEagerLiveSourceHistory(driver, history, { + reach: reach?.reach, + }) + } catch (error) { + const law = + error instanceof ConformanceViolation + ? error.law + : `failure outside the comparison` + target ??= law + if (law === target) throw error + } + }, + ) +} + +// Exported for calibration tests: the model's prediction for a history. +export const eagerLiveSourceModel = { + initial: initialModel, + step: stepModel, + expected: expectedObservation, +} diff --git a/packages/db-adapter-conformance/src/index.ts b/packages/db-adapter-conformance/src/index.ts new file mode 100644 index 0000000000..90de37d053 --- /dev/null +++ b/packages/db-adapter-conformance/src/index.ts @@ -0,0 +1,24 @@ +export { + checkEagerLiveSourceHistory, + ConformanceViolation, + eagerLiveSourceHistoryArb, + eagerLiveSourceModel, + eagerLiveSourceProperty, + eagerLiveSourceReachStates, + reachStatesFor, +} from './eager-live-source-oracle' +export type { + CheckOptions, + ConformanceRow, + ControlledProvider, + EagerLiveSourceCapability, + EagerLiveSourceDriver, + EagerLiveSourceFixture, + EagerLiveSourceHistory, + EagerLiveSourceReachState, + EagerLiveSourceStep, + ExpectedObservation, + ObservedCollection, + ProviderChange, + ReachCounter, +} from './eager-live-source-oracle' diff --git a/packages/db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts b/packages/db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts new file mode 100644 index 0000000000..420ca882a0 --- /dev/null +++ b/packages/db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts @@ -0,0 +1,79 @@ +import { fc } from '@fast-check/vitest' +import { describe, it } from 'vitest' +import { + oraclePropertyOptions, + oracleReplayTest, + oracleRuns, +} from '../../db/tests/oracle-config' +import { declareOracle } from '../../db/tests/oracle-declaration' +import { createOracleReach } from '../../db/tests/oracle-reach' +import { + checkEagerLiveSourceHistory, + eagerLiveSourceHistoryArb, + eagerLiveSourceModel, + eagerLiveSourceProperty, + reachStatesFor, +} from '../src' +import { neonRealtimeDriver } from './neon/neon-realtime-driver' + +/** + * # The eager live-source oracle on the Neon Realtime adapter + * + * A receiving driver for the shared oracle in + * `src/eager-live-source-oracle.ts`. It runs Neon's adapter source, vendored + * at a fixed commit, on this workspace's TanStack DB, with a controlled raw + * live-query subscription as its provider (see `neon/neon-realtime-driver.ts`). + * The law, model, grammar, and check are the shared ones; this file states no + * expected results. + * + * This adapter targets the 0.9 sync API, so it is expected to fail laws that + * depend on the current failure channel (READY-4, HEALTH-1) until it is + * updated. That is why the file has its own Vitest config and stays out of + * the default run. The claim is limited to the controlled provider; a claim + * about Neon's real client needs a receiving witness against it. + */ + +const property = `adapter-conformance.eager-live-source.neon-realtime` +const driver = neonRealtimeDriver() +const historyArb = eagerLiveSourceHistoryArb(driver.capabilities) +const runs = 200 +// Timed waits sleep past their timeout, so a passing campaign takes seconds. +const campaignTimeoutMs = 120_000 + +async function campaign(options: fc.Parameters<[unknown]>) { + const reach = createOracleReach( + property, + reachStatesFor(driver.capabilities), + { + minimum: 5, + }, + ) + await fc.assert(eagerLiveSourceProperty(driver, reach), options) + return reach +} + +describe(`eager live-source conformance: Neon Realtime adapter`, () => { + oracleReplayTest(it)( + `conforms on a fixed-seed campaign`, + async () => { + const reach = await campaign({ seed: 2118, numRuns: oracleRuns(runs) }) + reach.assertReached() + }, + campaignTimeoutMs, + ) + + oracleReplayTest(it, property)( + `conforms on a random campaign or a replayed seed`, + async () => { + const reach = await campaign(oraclePropertyOptions(runs, property)) + reach.assertReached() + }, + campaignTimeoutMs, + ) +}) + +declareOracle({ + check: [checkEagerLiveSourceHistory, eagerLiveSourceProperty], + model: [eagerLiveSourceModel.step, eagerLiveSourceModel.expected], + grammar: [historyArb], +}) diff --git a/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts b/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts new file mode 100644 index 0000000000..a1b61d58b7 --- /dev/null +++ b/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts @@ -0,0 +1,282 @@ +/** + * Production driver for the Neon Realtime adapter (`@neon/realtime-tanstack`). + * + * The adapter source is vendored unchanged in `vendor/` from + * `neondatabase/neon-pkgs` at commit 7ac55e3e7b9e491cc53fa6225bab29d700b729e2 + * (package version 0.1.1, Apache-2.0). It runs against this workspace's + * TanStack DB and the published `@neon/realtime` client, so the refresh + * controller is Neon's own. + * + * The provider is a controlled raw live-query subscription, in the shape of + * the `RawLiveQuerySubscription` interface that the adapter consumes. It maps + * the oracle's steps onto Neon's events: + * + * - a complete delivery is a `rows` update, with row ID `r`; + * - a change batch is a `changes` update that carries one txid; + * - a provider failure is an `error` state, which also rejects pending txid + * waits, as the interface documents; + * - a transport drop is a `connecting` state and its return is `live`; + * - renewal: the initial capability expires 10 s after creation, so Neon's + * refresh controller asks for a replacement at once. The driver holds that + * request until `completeRenewal`, then waits for `renew` on the open + * subscription. + * + * A real client catches an exception thrown by an update or state listener. + * The controlled subscription does the same and records it, so the oracle + * judges the Collection's public state after such an exception. + */ +import { createCollection } from '@tanstack/db' +import { realtimeCollectionOptions } from './vendor/collection-options' +import type { + LiveQueryChange, + LiveQueryState, + RawLiveQueryOptions, + RawLiveQueryRow, + RawLiveQuerySubscription, + RawLiveQueryUpdate, + RealtimeClient, + SealedLiveQuery, +} from '@neon/realtime/client' +import type { + ConformanceRow, + ControlledProvider, + EagerLiveSourceDriver, + EagerLiveSourceFixture, + ObservedCollection, + ProviderChange, +} from '../../src' + +class ControlledRawSubscription implements RawLiveQuerySubscription { + readonly renewals: Array> = [] + readonly listenerErrors: Array = [] + unsubscribed = false + unsubscribes = 0 + private state: LiveQueryState = { status: `connecting`, error: undefined } + private readonly updates = new Set< + (update: RawLiveQueryUpdate) => void + >() + private readonly states = new Set<(state: LiveQueryState) => void>() + private readonly applied = new Set() + private readonly waits = new Map< + string, + Array<{ resolve: () => void; reject: (error: Error) => void }> + >() + + getState = (): LiveQueryState => this.state + + onUpdate = ( + listener: (update: RawLiveQueryUpdate) => void, + ) => { + this.updates.add(listener) + return () => this.updates.delete(listener) + } + + onStateChange = (listener: (state: LiveQueryState) => void) => { + this.states.add(listener) + return () => this.states.delete(listener) + } + + // As the interface documents: a timeout rejects a wait that has no evidence. + awaitTxId = (txid: string, timeout?: number): Promise => { + if (this.applied.has(txid)) return Promise.resolve() + if (this.unsubscribed) { + return Promise.reject(new Error(`Live-query subscription is closed`)) + } + return new Promise((resolve, reject) => { + const list = this.waits.get(txid) ?? [] + const wait = { resolve, reject } + list.push(wait) + this.waits.set(txid, list) + if (timeout !== undefined) { + setTimeout(() => { + const index = list.indexOf(wait) + if (index === -1) return + list.splice(index, 1) + reject( + new Error(`Timed out waiting for live-query transaction ${txid}`), + ) + }, timeout) + } + }) + } + + renew = (query: SealedLiveQuery): Promise => { + this.renewals.push(query) + return Promise.resolve() + } + + unsubscribe = (): void => { + this.unsubscribes += 1 + this.unsubscribed = true + this.state = { status: `closed`, error: undefined } + this.rejectWaits(new Error(`Live-query subscription is closed`)) + } + + deliver( + update: RawLiveQueryUpdate, + txids: ReadonlyArray = [], + ) { + for (const listener of this.updates) this.call(() => listener(update)) + for (const txid of txids) { + this.applied.add(txid) + for (const wait of this.waits.get(txid) ?? []) wait.resolve() + this.waits.delete(txid) + } + } + + setState(state: LiveQueryState) { + this.state = state + for (const listener of this.states) this.call(() => listener(state)) + if (state.status === `error`) this.rejectWaits(state.error) + } + + private call(run: () => void) { + try { + run() + } catch (error) { + this.listenerErrors.push(error) + } + } + + private rejectWaits(error: Error) { + for (const list of this.waits.values()) { + for (const wait of list) wait.reject(error) + } + this.waits.clear() + } +} + +function sealed(expiresInMs: number): SealedLiveQuery { + const header = Buffer.from( + JSON.stringify({ alg: `dir`, enc: `A256GCM`, v: 1 }), + ).toString(`base64url`) + return { + capability: `${header}..a.${Buffer.from(`conformance`).toString(`base64url`)}.c`, + queryFingerprint: `11`.repeat(32), + expiresAt: Date.now() + expiresInMs, + } +} + +function rawRow(row: ConformanceRow): RawLiveQueryRow { + return { rowId: `r${row.id}`, row } +} + +function rawChange(change: ProviderChange): LiveQueryChange { + return change.type === `upsert` + ? { type: `upsert`, rowId: `r${change.row.id}`, row: change.row } + : { type: `remove`, rowId: `r${change.id}` } +} + +class NeonProvider implements ControlledProvider { + readonly subscriptions: Array = [] + private pendingRefresh: + ((query: SealedLiveQuery) => void) | undefined + + readonly client: RealtimeClient = { + subscribe: (( + _query: SealedLiveQuery, + options?: RawLiveQueryOptions, + ) => { + if (options?.materialize !== false) { + throw new Error( + `the Neon adapter is expected to open a raw subscription`, + ) + } + const subscription = new ControlledRawSubscription() + this.subscriptions.push(subscription) + return subscription + }) as unknown as RealtimeClient[`subscribe`], + close: () => {}, + } + + refreshQuery = () => + new Promise>((resolve) => { + this.pendingRefresh = resolve + }) + + private latest(): ControlledRawSubscription { + const subscription = this.subscriptions.at(-1) + if (!subscription) throw new Error(`no provider subscription`) + return subscription + } + + listenerErrors = () => this.subscriptions.flatMap((s) => s.listenerErrors) + releases = () => this.subscriptions.map((s) => s.unsubscribes) + deliverSnapshot = (rows: ReadonlyArray) => { + this.latest().setState({ status: `live`, error: undefined }) + this.latest().deliver({ type: `rows`, rows: rows.map(rawRow) }) + } + deliverChanges = (changes: ReadonlyArray, txid: string) => + this.latest().deliver( + { + type: `changes`, + changes: changes.map(rawChange), + batch: { txids: [txid] }, + }, + [txid], + ) + // Neon's error state carries a code and a retry flag. They are added to + // the oracle's failure value itself, so its identity is kept. + fail = (error: Error) => + this.latest().setState({ + status: `error`, + error: Object.assign(error, { + code: `CONFORMANCE_FAILURE`, + retryable: false, + }), + }) + disconnect = () => + this.latest().setState({ status: `connecting`, error: undefined }) + reconnect = () => this.latest().setState({ status: `live`, error: undefined }) + completeRenewal = async () => { + for (let turn = 0; turn < 20 && !this.pendingRefresh; turn++) { + await new Promise((settle) => setTimeout(settle, 0)) + } + const resolve = this.pendingRefresh + if (!resolve) + throw new Error(`the adapter did not request a replacement capability`) + this.pendingRefresh = undefined + const subscription = this.latest() + const before = subscription.renewals.length + resolve(sealed(3_000_000)) + for ( + let turn = 0; + turn < 20 && subscription.renewals.length === before; + turn++ + ) { + await new Promise((settle) => setTimeout(settle, 0)) + } + } + renewals = () => this.subscriptions.at(-1)?.renewals.length ?? 0 +} + +export function neonRealtimeDriver(): EagerLiveSourceDriver { + return { + adapter: `@neon/realtime-tanstack 0.1.1`, + changeCommits: `per-batch`, + capabilities: new Set([ + `complete-snapshot`, + `await-evidence`, + `transport-reconnect`, + `capability-renewal`, + ]), + create(): EagerLiveSourceFixture { + const provider = new NeonProvider() + const collection = createCollection( + realtimeCollectionOptions({ + id: `neon-conformance`, + client: provider.client, + // Expires within Neon's 10 s refresh lead, so a renewal is due at once. + query: sealed(10_000), + refreshQuery: provider.refreshQuery, + getKey: (row: ConformanceRow) => row.id, + }), + ) + return { + collection: collection as unknown as ObservedCollection, + provider, + awaitTxId: (txid, timeout) => collection.utils.awaitTxId(txid, timeout), + } + }, + } +} diff --git a/packages/db-adapter-conformance/tests/neon/vendor/LICENSE b/packages/db-adapter-conformance/tests/neon/vendor/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/packages/db-adapter-conformance/tests/neon/vendor/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/db-adapter-conformance/tests/neon/vendor/collection-options.ts b/packages/db-adapter-conformance/tests/neon/vendor/collection-options.ts new file mode 100644 index 0000000000..7d35f295d3 --- /dev/null +++ b/packages/db-adapter-conformance/tests/neon/vendor/collection-options.ts @@ -0,0 +1,335 @@ +// @ts-nocheck -- vendored unchanged from neondatabase/neon-pkgs@7ac55e3e; it targets the TanStack DB 0.9 sync API. +import { + type LiveQueryChange, + type LiveQueryState, + QueryRefreshController, + type RawLiveQueryRow, + type RawLiveQuerySubscription, + type RealtimeClient, + type SealedLiveQuery, +} from "@neon/realtime/client"; +import type { StandardSchemaV1 } from "@standard-schema/spec"; +import type { + BaseCollectionConfig, + CollectionConfig, + SyncAppliedReceipt, + UtilsRecord, +} from "@tanstack/db"; +import { withCollectionConfigFactory } from "@tanstack/db"; +import type { NoInferCompat } from "./no-infer.js"; + +/** Utilities attached to a Realtime-backed TanStack DB collection. */ +export interface RealtimeCollectionUtils extends UtilsRecord { + /** + * Wait until this query has processed a known committed PostgreSQL transaction. + * Covered row changes enter TanStack DB's causal sync queue before this resolves. + * + * @remarks + * This does not determine whether a transaction committed, so pass only IDs + * from transactions known to have committed. + * + * @param txid - PostgreSQL transaction ID as a decimal string. + * @param timeout - Optional maximum wait in milliseconds. By default, the + * promise remains pending until the transaction arrives or the collection is + * cleaned up. + * @throws If a supplied timeout elapses or the collection is cleaned up. + */ + awaitTxId(txid: string, timeout?: number): Promise; +} + +/** + * Configuration for a Realtime-backed TanStack DB collection. + * + * The adapter owns `sync`, uses eager synchronization, and supplies its own + * collection utilities. + * + * @typeParam Row - Object row synchronized into the collection. + * @typeParam Key - Stable key returned by `getKey`. + * @typeParam Schema - Optional Standard Schema used by TanStack DB. + */ +export interface RealtimeCollectionConfig< + Row extends object, + Key extends string | number = string | number, + Schema extends StandardSchemaV1 = never, +> extends Omit< + BaseCollectionConfig, + "syncMode" | "utils" + > { + /** Reusable low-level Realtime client. */ + readonly client: RealtimeClient; + /** Initial query for the exact query backing this collection. */ + readonly query: SealedLiveQuery; + /** + * Obtain a replacement capability for the same exact query before expiry. + * Transient failures are retried while the current capability remains valid. + */ + readonly refreshQuery?: () => Promise>; + /** Derive the stable unique TanStack DB key for a row. */ + readonly getKey: (row: Row) => Key; +} + +/** + * TanStack DB collection options produced by {@link realtimeCollectionOptions}. + * + * @typeParam Row - Object row synchronized into the collection. + * @typeParam Key - Stable key for a collection row. + * @typeParam Schema - Optional Standard Schema used by TanStack DB. + */ +export type RealtimeCollectionOptions< + Row extends object, + Key extends string | number, + Schema extends StandardSchemaV1, +> = Omit< + CollectionConfig, + "utils" +> & { + /** Realtime-specific transaction-confirmation utility. */ + readonly utils: RealtimeCollectionUtils; +}; + +/** + * Create TanStack DB collection options backed by a raw live-query subscription. + * Complete-row replacements and committed change batches are applied atomically. + * + * @typeParam Row - Object row synchronized into the collection. + * @typeParam Key - Stable key returned by `getKey`. + * @typeParam Schema - Optional Standard Schema used by TanStack DB. + * @param config - Live-query subscription settings and ordinary TanStack DB + * collection options. + * @returns Collection options to pass to TanStack DB's `createCollection()`. + */ +export function realtimeCollectionOptions< + Row extends object, + Key extends string | number = string | number, + Schema extends StandardSchemaV1 = never, +>( + // Infer rows from the query, then check collection options against that type. + config: RealtimeCollectionConfig, Key, Schema> & { + readonly query: SealedLiveQuery; + }, +): RealtimeCollectionOptions { + const createOptions = () => createRealtimeCollectionOptions(config); + return withCollectionConfigFactory(createOptions(), createOptions); +} + +function createRealtimeCollectionOptions< + Row extends object, + Key extends string | number, + Schema extends StandardSchemaV1, +>( + config: RealtimeCollectionConfig, +): RealtimeCollectionOptions { + const { client, query: initialQuery, refreshQuery, ...baseConfig } = config; + let activeSubscription: RawLiveQuerySubscription | undefined; + const utils: RealtimeCollectionUtils = Object.freeze({ + awaitTxId: async (txid, timeout) => { + const subscription = activeSubscription; + if (!subscription) { + throw new Error("Realtime collection is not syncing"); + } + try { + await subscription.awaitTxId(txid, timeout); + } catch (error) { + if (activeSubscription !== subscription) { + throw new Error("Realtime collection was cleaned up", { + cause: error, + }); + } + throw error; + } + return true; + }, + }); + + const options: RealtimeCollectionOptions = { + ...baseConfig, + syncMode: "eager", + utils, + sync: { + rowUpdateMode: "full", + sync: ({ + begin, + write, + commit, + truncate, + markReady, + markError, + }) => { + let cleaned = false; + let rowIds = new Map(); + let keys = new Map(); + const subscription = client.subscribe(initialQuery, { + materialize: false, + }); + activeSubscription = subscription; + + const reportError = (error: unknown) => { + if (!cleaned) markError(error); + }; + const queryRefresh = new QueryRefreshController({ + query: initialQuery, + refreshQuery, + renewSubscription: (replacement) => + subscription.renew(replacement), + onRefreshExhausted: reportError, + subscription, + }); + const observeReceipt = ( + receipt: SyncAppliedReceipt, + onApplied?: () => void, + ) => { + if (receipt === true) { + onApplied?.(); + } else { + void receipt.then( + () => { + if (!cleaned) onApplied?.(); + }, + (error) => reportError(error), + ); + } + }; + + const installReset = ( + rows: readonly RawLiveQueryRow[], + ) => { + try { + const nextRowIds = new Map(); + const nextKeys = new Map(); + for (const { rowId, row } of rows) { + const key = config.getKey(row); + if (nextRowIds.has(rowId)) { + throw new Error( + `Duplicate Realtime row ID: ${rowId}`, + ); + } + if (nextKeys.has(key)) { + throw new Error( + `Duplicate TanStack collection key: ${String(key)}`, + ); + } + nextRowIds.set(rowId, key); + nextKeys.set(key, rowId); + } + + begin(); + truncate(); + for (const { row } of rows) + write({ type: "insert", value: row }); + const receipt = commit(); + rowIds = nextRowIds; + keys = nextKeys; + observeReceipt(receipt, markReady); + } catch (error) { + reportError(error); + } + }; + + const applyBatch = ( + changes: readonly LiveQueryChange[], + ) => { + try { + const nextRowIds = new Map(rowIds); + const nextKeys = new Map(keys); + const writes: Array< + | { type: "insert" | "update"; value: Row } + | { type: "delete"; key: Key } + > = []; + + for (const change of changes) { + if (change.type === "upsert") { + const key = config.getKey(change.row); + const previousKey = nextRowIds.get( + change.rowId, + ); + const keyOwner = nextKeys.get(key); + if ( + keyOwner !== undefined && + keyOwner !== change.rowId + ) { + throw new Error( + `Realtime upserted an existing key: ${String(key)}`, + ); + } + if (previousKey === undefined) { + nextRowIds.set(change.rowId, key); + nextKeys.set(key, change.rowId); + writes.push({ + type: "insert", + value: change.row, + }); + } else if (key !== previousKey) { + nextKeys.delete(previousKey); + nextKeys.set(key, change.rowId); + nextRowIds.set(change.rowId, key); + writes.push({ + type: "delete", + key: previousKey, + }); + writes.push({ + type: "insert", + value: change.row, + }); + } else { + writes.push({ + type: "update", + value: change.row, + }); + } + } else { + const key = nextRowIds.get(change.rowId); + if (key === undefined) { + throw new Error( + `Realtime deleted an unknown row ID: ${change.rowId}`, + ); + } + nextRowIds.delete(change.rowId); + nextKeys.delete(key); + writes.push({ type: "delete", key }); + } + } + + begin(); + for (const message of writes) write(message); + const receipt = commit(); + rowIds = nextRowIds; + keys = nextKeys; + observeReceipt(receipt); + } catch (error) { + reportError(error); + } + }; + + const stateChanged = (state: LiveQueryState) => { + if (state.status === "error") markError(state.error); + // Initial connecting maps to loading; complete rows make the + // collection ready. Reconnection and stale capability states retain + // an already-ready collection, while closed uses normal sync cleanup. + }; + + const unsubscribeUpdate = subscription.onUpdate((update) => { + if (update.type === "rows") installReset(update.rows); + else applyBatch(update.changes); + }); + const unsubscribeState = + subscription.onStateChange(stateChanged); + queryRefresh.start(); + + return () => { + cleaned = true; + queryRefresh.stop(); + unsubscribeUpdate(); + unsubscribeState(); + subscription.unsubscribe(); + if (activeSubscription === subscription) { + activeSubscription = undefined; + } + rowIds.clear(); + keys.clear(); + }; + }, + }, + }; + + return options; +} diff --git a/packages/db-adapter-conformance/tests/neon/vendor/no-infer.ts b/packages/db-adapter-conformance/tests/neon/vendor/no-infer.ts new file mode 100644 index 0000000000..2e4e854247 --- /dev/null +++ b/packages/db-adapter-conformance/tests/neon/vendor/no-infer.ts @@ -0,0 +1,6 @@ +/** + * @internal Pre-5.4 inference barrier. Kept package-local so declarations work + * with every core SDK version allowed by this package's peer dependency. + */ +// biome-ignore lint/suspicious/noExplicitAny: This is the pre-5.4 deferred-conditional inference barrier. +export type NoInferCompat = [T][T extends any ? 0 : never]; diff --git a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts new file mode 100644 index 0000000000..ff6d1cedb7 --- /dev/null +++ b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts @@ -0,0 +1,385 @@ +import { fc } from '@fast-check/vitest' +import { describe, expect, it } from 'vitest' +import { + oraclePropertyOptions, + oracleReplayTest, + oracleRuns, +} from '../../db/tests/oracle-config' +import { declareOracle } from '../../db/tests/oracle-declaration' +import { createOracleReach } from '../../db/tests/oracle-reach' +import { + ConformanceViolation, + checkEagerLiveSourceHistory, + eagerLiveSourceHistoryArb, + eagerLiveSourceModel, + eagerLiveSourceProperty, + reachStatesFor, +} from '../src' +import { referenceDriver } from './reference-adapter' +import type { + EagerLiveSourceHistory, + EagerLiveSourceReachState, + EagerLiveSourceStep, +} from '../src' +import type { ReferenceMutant } from './reference-adapter' + +/** + * # The eager live-source oracle on a conforming adapter + * + * The shared oracle in `src/eager-live-source-oracle.ts` states the laws, the + * model, the history grammar, and the check. This file supplies one production + * driver: the reference push adapter in `reference-adapter.ts`, which declares + * every optional capability the oracle checks. A green run here shows that + * the check accepts a conforming adapter. It says nothing about a published + * adapter; each one needs its own driver. + * + * - **Campaigns.** The same property, grammar, and run budget run with a + * fixed seed and without one (ORC-007). A path replay runs only the + * requested history. Each campaign fails when fewer than five histories + * reach a declared state (ORC-015). + * - **Calibration.** Hostile mutants of the reference adapter each break one + * law, and the fixed campaign must reject each with that law (ORC-006). + * - **Grammar controls (ORC-004).** Known witnesses from the Neon and + * TrailBase suites are rebuilt as histories and replayed through the check + * (reconstruction). Nearby invalid histories are skipped without moving the + * model (exclusion). Removing each step kind loses the declared state that + * only it reaches (ablation). Range: keys 1..5 and three titles let batches + * revisit keys and complete deliveries omit held keys; histories hold up to + * 12 steps; waits look 0..2 txids ahead, so an off-by-one boundary is + * reachable on both sides. + */ + +const property = `adapter-conformance.eager-live-source.reference` +const driver = referenceDriver() +const historyArb = eagerLiveSourceHistoryArb(driver.capabilities) +// The reference adapter declares every capability, so its grammar spreads +// over the most step kinds. Across 20 seeds, 400 histories reached every +// declared state at least 11 times. +const runs = 400 +// Timed waits sleep past their timeout, so a campaign takes several seconds. +const campaignTimeoutMs = 120_000 + +async function campaign(options: fc.Parameters<[unknown]>) { + const reach = createOracleReach( + property, + reachStatesFor(driver.capabilities), + { + minimum: 5, + }, + ) + await fc.assert(eagerLiveSourceProperty(driver, reach), options) + return reach +} + +describe(`eager live-source conformance: reference adapter`, () => { + oracleReplayTest(it)( + `conforms on a fixed-seed campaign`, + async () => { + const reach = await campaign({ seed: 2118, numRuns: oracleRuns(runs) }) + reach.assertReached() + }, + campaignTimeoutMs, + ) + + oracleReplayTest(it, property)( + `conforms on a random campaign or a replayed seed`, + async () => { + const reach = await campaign(oraclePropertyOptions(runs, property)) + reach.assertReached() + }, + campaignTimeoutMs, + ) + + // Each mutant breaks one law; the check must report that law. + const mutants: Array<[ReferenceMutant, string]> = [ + [`ready-before-delivery`, `READY-2`], + [`wraps-loading-failure`, `READY-4`], + [`failure-clears-rows`, `HEALTH-1`], + [`transient-is-fatal`, `HEALTH-1`], + [`never-recovers`, `HEALTH-2`], + [`transient-fails-loading`, `RETRY-1`], + [`splits-batch`, `PUB-1`], + [`drops-deletes`, `PUB-3`], + [`snapshot-merges`, `PUB-7`], + [`leaks-subscription`, `LIFE-2`], + [`restart-replays-rows`, `LIFE-5`], + [`keeps-failed-source`, `LIFE-10`], + [`double-release`, `LIFE-10`], + [`detached-rejection`, `HEALTH-5`], + [`logs-failure`, `HEALTH-5`], + [`wait-off-by-one`, `EVID-1`], + [`wait-ignores-timeout`, `EVID-3`], + [`disconnect-fails`, `RECON-1`], + [`renew-resubscribes`, `RENEW-1`], + ] + oracleReplayTest(it).each(mutants)( + `rejects the %s mutant with %s`, + async (mutant, law) => { + const result = await fc.check( + eagerLiveSourceProperty(referenceDriver(mutant)), + { + seed: 2118, + numRuns: runs, + }, + ) + expect(result.failed).toBe(true) + expect(result.errorInstance).toBeInstanceOf(ConformanceViolation) + expect((result.errorInstance as ConformanceViolation).law).toContain(law) + }, + campaignTimeoutMs, + ) +}) + +// PUB-1 has two commit granularities. The per-change reference adapter +// commits each change of a batch alone; it must conform, and a variant that +// commits them out of order must be rejected by the per-change comparison. +describe(`eager live-source conformance: per-change commits`, () => { + oracleReplayTest(it)( + `the per-change reference adapter conforms`, + async () => { + const perChange = referenceDriver(undefined, { perChange: true }) + await fc.assert(eagerLiveSourceProperty(perChange), { + seed: 2118, + numRuns: runs, + }) + }, + campaignTimeoutMs, + ) + + oracleReplayTest(it)( + `rejects a batch committed out of order with PUB-1`, + async () => { + const reversed = referenceDriver(`reverses-batch`, { perChange: true }) + const result = await fc.check(eagerLiveSourceProperty(reversed), { + seed: 2118, + numRuns: runs, + }) + expect(result.failed).toBe(true) + expect((result.errorInstance as ConformanceViolation).law).toContain( + `PUB-1`, + ) + }, + campaignTimeoutMs, + ) +}) + +// --------------------------------------------------------------------------- +// Grammar controls (ORC-004) +// --------------------------------------------------------------------------- + +/** + * Reconstruction. Each known witness is a history from an existing adapter + * suite, translated into this grammar's steps. The model must accept every + * step, and the reference adapter must pass it. A witness cell the grammar + * cannot express is named with what it lacks. + */ +const witnesses: Array<[string, EagerLiveSourceHistory]> = [ + [ + // neon-pkgs realtime-tanstack collection-options.test.ts:119 + `Neon: installs rows and changes as authoritative state`, + [ + { + kind: `snapshot`, + rows: [ + { id: 1, title: `a` }, + { id: 2, title: `b` }, + ], + }, + { kind: `wait`, ahead: 1 }, + { + kind: `changes`, + changes: [ + { type: `upsert`, row: { id: 1, title: `c` } }, + { type: `delete`, id: 2 }, + { type: `upsert`, row: { id: 3, title: `c` } }, + ], + }, + ], + ], + [ + // collection-options.test.ts:164 + `Neon: remembers a transaction that arrives before awaitTxId`, + [ + { kind: `snapshot`, rows: [{ id: 1, title: `a` }] }, + { + kind: `changes`, + changes: [{ type: `upsert`, row: { id: 1, title: `b` } }], + }, + { kind: `wait`, ahead: 0 }, + ], + ], + [ + // collection-options.test.ts:227, the `connecting` cell + `Neon: stays ready while the transport reconnects`, + [ + { kind: `snapshot`, rows: [{ id: 1, title: `a` }] }, + { kind: `disconnect` }, + ], + ], + [ + // collection-options.test.ts:243 + `Neon: a provider error before ready is a Collection error`, + [{ kind: `fail` }], + ], + [ + // collection-options.test.ts:255 + `Neon: renews an expiring capability`, + [{ kind: `renew` }], + ], + [ + // collection-options.test.ts:364. Its timeout cell is outside the grammar + // (EVID-3 is not checked); its cleanup cell is the final retirement. + `Neon: cleanup rejects a pending wait`, + [{ kind: `wait`, ahead: 1 }], + ], + [ + // trailbase-db-collection lifecycle-oracle.property.test.ts, stream end + `TrailBase: a stream failure after ready keeps rows and reports`, + [ + { kind: `snapshot`, rows: [{ id: 1, title: `a` }] }, + { + kind: `changes`, + changes: [{ type: `upsert`, row: { id: 2, title: `b` } }], + }, + { kind: `fail` }, + ], + ], + [ + // A retried failure after ready, then a delivery that recovers. + `retry: a retried failure is reported, then cleared by a delivery`, + [ + { kind: `snapshot`, rows: [{ id: 1, title: `a` }] }, + { kind: `transient-failure` }, + { + kind: `changes`, + changes: [{ type: `upsert`, row: { id: 1, title: `b` } }], + }, + ], + ], + [ + // lifecycle-oracle.property.test.ts, restart fencing + `TrailBase: a restart is a fresh sync run`, + [ + { kind: `snapshot`, rows: [{ id: 1, title: `a` }] }, + { kind: `restart` }, + { kind: `snapshot`, rows: [{ id: 2, title: `b` }] }, + ], + ], +] + +/** Whether `history` lies in the generator's domain. */ +function withinGrammar(history: EagerLiveSourceHistory): boolean { + const row = (r: { id: number; title: string }) => + r.id >= 1 && r.id <= 5 && [`a`, `b`, `c`].includes(r.title) + return ( + history.length >= 1 && + history.length <= 12 && + history.every((step) => { + switch (step.kind) { + case `snapshot`: + return step.rows.length <= 4 && step.rows.every(row) + case `changes`: + return ( + step.changes.length <= 3 && + step.changes.every((change) => + change.type === `upsert` + ? row(change.row) + : change.id >= 1 && change.id <= 5, + ) + ) + case `wait`: + return step.ahead >= 0 && step.ahead <= 2 + default: + return true + } + }) + ) +} + +describe(`eager live-source grammar controls`, () => { + oracleReplayTest(it).each(witnesses)( + `reconstructs %s`, + async (_name, history) => { + expect(withinGrammar(history)).toBe(true) + const { skipped } = await checkEagerLiveSourceHistory(driver, history) + expect(skipped).toBe(0) + }, + ) + + // Exclusion: each history ends with one step that is illegal where it + // stands. The check skips it and leaves the model where it was. + const invalid: Array<[string, EagerLiveSourceHistory]> = [ + [`a batch before the first ready`, [{ kind: `changes`, changes: [] }]], + [ + `a second renewal in one sync run`, + [{ kind: `renew` }, { kind: `renew` }], + ], + [ + `a reconnect without a disconnect`, + [{ kind: `snapshot`, rows: [] }, { kind: `reconnect` }], + ], + [ + `a delivery after the provider failed`, + [{ kind: `fail` }, { kind: `snapshot`, rows: [] }], + ], + [ + `a batch while disconnected`, + [ + { kind: `snapshot`, rows: [] }, + { kind: `disconnect` }, + { kind: `changes`, changes: [] }, + ], + ], + ] + oracleReplayTest(it).each(invalid)(`excludes %s`, async (_name, history) => { + const { skipped } = await checkEagerLiveSourceHistory(driver, history) + expect(skipped).toBe(1) + }) + + // Ablation: removing a step kind loses the declared state that only it + // reaches, so every kind contributes to the laws the campaign enforces. + const ablations: Array< + [EagerLiveSourceStep[`kind`], EagerLiveSourceReachState] + > = [ + [`snapshot`, `first-ready`], + [`changes`, `change-batch`], + [`fail`, `failure-after-ready`], + [`restart`, `restart`], + [`wait`, `wait-before-evidence-fulfilled`], + [`disconnect`, `reconnect`], + [`reconnect`, `reconnect`], + [`renew`, `renewal`], + [`transient-failure`, `transient-failure-reported`], + ] + oracleReplayTest(it).each(ablations)( + `without %s steps, histories never reach %s`, + async (kind, lost) => { + const states = reachStatesFor(driver.capabilities) + const reach = createOracleReach(`${property}.ablation`, states, { + replaying: false, + }) + await fc.assert( + fc.asyncProperty( + historyArb.map((history) => + history.filter((step) => step.kind !== kind), + ), + async (history) => { + reach.beginHistory() + await checkEagerLiveSourceHistory(driver, history, { + reach: reach.reach, + }) + }, + ), + { seed: 2118, numRuns: 100 }, + ) + expect(reach.counts().get(lost)).toBe(0) + }, + campaignTimeoutMs, + ) +}) + +declareOracle({ + check: [checkEagerLiveSourceHistory, eagerLiveSourceProperty], + model: [eagerLiveSourceModel.step, eagerLiveSourceModel.expected], + grammar: [historyArb], +}) diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts new file mode 100644 index 0000000000..7c74756a1b --- /dev/null +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -0,0 +1,387 @@ +/** + * A small push adapter written for this package, and the controlled provider + * it reads. It exists to give the eager live-source oracle a production path + * that is expected to conform, and to host the hostile mutants that the + * calibration tests run (ORC-006). It is not a published adapter. + * + * The provider pushes a complete result, then committed batches that carry a + * txid. It can fail, report a failure it is retrying, drop and restore its + * transport, and ask for a capability renewal, which the adapter answers by + * renewing the same subscription. + */ +import { createCollection } from '@tanstack/db' +import type { + ConformanceRow, + ControlledProvider, + EagerLiveSourceDriver, + EagerLiveSourceFixture, + ProviderChange, +} from '../src' + +/** Wrong designs the calibration tests inject, one decision each. */ +export type ReferenceMutant = + /** READY-2: the sync run reports ready before its first delivery. */ + | `ready-before-delivery` + /** READY-4: a failure before ready reports a different error value. */ + | `wraps-loading-failure` + /** HEALTH-1: a failure after ready empties the rows. */ + | `failure-clears-rows` + /** HEALTH-1: a retried failure is reported as a given-up one. */ + | `transient-is-fatal` + /** HEALTH-2: a successful delivery does not clear a retried failure. */ + | `never-recovers` + /** RETRY-1: a retried failure before ready fails the first load. */ + | `transient-fails-loading` + /** PUB-1: one provider batch is committed as one transaction per change. */ + | `splits-batch` + /** PUB-3: a batch's deletes are not written. */ + | `drops-deletes` + /** PUB-7: a later complete delivery merges into the rows. */ + | `snapshot-merges` + /** LIFE-2: cleanup does not release the provider subscription. */ + | `leaks-subscription` + /** LIFE-5: a new sync run starts from the previous run's rows. */ + | `restart-replays-rows` + /** LIFE-10: a failed provider's subscription stays open until cleanup. */ + | `keeps-failed-source` + /** LIFE-10: a failed provider's subscription is released twice. */ + | `double-release` + /** HEALTH-5: a failure also leaves a detached rejection. */ + | `detached-rejection` + /** HEALTH-5: a failure is also logged with console.error. */ + | `logs-failure` + /** EVID-1: a wait for the next txid fulfills before that batch arrives. */ + | `wait-off-by-one` + /** EVID-3: a wait ignores its timeout. */ + | `wait-ignores-timeout` + /** RECON-1: a transport drop is reported as a failure. */ + | `disconnect-fails` + /** RENEW-1: renewal opens a second subscription instead of renewing. */ + | `renew-resubscribes` + /** PUB-1, per-change commits: a batch of distinct keys commits in reverse. */ + | `reverses-batch` + +type ProviderEvent = + | { type: `snapshot`; rows: ReadonlyArray } + | { type: `changes`; changes: ReadonlyArray; txid: string } + | { type: `failed`; error: Error } + | { type: `transient`; error: Error } + | { type: `transport`; connected: boolean } + +interface ProviderSubscription { + releases: number + renewed: number + listener: (event: ProviderEvent) => void +} + +/** The controlled provider: one live subscription per sync run. */ +class ControlledPushProvider implements ControlledProvider { + readonly subscriptions: Array = [] + private pendingRenewal: ((capability: string) => void) | undefined + + subscribe( + listener: ProviderSubscription[`listener`], + onRenewalDue: (renew: (capability: string) => void) => void, + ): ProviderSubscription { + const subscription: ProviderSubscription = { + releases: 0, + renewed: 0, + listener, + } + this.subscriptions.push(subscription) + // The capability is due for renewal as soon as the subscription opens; + // the adapter's renewal stays pending until `completeRenewal`. + onRenewalDue((capability) => { + // The provider accepts any replacement for an open subscription. + if (subscription.releases === 0 && capability.length > 0) { + this.pendingRenewal = undefined + subscription.renewed += 1 + } + }) + return subscription + } + + requestReplacement(): Promise { + return new Promise((resolve) => { + this.pendingRenewal = resolve + }) + } + + private latest(): ProviderSubscription { + const subscription = this.subscriptions.at(-1) + if (!subscription) throw new Error(`no provider subscription`) + return subscription + } + + releases = () => + this.subscriptions.map((subscription) => subscription.releases) + deliverSnapshot = (rows: ReadonlyArray) => + this.latest().listener({ type: `snapshot`, rows }) + deliverChanges = (changes: ReadonlyArray, txid: string) => + this.latest().listener({ type: `changes`, changes, txid }) + fail = (error: Error) => this.latest().listener({ type: `failed`, error }) + transientFailure = (error: Error) => + this.latest().listener({ type: `transient`, error }) + // The provider tells its subscription about transport loss and return, as + // a real client's state events do. + disconnect = () => + this.latest().listener({ type: `transport`, connected: false }) + reconnect = () => + this.latest().listener({ type: `transport`, connected: true }) + completeRenewal = async () => { + const resolve = this.pendingRenewal + if (!resolve) throw new Error(`the adapter did not request a renewal`) + const before = this.latest().renewed + resolve(`capability-${this.subscriptions.length}`) + for (let turn = 0; turn < 10 && this.latest().renewed === before; turn++) { + await new Promise((settle) => setTimeout(settle, 0)) + } + } + renewals = () => this.subscriptions.at(-1)?.renewed ?? 0 +} + +type Waiter = { resolve: () => void; reject: (error: Error) => void } + +const changeKey = (change: ProviderChange) => + change.type === `upsert` ? change.row.id : change.id + +/** Builds the reference adapter's Collection over `provider`. */ +function referenceCollection( + provider: ControlledPushProvider, + mutant: ReferenceMutant | undefined, + perChange: boolean, +) { + let applied = new Set() + let waiters = new Map>() + let highestTxid = 0 + // Only the `restart-replays-rows` mutant reads this across sync runs. + let previousRows: ReadonlyArray = [] + + const settleWaits = (txid: string) => { + applied.add(txid) + highestTxid = Math.max(highestTxid, Number(txid)) + for (const waiter of waiters.get(txid) ?? []) waiter.resolve() + waiters.delete(txid) + } + + const collection = createCollection({ + id: `reference-push`, + getKey: (row) => row.id, + syncMode: `eager`, + sync: { + sync: ({ + begin, + write, + commit, + markReady, + fail, + reportError, + reportHealthy, + }) => { + let ready = false + let failing = false + let keys = new Set() + const opened: Array = [] + applied = new Set() + waiters = new Map() + highestTxid = 0 + + const release = (subscription: ProviderSubscription) => { + if (subscription.releases === 0 || mutant === `double-release`) { + subscription.releases += 1 + } + } + if (mutant === `restart-replays-rows` && previousRows.length > 0) { + begin() + for (const row of previousRows) write({ type: `insert`, value: row }) + void commit() + keys = new Set(previousRows.map((row) => row.id)) + } + if (mutant === `ready-before-delivery`) { + ready = true + markReady() + } + + const writeChange = (change: ProviderChange) => { + if (change.type === `upsert`) { + write({ + type: keys.has(change.row.id) ? `update` : `insert`, + value: change.row, + }) + keys.add(change.row.id) + } else if (mutant !== `drops-deletes`) { + write({ type: `delete`, key: change.id }) + keys.delete(change.id) + } + } + + // A successful delivery after a retried failure reports recovery. + const delivered = () => { + if (failing && mutant !== `never-recovers`) reportHealthy() + failing = false + } + + const stop = (subscription: ProviderSubscription, error: Error) => { + if (mutant === `detached-rejection`) void Promise.reject(error) + if (mutant === `logs-failure`) console.error(`source failed`, error) + if (mutant !== `keeps-failed-source`) release(subscription) + fail(error) + } + + const subscribe = ( + renewalDue: (renew: (capability: string) => void) => void, + ) => { + const subscription = provider.subscribe((event) => { + if (event.type === `snapshot`) { + if (mutant === `snapshot-merges` && ready) { + begin() + for (const row of event.rows) { + writeChange({ type: `upsert`, row }) + } + void commit() + } else { + begin({ snapshot: true }) + for (const row of event.rows) + write({ type: `insert`, value: row }) + void commit() + keys = new Set(event.rows.map((row) => row.id)) + } + previousRows = event.rows + if (!ready) { + ready = true + markReady() + } + delivered() + } else if (event.type === `changes`) { + let receipt: true | Promise = true + if (mutant === `splits-batch` || perChange) { + // The mutant reverses only batches of distinct keys, so the + // final rows match and only the intermediate commits differ. + const ordered = + mutant === `reverses-batch` && + new Set(event.changes.map(changeKey)).size === + event.changes.length + ? [...event.changes].reverse() + : event.changes + for (const change of ordered) { + begin() + writeChange(change) + receipt = commit() + } + } else { + begin() + for (const change of event.changes) writeChange(change) + receipt = commit() + } + if (receipt === true) settleWaits(event.txid) + else void receipt.then(() => settleWaits(event.txid)) + delivered() + } else if (event.type === `transport`) { + if (mutant === `disconnect-fails` && !event.connected) { + stop(subscription, new Error(`transport lost`)) + } + } else if (event.type === `transient`) { + if (!ready) { + if (mutant === `transient-fails-loading`) { + stop(subscription, event.error) + } + return + } + if (mutant === `transient-is-fatal`) { + stop(subscription, event.error) + return + } + failing = true + reportError(event.error) + } else if (!ready) { + stop( + subscription, + mutant === `wraps-loading-failure` + ? new Error(`wrapped`, { cause: event.error }) + : event.error, + ) + } else { + if (mutant === `failure-clears-rows`) { + begin({ snapshot: true }) + void commit() + } + stop(subscription, event.error) + } + }, renewalDue) + opened.push(subscription) + return subscription + } + + subscribe((renew) => { + void provider.requestReplacement().then((capability) => { + if (mutant === `renew-resubscribes`) subscribe(() => {}) + renew(capability) + }) + }) + + return () => { + if (mutant !== `leaks-subscription`) { + for (const subscription of opened) release(subscription) + } + const error = new Error(`reference collection was cleaned up`) + for (const set of waiters.values()) { + for (const waiter of set) waiter.reject(error) + } + waiters.clear() + } + }, + }, + }) + + const awaitTxId = (txid: string, timeout?: number): Promise => { + if (applied.has(txid)) return Promise.resolve(true) + if (mutant === `wait-off-by-one` && Number(txid) <= highestTxid + 1) { + return Promise.resolve(true) + } + return new Promise((resolve, reject) => { + const set = waiters.get(txid) ?? new Set() + const waiter: Waiter = { resolve: () => resolve(true), reject } + set.add(waiter) + waiters.set(txid, set) + if (timeout !== undefined && mutant !== `wait-ignores-timeout`) { + setTimeout(() => { + if (!set.delete(waiter)) return + reject(new Error(`timed out waiting for txid ${txid}`)) + }, timeout) + } + }) + } + + return { collection, awaitTxId } +} + +/** + * A production driver for the reference adapter, optionally mutated. With + * `perChange`, the adapter commits each change of a batch on its own and + * declares `per-change` commits, as TrailBase does. + */ +export function referenceDriver( + mutant?: ReferenceMutant, + { perChange = false }: { perChange?: boolean } = {}, +): EagerLiveSourceDriver { + return { + adapter: `reference${perChange ? ` per-change` : ``}${mutant ? ` (${mutant})` : ``}`, + capabilities: new Set([ + `complete-snapshot`, + `await-evidence`, + `retry-policy`, + `transport-reconnect`, + `capability-renewal`, + ]), + changeCommits: perChange ? `per-change` : `per-batch`, + create(): EagerLiveSourceFixture { + const provider = new ControlledPushProvider() + const { collection, awaitTxId } = referenceCollection( + provider, + mutant, + perChange, + ) + return { collection, provider, awaitTxId } + }, + } +} diff --git a/packages/db-adapter-conformance/tsconfig.json b/packages/db-adapter-conformance/tsconfig.json new file mode 100644 index 0000000000..e73985245b --- /dev/null +++ b/packages/db-adapter-conformance/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "target": "ES2020", + "module": "ESNext", + "lib": ["ES2024", "DOM"], + "moduleResolution": "Bundler", + "declaration": true, + "outDir": "dist", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "paths": { + "@tanstack/store": ["../store/src"], + "@tanstack/db-ivm": ["../db-ivm/src"], + "@tanstack/db": ["../db/src"] + } + }, + "include": ["src", "tests", "vite.config.ts", "vitest.neon.config.ts"], + "exclude": ["node_modules", "dist", "tests/neon/vendor"] +} diff --git a/packages/db-adapter-conformance/vite.config.ts b/packages/db-adapter-conformance/vite.config.ts new file mode 100644 index 0000000000..e94ee7e1f7 --- /dev/null +++ b/packages/db-adapter-conformance/vite.config.ts @@ -0,0 +1,24 @@ +import { defineConfig, mergeConfig } from 'vitest/config' +import { tanstackViteConfig } from '@tanstack/vite-config' +import packageJson from './package.json' + +const config = defineConfig({ + test: { + name: packageJson.name, + dir: `./tests`, + environment: `node`, + // The Neon receiving driver runs a vendored adapter that targets an older + // TanStack DB sync API. It has its own config and is expected to fail + // until that adapter is updated (see tests/neon/README.md). + exclude: [`**/node_modules/**`, `tests/neon/**`, `**/neon-*.test.ts`], + coverage: { enabled: true, provider: `istanbul`, include: [`src/**/*`] }, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ + entry: `./src/index.ts`, + srcDir: `./src`, + }), +) diff --git a/packages/db-adapter-conformance/vitest.neon.config.ts b/packages/db-adapter-conformance/vitest.neon.config.ts new file mode 100644 index 0000000000..6619d4ca21 --- /dev/null +++ b/packages/db-adapter-conformance/vitest.neon.config.ts @@ -0,0 +1,34 @@ +import { fileURLToPath } from 'node:url' +import { defineConfig, mergeConfig } from 'vitest/config' +import { tanstackViteConfig } from '@tanstack/vite-config' + +// The Neon Realtime receiving driver. It runs the shared eager live-source +// oracle against Neon's adapter source, vendored at a fixed commit, on the +// workspace TanStack DB. It is kept out of the default run because that +// adapter targets the 0.9 sync API and is expected to fail several laws until +// it is updated. Run it with `pnpm --filter @tanstack/db-adapter-conformance +// test:neon`. +const config = defineConfig({ + // The vendored adapter sits outside this package's tsconfig, so its + // imports need the workspace sources named directly. + resolve: { + alias: { + '@tanstack/db': fileURLToPath( + new URL(`../db/src/index.ts`, import.meta.url), + ), + '@tanstack/db-ivm': fileURLToPath( + new URL(`../db-ivm/src/index.ts`, import.meta.url), + ), + }, + }, + test: { + name: `@tanstack/db-adapter-conformance (neon)`, + include: [`tests/neon-realtime-oracle.property.test.ts`], + environment: `node`, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ entry: `./src/index.ts`, srcDir: `./src` }), +) diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index ca9c1a8927..abb26d1ce0 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -142,6 +142,9 @@ const staticOracleProperties = [ `sqlite-persistence.open-transaction-boundary`, `sqlite-persistence.request-local-failure-buffered-source`, `sqlite-persistence.optimistic-history`, + `adapter-conformance.eager-live-source.reference`, + `adapter-conformance.eager-live-source.neon-realtime`, + `adapter-conformance.eager-live-source.trailbase`, `trailbase.lifecycle`, `source-health.history`, `source-health.required-history`, diff --git a/packages/trailbase-db-collection/tests/ORACLE.md b/packages/trailbase-db-collection/tests/ORACLE.md index a7d57a4681..0415c6057e 100644 --- a/packages/trailbase-db-collection/tests/ORACLE.md +++ b/packages/trailbase-db-collection/tests/ORACLE.md @@ -250,3 +250,14 @@ These targeted mutations test known failure classes, not overall mutation coverage or a proof of completeness. The 10× campaign passed 800 generated histories (fixed seed 714203, random seed 127535183) plus the 32 corpus cases. No service-backed E2E run is claimed. + +## Shared conformance driver + +`eager-live-source-conformance-oracle.property.test.ts` runs the shared eager +live-source oracle from `packages/db-adapter-conformance` through +`conformance-driver.ts`, with the same controlled RecordApi. It checks +READY-2/3/4, HEALTH-1, PUB-1 (per-change commits), PUB-2/3, FID-4, +LIFE-2/5/10, and HEALTH-5 in eager mode. TrailBase declares no optional +capability, so it is not checked for PUB-7, waits, retried failures, +reconnect, or renewal. This file's +lifecycle oracle keeps its own laws; see `packages/db-adapter-conformance/MIGRATION.md`. diff --git a/packages/trailbase-db-collection/tests/conformance-driver.ts b/packages/trailbase-db-collection/tests/conformance-driver.ts new file mode 100644 index 0000000000..0ca4c94734 --- /dev/null +++ b/packages/trailbase-db-collection/tests/conformance-driver.ts @@ -0,0 +1,162 @@ +/** + * Production driver for the shared eager live-source conformance oracle + * (`packages/db-adapter-conformance`). It runs the real TrailBase adapter in + * eager mode against a controlled RecordApi, as `lifecycle-oracle` does: the + * driver supplies `subscribe` and `list` results and already-decoded events, + * and bypasses the SDK's fetch and SSE decoding. + * + * The oracle's steps map onto TrailBase like this: + * + * - a complete delivery before ready is the eager `list` page. TrailBase lists + * once per sync run and does not re-deliver a complete result, so the + * driver does not declare `complete-snapshot`; + * - a change batch is one change event per change: `Insert` for a key the + * provider does not hold, `Update` for one it does, `Delete` otherwise. + * TrailBase carries no txid, so the driver does not declare + * `await-evidence`; + * - a failure before ready rejects the pending `list`; after ready it errors + * the change stream; + * - TrailBase commits each change event on its own, so the driver declares + * `per-change` commits (PUB-1); + * - a release is the adapter cancelling the stream or releasing its reader + * lock (LIFE-2, LIFE-10). An errored stream does not call its source's + * `cancel`, so for it only the lock release is seen. + * + * The provider's own rows decide `Insert` versus `Update`, as a real service + * would. The driver never reads the Collection. + */ +import { createCollection } from '@tanstack/db' +import { trailBaseCollectionOptions } from '../src/trailbase' +import { MockRecordApi } from './mock-record-api' +import type { Event, ListResponse } from 'trailbase' +import type { + ConformanceRow, + ControlledProvider, + EagerLiveSourceDriver, + EagerLiveSourceFixture, + ObservedCollection, + ProviderChange, +} from '../../db-adapter-conformance/src' + +interface ProviderSession { + stream: ReadableStream + controller: ReadableStreamDefaultController + cancels: number + locked: boolean + list?: { + resolve: (response: ListResponse) => void + reject: (error: unknown) => void + } +} + +class TrailBaseProvider implements ControlledProvider { + readonly api = new MockRecordApi() + readonly sessions: Array = [] + private rows = new Map() + + constructor() { + this.api.subscribe.mockImplementation(() => { + let controller!: ReadableStreamDefaultController + const session = { cancels: 0, locked: false } as ProviderSession + const stream = new ReadableStream({ + start: (value) => { + controller = value + }, + cancel: () => { + session.cancels += 1 + }, + }) + session.stream = stream + session.controller = controller + this.sessions.push(session) + // A new sync run lists the source again from its current rows. + this.rows = new Map() + return Promise.resolve(stream) + }) + this.api.list.mockImplementation( + () => + new Promise>((resolve, reject) => { + this.latest().list = { resolve, reject } + }), + ) + } + + private latest(): ProviderSession { + const session = this.sessions.at(-1) + if (!session) throw new Error(`the adapter has not subscribed`) + return session + } + + private pendingList() { + const list = this.latest().list + if (!list) throw new Error(`the adapter has not requested its list`) + this.latest().list = undefined + return list + } + + // A session counts as released once the adapter cancelled it, or once it + // held the reader lock and gave it back. + releases = () => + this.sessions.map((session) => { + if (session.stream.locked) session.locked = true + const unlocked = session.locked && !session.stream.locked + return Math.max(session.cancels, unlocked ? 1 : 0) + }) + + deliverSnapshot = (rows: ReadonlyArray) => { + this.rows = new Map(rows.map((row) => [row.id, row])) + // One short page drains TrailBase's paged list. + this.pendingList().resolve({ records: [...rows] }) + } + + deliverChanges = (changes: ReadonlyArray) => { + const { controller } = this.latest() + for (const change of changes) { + if (change.type === `upsert`) { + const held = this.rows.has(change.row.id) + this.rows.set(change.row.id, change.row) + controller.enqueue( + held ? { Update: { ...change.row } } : { Insert: { ...change.row } }, + ) + } else { + const row = this.rows.get(change.id) + if (!row) throw new Error(`provider does not hold row ${change.id}`) + this.rows.delete(change.id) + controller.enqueue({ Delete: { ...row } }) + } + } + } + + fail = (error: Error) => { + const session = this.latest() + if (session.list) { + this.pendingList().reject(error) + } else { + session.controller.error(error) + } + } +} + +export function trailBaseConformanceDriver(): EagerLiveSourceDriver { + return { + adapter: `@tanstack/trailbase-db-collection (eager)`, + capabilities: new Set(), + changeCommits: `per-change`, + create(): EagerLiveSourceFixture { + const provider = new TrailBaseProvider() + const collection = createCollection( + trailBaseCollectionOptions({ + recordApi: provider.api, + getKey: (row: ConformanceRow) => row.id, + syncMode: `eager`, + parse: {}, + serialize: {}, + }), + ) + return { + collection: collection as unknown as ObservedCollection, + provider, + } + }, + } +} diff --git a/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts new file mode 100644 index 0000000000..ee412c2d35 --- /dev/null +++ b/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -0,0 +1,71 @@ +import { fc } from '@fast-check/vitest' +import { describe, it } from 'vitest' +import { + oraclePropertyOptions, + oracleReplayTest, + oracleRuns, +} from '../../db/tests/oracle-config' +import { declareOracle } from '../../db/tests/oracle-declaration' +import { createOracleReach } from '../../db/tests/oracle-reach' +import { + checkEagerLiveSourceHistory, + eagerLiveSourceHistoryArb, + eagerLiveSourceModel, + eagerLiveSourceProperty, + reachStatesFor, +} from '../../db-adapter-conformance/src' +import { trailBaseConformanceDriver } from './conformance-driver' + +/** + * # TrailBase on the shared eager live-source conformance oracle + * + * A receiving driver for `packages/db-adapter-conformance`. The shared + * module states the laws (READY-2/3/4, HEALTH-1, PUB-2/3, FID-4, LIFE-2/5), + * the model, the history grammar, and the check. This file supplies only the + * TrailBase production driver in `conformance-driver.ts` and the campaigns; + * it states no expected results. + * + * TrailBase declares none of the oracle's optional capabilities: it lists + * once per sync run, carries no txid, and does not resubscribe. Those laws + * stay with `lifecycle-oracle.property.test.ts`, which also owns on-demand + * loading, page and event fencing, cancellation, and timers. The claim is + * limited to the controlled RecordApi. + */ + +const property = `adapter-conformance.eager-live-source.trailbase` +const driver = trailBaseConformanceDriver() +const historyArb = eagerLiveSourceHistoryArb(driver.capabilities) +const runs = 200 + +async function campaign(options: fc.Parameters<[unknown]>) { + const reach = createOracleReach( + property, + reachStatesFor(driver.capabilities), + { + minimum: 5, + }, + ) + await fc.assert(eagerLiveSourceProperty(driver, reach), options) + return reach +} + +describe(`eager live-source conformance: TrailBase`, () => { + oracleReplayTest(it)(`conforms on a fixed-seed campaign`, async () => { + const reach = await campaign({ seed: 2118, numRuns: oracleRuns(runs) }) + reach.assertReached() + }) + + oracleReplayTest(it, property)( + `conforms on a random campaign or a replayed seed`, + async () => { + const reach = await campaign(oraclePropertyOptions(runs, property)) + reach.assertReached() + }, + ) +}) + +declareOracle({ + check: [checkEagerLiveSourceHistory, eagerLiveSourceProperty], + model: [eagerLiveSourceModel.step, eagerLiveSourceModel.expected], + grammar: [historyArb], +}) From 6e19f7677f1ecc5f196e9f6cd049560ade3c1fa7 Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 29 Oct 2026 11:56:55 -0600 Subject: [PATCH 02/64] test(rxdb): run RxDB on the eager live-source conformance oracle Add an RxDB production driver for the shared eager live-source oracle. The controlled RxCollection keeps a real RxSchema and answers the adapter's prepared queries with RxDB's getQueryMatcher and getSortComparator, so the paged initial fetch and its _meta.lwt cursor run against RxDB's query semantics. Each delivery is one bulk write whose documents share an lwt, as RxDB storage writes do. - Law: READY-2/3/4, HEALTH-1, PUB-1/2/3, FID-4, LIFE-2/5/10, HEALTH-5 from the adapter law set v18; RxDB declares no optional capability. Authority: the sync callback contract in packages/db/src/types.ts. - Reach: both campaigns (200 histories) assert every declared state at least five times. - RED: temporary edits of src/rxdb.ts, each reverted. No release when the initial fetch fails: LIFE-10. A `$` error ignored: HEALTH-1. The cursor's primary-key tie-break removed: READY-3 (rows lost on the second page). A live event committed without its write: PUB-3/FID-4. - GREEN: RxDB suite 21/21, including both conformance campaigns. - Mutant: the four edits above. Setting the cursor to a page's first document survives and is equivalent: it re-reads rows it already holds. - Limits: the grammar never sends events during RxDB's initial fetch, so its subscribe-before-read buffer is not generated (recorded in the coverage map and MIGRATION.md); the claim covers the controlled RxCollection, not a real RxDB storage. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 8 +- packages/db-adapter-conformance/MIGRATION.md | 57 +++- packages/db/tests/oracle-config.ts | 1 + .../tests/conformance-driver.ts | 293 ++++++++++++++++++ ...source-conformance-oracle.property.test.ts | 72 +++++ ...source-conformance-oracle.property.test.ts | 4 +- 6 files changed, 418 insertions(+), 17 deletions(-) create mode 100644 packages/rxdb-db-collection/tests/conformance-driver.ts create mode 100644 packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 756a603966..06d6bb5681 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -5998,4 +5998,10 @@ adapter oracles keep their laws until a shared family checks the same law, history, path, and observation. TrailBase (eager mode, per-change commits, `adapter-conformance.eager-live-source.trailbase`) is GREEN; its lifecycle oracle keeps its laws, so eager startup failure and stream-end health have -two owners until this entry moves them. +two owners until this entry moves them. RxDB +(`adapter-conformance.eager-live-source.rxdb`, per-change commits, a +controlled RxCollection with RxDB's own query matcher and sort comparator) is +GREEN. Open: events during RxDB's initial fetch, which the eager grammar does +not generate (owner: `packages/db-adapter-conformance`; witness needed: a +provider-state history with events before the first ready); `rxdb.test.ts` +keeps examples. diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index d0bd048b52..5b9d2b3a79 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -27,23 +27,23 @@ reviewed change. ## Adapters -| Adapter | Families that apply (section A) | Driver status | -| ----------------- | ------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Neon Realtime | eager live source, mutations | Eager: RED (READY-4, HEALTH-1, and LIFE-10 behind them); adapter update pending | -| Electric | eager live source, mutations, on-demand, source shapes, reload and retry, persistence | Not started | -| PowerSync | eager live source, mutations, on-demand | Not started | -| RxDB | eager live source, mutations | Not started | -| TrailBase | eager live source, mutations, on-demand | Eager: GREEN (2026-10-29), per-change commits. Driver `trailbase-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.trailbase`; declares no optional capability. Mutations, on-demand: not started | -| Convex | eager live source, mutations, on-demand, source shapes | Not started | -| query-db | client cache and refetch, on-demand, mutations, reload and retry, persistence | Not started; not a push source, so the eager family does not apply | -| local-only | local and shared stores, mutations | Not started | -| localStorage | local and shared stores, mutations | Not started | -| Persisted wrapper | persistence wrapper | Not started | -| Storage hosts (8) | storage hosts | Not started | +| Adapter | Families that apply (section A) | Driver status | +| ----------------- | ------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Neon Realtime | eager live source, mutations | Eager: RED (READY-4, HEALTH-1, and LIFE-10 behind them); adapter update pending | +| Electric | eager live source, mutations, on-demand, source shapes, reload and retry, persistence | Not started | +| PowerSync | eager live source, mutations, on-demand | Not started | +| RxDB | eager live source, mutations | Eager: GREEN (2026-10-29), per-change commits. Driver `rxdb-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.rxdb`; declares no optional capability. Open: events during the initial fetch (see log). Mutations: not started | +| TrailBase | eager live source, mutations, on-demand | Eager: GREEN (2026-10-29), per-change commits. Driver `trailbase-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.trailbase`; declares no optional capability. Mutations, on-demand: not started | +| Convex | eager live source, mutations, on-demand, source shapes | Not started | +| query-db | client cache and refetch, on-demand, mutations, reload and retry, persistence | Not started; not a push source, so the eager family does not apply | +| local-only | local and shared stores, mutations | Not started | +| localStorage | local and shared stores, mutations | Not started | +| Persisted wrapper | persistence wrapper | Not started | +| Storage hosts (8) | storage hosts | Not started | ## Suggested order -1. **Push adapters on the existing family:** RxDB, TrailBase (done), PowerSync, Electric (eager mode), and Convex. Each needs only a controlled-provider driver. A RED result is a finding against that adapter. +1. **Push adapters on the existing family:** RxDB (done), TrailBase (done), PowerSync, Electric (eager mode), and Convex. Each needs only a controlled-provider driver. A RED result is a finding against that adapter. 2. **The Neon update, run against its driver.** This shows an adapter moving from RED to GREEN. 3. **The mutations family.** It applies to every adapter except the storage hosts. 4. **The other families,** one at a time. Each family's new driver work goes to the adapters that declare it. @@ -72,3 +72,32 @@ reviewed change. LIFE-10: it keeps the raw subscription open after an error state until cleanup. With that subscription also released once at the error state, 300 histories pass. These two changes are the Neon update's scope. +- **2026-10-29, RxDB, eager live source.** Both campaigns are GREEN; it + found no RxDB defect. + - **Driver:** a controlled RxCollection with a real `RxSchema`. Its + `storageInstance.query` answers the adapter's prepared queries with + RxDB's `getQueryMatcher` and `getSortComparator`, so the adapter's paged + initial fetch (`syncBatchSize` 2) and `_meta.lwt` cursor run against + RxDB's query semantics. Each delivery is one bulk write whose documents + share an `lwt`, as RxDB storage writes do. `$` is an RxJS Observable that + counts teardowns per subscription. + - **Driver evidence:** temporary edits of `src/rxdb.ts`, each reverted: + - no release when the initial fetch fails: LIFE-10 (open provider + subscriptions), both campaigns; + - a `$` error ignored: HEALTH-1 (health error cause), both campaigns; + - the cursor's primary-key tie-break removed: READY-3 (rows), both + campaigns. The first driver gave each document its own `lwt` and + missed this; RxDB's bulk writes share one, so the driver was changed; + - a live event committed without its write: PUB-3/FID-4 (rows), both + campaigns; + - the cursor set to a page's first document: survives. It is + equivalent: the adapter re-reads and re-inserts rows it already holds, + and the rows are unchanged. + - **Open (in scope, not generated):** RxDB subscribes to `$` before its + initial fetch and buffers events until the fetch completes. The eager + grammar treats a batch before the first ready as illegal, so it never + sends events during the fetch. That needs a provider-state grammar, in + which the complete delivery reads the provider's current state and + earlier events are a consistent log. `rxdb.test.ts` keeps examples for + it. Owner: this package; witness needed: a generated history with + events between the subscription and the first ready. diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index abb26d1ce0..a85dbb43c5 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -145,6 +145,7 @@ const staticOracleProperties = [ `adapter-conformance.eager-live-source.reference`, `adapter-conformance.eager-live-source.neon-realtime`, `adapter-conformance.eager-live-source.trailbase`, + `adapter-conformance.eager-live-source.rxdb`, `trailbase.lifecycle`, `source-health.history`, `source-health.required-history`, diff --git a/packages/rxdb-db-collection/tests/conformance-driver.ts b/packages/rxdb-db-collection/tests/conformance-driver.ts new file mode 100644 index 0000000000..8e37b3e774 --- /dev/null +++ b/packages/rxdb-db-collection/tests/conformance-driver.ts @@ -0,0 +1,293 @@ +/** + * Production driver for the shared eager live-source conformance oracle + * (`packages/db-adapter-conformance`). It runs the real RxDB adapter against a + * controlled RxCollection. The controlled collection keeps the four parts the + * adapter's sync reads: `schema.primaryPath`, `storageInstance.schema`, + * `storageInstance.query`, and the `$` change stream. The schema is a real + * `RxSchema`, and `query` answers the adapter's own prepared queries with + * RxDB's query matcher and sort comparator, so the adapter's paged initial + * fetch and its `_meta.lwt` cursor run against RxDB's query semantics. + * + * The oracle's steps map onto RxDB like this: + * + * - a complete delivery is the initial fetch. The adapter's first query of a + * sync run waits until the oracle delivers; the provider then holds exactly + * those documents and answers every page from them. RxDB reads storage + * once per sync run, so the driver does not declare `complete-snapshot`; + * - a change batch is one change event per change on `$`: `INSERT` for a key + * the provider does not hold, `UPDATE` for one it does, `DELETE` otherwise. + * The adapter commits each event alone, so the driver declares `per-change` + * commits. RxDB events carry no txid, so the driver does not declare + * `await-evidence`; + * - a failure before ready rejects the pending query; after ready it errors + * the `$` stream; + * - a release is the teardown of the adapter's `$` subscription (LIFE-2, + * LIFE-10). An errored RxJS stream tears its subscriber down itself. + * + * `syncBatchSize` is 2, so a complete delivery of up to four rows takes up to + * three pages. Each delivery is one bulk write whose documents share a + * `_meta.lwt`, as RxDB's storage writes do, so pages after the first are + * found by the cursor's primary-key tie-break. + * + * RxDB primary keys are strings. The driver stores row `n` as document id + * `"n"`, and its observed view maps each key and row id back with + * `conformanceId`, which rejects anything that is not such a key. The view + * passes status, health, `preload`, and `cleanup` through unchanged. + */ +import { createCollection } from '@tanstack/db' +import { + createRxSchema, + defaultHashSha256, + getQueryMatcher, + getSortComparator, +} from 'rxdb/plugins/core' +import { Observable } from 'rxjs' +import { rxdbCollectionOptions } from '../src/rxdb' +import type { ChangeMessage } from '@tanstack/db' +import type { + PreparedQuery, + RxChangeEvent, + RxCollection, + RxDocumentData, +} from 'rxdb/plugins/core' +import type { Subscriber } from 'rxjs' +import type { + ConformanceRow, + ControlledProvider, + EagerLiveSourceDriver, + EagerLiveSourceFixture, + ObservedCollection, + ProviderChange, +} from '../../db-adapter-conformance/src' + +type Doc = { id: string; title: string } + +const schema = createRxSchema( + { + version: 0, + type: `object`, + primaryKey: `id`, + properties: { + id: { type: `string`, maxLength: 10 }, + title: { type: `string` }, + }, + required: [`id`, `title`], + }, + defaultHashSha256, +) + +interface ProviderSession { + subscriber: Subscriber> + releases: number + delivered: boolean + firstQuery?: { + resolve: () => void + reject: (error: unknown) => void + } +} + +class RxdbProvider implements ControlledProvider { + readonly sessions: Array = [] + private docs = new Map>() + private lwt = 0 + + /** The controlled RxCollection that the adapter reads. */ + readonly rxCollection = { + schema, + storageInstance: { + schema: schema.jsonSchema, + query: (prepared: PreparedQuery) => this.query(prepared), + }, + $: new Observable>((subscriber) => { + const session: ProviderSession = { + subscriber, + releases: 0, + delivered: false, + } + this.sessions.push(session) + return () => { + session.releases += 1 + } + }), + } as unknown as RxCollection + + private latest(): ProviderSession { + const session = this.sessions.at(-1) + if (!session) throw new Error(`the adapter has not subscribed`) + return session + } + + // The first query of a sync run waits for the complete delivery; later + // pages answer from the documents the provider then holds. + private async query(prepared: PreparedQuery) { + const session = this.latest() + if (!session.delivered) { + if (session.firstQuery) throw new Error(`a second query before delivery`) + await new Promise((resolve, reject) => { + session.firstQuery = { resolve, reject } + }) + } + const matches = getQueryMatcher(schema.jsonSchema, prepared.query) + const order = getSortComparator(schema.jsonSchema, prepared.query) + const { skip = 0, limit = Infinity } = prepared.query + const documents = [...this.docs.values()] + .filter((doc) => matches(doc)) + .sort(order) + .slice(skip, skip + limit) + return { documents } + } + + // One delivery is one RxDB bulk write, and every document a bulk write + // stores shares its `_meta.lwt`. The adapter's cursor breaks those ties on + // the primary key, so a complete delivery of several rows reaches that + // tie-break. + private document( + row: ConformanceRow, + lwt: number, + deleted = false, + ): RxDocumentData { + return { + id: String(row.id), + title: row.title, + _deleted: deleted, + _rev: `${lwt}-conformance`, + _attachments: {}, + _meta: { lwt }, + } + } + + private emit( + operation: RxChangeEvent[`operation`], + documentData: RxDocumentData, + previousDocumentData: RxDocumentData | undefined, + ) { + this.latest().subscriber.next({ + operation, + documentId: documentData.id, + documentData, + previousDocumentData, + isLocal: false, + collectionName: `conformance`, + } as RxChangeEvent) + } + + releases = () => this.sessions.map((session) => session.releases) + + deliverSnapshot = (rows: ReadonlyArray) => { + const session = this.latest() + const firstQuery = session.firstQuery + if (!firstQuery) throw new Error(`the adapter has not queried`) + const lwt = ++this.lwt + this.docs = new Map( + rows.map((row) => [String(row.id), this.document(row, lwt)]), + ) + session.delivered = true + session.firstQuery = undefined + firstQuery.resolve() + } + + deliverChanges = (changes: ReadonlyArray) => { + const lwt = ++this.lwt + for (const change of changes) { + if (change.type === `upsert`) { + const id = String(change.row.id) + const previous = this.docs.get(id) + const doc = this.document(change.row, lwt) + this.docs.set(id, doc) + this.emit(previous ? `UPDATE` : `INSERT`, doc, previous) + } else { + const id = String(change.id) + const previous = this.docs.get(id) + if (!previous) throw new Error(`provider does not hold row ${id}`) + this.docs.delete(id) + const doc = this.document( + { id: change.id, title: previous.title }, + lwt, + true, + ) + this.emit(`DELETE`, doc, previous) + } + } + } + + fail = (error: Error) => { + const session = this.latest() + if (session.firstQuery) { + session.firstQuery.reject(error) + session.firstQuery = undefined + } else { + session.subscriber.error(error) + } + } +} + +/** Maps an RxDB key back to its conformance row id; rejects anything else. */ +function conformanceId(key: unknown): number { + const id = Number(key) + if (typeof key !== `string` || !Number.isInteger(id) || String(id) !== key) { + throw new Error(`RxDB key ${JSON.stringify(key)} is not a conformance id`) + } + return id +} + +function conformanceRow(doc: Doc): ConformanceRow { + return { ...doc, id: conformanceId(doc.id) } +} + +/** The adapter's Collection, seen with conformance row ids. */ +function observedView( + collection: ReturnType, +): ObservedCollection { + return { + get status() { + return collection.status + }, + get health() { + return collection.health + }, + *values() { + for (const doc of collection.values()) yield conformanceRow(doc) + }, + preload: () => collection.preload(), + cleanup: () => collection.cleanup(), + subscribeChanges: (callback, options) => + collection.subscribeChanges( + (changes) => + callback( + changes.map((change): ChangeMessage => ({ + ...change, + key: conformanceId(change.key), + value: conformanceRow(change.value), + previousValue: + change.previousValue && conformanceRow(change.previousValue), + })), + ), + options, + ), + } +} + +function createRxdbCollection(provider: RxdbProvider) { + return createCollection( + rxdbCollectionOptions({ + rxCollection: provider.rxCollection, + syncBatchSize: 2, + syncMode: `eager`, + }), + ) +} + +export function rxdbConformanceDriver(): EagerLiveSourceDriver { + return { + adapter: `@tanstack/rxdb-db-collection`, + capabilities: new Set(), + changeCommits: `per-change`, + create(): EagerLiveSourceFixture { + const provider = new RxdbProvider() + return { + collection: observedView(createRxdbCollection(provider)), + provider, + } + }, + } +} diff --git a/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts new file mode 100644 index 0000000000..84fdded398 --- /dev/null +++ b/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -0,0 +1,72 @@ +import { fc } from '@fast-check/vitest' +import { describe, it } from 'vitest' +import { + oraclePropertyOptions, + oracleReplayTest, + oracleRuns, +} from '../../db/tests/oracle-config' +import { declareOracle } from '../../db/tests/oracle-declaration' +import { createOracleReach } from '../../db/tests/oracle-reach' +import { + checkEagerLiveSourceHistory, + eagerLiveSourceHistoryArb, + eagerLiveSourceModel, + eagerLiveSourceProperty, + reachStatesFor, +} from '../../db-adapter-conformance/src' +import { rxdbConformanceDriver } from './conformance-driver' + +/** + * # RxDB on the shared eager live-source conformance oracle + * + * A receiving driver for `packages/db-adapter-conformance`. The shared + * module states the laws (READY-2/3/4, HEALTH-1, PUB-1/2/3, FID-4, + * LIFE-2/5/10, HEALTH-5), the model, the history grammar, and the check. This + * file supplies only the RxDB production driver in `conformance-driver.ts` + * and the campaigns; it states no expected results. + * + * RxDB declares none of the oracle's optional capabilities: it reads storage + * once per sync run, its change events carry no txid, and it has no + * transport or capability to renew. The claim is limited to the controlled + * RxCollection; a claim about a real RxDB storage needs a receiving witness + * against it. `rxdb.test.ts` keeps its examples, including changes that + * arrive during the initial fetch, which this grammar does not generate. + */ + +const property = `adapter-conformance.eager-live-source.rxdb` +const driver = rxdbConformanceDriver() +const historyArb = eagerLiveSourceHistoryArb(driver.capabilities) +const runs = 200 + +async function campaign(options: fc.Parameters<[unknown]>) { + const reach = createOracleReach( + property, + reachStatesFor(driver.capabilities), + { + minimum: 5, + }, + ) + await fc.assert(eagerLiveSourceProperty(driver, reach), options) + return reach +} + +describe(`eager live-source conformance: RxDB`, () => { + oracleReplayTest(it)(`conforms on a fixed-seed campaign`, async () => { + const reach = await campaign({ seed: 2118, numRuns: oracleRuns(runs) }) + reach.assertReached() + }) + + oracleReplayTest(it, property)( + `conforms on a random campaign or a replayed seed`, + async () => { + const reach = await campaign(oraclePropertyOptions(runs, property)) + reach.assertReached() + }, + ) +}) + +declareOracle({ + check: [checkEagerLiveSourceHistory, eagerLiveSourceProperty], + model: [eagerLiveSourceModel.step, eagerLiveSourceModel.expected], + grammar: [historyArb], +}) diff --git a/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts index ee412c2d35..d225befe77 100644 --- a/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts +++ b/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -20,8 +20,8 @@ import { trailBaseConformanceDriver } from './conformance-driver' * # TrailBase on the shared eager live-source conformance oracle * * A receiving driver for `packages/db-adapter-conformance`. The shared - * module states the laws (READY-2/3/4, HEALTH-1, PUB-2/3, FID-4, LIFE-2/5), - * the model, the history grammar, and the check. This file supplies only the + * module states the laws (READY-2/3/4, HEALTH-1, PUB-1/2/3, FID-4, + * LIFE-2/5/10, HEALTH-5), the model, the history grammar, and the check. This file supplies only the * TrailBase production driver in `conformance-driver.ts` and the campaigns; * it states no expected results. * From 844b97b57a1934bc7c93575f97e613184922972c Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 29 Oct 2026 12:24:18 -0600 Subject: [PATCH 03/64] test(conformance): generate batches committed during the initial fetch Add the `listen-before-fetch` capability to the eager live-source oracle. The model gains backend state: a seed of rows committed before the subscription, batches committed while the first fetch is outstanding (a generated startup burst), and a read point `seen` on the first complete delivery. The initial cut folds every buffered batch, in order, onto the fetch result. Every driver is also checked for the rows at the moment status becomes `ready`, and a provider may count fetches issued before its listener was live. - Law: READY-8, READY-10, PUB-5, PUB-9 from the adapter law set v18 (CAP:live-changes), checked for drivers that declare `listen-before-fetch`; READY-3/READY-10 rows at ready for every driver. - Reach: fixed campaign (seed 2118, 400 histories) reaches change-before-ready 341, read-missed-change 70, seen-change-superseded 21; the lowest count of any state across 20 other seeds is 11. Without the startup burst, seen-change-superseded was reached 0 and 2 times, and the reach gate failed the campaign. - RED: the five new reference-adapter mutants fail the oracle. - GREEN: reference oracle 54/54; TrailBase conformance 2/2; Neon keeps its READY-4 and HEALTH-1 RED. - Mutant: drops-buffered (READY-10), buffer-before-rows (READY-10), replays-per-batch (PUB-5), ready-before-buffer (READY-10 rows when ready), shows-buffered (READY-10/PUB-9). An early-fulfilled wait for a still-buffered batch is now labeled READY-10/PUB-9, not EVID-1. - Limits: rows carry no optional field (FID-1 field removal unseen); the event fold does not flag an update of an absent key; commit receipts resolve at once, so a pending live receipt at the cut is not generated. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 12 +- packages/db-adapter-conformance/MIGRATION.md | 30 +- packages/db-adapter-conformance/README.md | 135 +++--- .../src/eager-live-source-oracle.ts | 411 +++++++++++++++--- .../reference-adapter-oracle.property.test.ts | 91 +++- .../tests/reference-adapter.ts | 50 ++- 6 files changed, 592 insertions(+), 137 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 06d6bb5681..3e77101413 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -5972,14 +5972,16 @@ mutants remain required receiving checks. Generated docs remain CI-owned. shared eager live-source laws from the adapter conformance law set (v18): READY-2/3/4, HEALTH-1, PUB-1/2/3/7, FID-4, LIFE-2/5/10, HEALTH-5, EVID-1/2/3 (`await-evidence`), HEALTH-2 and RETRY-1 for retried failures -(`retry-policy`), RECON-1 (`transport-reconnect`), and RENEW-1 -(`capability-renewal`). Its model is an independent row Map and sync-run +(`retry-policy`), RECON-1 (`transport-reconnect`), RENEW-1 +(`capability-renewal`), and READY-8/10 with PUB-5/9 for batches committed +during the initial fetch (`listen-before-fetch`). Every driver is also +checked for the rows at the moment status becomes `ready`. Its model is an independent row Map and sync-run phase; drivers supply only a controlled provider and declare whether they commit per batch or per change. PUB-1 checks every change publication during a step, not only the settled checkpoint. The reference push adapter (`tests/reference-adapter-oracle.property.test.ts`, `adapter-conformance.eager-live-source.reference`) runs fixed and random -campaigns of 400 histories with declared reach, a per-change variant, and 20 +campaigns of 400 histories with declared reach, a per-change variant, and 25 law-named hostile mutants, with ORC-004 reconstruction, exclusion, and ablation controls. The Neon Realtime receiving driver (`adapter-conformance.eager-live-source.neon-realtime`, `test:neon`) runs @@ -5988,7 +5990,9 @@ READY-4 and HEALTH-1 until that adapter moves to `fail()`; probes show LIFE-10 (subscription kept open after an error state) behind them. Limits: no on-demand loading, mutation handlers, or reload; publications are observed through `subscribeChanges` only; EVID-3 does not distinguish a timed wait -that rejects before its timeout; the Neon and TrailBase claims cover +that rejects before its timeout; rows carry no optional field, so FID-1 field +removal is not observed; the event fold does not flag an update of an absent +key; the Neon and TrailBase claims cover controlled providers, not real clients. Owner: `packages/db-adapter-conformance`. Evidence: the package README. Migration goal (maintainer, 2026-10-29): every collection adapter gets a diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index 5b9d2b3a79..fce70ddc36 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -14,16 +14,16 @@ reviewed change. ## Oracle families -| Family | Status | Laws | -| ------------------------------------- | ----------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| Eager live source | Built (`src/eager-live-source-oracle.ts`) | READY-2/3/4, HEALTH-1/2/5, PUB-1/2/3/7, FID-4, LIFE-2/5/10, EVID-1/2/3, RETRY-1 (eager), RECON-1, RENEW-1 | -| Mutations and confirmation | Needed | MUT-1/2/4/5/7/10/13, FID-5 | -| On-demand loading | Needed | DEM-1 to DEM-23, MAP-1 to MAP-4 | -| Reload and retry | Needed | RELOAD-1 to RELOAD-5; RETRY-1 and HEALTH-2 beyond the eager source's retried failures | -| Client cache and refetch | Needed | CACHE-1 to CACHE-8, REF-1 to REF-7, DIR-1 to DIR-5 | -| Source shapes | Needed | RESET-1, RESUME-1 to RESUME-3, PROG-1, MEMB-1/2, PAGE-1 to PAGE-3, OWN-1/2, META-1 | -| Local and shared stores | Needed | SELF-1 to SELF-3, STORE-1 to STORE-4, PEER-1 | -| Persistence wrapper and storage hosts | Needed | P-_, S-_ | +| Family | Status | Laws | +| ------------------------------------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | +| Eager live source | Built (`src/eager-live-source-oracle.ts`) | READY-2/3/4/8/10, HEALTH-1/2/5, PUB-1/2/3/5/7/9, FID-4, LIFE-2/5/10, EVID-1/2/3, RETRY-1 (eager), RECON-1, RENEW-1 | +| Mutations and confirmation | Needed | MUT-1/2/4/5/7/10/13, FID-5 | +| On-demand loading | Needed | DEM-1 to DEM-23, MAP-1 to MAP-4 | +| Reload and retry | Needed | RELOAD-1 to RELOAD-5; RETRY-1 and HEALTH-2 beyond the eager source's retried failures | +| Client cache and refetch | Needed | CACHE-1 to CACHE-8, REF-1 to REF-7, DIR-1 to DIR-5 | +| Source shapes | Needed | RESET-1, RESUME-1 to RESUME-3, PROG-1, MEMB-1/2, PAGE-1 to PAGE-3, OWN-1/2, META-1 | +| Local and shared stores | Needed | SELF-1 to SELF-3, STORE-1 to STORE-4, PEER-1 | +| Persistence wrapper and storage hosts | Needed | P-_, S-_ | ## Adapters @@ -101,3 +101,13 @@ reviewed change. earlier events are a consistent log. `rxdb.test.ts` keeps examples for it. Owner: this package; witness needed: a generated history with events between the subscription and the first ready. +- **2026-10-29, eager live source, `listen-before-fetch` (from the RxDB + migration).** The RxDB open item became an oracle extension. A driver + that declares `listen-before-fetch` gets backend state, a seed of rows + committed before the subscription, a startup burst of batches committed + during the first fetch, and a read point (`seen`). The model folds the + buffered batches onto the fetch result at the initial cut; publications + must be backend states at or after the read point (PUB-5, PUB-9). Every + driver also gets a check of the rows at the moment status becomes `ready` + (READY-3, READY-10). Five reference mutants calibrate it, and the reach + of the three new states is declared. Evidence: `README.md`. diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md index 719ae519cb..504997145f 100644 --- a/packages/db-adapter-conformance/README.md +++ b/packages/db-adapter-conformance/README.md @@ -13,20 +13,23 @@ The package is private while its first oracle settles. pushes a complete result and then committed change batches. Its opening comment states the laws and their limits. In short: -| Law | Checked behavior | -| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| READY-2, READY-3 | `loading` until the first complete delivery; then `ready` with exactly its rows | -| READY-4 | A failure before ready: status `error`, `preload()` rejects with the value, `health.error.cause` is the value | -| HEALTH-1 | A failure after ready: status stays `ready`, rows kept, `health.error.cause` is the value | -| PUB-1 | Every change publication during a step is one whole committed state, in commit order; per batch or per change, as the driver declares | -| PUB-2, PUB-3, PUB-7, FID-4 | Change events fold to the rows; batches fold onto rows; a complete delivery replaces (PUB-7 with `complete-snapshot`) | -| LIFE-2, LIFE-5 | Cleanup releases the provider subscription; a restart is a fresh, empty sync run | -| LIFE-10 | A failed provider's subscription is released at the failure, and no subscription is released twice | -| HEALTH-5 | No unhandled rejection and no undeclared `console.error` or `console.warn` | -| EVID-1, EVID-2, EVID-3 | `awaitTxId` fulfills after its batch applies, stays pending before, rejects after its timeout, and rejects at retirement | -| HEALTH-2, RETRY-1 | A retried failure stays hidden before ready, joins `failureReason` after it, and clears on the next delivery (`retry-policy`) | -| RECON-1 | Disconnect and reconnect keep status and rows | -| RENEW-1 | Capability renewal reuses the provider subscription and changes nothing visible | +| Law | Checked behavior | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| READY-2, READY-3 | `loading` until the first complete delivery; then `ready` with exactly its rows | +| READY-4 | A failure before ready: status `error`, `preload()` rejects with the value, `health.error.cause` is the value | +| HEALTH-1 | A failure after ready: status stays `ready`, rows kept, `health.error.cause` is the value | +| PUB-1 | Every change publication during a step is one whole committed state, in commit order; per batch or per change, as the driver declares | +| PUB-2, PUB-3, PUB-7, FID-4 | Change events fold to the rows; batches fold onto rows; a complete delivery replaces (PUB-7 with `complete-snapshot`) | +| LIFE-2, LIFE-5 | Cleanup releases the provider subscription; a restart is a fresh, empty sync run | +| LIFE-10 | A failed provider's subscription is released at the failure, and no subscription is released twice | +| HEALTH-5 | No unhandled rejection and no undeclared `console.error` or `console.warn` | +| EVID-1, EVID-2, EVID-3 | `awaitTxId` fulfills after its batch applies, stays pending before, rejects after its timeout, and rejects at retirement | +| HEALTH-2, RETRY-1 | A retried failure stays hidden before ready, joins `failureReason` after it, and clears on the next delivery (`retry-policy`) | +| RECON-1 | Disconnect and reconnect keep status and rows | +| RENEW-1 | Capability renewal reuses the provider subscription and changes nothing visible | +| READY-8, READY-10 | With `listen-before-fetch`: no fetch before the listener is live; batches committed during the fetch apply after its result, before ready | +| PUB-5, PUB-9 | With `listen-before-fetch`: the cut publishes only backend states at or after the read point, in order; no replay regresses a row | +| READY-3, READY-10 | When status becomes `ready`, the rows already equal the initial cut | The law IDs come from the adapter conformance law set, version 18. @@ -77,53 +80,61 @@ reach counter, so the run fails when histories miss a state the laws need. | Declared state | Histories | | ------------------------------ | --------- | - | first-ready | 184 | - | replacing-snapshot | 58 | - | change-batch | 77 | - | multi-change-batch | 28 | - | failure-before-ready | 83 | - | failure-after-ready | 24 | - | restart | 73 | - | wait-before-evidence-fulfilled | 23 | - | wait-timed-out | 60 | - | wait-rejected-at-retirement | 149 | - | transient-failure-reported | 48 | - | transient-failure-recovered | 22 | - | reconnect | 34 | - | renewal | 94 | - - The campaign executed 1,353 steps and skipped 1,101 as illegal at their - position, for example a batch before the first ready. Each campaign fails + | first-ready | 174 | + | replacing-snapshot | 48 | + | change-batch | 67 | + | multi-change-batch | 209 | + | failure-before-ready | 88 | + | failure-after-ready | 36 | + | restart | 87 | + | wait-before-evidence-fulfilled | 30 | + | wait-timed-out | 68 | + | wait-rejected-at-retirement | 128 | + | transient-failure-reported | 43 | + | transient-failure-recovered | 25 | + | reconnect | 31 | + | renewal | 101 | + | change-before-ready | 341 | + | read-missed-change | 70 | + | seen-change-superseded | 21 | + + The campaign executed 2,147 steps and skipped 938 as illegal at their + position, for example a batch after a failure. Each campaign fails when a state is reached fewer than five times. Across 20 other seeds, the - lowest count for any state was 11. + lowest count for any state was 11 (`seen-change-superseded`). - **GREEN:** `pnpm exec vitest run tests/reference-adapter-oracle.property.test.ts`: - 46 tests pass (both campaigns, the per-change pair, 19 calibrations, and the + 54 tests pass (both campaigns, the per-change pair, 24 calibrations, and the grammar controls). TrailBase: its full suite, 151 tests, passes. - **Mutants:** the fixed campaign rejects each with the law it breaks: - | Mutant | Law | - | ----------------------------- | -------- | - | `ready-before-delivery` | READY-2 | - | `wraps-loading-failure` | READY-4 | - | `failure-clears-rows` | HEALTH-1 | - | `transient-is-fatal` | HEALTH-1 | - | `never-recovers` | HEALTH-2 | - | `transient-fails-loading` | RETRY-1 | - | `splits-batch` | PUB-1 | - | `reverses-batch` (per-change) | PUB-1 | - | `drops-deletes` | PUB-3 | - | `snapshot-merges` | PUB-7 | - | `leaks-subscription` | LIFE-2 | - | `restart-replays-rows` | LIFE-5 | - | `keeps-failed-source` | LIFE-10 | - | `double-release` | LIFE-10 | - | `detached-rejection` | HEALTH-5 | - | `logs-failure` | HEALTH-5 | - | `wait-off-by-one` | EVID-1 | - | `wait-ignores-timeout` | EVID-3 | - | `disconnect-fails` | RECON-1 | - | `renew-resubscribes` | RENEW-1 | + | Mutant | Law | + | ----------------------------- | -------------------------- | + | `ready-before-delivery` | READY-2 | + | `wraps-loading-failure` | READY-4 | + | `failure-clears-rows` | HEALTH-1 | + | `transient-is-fatal` | HEALTH-1 | + | `never-recovers` | HEALTH-2 | + | `transient-fails-loading` | RETRY-1 | + | `splits-batch` | PUB-1 | + | `reverses-batch` (per-change) | PUB-1 | + | `drops-deletes` | PUB-3 | + | `snapshot-merges` | PUB-7 | + | `leaks-subscription` | LIFE-2 | + | `restart-replays-rows` | LIFE-5 | + | `keeps-failed-source` | LIFE-10 | + | `double-release` | LIFE-10 | + | `detached-rejection` | HEALTH-5 | + | `logs-failure` | HEALTH-5 | + | `wait-off-by-one` | EVID-1 | + | `wait-ignores-timeout` | EVID-3 | + | `disconnect-fails` | RECON-1 | + | `renew-resubscribes` | RENEW-1 | + | `drops-buffered` | READY-10 | + | `buffer-before-rows` | READY-10 | + | `replays-per-batch` | PUB-5 | + | `ready-before-buffer` | READY-10 (rows when ready) | + | `shows-buffered` | READY-10/PUB-9 | `reverses-batch` reverses only batches of distinct keys, so its final rows are right and only the intermediate publications are wrong. An earlier @@ -132,9 +143,9 @@ reach counter, so the run fails when histories miss a state the laws need. - **Shrinking keeps the first violation (ORC-010):** `eagerLiveSourceProperty` treats a smaller history that fails a different law as passing. A shrunk counterexample therefore violates the law the campaign found. A cleanup error after a violation is kept with it in an `AggregateError`. - **Grammar controls (ORC-004):** these run in the reference oracle file. - - **Reconstruction:** nine witnesses from the Neon and TrailBase suites and the retry contract are rebuilt as histories. Each is accepted with no skipped step and passes. - - **Exclusion:** five nearby invalid histories are each skipped by exactly one step. - - **Ablation:** removing each step kind loses the declared state that only it reaches. + - **Reconstruction:** ten witnesses from the Neon, TrailBase, and RxDB suites and the retry contract are rebuilt as histories. Each is accepted with no skipped step and passes. + - **Exclusion:** six nearby invalid histories are each skipped by exactly one step; a batch before the first ready is invalid only without `listen-before-fetch`. + - **Ablation:** removing each step kind loses the declared state that only it reaches, and reading after every buffered batch loses `read-missed-change`. - **Replay:** during a path replay, fixed campaigns and calibrations stand aside (`oracleReplayTest`). - **RED (Neon):** `pnpm test:neon`. Both campaigns fail. The fixed campaign shrinks to `[fail, wait]`: READY-4, status `loading` where `error` is owed. @@ -157,5 +168,13 @@ reach counter, so the run fails when histories miss a state the laws need. timed wait settles within its own step, so no evidence arrives during it. An adapter that rejects a timed wait at once, before its timeout, is not distinguished from one that waits. + - Rows carry only an id and a title, so a field that an update removes is + not observed (FID-1). + - The change-event fold flags a duplicate insert and a delete of an absent + key, but not an update of an absent key. + - READY-8's fetch order is checked only by a provider that implements + `fetchesBeforeListening`. Commit receipts resolve at once in every + driver, so a live commit whose receipt is still pending at the cut + (READY-10's "does not delay it") is not generated. - The Neon and TrailBase claims are limited to the controlled providers. A claim about a real client needs a receiving witness against it (ORC-014). diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 21aca4b23c..55989b4f70 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -50,12 +50,27 @@ * - **RENEW-1** (capability `capability-renewal`). While the source's * capability awaits renewal, status and rows do not change. Renewal reuses * the existing provider subscription: no new subscription, no reset. + * - **READY-8, READY-10, PUB-5, PUB-9** (capability `listen-before-fetch`). + * The adapter's change listener is live before it fetches. The backend can + * commit batches while the fetch is outstanding; the listener sees them, + * and the fetch result reflects only those committed before its read point. + * Until the initial cut applies, status stays `loading` and no row shows. + * The cut is the fetch result followed by the batches the listener saw, in + * order, and status becomes `ready` only after it applied, with the rows + * equal to the backend's current state. A batch the read already reflects + * is never replayed so that an older value overtakes a newer one: every + * publication is a backend state at or after the read point, in order. + * - **READY-3, READY-10.** When status becomes `ready`, the rows already + * equal the initial cut. A `status:ready` listener never sees fewer rows. * * Authority: the sync callback contract in `packages/db/src/types.ts` * (`begin`, `commit`, `markReady`, `fail`, `reportError`, `reportHealthy`), * the Collection status and health terms in `docs/contributing/glossary.md`, * and the adapter conformance law set (issue #2118, version 18), whose law - * IDs are used here. + * IDs are used here. READY-8 and READY-10 are that law set's laws for + * capability `live-changes`; this oracle checks them for adapters that + * declare `listen-before-fetch`, whose provider can deliver changes while + * the initial fetch is outstanding. * * Boundary. The oracle checks an eager Collection whose provider pushes one * complete result at a time. On-demand loading, mutation handlers, and @@ -94,6 +109,7 @@ export type EagerLiveSourceCapability = | `transport-reconnect` | `capability-renewal` | `retry-policy` + | `listen-before-fetch` /** * The public Collection surface the oracle reads. A driver passes the @@ -110,6 +126,8 @@ export interface ObservedCollection { callback: (changes: Array>) => void, options?: { includeInitialState?: boolean }, ) => { unsubscribe: () => void } + /** Core's status event; `cleanup()` removes its listeners. */ + on: (event: `status:ready`, callback: () => void) => () => void } /** @@ -143,6 +161,16 @@ export interface ControlledProvider { completeRenewal?: () => Promise /** Renewals the adapter completed on its latest subscription. */ renewals?: () => number + /** + * Sets the backend's rows before the adapter subscribes: rows committed + * before the subscription, which no change event announces (capability + * `listen-before-fetch`). A later `deliverChanges` before the first ready + * is a backend commit on top of them; `deliverSnapshot` then delivers the + * fetch result the oracle computed. + */ + seed?: (rows: ReadonlyArray) => void + /** Fetches the adapter issued while its change listener was not live. */ + fetchesBeforeListening?: () => number } /** One adapter under test. */ @@ -186,10 +214,14 @@ export interface EagerLiveSourceDriver { * counted, so a generated list stays meaningful after shrinking. * * - `snapshot`: a complete delivery. Legal while the sync run is active and - * connected; after the first ready only with `complete-snapshot`. - * - `changes`: one committed batch, legal after the first ready. Its txid is - * the next in the sync run. A delete of a key the source does not hold is - * dropped, because a provider never sends one. + * connected; after the first ready only with `complete-snapshot`. With + * `listen-before-fetch`, the first one is the initial fetch's result: its + * rows are the backend's rows before the subscription, and `seen` places + * the fetch's read point after that many of the batches committed since. + * - `changes`: one committed batch, legal after the first ready, and before + * it with `listen-before-fetch`. Its txid is the next in the sync run. A + * delete of a key the source does not hold is dropped, because a provider + * never sends one. * - `wait`: `awaitTxId` for the last txid plus `ahead`, so a wait can target * an applied batch (`ahead` 0) or a future one. With `timeout`, the wait * carries a short timeout. @@ -201,10 +233,17 @@ export interface EagerLiveSourceDriver { * - `restart`: cleanup, then `preload()` starts a new sync run. * * Keys range over 1..5 and titles over three values, so batches revisit - * keys, deletes hit live rows, and complete deliveries omit held keys. + * keys, deletes hit live rows, and complete deliveries omit held keys. With + * `listen-before-fetch`, a history starts with up to three batches committed + * during the first fetch. */ export type EagerLiveSourceStep = - | { readonly kind: `snapshot`; readonly rows: ReadonlyArray } + | { + readonly kind: `snapshot` + readonly rows: ReadonlyArray + /** The read point (`listen-before-fetch`); all batches when absent. */ + readonly seen?: number + } | { readonly kind: `changes` readonly changes: ReadonlyArray @@ -244,8 +283,18 @@ export function eagerLiveSourceHistoryArb( { weight: 4, arbitrary: fc - .uniqueArray(rowArb, { selector: (row) => row.id, maxLength: 4 }) - .map((rows) => ({ kind: `snapshot` as const, rows })), + .record({ + rows: fc.uniqueArray(rowArb, { + selector: (row) => row.id, + maxLength: 4, + }), + seen: fc.integer({ min: 0, max: 3 }), + }) + .map(({ rows, seen }) => + capabilities.has(`listen-before-fetch`) + ? { kind: `snapshot` as const, rows, seen } + : { kind: `snapshot` as const, rows }, + ), }, { weight: 6, @@ -293,7 +342,17 @@ export function eagerLiveSourceHistoryArb( arbitrary: fc.constant({ kind: `renew` as const }), }) } - return fc.array(fc.oneof(...steps), { minLength: 1, maxLength: 12 }) + const history = fc.array(fc.oneof(...steps), { minLength: 1, maxLength: 12 }) + if (!capabilities.has(`listen-before-fetch`)) return history + // A startup burst: batches the backend commits while the first fetch is + // outstanding. Without it, two buffered batches that touch one key before + // the read point are too rare for PUB-5 to be checked. + const startup = fc + .array(changeArb, { minLength: 1, maxLength: 3 }) + .map((changes) => ({ kind: `changes` as const, changes })) + return fc + .tuple(fc.array(startup, { maxLength: 3 }), history) + .map(([burst, rest]) => [...burst, ...rest]) } // --------------------------------------------------------------------------- @@ -310,6 +369,10 @@ export function eagerLiveSourceHistoryArb( * a flag. * - `rows` is the source's result as the provider delivered it: an * independent Map, folded from steps, never read from the adapter. + * - `backend` is the provider's current rows. It equals `rows` after the + * first ready. Before it, with `listen-before-fetch`, it starts at `seed` + * and `buffered` logs the batches committed since the subscription; the + * initial cut folds them onto the fetch result. * - `lastTxid` numbers the sync run's batches. A wait targets it. * - `transient` lists the open transient failures, oldest first. Only a * failure after the first ready is listed (RETRY-1). @@ -331,6 +394,9 @@ interface ModelState { run: number phase: Phase rows: Map + backend: Map + seed: Map + buffered: Array<{ changes: ReadonlyArray; txid: number }> connected: boolean lastTxid: number committed: Set @@ -357,11 +423,18 @@ export interface ExpectedObservation { renewalsThisRun: number } -function initialModel(): ModelState { +function rowMap(rows: ReadonlyArray): Map { + return new Map(rows.map((row) => [row.id, row.title])) +} + +function initialModel(seed: ReadonlyArray = []): ModelState { return { run: 1, phase: `loading`, rows: new Map(), + backend: rowMap(seed), + seed: rowMap(seed), + buffered: [], connected: true, lastTxid: 0, committed: new Set(), @@ -392,10 +465,59 @@ function rowState(rows: ReadonlyMap): RowState { return [...rows].sort(([a], [b]) => a - b) } +/** + * Folds one batch onto `rows` in order and returns the changes a provider + * delivers: a delete of a key it does not hold is dropped. `onChange` sees + * the rows after each delivered change. + */ +function foldBatch( + rows: Map, + changes: ReadonlyArray, + onChange: () => void = () => {}, +): Array { + const delivered: Array = [] + for (const change of changes) { + if (change.type === `upsert`) { + rows.set(change.row.id, change.row.title) + } else if (rows.has(change.id)) { + rows.delete(change.id) + } else { + continue + } + delivered.push(change) + onChange() + } + return delivered +} + +/** Whether replaying `changes` onto `rows` would change them. */ +function replayChanges( + rows: ReadonlyMap, + changes: ReadonlyArray, +): boolean { + const replayed = new Map(rows) + foldBatch(replayed, changes) + return !sameRows(rowState(replayed), rowState(rows)) +} + /** A step the model accepts, with what it actually delivers. */ type AcceptedStep = - | { kind: `snapshot`; rows: ReadonlyArray } - | { kind: `changes`; changes: Array; txid: number } + | { + kind: `snapshot` + rows: ReadonlyArray + /** Batches the initial cut folds after the fetch result. */ + buffered: number + /** Buffered batches the fetch result does not reflect. */ + missed: number + /** Whether replaying a batch the read reflects would change the rows. */ + superseded: boolean + } + | { + kind: `changes` + changes: Array + txid: number + beforeReady: boolean + } | { kind: `wait`; txid: number; timeout: boolean } | { kind: `fail`; failure: number } | { kind: `transient-failure`; failure: number } @@ -409,62 +531,153 @@ type AcceptedStep = * states that a publication during the step may show (PUB-1), or * `undefined` when the step is illegal now. */ +/** The driver facts the model depends on. */ +interface ModelOptions { + changeCommits: EagerLiveSourceDriver[`changeCommits`] + listenBeforeFetch: boolean +} + +/** EVID-1: a wait for `txid` fulfills once its batch applied. */ +function applyEvidence(state: ModelState, txid: number): void { + state.committed.add(txid) + for (const wait of state.waits) { + if ( + wait.run === state.run && + wait.txid === txid && + wait.expected === `pending` + ) { + wait.expected = `fulfilled` + } + } +} + +/** + * READY-10, PUB-5, PUB-9: the initial cut of a `listen-before-fetch` sync + * run. The fetch read the backend after `seen` of the buffered batches, so + * its result is the seed with those folded on. The cut then folds every + * buffered batch, in order, onto the result. A publication may show the + * result or any backend state after it, in order; with per-change commits + * also each change's prefix of a batch the read did not reflect. A replayed + * batch the read already reflects is never a publishable state of its own. + */ +function initialCut( + state: ModelState, + seen: number, + changeCommits: ModelOptions[`changeCommits`], +) { + const backend = new Map(state.seed) + const publishable: Array = [] + let read = rowState(backend) + state.buffered.forEach((batch, index) => { + if (index === seen) { + read = rowState(backend) + publishable.push(read) + } + const prefixes: Array = [] + foldBatch(backend, batch.changes, () => prefixes.push(rowState(backend))) + if (index < seen) return + if (changeCommits === `per-change`) publishable.push(...prefixes) + else publishable.push(rowState(backend)) + }) + if (seen === state.buffered.length) { + read = rowState(backend) + publishable.push(read) + } + const readRows = new Map(read) + const superseded = state.buffered + .slice(0, seen) + .some((batch) => replayChanges(readRows, batch.changes)) + return { backend, read, publishable, superseded } +} + function stepModel( state: ModelState, step: EagerLiveSourceStep, failures: { next: number }, - changeCommits: EagerLiveSourceDriver[`changeCommits`], + options: ModelOptions, + nextSeed: ReadonlyArray = [], ): { done: AcceptedStep; publishable: ReadonlyArray } | undefined { switch (step.kind) { case `snapshot`: { if (!canDeliver(state)) return undefined + // HEALTH-2: a successful delivery clears the open transient failures. + state.transient = [] + if (options.listenBeforeFetch && state.phase === `loading`) { + const buffered = state.buffered.length + const seen = Math.min(step.seen ?? buffered, buffered) + const cut = initialCut(state, seen, options.changeCommits) + state.rows = cut.backend + state.backend = new Map(cut.backend) + state.phase = `ready` + for (const batch of state.buffered) applyEvidence(state, batch.txid) + state.buffered = [] + return { + done: { + kind: `snapshot`, + rows: cut.read.map(([id, title]) => ({ id, title })), + buffered, + missed: buffered - seen, + superseded: cut.superseded, + }, + publishable: cut.publishable, + } + } // PUB-7: a complete delivery replaces the rows. READY-3: the first one - // opens the initial gate. HEALTH-2: a successful delivery clears the - // open transient failures. - state.rows = new Map(step.rows.map((row) => [row.id, row.title])) + // opens the initial gate. + state.rows = rowMap(step.rows) + state.backend = rowMap(step.rows) state.phase = `ready` - state.transient = [] return { - done: { kind: `snapshot`, rows: step.rows }, + done: { + kind: `snapshot`, + rows: step.rows, + buffered: 0, + missed: 0, + superseded: false, + }, publishable: [rowState(state.rows)], } } case `changes`: { - if (state.phase !== `ready` || !state.connected) return undefined - // PUB-3: fold the batch in order. A provider never deletes a key it - // does not hold, so such a delete is not delivered. PUB-1: each prefix - // is a committed state when the provider commits each change alone. - const delivered: Array = [] - const prefixes: Array = [] - for (const change of step.changes) { - if (change.type === `upsert`) { - state.rows.set(change.row.id, change.row.title) - delivered.push(change) - } else if (state.rows.has(change.id)) { - state.rows.delete(change.id) - delivered.push(change) - } else { - continue + if (!state.connected) return undefined + if (state.phase === `loading` && options.listenBeforeFetch) { + // READY-8: the listener is live, so a batch committed during the + // fetch reaches the adapter. It shows nothing until the cut. + const delivered = foldBatch(state.backend, step.changes) + state.lastTxid += 1 + state.buffered.push({ changes: delivered, txid: state.lastTxid }) + return { + done: { + kind: `changes`, + changes: delivered, + txid: state.lastTxid, + beforeReady: true, + }, + publishable: [], } - prefixes.push(rowState(state.rows)) } + if (state.phase !== `ready`) return undefined + // PUB-3: fold the batch in order. PUB-1: each prefix is a committed + // state when the provider commits each change alone. + const prefixes: Array = [] + const delivered = foldBatch(state.rows, step.changes, () => + prefixes.push(rowState(state.rows)), + ) + foldBatch(state.backend, delivered) state.lastTxid += 1 - state.committed.add(state.lastTxid) state.transient = [] - // EVID-1: a wait for this txid fulfills once the batch applied. - for (const wait of state.waits) { - if ( - wait.run === state.run && - wait.txid === state.lastTxid && - wait.expected === `pending` - ) { - wait.expected = `fulfilled` - } - } + applyEvidence(state, state.lastTxid) return { - done: { kind: `changes`, changes: delivered, txid: state.lastTxid }, + done: { + kind: `changes`, + changes: delivered, + txid: state.lastTxid, + beforeReady: false, + }, publishable: - changeCommits === `per-change` ? prefixes : [rowState(state.rows)], + options.changeCommits === `per-change` + ? prefixes + : [rowState(state.rows)], } } case `wait`: { @@ -527,11 +740,16 @@ function stepModel( return { done: { kind: `renew` }, publishable: [] } } case `restart`: { - // LIFE-5: cleanup retires the sync run; the next one starts empty. + // LIFE-5: cleanup retires the sync run; the next one starts empty. The + // backend may change while no sync run listens: it then holds + // `nextSeed`. retireWaits(state) state.run += 1 state.phase = `loading` state.rows = new Map() + state.backend = rowMap(nextSeed) + state.seed = rowMap(nextSeed) + state.buffered = [] state.connected = true state.lastTxid = 0 state.committed = new Set() @@ -590,6 +808,9 @@ export const eagerLiveSourceReachStates = [ `transient-failure-recovered`, `reconnect`, `renewal`, + `change-before-ready`, + `read-missed-change`, + `seen-change-superseded`, ] as const export type EagerLiveSourceReachState = @@ -606,6 +827,9 @@ const capabilityOfState: Partial< 'transient-failure-recovered': `retry-policy`, reconnect: `transport-reconnect`, renewal: `capability-renewal`, + 'change-before-ready': `listen-before-fetch`, + 'read-missed-change': `listen-before-fetch`, + 'seen-change-superseded': `listen-before-fetch`, } /** The declared states a driver's capabilities let histories reach. */ @@ -669,9 +893,9 @@ const timeoutSettleMs = 30 function lawFor(done: AcceptedStep, wasReady: boolean): string { switch (done.kind) { case `snapshot`: - return wasReady ? `PUB-7` : `READY-3` + return wasReady ? `PUB-7` : done.buffered > 0 ? `READY-10` : `READY-3` case `changes`: - return `PUB-3/FID-4` + return done.beforeReady ? `READY-10/PUB-9` : `PUB-3/FID-4` case `wait`: return done.timeout ? `EVID-3` : `EVID-1` case `fail`: @@ -734,13 +958,38 @@ export async function checkEagerLiveSourceHistory( options: CheckOptions = {}, ): Promise<{ accepted: number; skipped: number }> { const reach = options.reach ?? (() => {}) + const listenBeforeFetch = driver.capabilities.has(`listen-before-fetch`) + const modelOptions: ModelOptions = { + changeCommits: driver.changeCommits, + listenBeforeFetch, + } + // With `listen-before-fetch`, a sync run's backend starts with the rows of + // its first complete delivery, committed before the adapter subscribes. + const seedFrom = (start: number): ReadonlyArray => { + if (!listenBeforeFetch) return [] + for (const step of history.slice(start)) { + if (step.kind === `restart`) break + if (step.kind === `snapshot`) return step.rows + } + return [] + } const detached = watchDetachedFailures(driver) const fixture = driver.create() const { collection, provider } = fixture - const state = initialModel() + const state = initialModel(seedFrom(0)) const failures: Array = [] const failureCounter = { next: 0 } const waits: Array<{ current: Settlement }> = [] + if (listenBeforeFetch) provider.seed?.(seedFrom(0)) + // READY-3, READY-10: the rows at the moment status becomes `ready`. + let readyRows: RowState | undefined + const watchReady = () => + collection.on(`status:ready`, () => { + readyRows = rowState( + new Map([...collection.values()].map((row) => [row.id, row.title])), + ) + }) + watchReady() let preload = recordSettlement(collection.preload()) let events = foldChanges(collection) let accepted = 0 @@ -763,13 +1012,19 @@ export async function checkEagerLiveSourceHistory( stepIndex: number, publishable: ReadonlyArray, becameReady: boolean, + law: string, ) => { let last = -1 let readiness = becameReady ? 1 : 0 for (const publication of events.take()) { if (publication === `empty`) { if (readiness-- > 0) continue - throw violation(`PUB-1 (empty publication)`, stepIndex, `none`, `empty`) + throw violation( + `${law} (empty publication)`, + stepIndex, + `none`, + `empty`, + ) } // A publication shows a commit after the last one shown. Prefix states // can repeat within a batch, so the search starts after that commit. @@ -778,7 +1033,7 @@ export async function checkEagerLiveSourceHistory( ) if (index === -1) { throw violation( - `PUB-1 (publication)`, + `${law} (publication)`, stepIndex, publishable, publication, @@ -790,7 +1045,8 @@ export async function checkEagerLiveSourceHistory( // `law` names the step's law. Observations whose law does not depend on // the step name their own: HEALTH-2 when a delivery should have cleared a - // retried failure, EVID-1 for a wait that fulfilled without evidence, + // retried failure, EVID-1 for a wait that fulfilled without evidence + // (READY-10/PUB-9 when its batch should still be buffered), // EVID-3 for a timed wait that did not reject, and LIFE-10 for a failed // provider's subscription that stays open or is released twice. const compare = async ( @@ -817,6 +1073,19 @@ export async function checkEagerLiveSourceHistory( collection.status, ) check(`rows`, sameRows(rows, expected.rows), expected.rows, rows) + if (readyRows !== undefined) { + check( + `rows when ready`, + sameRows(readyRows, expected.rows), + expected.rows, + readyRows, + ) + readyRows = undefined + } + const early = provider.fetchesBeforeListening?.() ?? 0 + if (early > 0) { + throw violation(`READY-8 (fetch before listening)`, stepIndex, 0, early) + } check(`change events`, sameRows(events.rows(), rows), rows, events.rows()) const health = collection.health if (expected.healthError === null) { @@ -866,9 +1135,18 @@ export async function checkEagerLiveSourceHistory( const gotWait = waits[index]!.current const gotKind = typeof gotWait === `object` ? `rejected` : gotWait if (gotKind === wantWait) return + // A wait for a batch still buffered before the cut fulfilled because + // the adapter applied that batch early. + const buffered = state.buffered.some( + (batch) => + batch.txid === state.waits[index]!.txid && + state.waits[index]!.run === state.run, + ) const waitLaw = gotKind === `fulfilled` - ? `EVID-1` + ? buffered + ? `READY-10/PUB-9` + : `EVID-1` : wantWait === `rejected` && state.waits[index]!.run === state.run ? `EVID-3` : law @@ -923,7 +1201,13 @@ export async function checkEagerLiveSourceHistory( // Support is decided before the model moves, so a skipped step leaves // the model unchanged. const result = supports(driver, fixture, step, wasReady) - ? stepModel(state, step, failureCounter, driver.changeCommits) + ? stepModel( + state, + step, + failureCounter, + modelOptions, + seedFrom(index + 1), + ) : undefined if (result === undefined) { skipped++ @@ -935,11 +1219,13 @@ export async function checkEagerLiveSourceHistory( case `snapshot`: provider.deliverSnapshot(done.rows) reach(wasReady ? `replacing-snapshot` : `first-ready`) + if (done.missed > 0) reach(`read-missed-change`) + if (done.superseded) reach(`seen-change-superseded`) if (transientBefore > 0) reach(`transient-failure-recovered`) break case `changes`: provider.deliverChanges(done.changes, String(done.txid)) - reach(`change-batch`) + reach(done.beforeReady ? `change-before-ready` : `change-batch`) if (done.changes.length > 1) reach(`multi-change-batch`) if (transientBefore > 0) reach(`transient-failure-recovered`) if ( @@ -999,6 +1285,10 @@ export async function checkEagerLiveSourceHistory( const pendingBefore = waits.some((wait) => wait.current === `pending`) events.stop() await collection.cleanup() + // The backend may change while no sync run listens. + if (listenBeforeFetch) provider.seed?.(seedFrom(index + 1)) + readyRows = undefined + watchReady() preload = recordSettlement(collection.preload()) events = foldChanges(collection) reach(`restart`) @@ -1016,6 +1306,11 @@ export async function checkEagerLiveSourceHistory( index, publishable, !wasReady && state.phase === `ready`, + done.kind === `snapshot` && done.buffered > 0 + ? `PUB-5/PUB-9` + : done.kind === `changes` && done.beforeReady + ? `READY-10/PUB-9` + : `PUB-1`, ) } else { events.take() diff --git a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts index ff6d1cedb7..c1a1fae116 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts @@ -45,7 +45,7 @@ import type { ReferenceMutant } from './reference-adapter' * model (exclusion). Removing each step kind loses the declared state that * only it reaches (ablation). Range: keys 1..5 and three titles let batches * revisit keys and complete deliveries omit held keys; histories hold up to - * 12 steps; waits look 0..2 txids ahead, so an off-by-one boundary is + * 12 steps after a startup burst of up to three batches; waits look 0..2 txids ahead, so an off-by-one boundary is * reachable on both sides. */ @@ -111,6 +111,11 @@ describe(`eager live-source conformance: reference adapter`, () => { [`wait-ignores-timeout`, `EVID-3`], [`disconnect-fails`, `RECON-1`], [`renew-resubscribes`, `RENEW-1`], + [`drops-buffered`, `READY-10`], + [`buffer-before-rows`, `READY-10`], + [`replays-per-batch`, `PUB-5`], + [`ready-before-buffer`, `READY-10 (rows when ready)`], + [`shows-buffered`, `READY-10/PUB-9`], ] oracleReplayTest(it).each(mutants)( `rejects the %s mutant with %s`, @@ -256,6 +261,23 @@ const witnesses: Array<[string, EagerLiveSourceHistory]> = [ }, ], ], + [ + // rxdb-db-collection rxdb.test.ts, `marks initial sync ready only after + // its rows are applied`: a document inserted after the initial query read + // and then patched is buffered; the newer value wins over the older one. + `RxDB: buffered startup changes apply after the initial rows, in order`, + [ + { + kind: `changes`, + changes: [{ type: `upsert`, row: { id: 2, title: `b` } }], + }, + { + kind: `changes`, + changes: [{ type: `upsert`, row: { id: 2, title: `c` } }], + }, + { kind: `snapshot`, rows: [{ id: 1, title: `a` }], seen: 0 }, + ], + ], [ // lifecycle-oracle.property.test.ts, restart fencing `TrailBase: a restart is a fresh sync run`, @@ -273,11 +295,9 @@ function withinGrammar(history: EagerLiveSourceHistory): boolean { r.id >= 1 && r.id <= 5 && [`a`, `b`, `c`].includes(r.title) return ( history.length >= 1 && - history.length <= 12 && + history.length <= 15 && history.every((step) => { switch (step.kind) { - case `snapshot`: - return step.rows.length <= 4 && step.rows.every(row) case `changes`: return ( step.changes.length <= 3 && @@ -289,6 +309,13 @@ function withinGrammar(history: EagerLiveSourceHistory): boolean { ) case `wait`: return step.ahead >= 0 && step.ahead <= 2 + case `snapshot`: + return ( + step.rows.length <= 4 && + step.rows.every(row) && + (step.seen ?? 0) >= 0 && + (step.seen ?? 0) <= 3 + ) default: return true } @@ -307,9 +334,27 @@ describe(`eager live-source grammar controls`, () => { ) // Exclusion: each history ends with one step that is illegal where it - // stands. The check skips it and leaves the model where it was. + // stands. The check skips it and leaves the model where it was. A batch + // before the first ready is legal only with `listen-before-fetch`, so its + // case runs on a driver that does not declare it. + const lateListener = { + ...driver, + capabilities: new Set( + [...driver.capabilities].filter((c) => c !== `listen-before-fetch`), + ), + } + oracleReplayTest(it)( + `excludes a batch before the first ready without listen-before-fetch`, + async () => { + const history = [{ kind: `changes` as const, changes: [] }] + const { skipped } = await checkEagerLiveSourceHistory( + lateListener, + history, + ) + expect(skipped).toBe(1) + }, + ) const invalid: Array<[string, EagerLiveSourceHistory]> = [ - [`a batch before the first ready`, [{ kind: `changes`, changes: [] }]], [ `a second renewal in one sync run`, [{ kind: `renew` }, { kind: `renew` }], @@ -350,6 +395,7 @@ describe(`eager live-source grammar controls`, () => { [`reconnect`, `reconnect`], [`renew`, `renewal`], [`transient-failure`, `transient-failure-reported`], + [`changes`, `change-before-ready`], ] oracleReplayTest(it).each(ablations)( `without %s steps, histories never reach %s`, @@ -376,6 +422,39 @@ describe(`eager live-source grammar controls`, () => { }, campaignTimeoutMs, ) + + // Ablation of the read point: when every fetch reads after all buffered + // batches, no history reaches a cut whose result missed one. + oracleReplayTest(it)( + `with every read after all buffered batches, histories never reach read-missed-change`, + async () => { + const reach = createOracleReach( + `${property}.ablation`, + reachStatesFor(driver.capabilities), + { replaying: false }, + ) + await fc.assert( + fc.asyncProperty( + historyArb.map((history) => + history.map((step) => + step.kind === `snapshot` + ? { kind: step.kind, rows: step.rows } + : step, + ), + ), + async (history) => { + reach.beginHistory() + await checkEagerLiveSourceHistory(driver, history, { + reach: reach.reach, + }) + }, + ), + { seed: 2118, numRuns: 100 }, + ) + expect(reach.counts().get(`read-missed-change`)).toBe(0) + }, + campaignTimeoutMs, + ) }) declareOracle({ diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index 7c74756a1b..301cff5d24 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -5,7 +5,9 @@ * calibration tests run (ORC-006). It is not a published adapter. * * The provider pushes a complete result, then committed batches that carry a - * txid. It can fail, report a failure it is retrying, drop and restore its + * txid. Its subscription is live before the result arrives, so batches can + * arrive first; the adapter buffers them and applies them, in one + * transaction after the result, before it reports ready. It can fail, report a failure it is retrying, drop and restore its * transport, and ask for a capability renewal, which the adapter answers by * renewing the same subscription. */ @@ -60,6 +62,16 @@ export type ReferenceMutant = | `renew-resubscribes` /** PUB-1, per-change commits: a batch of distinct keys commits in reverse. */ | `reverses-batch` + /** READY-10: batches committed during the initial fetch are dropped. */ + | `drops-buffered` + /** PUB-9: buffered batches apply before the fetch result, which erases them. */ + | `buffer-before-rows` + /** PUB-5: each buffered batch commits alone, so a replay can regress a row. */ + | `replays-per-batch` + /** READY-10: status becomes `ready` before the buffered batches apply. */ + | `ready-before-buffer` + /** READY-10, PUB-9: a batch during the initial fetch shows before ready. */ + | `shows-buffered` type ProviderEvent = | { type: `snapshot`; rows: ReadonlyArray } @@ -180,6 +192,11 @@ function referenceCollection( }) => { let ready = false let failing = false + // Batches that arrived before the first complete result. + const buffered: Array<{ + changes: ReadonlyArray + txid: string + }> = [] let keys = new Set() const opened: Array = [] applied = new Set() @@ -215,6 +232,25 @@ function referenceCollection( } } + // READY-10: the buffered batches follow the result in one commit, so + // a batch the result already reflects never shows on its own. + const applyBuffered = (batches: typeof buffered) => { + if (batches.length === 0) return + const commits: Array> = + mutant === `replays-per-batch` + ? batches.map((batch) => [batch]) + : [batches] + for (const group of commits) { + begin() + for (const batch of group) batch.changes.forEach(writeChange) + const receipt = commit() + for (const { txid } of group) { + if (receipt === true) settleWaits(txid) + else void receipt.then(() => settleWaits(txid)) + } + } + } + // A successful delivery after a retried failure reports recovery. const delivered = () => { if (failing && mutant !== `never-recovers`) reportHealthy() @@ -233,6 +269,8 @@ function referenceCollection( ) => { const subscription = provider.subscribe((event) => { if (event.type === `snapshot`) { + const cut = ready ? [] : buffered.splice(0) + if (mutant === `buffer-before-rows`) applyBuffered(cut) if (mutant === `snapshot-merges` && ready) { begin() for (const row of event.rows) { @@ -246,6 +284,11 @@ function referenceCollection( void commit() keys = new Set(event.rows.map((row) => row.id)) } + if (mutant === `ready-before-buffer` && !ready) { + ready = true + markReady() + } + if (mutant !== `buffer-before-rows`) applyBuffered(cut) previousRows = event.rows if (!ready) { ready = true @@ -253,6 +296,10 @@ function referenceCollection( } delivered() } else if (event.type === `changes`) { + if (!ready && mutant !== `shows-buffered`) { + if (mutant !== `drops-buffered`) buffered.push(event) + return + } let receipt: true | Promise = true if (mutant === `splits-batch` || perChange) { // The mutant reverses only batches of distinct keys, so the @@ -372,6 +419,7 @@ export function referenceDriver( `retry-policy`, `transport-reconnect`, `capability-renewal`, + `listen-before-fetch`, ]), changeCommits: perChange ? `per-change` : `per-batch`, create(): EagerLiveSourceFixture { From f15cd7344db704ded428ce8fc51ebf0a54b5c3cf Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 29 Oct 2026 12:31:26 -0600 Subject: [PATCH 04/64] fix(rxdb): apply a buffered insert over the row the initial read holds RxDB subscribes to `$` before its initial fetch and buffers the events that arrive during it. When a document was inserted and then updated during the fetch and the read already returned its newer version, the buffered INSERT replayed as a sync `insert` of a key the Collection held with a different value, and initial sync failed with a duplicate key error. RxDB now writes INSERT and UPDATE events as a sync `update`; core inserts an updated key it does not hold. The RxDB conformance driver now declares `listen-before-fetch`: it seeds the backend, answers the initial fetch from its read point, and counts queries issued without a live `$` subscription. The oracle checks READY-8 before any other observation. - Law: READY-10 (ready follows the initial cut) and PUB-9 (transcript order) from the adapter law set v18, CAP:live-changes. - Reach: both RxDB campaigns assert every declared state at least five times, including change-before-ready, read-missed-change, and seen-change-superseded. - RED: on the parent, both campaigns failed READY-10 (status `error` where `ready` is owed), shrunk to [changes [upsert 3 c, upsert 3 a], snapshot seen 1]. The memory-storage witness in rxdb.test.ts failed with "Cannot insert document with key "1" from sync because it already exists". - GREEN: RxDB suite 22/22; reference oracle 54/54; TrailBase 2/2. - Mutant: the original insert write (READY-10 status), fetch before listen (READY-8), buffer dropped (READY-10 rows), ready before the buffer applies (READY-10 rows when ready), no release on a failed fetch (LIFE-10), `$` error ignored (HEALTH-1), no cursor tie-break (READY-10 rows), live write dropped (PUB-3/FID-4). - Size: rxdb-db-collection -1 byte (gzip), one switch case removed. - Limits: the claim covers the controlled RxCollection plus one memory-storage witness; partial-update merging of whole documents (field removal, FID-1) is not observed by the oracle yet. Co-authored-by: Isaac --- .changeset/rxdb-buffered-insert-replay.md | 5 ++ docs/contributing/oracle-coverage.md | 9 ++-- packages/db-adapter-conformance/MIGRATION.md | 48 ++++++++++++++----- .../src/eager-live-source-oracle.ts | 9 ++-- packages/rxdb-db-collection/src/rxdb.ts | 5 +- .../tests/conformance-driver.ts | 46 ++++++++++++++---- ...source-conformance-oracle.property.test.ts | 28 +++++++---- .../rxdb-db-collection/tests/rxdb.test.ts | 45 +++++++++++++++++ 8 files changed, 153 insertions(+), 42 deletions(-) create mode 100644 .changeset/rxdb-buffered-insert-replay.md diff --git a/.changeset/rxdb-buffered-insert-replay.md b/.changeset/rxdb-buffered-insert-replay.md new file mode 100644 index 0000000000..e1b8e1d108 --- /dev/null +++ b/.changeset/rxdb-buffered-insert-replay.md @@ -0,0 +1,5 @@ +--- +'@tanstack/rxdb-db-collection': patch +--- + +Fix initial sync failing with "Cannot insert document ... from sync because it already exists" when a document is inserted and then updated while the initial fetch is running. The buffered insert event is now applied over the row that the initial read already returned. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 3e77101413..e9a1f66282 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6004,8 +6004,7 @@ history, path, and observation. TrailBase (eager mode, per-change commits, oracle keeps its laws, so eager startup failure and stream-end health have two owners until this entry moves them. RxDB (`adapter-conformance.eager-live-source.rxdb`, per-change commits, a -controlled RxCollection with RxDB's own query matcher and sort comparator) is -GREEN. Open: events during RxDB's initial fetch, which the eager grammar does -not generate (owner: `packages/db-adapter-conformance`; witness needed: a -provider-state history with events before the first ready); `rxdb.test.ts` -keeps examples. +controlled RxCollection with RxDB's own query matcher and sort comparator, +`listen-before-fetch`) is GREEN after a fix for a buffered insert that the +initial read already reflects; `rxdb.test.ts` holds a receiving witness for +it on memory storage. diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index fce70ddc36..80f32c32d0 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -27,19 +27,19 @@ reviewed change. ## Adapters -| Adapter | Families that apply (section A) | Driver status | -| ----------------- | ------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Neon Realtime | eager live source, mutations | Eager: RED (READY-4, HEALTH-1, and LIFE-10 behind them); adapter update pending | -| Electric | eager live source, mutations, on-demand, source shapes, reload and retry, persistence | Not started | -| PowerSync | eager live source, mutations, on-demand | Not started | -| RxDB | eager live source, mutations | Eager: GREEN (2026-10-29), per-change commits. Driver `rxdb-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.rxdb`; declares no optional capability. Open: events during the initial fetch (see log). Mutations: not started | -| TrailBase | eager live source, mutations, on-demand | Eager: GREEN (2026-10-29), per-change commits. Driver `trailbase-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.trailbase`; declares no optional capability. Mutations, on-demand: not started | -| Convex | eager live source, mutations, on-demand, source shapes | Not started | -| query-db | client cache and refetch, on-demand, mutations, reload and retry, persistence | Not started; not a push source, so the eager family does not apply | -| local-only | local and shared stores, mutations | Not started | -| localStorage | local and shared stores, mutations | Not started | -| Persisted wrapper | persistence wrapper | Not started | -| Storage hosts (8) | storage hosts | Not started | +| Adapter | Families that apply (section A) | Driver status | +| ----------------- | ------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Neon Realtime | eager live source, mutations | Eager: RED (READY-4, HEALTH-1, and LIFE-10 behind them); adapter update pending | +| Electric | eager live source, mutations, on-demand, source shapes, reload and retry, persistence | Not started | +| PowerSync | eager live source, mutations, on-demand | Not started | +| RxDB | eager live source, mutations | Eager: GREEN (2026-10-29) after a fix, per-change commits, `listen-before-fetch`. Driver `rxdb-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.rxdb`. Mutations: not started | +| TrailBase | eager live source, mutations, on-demand | Eager: GREEN (2026-10-29), per-change commits. Driver `trailbase-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.trailbase`; declares no optional capability. Mutations, on-demand: not started | +| Convex | eager live source, mutations, on-demand, source shapes | Not started | +| query-db | client cache and refetch, on-demand, mutations, reload and retry, persistence | Not started; not a push source, so the eager family does not apply | +| local-only | local and shared stores, mutations | Not started | +| localStorage | local and shared stores, mutations | Not started | +| Persisted wrapper | persistence wrapper | Not started | +| Storage hosts (8) | storage hosts | Not started | ## Suggested order @@ -111,3 +111,25 @@ reviewed change. driver also gets a check of the rows at the moment status becomes `ready` (READY-3, READY-10). Five reference mutants calibrate it, and the reach of the three new states is declared. Evidence: `README.md`. +- **2026-10-29, RxDB on `listen-before-fetch`: one RxDB defect.** The + driver now seeds the backend, answers the initial fetch from its read + point, and counts queries issued without a live `$` subscription. + - **RED:** both campaigns failed READY-10 (status `error` where `ready` is + owed). Shrunk: a document inserted and then updated during the fetch, + which the read already reflects. RxDB replayed the buffered `INSERT` as + a sync `insert`, and core rejected it as a duplicate key with a + different value. A receiving witness on memory storage + (`rxdb.test.ts`, `becomes ready when the initial read already reflects +a buffered insert and update`) failed the same way. + - **Fix:** RxDB writes `INSERT` and `UPDATE` events as a sync `update`; + core inserts an updated key it does not hold. One case less. + - **GREEN:** the RxDB suite, 22 tests, including both campaigns and the + witness. + - **Mutants of `src/rxdb.ts`:** the original insert write (READY-10 + status), fetch before listen (READY-8), buffer dropped (READY-10 rows), + ready before the buffer applies (READY-10 rows when ready), no release + on a failed fetch (LIFE-10), a `$` error ignored (HEALTH-1), no cursor + tie-break (READY-10 rows), a live write dropped (PUB-3/FID-4). Each was + rejected by both campaigns and reverted. + - **Oracle change this exposed:** READY-8 is checked before any other + observation, since a fetch before listening explains what follows. diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 55989b4f70..882355dd12 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -1062,6 +1062,11 @@ export async function checkEagerLiveSourceHistory( const check = (name: string, ok: boolean, want: unknown, got: unknown) => { if (!ok) throw violation(`${law} (${name})`, stepIndex, want, got) } + // READY-8 comes first: a fetch before listening explains what follows. + const early = provider.fetchesBeforeListening?.() ?? 0 + if (early > 0) { + throw violation(`READY-8 (fetch before listening)`, stepIndex, 0, early) + } const detachedNow = detached.found() if (detachedNow.length > 0) { throw violation(`HEALTH-5`, stepIndex, `no detached failure`, detachedNow) @@ -1082,10 +1087,6 @@ export async function checkEagerLiveSourceHistory( ) readyRows = undefined } - const early = provider.fetchesBeforeListening?.() ?? 0 - if (early > 0) { - throw violation(`READY-8 (fetch before listening)`, stepIndex, 0, early) - } check(`change events`, sameRows(events.rows(), rows), rows, events.rows()) const health = collection.health if (expected.healthError === null) { diff --git a/packages/rxdb-db-collection/src/rxdb.ts b/packages/rxdb-db-collection/src/rxdb.ts index 2e7fe00bf2..064cf51198 100644 --- a/packages/rxdb-db-collection/src/rxdb.ts +++ b/packages/rxdb-db-collection/src/rxdb.ts @@ -234,9 +234,10 @@ export function rxdbCollectionOptions( next: (ev) => { const cur: Row = stripRxdbFields(clone(ev.documentData as Row)) switch (ev.operation) { + // The initial read can already hold a document whose insert + // event was buffered, so an insert writes as an update. Core + // inserts an updated key it does not hold. case `INSERT`: - queue({ type: `insert`, value: cur }) - break case `UPDATE`: queue({ type: `update`, value: cur }) break diff --git a/packages/rxdb-db-collection/tests/conformance-driver.ts b/packages/rxdb-db-collection/tests/conformance-driver.ts index 8e37b3e774..9ce35b0a72 100644 --- a/packages/rxdb-db-collection/tests/conformance-driver.ts +++ b/packages/rxdb-db-collection/tests/conformance-driver.ts @@ -10,10 +10,15 @@ * * The oracle's steps map onto RxDB like this: * - * - a complete delivery is the initial fetch. The adapter's first query of a - * sync run waits until the oracle delivers; the provider then holds exactly - * those documents and answers every page from them. RxDB reads storage - * once per sync run, so the driver does not declare `complete-snapshot`; + * - the provider holds the backend's documents. `seed` sets them before a + * sync run subscribes. A complete delivery is the initial fetch's result: + * the adapter's first query of a sync run waits until the oracle delivers + * it, and every page answers from it. RxDB reads storage once per sync + * run, so the driver does not declare `complete-snapshot`; + * - RxDB subscribes to `$` before it queries, so the driver declares + * `listen-before-fetch`: a change batch before the first ready is a + * backend write during the fetch, announced on `$`. A query issued while + * no `$` subscription is live counts as a fetch before listening (READY-8); * - a change batch is one change event per change on `$`: `INSERT` for a key * the provider does not hold, `UPDATE` for one it does, `DELETE` otherwise. * The adapter commits each event alone, so the driver declares `per-change` @@ -80,6 +85,8 @@ interface ProviderSession { subscriber: Subscriber> releases: number delivered: boolean + /** The documents the initial fetch reads: the result at its read point. */ + read: Map> firstQuery?: { resolve: () => void reject: (error: unknown) => void @@ -90,6 +97,7 @@ class RxdbProvider implements ControlledProvider { readonly sessions: Array = [] private docs = new Map>() private lwt = 0 + private early = 0 /** The controlled RxCollection that the adapter reads. */ readonly rxCollection = { @@ -103,6 +111,7 @@ class RxdbProvider implements ControlledProvider { subscriber, releases: 0, delivered: false, + read: new Map(), } this.sessions.push(session) return () => { @@ -120,6 +129,8 @@ class RxdbProvider implements ControlledProvider { // The first query of a sync run waits for the complete delivery; later // pages answer from the documents the provider then holds. private async query(prepared: PreparedQuery) { + const listening = this.sessions.at(-1) + if (!listening || listening.releases > 0) this.early += 1 const session = this.latest() if (!session.delivered) { if (session.firstQuery) throw new Error(`a second query before delivery`) @@ -130,7 +141,7 @@ class RxdbProvider implements ControlledProvider { const matches = getQueryMatcher(schema.jsonSchema, prepared.query) const order = getSortComparator(schema.jsonSchema, prepared.query) const { skip = 0, limit = Infinity } = prepared.query - const documents = [...this.docs.values()] + const documents = [...session.read.values()] .filter((doc) => matches(doc)) .sort(order) .slice(skip, skip + limit) @@ -172,14 +183,32 @@ class RxdbProvider implements ControlledProvider { } releases = () => this.sessions.map((session) => session.releases) + fetchesBeforeListening = () => this.early + // The backend's rows before the adapter subscribes, in one bulk write. + seed = (rows: ReadonlyArray) => { + const lwt = ++this.lwt + this.docs = new Map( + rows.map((row) => [String(row.id), this.document(row, lwt)]), + ) + } + + // The fetch result at its read point. Its documents keep the backend's + // current versions where they agree, and the read's own otherwise. deliverSnapshot = (rows: ReadonlyArray) => { const session = this.latest() const firstQuery = session.firstQuery if (!firstQuery) throw new Error(`the adapter has not queried`) const lwt = ++this.lwt - this.docs = new Map( - rows.map((row) => [String(row.id), this.document(row, lwt)]), + session.read = new Map( + rows.map((row) => { + const id = String(row.id) + const current = this.docs.get(id) + return [ + id, + current?.title === row.title ? current : this.document(row, lwt), + ] + }), ) session.delivered = true session.firstQuery = undefined @@ -250,6 +279,7 @@ function observedView( }, preload: () => collection.preload(), cleanup: () => collection.cleanup(), + on: (event, callback) => collection.on(event, callback), subscribeChanges: (callback, options) => collection.subscribeChanges( (changes) => @@ -280,7 +310,7 @@ function createRxdbCollection(provider: RxdbProvider) { export function rxdbConformanceDriver(): EagerLiveSourceDriver { return { adapter: `@tanstack/rxdb-db-collection`, - capabilities: new Set(), + capabilities: new Set([`listen-before-fetch`]), changeCommits: `per-change`, create(): EagerLiveSourceFixture { const provider = new RxdbProvider() diff --git a/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts index 84fdded398..94cd33b069 100644 --- a/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts +++ b/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -20,23 +20,26 @@ import { rxdbConformanceDriver } from './conformance-driver' * # RxDB on the shared eager live-source conformance oracle * * A receiving driver for `packages/db-adapter-conformance`. The shared - * module states the laws (READY-2/3/4, HEALTH-1, PUB-1/2/3, FID-4, + * module states the laws (READY-2/3/4/8/10, HEALTH-1, PUB-1/2/3/5/9, FID-4, * LIFE-2/5/10, HEALTH-5), the model, the history grammar, and the check. This * file supplies only the RxDB production driver in `conformance-driver.ts` * and the campaigns; it states no expected results. * - * RxDB declares none of the oracle's optional capabilities: it reads storage - * once per sync run, its change events carry no txid, and it has no - * transport or capability to renew. The claim is limited to the controlled + * RxDB declares `listen-before-fetch`: it subscribes to `$` before its + * initial fetch and buffers the events that arrive during it. It declares no + * other optional capability: it reads storage once per sync run, its change + * events carry no txid, and it has no transport or capability to renew. The claim is limited to the controlled * RxCollection; a claim about a real RxDB storage needs a receiving witness - * against it. `rxdb.test.ts` keeps its examples, including changes that - * arrive during the initial fetch, which this grammar does not generate. + * against it; `rxdb.test.ts` holds one on memory storage for a buffered + * insert that the initial read already reflects. */ const property = `adapter-conformance.eager-live-source.rxdb` const driver = rxdbConformanceDriver() const historyArb = eagerLiveSourceHistoryArb(driver.capabilities) const runs = 200 +// Startup bursts lengthen histories, so a campaign takes several seconds. +const campaignTimeoutMs = 120_000 async function campaign(options: fc.Parameters<[unknown]>) { const reach = createOracleReach( @@ -51,10 +54,14 @@ async function campaign(options: fc.Parameters<[unknown]>) { } describe(`eager live-source conformance: RxDB`, () => { - oracleReplayTest(it)(`conforms on a fixed-seed campaign`, async () => { - const reach = await campaign({ seed: 2118, numRuns: oracleRuns(runs) }) - reach.assertReached() - }) + oracleReplayTest(it)( + `conforms on a fixed-seed campaign`, + async () => { + const reach = await campaign({ seed: 2118, numRuns: oracleRuns(runs) }) + reach.assertReached() + }, + campaignTimeoutMs, + ) oracleReplayTest(it, property)( `conforms on a random campaign or a replayed seed`, @@ -62,6 +69,7 @@ describe(`eager live-source conformance: RxDB`, () => { const reach = await campaign(oraclePropertyOptions(runs, property)) reach.assertReached() }, + campaignTimeoutMs, ) }) diff --git a/packages/rxdb-db-collection/tests/rxdb.test.ts b/packages/rxdb-db-collection/tests/rxdb.test.ts index 6e4ffe7a56..7498018175 100644 --- a/packages/rxdb-db-collection/tests/rxdb.test.ts +++ b/packages/rxdb-db-collection/tests/rxdb.test.ts @@ -392,6 +392,51 @@ describe(`RxDB Integration`, () => { } }) + // A receiving witness for the conformance oracle's `listen-before-fetch` + // lane on real memory storage: the initial query reads a document that + // was inserted and then updated while the query waited, so the buffered + // insert event is already reflected in the result. + it(`becomes ready when the initial read already reflects a buffered insert and update`, async () => { + const db = await getDatababase() + const rxCollection: RxCollection = db.test + const queryWaiting = createDeferred() + const releaseRead = createDeferred() + const storageQuery = rxCollection.storageInstance.query.bind( + rxCollection.storageInstance, + ) + const query = vi + .spyOn(rxCollection.storageInstance, `query`) + .mockImplementationOnce(async (preparedQuery) => { + queryWaiting.resolve() + await releaseRead.promise + return storageQuery(preparedQuery) + }) + const collection = createCollection( + rxdbCollectionOptions({ rxCollection, syncBatchSize: 10 }), + ) + + try { + const ready = collection.preload() + await queryWaiting.promise + const doc = await rxCollection.insert({ id: `1`, name: `First` }) + await doc.patch({ name: `Second` }) + await flushPromises() + releaseRead.resolve() + await ready + + expect(collection.status).toBe(`ready`) + expect(stripVirtualProps(collection.get(`1`))).toEqual({ + id: `1`, + name: `Second`, + }) + } finally { + releaseRead.resolve() + query.mockRestore() + await collection.cleanup() + await db.remove() + } + }) + it(`does not let later live traffic extend the startup readiness boundary`, async () => { const db = await getDatababase() const rxCollection: RxCollection = db.test From f82985af8ddec05c2872df7785c1cd091da4480c Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 29 Oct 2026 13:10:40 -0600 Subject: [PATCH 05/64] test(conformance): generate field removal and check update events Give the conformance row an optional `note`, so a later row can remove a field. A delivered row is the whole backend row (FID-1): a mismatch only in fields is labeled FID-1, and the declared state `field-removed` covers live batches, replacing complete deliveries, and the initial cut. The change-event fold now also flags an update of a key it does not hold (PUB-2). Drivers share `observedView` for one-to-one row and key mappings. TrailBase and Neon carry an absent `note` as SQL NULL (`toSqlRow`, `fromSqlRow`), as their backends do. - Law: FID-1 (values round-trip, null versus absent) and PUB-2 (change events fold to the rows) from the adapter law set v18. - Reach: reference fixed campaign (seed 2118, 600 histories) reaches field-removed 28 times; the lowest count of any state across 20 seeds is 15. At 200 histories TrailBase reached field-removed as few as 2 times, so drivers now run 400 histories (lowest 11 for TrailBase). - RED: merges-updates (a partial row merge) fails FID-1; relabels-inserts, a checker calibration that relabels insert events as updates, fails PUB-2. - GREEN: reference 58/58; TrailBase conformance 2/2 over four repeats; Neon keeps its READY-4 and HEALTH-1 RED, and passes 300 histories with notes when failure steps are removed. - Mutant: merges-updates (FID-1), relabels-inserts (PUB-2). drops-deletes now drops only live deletes, so it still names PUB-3. - Limits: FID-1 covers one optional string field, not value domains. RxDB is not moved to the new rows here; it fails FID-1 and is fixed in the next commit. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 12 +- packages/db-adapter-conformance/MIGRATION.md | 38 +++-- packages/db-adapter-conformance/README.md | 150 +++++++++--------- .../src/eager-live-source-oracle.ts | 137 +++++++++++++--- packages/db-adapter-conformance/src/index.ts | 7 + .../src/observed-view.ts | 96 +++++++++++ .../neon-realtime-oracle.property.test.ts | 4 +- .../tests/neon/neon-realtime-driver.ts | 42 ++--- .../reference-adapter-oracle.property.test.ts | 70 +++++++- .../tests/reference-adapter.ts | 44 ++++- .../trailbase-db-collection/tests/ORACLE.md | 2 +- .../tests/conformance-driver.ts | 35 ++-- ...source-conformance-oracle.property.test.ts | 18 ++- 13 files changed, 497 insertions(+), 158 deletions(-) create mode 100644 packages/db-adapter-conformance/src/observed-view.ts diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index e9a1f66282..b2ce11637d 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -5973,15 +5973,16 @@ shared eager live-source laws from the adapter conformance law set (v18): READY-2/3/4, HEALTH-1, PUB-1/2/3/7, FID-4, LIFE-2/5/10, HEALTH-5, EVID-1/2/3 (`await-evidence`), HEALTH-2 and RETRY-1 for retried failures (`retry-policy`), RECON-1 (`transport-reconnect`), RENEW-1 -(`capability-renewal`), and READY-8/10 with PUB-5/9 for batches committed -during the initial fetch (`listen-before-fetch`). Every driver is also +(`capability-renewal`), READY-8/10 with PUB-5/9 for batches committed +during the initial fetch (`listen-before-fetch`), and FID-1 removal of an +optional field. Every driver is also checked for the rows at the moment status becomes `ready`. Its model is an independent row Map and sync-run phase; drivers supply only a controlled provider and declare whether they commit per batch or per change. PUB-1 checks every change publication during a step, not only the settled checkpoint. The reference push adapter (`tests/reference-adapter-oracle.property.test.ts`, `adapter-conformance.eager-live-source.reference`) runs fixed and random -campaigns of 400 histories with declared reach, a per-change variant, and 25 +campaigns of 600 histories with declared reach, a per-change variant, and 27 law-named hostile mutants, with ORC-004 reconstruction, exclusion, and ablation controls. The Neon Realtime receiving driver (`adapter-conformance.eager-live-source.neon-realtime`, `test:neon`) runs @@ -5990,9 +5991,8 @@ READY-4 and HEALTH-1 until that adapter moves to `fail()`; probes show LIFE-10 (subscription kept open after an error state) behind them. Limits: no on-demand loading, mutation handlers, or reload; publications are observed through `subscribeChanges` only; EVID-3 does not distinguish a timed wait -that rejects before its timeout; rows carry no optional field, so FID-1 field -removal is not observed; the event fold does not flag an update of an absent -key; the Neon and TrailBase claims cover +that rejects before its timeout; FID-1 covers one optional string field, not +value domains; the Neon and TrailBase claims cover controlled providers, not real clients. Owner: `packages/db-adapter-conformance`. Evidence: the package README. Migration goal (maintainer, 2026-10-29): every collection adapter gets a diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index 80f32c32d0..598224707a 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -14,16 +14,16 @@ reviewed change. ## Oracle families -| Family | Status | Laws | -| ------------------------------------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | -| Eager live source | Built (`src/eager-live-source-oracle.ts`) | READY-2/3/4/8/10, HEALTH-1/2/5, PUB-1/2/3/5/7/9, FID-4, LIFE-2/5/10, EVID-1/2/3, RETRY-1 (eager), RECON-1, RENEW-1 | -| Mutations and confirmation | Needed | MUT-1/2/4/5/7/10/13, FID-5 | -| On-demand loading | Needed | DEM-1 to DEM-23, MAP-1 to MAP-4 | -| Reload and retry | Needed | RELOAD-1 to RELOAD-5; RETRY-1 and HEALTH-2 beyond the eager source's retried failures | -| Client cache and refetch | Needed | CACHE-1 to CACHE-8, REF-1 to REF-7, DIR-1 to DIR-5 | -| Source shapes | Needed | RESET-1, RESUME-1 to RESUME-3, PROG-1, MEMB-1/2, PAGE-1 to PAGE-3, OWN-1/2, META-1 | -| Local and shared stores | Needed | SELF-1 to SELF-3, STORE-1 to STORE-4, PEER-1 | -| Persistence wrapper and storage hosts | Needed | P-_, S-_ | +| Family | Status | Laws | +| ------------------------------------- | ----------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | +| Eager live source | Built (`src/eager-live-source-oracle.ts`) | READY-2/3/4/8/10, HEALTH-1/2/5, PUB-1/2/3/5/7/9, FID-1/4, LIFE-2/5/10, EVID-1/2/3, RETRY-1 (eager), RECON-1, RENEW-1 | +| Mutations and confirmation | Needed | MUT-1/2/4/5/7/10/13, FID-5 | +| On-demand loading | Needed | DEM-1 to DEM-23, MAP-1 to MAP-4 | +| Reload and retry | Needed | RELOAD-1 to RELOAD-5; RETRY-1 and HEALTH-2 beyond the eager source's retried failures | +| Client cache and refetch | Needed | CACHE-1 to CACHE-8, REF-1 to REF-7, DIR-1 to DIR-5 | +| Source shapes | Needed | RESET-1, RESUME-1 to RESUME-3, PROG-1, MEMB-1/2, PAGE-1 to PAGE-3, OWN-1/2, META-1 | +| Local and shared stores | Needed | SELF-1 to SELF-3, STORE-1 to STORE-4, PEER-1 | +| Persistence wrapper and storage hosts | Needed | P-_, S-_ | ## Adapters @@ -133,3 +133,21 @@ a buffered insert and update`) failed the same way. rejected by both campaigns and reverted. - **Oracle change this exposed:** READY-8 is checked before any other observation, since a fetch before listening explains what follows. +- **2026-10-29, eager live source, field removal and event fold (from the + RxDB migration).** A row now has an optional `note`, so a later row can + remove a field (FID-1); a mismatch only in fields is labeled FID-1, and + the declared state `field-removed` covers live batches, replacing + complete deliveries, and the initial cut. The change-event fold now also + flags an update of a key it does not hold (PUB-2). Drivers share + `observedView`; TrailBase and Neon, whose backends are SQL, carry an + absent `note` as `NULL` (`toSqlRow`, `fromSqlRow`). + - **Calibration:** `merges-updates` (FID-1) and `relabels-inserts`, a + checker calibration that relabels insert events as updates (PUB-2). + - **Budgets:** `field-removed` needs a held note and a later row without + it. At 200 histories TrailBase reached it as few as 2 times; drivers now + run 400 histories and the reference 600. + - **TrailBase:** GREEN; no defect. Its records carry every column, so a + removed note arrives as `NULL` and replaces the old value. + - **Neon:** still RED on READY-4 and HEALTH-1 only; a probe without + failure steps passed 300 histories with notes. + - **RxDB:** RED on FID-1; see the next entry. diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md index 504997145f..85d4d385c3 100644 --- a/packages/db-adapter-conformance/README.md +++ b/packages/db-adapter-conformance/README.md @@ -13,23 +13,24 @@ The package is private while its first oracle settles. pushes a complete result and then committed change batches. Its opening comment states the laws and their limits. In short: -| Law | Checked behavior | -| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -| READY-2, READY-3 | `loading` until the first complete delivery; then `ready` with exactly its rows | -| READY-4 | A failure before ready: status `error`, `preload()` rejects with the value, `health.error.cause` is the value | -| HEALTH-1 | A failure after ready: status stays `ready`, rows kept, `health.error.cause` is the value | -| PUB-1 | Every change publication during a step is one whole committed state, in commit order; per batch or per change, as the driver declares | -| PUB-2, PUB-3, PUB-7, FID-4 | Change events fold to the rows; batches fold onto rows; a complete delivery replaces (PUB-7 with `complete-snapshot`) | -| LIFE-2, LIFE-5 | Cleanup releases the provider subscription; a restart is a fresh, empty sync run | -| LIFE-10 | A failed provider's subscription is released at the failure, and no subscription is released twice | -| HEALTH-5 | No unhandled rejection and no undeclared `console.error` or `console.warn` | -| EVID-1, EVID-2, EVID-3 | `awaitTxId` fulfills after its batch applies, stays pending before, rejects after its timeout, and rejects at retirement | -| HEALTH-2, RETRY-1 | A retried failure stays hidden before ready, joins `failureReason` after it, and clears on the next delivery (`retry-policy`) | -| RECON-1 | Disconnect and reconnect keep status and rows | -| RENEW-1 | Capability renewal reuses the provider subscription and changes nothing visible | -| READY-8, READY-10 | With `listen-before-fetch`: no fetch before the listener is live; batches committed during the fetch apply after its result, before ready | -| PUB-5, PUB-9 | With `listen-before-fetch`: the cut publishes only backend states at or after the read point, in order; no replay regresses a row | -| READY-3, READY-10 | When status becomes `ready`, the rows already equal the initial cut | +| Law | Checked behavior | +| -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| READY-2, READY-3 | `loading` until the first complete delivery; then `ready` with exactly its rows | +| READY-4 | A failure before ready: status `error`, `preload()` rejects with the value, `health.error.cause` is the value | +| HEALTH-1 | A failure after ready: status stays `ready`, rows kept, `health.error.cause` is the value | +| PUB-1 | Every change publication during a step is one whole committed state, in commit order; per batch or per change, as the driver declares | +| PUB-2, PUB-3, PUB-7, FID-4 | Change events fold to the rows, with no insert of a held key and no update or delete of a missing one; batches fold onto rows; a complete delivery replaces (PUB-7 with `complete-snapshot`) | +| FID-1 | A delivered row is the whole backend row: a row without the optional `note` removes a `note` the Collection held | +| LIFE-2, LIFE-5 | Cleanup releases the provider subscription; a restart is a fresh, empty sync run | +| LIFE-10 | A failed provider's subscription is released at the failure, and no subscription is released twice | +| HEALTH-5 | No unhandled rejection and no undeclared `console.error` or `console.warn` | +| EVID-1, EVID-2, EVID-3 | `awaitTxId` fulfills after its batch applies, stays pending before, rejects after its timeout, and rejects at retirement | +| HEALTH-2, RETRY-1 | A retried failure stays hidden before ready, joins `failureReason` after it, and clears on the next delivery (`retry-policy`) | +| RECON-1 | Disconnect and reconnect keep status and rows | +| RENEW-1 | Capability renewal reuses the provider subscription and changes nothing visible | +| READY-8, READY-10 | With `listen-before-fetch`: no fetch before the listener is live; batches committed during the fetch apply after its result, before ready | +| PUB-5, PUB-9 | With `listen-before-fetch`: the cut publishes only backend states at or after the read point, in order; no replay regresses a row | +| READY-3, READY-10 | When status becomes `ready`, the rows already equal the initial cut | The law IDs come from the adapter conformance law set, version 18. @@ -42,6 +43,8 @@ Implement `EagerLiveSourceDriver`: - **`awaitTxId`:** your adapter's utility, if you declare `await-evidence`. It receives the oracle's timeout. - **`changeCommits`:** `per-batch` if your adapter commits a provider batch as one transaction, `per-change` if it commits each change alone. - **`declaredDiagnostic`:** optional; accepts console diagnostics your adapter documents, so HEALTH-5 does not reject them. +- **`seed` and `fetchesBeforeListening`:** with `listen-before-fetch`, the provider holds backend rows set by `seed`, treats a batch before the first ready as a backend commit during the fetch, and counts fetches issued while its listener was not live (READY-8). +- **`observedView`:** if your rows or keys differ from the conformance row, wrap the Collection with `observedView` and a one-to-one mapping. `conformanceKey` maps string keys; `toSqlRow` and `fromSqlRow` map an absent `note` to and from SQL `NULL`. Declare only the capabilities your adapter supports; the grammar generates only the steps they allow. Then run `checkEagerLiveSourceHistory` over @@ -75,66 +78,67 @@ reach counter, so the run fails when histories miss a state the laws need. - **Law:** the table above. Authority: the sync callback contract in `packages/db/src/types.ts`, the status and health terms in `docs/contributing/glossary.md`, and the adapter law set v18. -- **Reach:** in the fixed campaign (seed 2118, 400 histories) on the reference +- **Reach:** in the fixed campaign (seed 2118, 600 histories) on the reference adapter, the histories that reached each declared state: | Declared state | Histories | | ------------------------------ | --------- | - | first-ready | 174 | - | replacing-snapshot | 48 | - | change-batch | 67 | - | multi-change-batch | 209 | - | failure-before-ready | 88 | - | failure-after-ready | 36 | - | restart | 87 | - | wait-before-evidence-fulfilled | 30 | - | wait-timed-out | 68 | - | wait-rejected-at-retirement | 128 | - | transient-failure-reported | 43 | - | transient-failure-recovered | 25 | - | reconnect | 31 | - | renewal | 101 | - | change-before-ready | 341 | - | read-missed-change | 70 | - | seen-change-superseded | 21 | - - The campaign executed 2,147 steps and skipped 938 as illegal at their - position, for example a batch after a failure. Each campaign fails - when a state is reached fewer than five times. Across 20 other seeds, the - lowest count for any state was 11 (`seen-change-superseded`). + | first-ready | 230 | + | replacing-snapshot | 74 | + | change-batch | 105 | + | multi-change-batch | 317 | + | failure-before-ready | 132 | + | failure-after-ready | 41 | + | restart | 136 | + | wait-before-evidence-fulfilled | 54 | + | wait-timed-out | 91 | + | wait-rejected-at-retirement | 199 | + | transient-failure-reported | 64 | + | transient-failure-recovered | 31 | + | reconnect | 65 | + | renewal | 154 | + | change-before-ready | 514 | + | read-missed-change | 116 | + | seen-change-superseded | 31 | + | field-removed | 28 | + + Each campaign fails when a state is reached fewer than five times. Across + 20 other seeds, the lowest count for any state was 15 (`field-removed`). - **GREEN:** `pnpm exec vitest run tests/reference-adapter-oracle.property.test.ts`: - 54 tests pass (both campaigns, the per-change pair, 24 calibrations, and the + 58 tests pass (both campaigns, the per-change pair, 26 calibrations, and the grammar controls). TrailBase: its full suite, 151 tests, passes. - **Mutants:** the fixed campaign rejects each with the law it breaks: - | Mutant | Law | - | ----------------------------- | -------------------------- | - | `ready-before-delivery` | READY-2 | - | `wraps-loading-failure` | READY-4 | - | `failure-clears-rows` | HEALTH-1 | - | `transient-is-fatal` | HEALTH-1 | - | `never-recovers` | HEALTH-2 | - | `transient-fails-loading` | RETRY-1 | - | `splits-batch` | PUB-1 | - | `reverses-batch` (per-change) | PUB-1 | - | `drops-deletes` | PUB-3 | - | `snapshot-merges` | PUB-7 | - | `leaks-subscription` | LIFE-2 | - | `restart-replays-rows` | LIFE-5 | - | `keeps-failed-source` | LIFE-10 | - | `double-release` | LIFE-10 | - | `detached-rejection` | HEALTH-5 | - | `logs-failure` | HEALTH-5 | - | `wait-off-by-one` | EVID-1 | - | `wait-ignores-timeout` | EVID-3 | - | `disconnect-fails` | RECON-1 | - | `renew-resubscribes` | RENEW-1 | - | `drops-buffered` | READY-10 | - | `buffer-before-rows` | READY-10 | - | `replays-per-batch` | PUB-5 | - | `ready-before-buffer` | READY-10 (rows when ready) | - | `shows-buffered` | READY-10/PUB-9 | + | Mutant | Law | + | ---------------------------------------- | -------------------------- | + | `ready-before-delivery` | READY-2 | + | `wraps-loading-failure` | READY-4 | + | `failure-clears-rows` | HEALTH-1 | + | `transient-is-fatal` | HEALTH-1 | + | `never-recovers` | HEALTH-2 | + | `transient-fails-loading` | RETRY-1 | + | `splits-batch` | PUB-1 | + | `reverses-batch` (per-change) | PUB-1 | + | `drops-deletes` | PUB-3 | + | `snapshot-merges` | PUB-7 | + | `leaks-subscription` | LIFE-2 | + | `restart-replays-rows` | LIFE-5 | + | `keeps-failed-source` | LIFE-10 | + | `double-release` | LIFE-10 | + | `detached-rejection` | HEALTH-5 | + | `logs-failure` | HEALTH-5 | + | `wait-off-by-one` | EVID-1 | + | `wait-ignores-timeout` | EVID-3 | + | `disconnect-fails` | RECON-1 | + | `renew-resubscribes` | RENEW-1 | + | `drops-buffered` | READY-10 | + | `buffer-before-rows` | READY-10 | + | `replays-per-batch` | PUB-5 | + | `ready-before-buffer` | READY-10 (rows when ready) | + | `merges-updates` | FID-1 | + | `relabels-inserts` (checker calibration) | PUB-2 | + | `shows-buffered` | READY-10/PUB-9 | `reverses-batch` reverses only batches of distinct keys, so its final rows are right and only the intermediate publications are wrong. An earlier @@ -143,9 +147,9 @@ reach counter, so the run fails when histories miss a state the laws need. - **Shrinking keeps the first violation (ORC-010):** `eagerLiveSourceProperty` treats a smaller history that fails a different law as passing. A shrunk counterexample therefore violates the law the campaign found. A cleanup error after a violation is kept with it in an `AggregateError`. - **Grammar controls (ORC-004):** these run in the reference oracle file. - - **Reconstruction:** ten witnesses from the Neon, TrailBase, and RxDB suites and the retry contract are rebuilt as histories. Each is accepted with no skipped step and passes. + - **Reconstruction:** eleven witnesses from the Neon, TrailBase, and RxDB suites and the retry contract are rebuilt as histories. Each is accepted with no skipped step and passes. - **Exclusion:** six nearby invalid histories are each skipped by exactly one step; a batch before the first ready is invalid only without `listen-before-fetch`. - - **Ablation:** removing each step kind loses the declared state that only it reaches, and reading after every buffered batch loses `read-missed-change`. + - **Ablation:** removing each step kind loses the declared state that only it reaches, reading after every buffered batch loses `read-missed-change`, and rows without notes lose `field-removed`. - **Replay:** during a path replay, fixed campaigns and calibrations stand aside (`oracleReplayTest`). - **RED (Neon):** `pnpm test:neon`. Both campaigns fail. The fixed campaign shrinks to `[fail, wait]`: READY-4, status `loading` where `error` is owed. @@ -168,10 +172,8 @@ reach counter, so the run fails when histories miss a state the laws need. timed wait settles within its own step, so no evidence arrives during it. An adapter that rejects a timed wait at once, before its timeout, is not distinguished from one that waits. - - Rows carry only an id and a title, so a field that an update removes is - not observed (FID-1). - - The change-event fold flags a duplicate insert and a delete of an absent - key, but not an update of an absent key. + - FID-1 covers one optional string field. Value domains (Date, bigint, + nested objects) are not generated. - READY-8's fetch order is checked only by a provider that implements `fetchesBeforeListening`. Commit receipts resolve at once in every driver, so a live commit whose receipt is still pending at the cut diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 882355dd12..6cb6be11b8 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -22,8 +22,14 @@ * ready (`emitEmptyReadyEvent` in `packages/db/src/collection/changes.ts`) * is a readiness notification, not a commit publication. * - **PUB-2.** Folding the Collection's change events gives the current rows. + * An insert of a held key, an update of a key not held, and a delete of a + * key not held are each inconsistent. * - **PUB-3, FID-4.** After a committed change batch applies, the rows equal * the batch folded onto the previous rows. + * - **FID-1.** A delivered row is the complete backend row: a row without + * the optional `note` has no `note` in the Collection, also when the row + * it replaces had one. Fields compare by own property, so an own + * `undefined` differs from an absent field. * - **PUB-7** (capability `complete-snapshot`). A later complete delivery * replaces the rows: keys it omits are gone. * - **LIFE-2, LIFE-5.** Cleanup releases the provider subscription. A restart @@ -91,10 +97,14 @@ import type { ChangeMessage, HealthNode } from '@tanstack/db' // Production driver contract // --------------------------------------------------------------------------- -/** The row every conformance history uses. A driver maps it to its provider. */ +/** + * The row every conformance history uses. A driver maps it to its provider. + * `note` is optional, so a later row can remove it. + */ export interface ConformanceRow { readonly id: number readonly title: string + readonly note?: string } /** One change in a provider's committed batch. */ @@ -233,7 +243,8 @@ export interface EagerLiveSourceDriver { * - `restart`: cleanup, then `preload()` starts a new sync run. * * Keys range over 1..5 and titles over three values, so batches revisit - * keys, deletes hit live rows, and complete deliveries omit held keys. With + * keys, deletes hit live rows, and complete deliveries omit held keys. A row + * carries `note` half the time, so an update can add or remove a field. With * `listen-before-fetch`, a history starts with up to three batches committed * during the first fetch. */ @@ -262,10 +273,14 @@ export type EagerLiveSourceStep = export type EagerLiveSourceHistory = ReadonlyArray -const rowArb: fc.Arbitrary = fc.record({ - id: fc.integer({ min: 1, max: 5 }), - title: fc.constantFrom(`a`, `b`, `c`), -}) +const rowArb: fc.Arbitrary = fc.record( + { + id: fc.integer({ min: 1, max: 5 }), + title: fc.constantFrom(`a`, `b`, `c`), + note: fc.constantFrom(`x`, `y`), + }, + { requiredKeys: [`id`, `title`] }, +) const changeArb: fc.Arbitrary = fc.oneof( rowArb.map((row) => ({ type: `upsert` as const, row })), @@ -423,8 +438,30 @@ export interface ExpectedObservation { renewalsThisRun: number } +/** + * A row's value as the model compares it: its title, then its note when the + * row has its own `note` property. An own `undefined` shows as `undefined`. + */ +function rowValue(row: { readonly title: string; readonly note?: unknown }) { + return Object.hasOwn(row, `note`) + ? `${row.title}/${String(row.note)}` + : row.title +} + +function rowFromValue(id: number, value: string): ConformanceRow { + const [title = ``, note] = value.split(`/`) + return note === undefined ? { id, title } : { id, title, note } +} + function rowMap(rows: ReadonlyArray): Map { - return new Map(rows.map((row) => [row.id, row.title])) + return new Map(rows.map((row) => [row.id, rowValue(row)])) +} + +/** The rows a Collection holds, as the model compares them. */ +function observedRows(collection: ObservedCollection): RowState { + return rowState( + new Map([...collection.values()].map((row) => [row.id, rowValue(row)])), + ) } function initialModel(seed: ReadonlyArray = []): ModelState { @@ -478,7 +515,7 @@ function foldBatch( const delivered: Array = [] for (const change of changes) { if (change.type === `upsert`) { - rows.set(change.row.id, change.row.title) + rows.set(change.row.id, rowValue(change.row)) } else if (rows.has(change.id)) { rows.delete(change.id) } else { @@ -490,6 +527,17 @@ function foldBatch( return delivered } +/** Whether a key held in both lost the `note` it had in `before`. */ +function lostField( + before: ReadonlyMap, + after: ReadonlyMap, +): boolean { + return [...before].some( + ([id, value]) => + value.includes(`/`) && after.get(id)?.includes(`/`) === false, + ) +} + /** Whether replaying `changes` onto `rows` would change them. */ function replayChanges( rows: ReadonlyMap, @@ -511,12 +559,16 @@ type AcceptedStep = missed: number /** Whether replaying a batch the read reflects would change the rows. */ superseded: boolean + /** Whether a held row, or the read's row at the cut, lost a field. */ + removesField: boolean } | { kind: `changes` changes: Array txid: number beforeReady: boolean + /** Whether an upsert removed a field the row it replaced had. */ + removesField: boolean } | { kind: `wait`; txid: number; timeout: boolean } | { kind: `fail`; failure: number } @@ -614,16 +666,18 @@ function stepModel( return { done: { kind: `snapshot`, - rows: cut.read.map(([id, title]) => ({ id, title })), + rows: cut.read.map(([id, value]) => rowFromValue(id, value)), buffered, missed: buffered - seen, superseded: cut.superseded, + removesField: lostField(new Map(cut.read), cut.backend), }, publishable: cut.publishable, } } - // PUB-7: a complete delivery replaces the rows. READY-3: the first one - // opens the initial gate. + // PUB-7: a complete delivery replaces the rows, fields included. + // READY-3: the first one opens the initial gate. + const removesField = lostField(state.rows, rowMap(step.rows)) state.rows = rowMap(step.rows) state.backend = rowMap(step.rows) state.phase = `ready` @@ -634,6 +688,7 @@ function stepModel( buffered: 0, missed: 0, superseded: false, + removesField, }, publishable: [rowState(state.rows)], } @@ -652,6 +707,7 @@ function stepModel( changes: delivered, txid: state.lastTxid, beforeReady: true, + removesField: false, }, publishable: [], } @@ -663,6 +719,23 @@ function stepModel( const delivered = foldBatch(state.rows, step.changes, () => prefixes.push(rowState(state.rows)), ) + // Reach: an upsert without `note` replaced a row that had one, before + // or within this batch. + const held = new Map(state.backend) + let removesField = false + for (const change of delivered) { + if (change.type === `delete`) { + held.delete(change.id) + continue + } + if ( + !Object.hasOwn(change.row, `note`) && + held.get(change.row.id)?.includes(`/`) === true + ) { + removesField = true + } + held.set(change.row.id, rowValue(change.row)) + } foldBatch(state.backend, delivered) state.lastTxid += 1 state.transient = [] @@ -673,6 +746,7 @@ function stepModel( changes: delivered, txid: state.lastTxid, beforeReady: false, + removesField, }, publishable: options.changeCommits === `per-change` @@ -811,6 +885,7 @@ export const eagerLiveSourceReachStates = [ `change-before-ready`, `read-missed-change`, `seen-change-superseded`, + `field-removed`, ] as const export type EagerLiveSourceReachState = @@ -985,9 +1060,7 @@ export async function checkEagerLiveSourceHistory( let readyRows: RowState | undefined const watchReady = () => collection.on(`status:ready`, () => { - readyRows = rowState( - new Map([...collection.values()].map((row) => [row.id, row.title])), - ) + readyRows = observedRows(collection) }) watchReady() let preload = recordSettlement(collection.preload()) @@ -1056,9 +1129,7 @@ export async function checkEagerLiveSourceHistory( ) => { await settle() const expected = expectedObservation(state) - const rows = rowState( - new Map([...collection.values()].map((row) => [row.id, row.title])), - ) + const rows = observedRows(collection) const check = (name: string, ok: boolean, want: unknown, got: unknown) => { if (!ok) throw violation(`${law} (${name})`, stepIndex, want, got) } @@ -1077,6 +1148,11 @@ export async function checkEagerLiveSourceHistory( expected.status, collection.status, ) + // FID-1: rows that differ only in a field, with the right keys and + // titles, name the field law. + if (!sameRows(rows, expected.rows) && sameTitles(rows, expected.rows)) { + throw violation(`FID-1 (fields)`, stepIndex, expected.rows, rows) + } check(`rows`, sameRows(rows, expected.rows), expected.rows, rows) if (readyRows !== undefined) { check( @@ -1087,7 +1163,9 @@ export async function checkEagerLiveSourceHistory( ) readyRows = undefined } - check(`change events`, sameRows(events.rows(), rows), rows, events.rows()) + if (!sameRows(events.rows(), rows)) { + throw violation(`PUB-2 (change events)`, stepIndex, rows, events.rows()) + } const health = collection.health if (expected.healthError === null) { check(`health error`, health.error === null, null, health.error?.message) @@ -1222,11 +1300,13 @@ export async function checkEagerLiveSourceHistory( reach(wasReady ? `replacing-snapshot` : `first-ready`) if (done.missed > 0) reach(`read-missed-change`) if (done.superseded) reach(`seen-change-superseded`) + if (done.removesField) reach(`field-removed`) if (transientBefore > 0) reach(`transient-failure-recovered`) break case `changes`: provider.deliverChanges(done.changes, String(done.txid)) reach(done.beforeReady ? `change-before-ready` : `change-batch`) + if (done.removesField) reach(`field-removed`) if (done.changes.length > 1) reach(`multi-change-batch`) if (transientBefore > 0) reach(`transient-failure-recovered`) if ( @@ -1428,6 +1508,18 @@ function supports( } } +/** Same keys and titles, whatever the other fields. */ +function sameTitles(a: RowState, b: RowState): boolean { + const title = (value: string) => value.split(`/`)[0] + return ( + a.length === b.length && + a.every( + ([id, value], index) => + b[index]![0] === id && title(b[index]![1]) === title(value), + ) + ) +} + function sameRows(a: RowState, b: RowState): boolean { return ( a.length === b.length && @@ -1440,8 +1532,8 @@ function sameRows(a: RowState, b: RowState): boolean { /** * Folds the Collection's change events into rows (PUB-2) and records the row * state after each publication (PUB-1). The fold keeps every event, so a - * duplicate insert or a delete of an absent key shows as a mismatch rather - * than disappearing. It subscribes right after `preload()`, while the sync + * duplicate insert, an update of an absent key, or a delete of an absent key + * shows as a mismatch rather than disappearing. It subscribes right after `preload()`, while the sync * run holds no rows, so it needs no initial state. */ function foldChanges(collection: ObservedCollection) { @@ -1457,9 +1549,10 @@ function foldChanges(collection: ObservedCollection) { if (change.type === `delete`) { if (!rows.delete(change.key)) inconsistent = true } else { - if (change.type === `insert` && rows.has(change.key)) + // An insert of a held key, or an update of a key not held. + if ((change.type === `insert`) === rows.has(change.key)) inconsistent = true - rows.set(change.key, change.value.title) + rows.set(change.key, rowValue(change.value)) } } publications.push(rowState(rows)) diff --git a/packages/db-adapter-conformance/src/index.ts b/packages/db-adapter-conformance/src/index.ts index 90de37d053..5ef40bcaaa 100644 --- a/packages/db-adapter-conformance/src/index.ts +++ b/packages/db-adapter-conformance/src/index.ts @@ -22,3 +22,10 @@ export type { ProviderChange, ReachCounter, } from './eager-live-source-oracle' +export { + conformanceKey, + fromSqlRow, + observedView, + toSqlRow, +} from './observed-view' +export type { AdapterCollection } from './observed-view' diff --git a/packages/db-adapter-conformance/src/observed-view.ts b/packages/db-adapter-conformance/src/observed-view.ts new file mode 100644 index 0000000000..54412e9912 --- /dev/null +++ b/packages/db-adapter-conformance/src/observed-view.ts @@ -0,0 +1,96 @@ +/** + * Driver support for the conformance oracles. An adapter's rows or keys may + * differ from the conformance row: RxDB keys are strings, and a SQL backend + * stores an absent field as `NULL`. A driver states that mapping once with + * `observedView`, and the oracle reads the Collection through it. The mapping + * must be one-to-one on the rows a history can produce, so it cannot hide a + * defect: `conformanceKey` and `fromSqlRow` reject anything outside it. + */ +import type { ChangeMessage, HealthNode } from '@tanstack/db' +import type { + ConformanceRow, + ObservedCollection, +} from './eager-live-source-oracle' + +/** The public Collection surface `observedView` reads. */ +export interface AdapterCollection< + TRow extends object, + TKey extends string | number, +> { + readonly status: string + readonly health: HealthNode + values: () => IterableIterator + preload: () => Promise + cleanup: () => Promise + subscribeChanges: ( + callback: (changes: Array>) => void, + options?: { includeInitialState?: boolean }, + ) => { unsubscribe: () => void } + on: (event: `status:ready`, callback: () => void) => () => void +} + +/** The Collection seen with conformance rows and keys. */ +export function observedView( + collection: AdapterCollection, + mapping: { + row: (row: TRow) => ConformanceRow + key: (key: TKey) => number + }, +): ObservedCollection { + return { + get status() { + return collection.status + }, + get health() { + return collection.health + }, + *values() { + for (const row of collection.values()) yield mapping.row(row) + }, + preload: () => collection.preload(), + cleanup: () => collection.cleanup(), + on: (event, callback) => collection.on(event, callback), + subscribeChanges: (callback, options) => + collection.subscribeChanges( + (changes) => + callback( + changes.map((change): ChangeMessage => ({ + ...change, + key: mapping.key(change.key), + value: mapping.row(change.value), + previousValue: + change.previousValue && mapping.row(change.previousValue), + })), + ), + options, + ), + } +} + +/** Maps a string key `"n"` back to conformance id `n`; rejects anything else. */ +export function conformanceKey(key: unknown): number { + const id = Number(key) + if (typeof key !== `string` || !Number.isInteger(id) || String(id) !== key) { + throw new Error(`key ${JSON.stringify(key)} is not a conformance id`) + } + return id +} + +/** A conformance row as a SQL row: an absent `note` is `NULL`. */ +export function toSqlRow(row: ConformanceRow) { + return { id: row.id, title: row.title, note: row.note ?? null } +} + +/** + * A SQL row back as a conformance row: `NULL` is an absent `note`. Every + * other field passes through, so an extra or missing column still shows. + */ +export function fromSqlRow( + row: TRow, +): ConformanceRow { + if (!Object.hasOwn(row, `note`)) { + throw new Error(`SQL row ${JSON.stringify(row)} has no note column`) + } + const { note, ...rest } = row + return (note === null ? rest : { ...rest, note }) as unknown as ConformanceRow +} diff --git a/packages/db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts b/packages/db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts index 420ca882a0..4b234a5802 100644 --- a/packages/db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts @@ -36,7 +36,9 @@ import { neonRealtimeDriver } from './neon/neon-realtime-driver' const property = `adapter-conformance.eager-live-source.neon-realtime` const driver = neonRealtimeDriver() const historyArb = eagerLiveSourceHistoryArb(driver.capabilities) -const runs = 200 +// `field-removed` needs a held row with a note and a later upsert without +// one; at 400 histories every seed tried reached it at least 11 times. +const runs = 400 // Timed waits sleep past their timeout, so a passing campaign takes seconds. const campaignTimeoutMs = 120_000 diff --git a/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts b/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts index a1b61d58b7..b3fcf47f50 100644 --- a/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts +++ b/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts @@ -26,6 +26,7 @@ * judges the Collection's public state after such an exception. */ import { createCollection } from '@tanstack/db' +import { fromSqlRow, observedView, toSqlRow } from '../../src' import { realtimeCollectionOptions } from './vendor/collection-options' import type { LiveQueryChange, @@ -42,18 +43,20 @@ import type { ControlledProvider, EagerLiveSourceDriver, EagerLiveSourceFixture, - ObservedCollection, ProviderChange, } from '../../src' -class ControlledRawSubscription implements RawLiveQuerySubscription { - readonly renewals: Array> = [] +/** A Neon row is a Postgres row, so an absent `note` is `NULL`. */ +type NeonRow = ReturnType + +class ControlledRawSubscription implements RawLiveQuerySubscription { + readonly renewals: Array> = [] readonly listenerErrors: Array = [] unsubscribed = false unsubscribes = 0 private state: LiveQueryState = { status: `connecting`, error: undefined } private readonly updates = new Set< - (update: RawLiveQueryUpdate) => void + (update: RawLiveQueryUpdate) => void >() private readonly states = new Set<(state: LiveQueryState) => void>() private readonly applied = new Set() @@ -64,9 +67,7 @@ class ControlledRawSubscription implements RawLiveQuerySubscription this.state - onUpdate = ( - listener: (update: RawLiveQueryUpdate) => void, - ) => { + onUpdate = (listener: (update: RawLiveQueryUpdate) => void) => { this.updates.add(listener) return () => this.updates.delete(listener) } @@ -100,7 +101,7 @@ class ControlledRawSubscription implements RawLiveQuerySubscription): Promise => { + renew = (query: SealedLiveQuery): Promise => { this.renewals.push(query) return Promise.resolve() } @@ -113,7 +114,7 @@ class ControlledRawSubscription implements RawLiveQuerySubscription, + update: RawLiveQueryUpdate, txids: ReadonlyArray = [], ) { for (const listener of this.updates) this.call(() => listener(update)) @@ -146,7 +147,7 @@ class ControlledRawSubscription implements RawLiveQuerySubscription { +function sealed(expiresInMs: number): SealedLiveQuery { const header = Buffer.from( JSON.stringify({ alg: `dir`, enc: `A256GCM`, v: 1 }), ).toString(`base64url`) @@ -157,24 +158,24 @@ function sealed(expiresInMs: number): SealedLiveQuery { } } -function rawRow(row: ConformanceRow): RawLiveQueryRow { - return { rowId: `r${row.id}`, row } +function rawRow(row: ConformanceRow): RawLiveQueryRow { + return { rowId: `r${row.id}`, row: toSqlRow(row) } } -function rawChange(change: ProviderChange): LiveQueryChange { +function rawChange(change: ProviderChange): LiveQueryChange { return change.type === `upsert` - ? { type: `upsert`, rowId: `r${change.row.id}`, row: change.row } + ? { type: `upsert`, rowId: `r${change.row.id}`, row: toSqlRow(change.row) } : { type: `remove`, rowId: `r${change.id}` } } class NeonProvider implements ControlledProvider { readonly subscriptions: Array = [] private pendingRefresh: - ((query: SealedLiveQuery) => void) | undefined + ((query: SealedLiveQuery) => void) | undefined readonly client: RealtimeClient = { subscribe: (( - _query: SealedLiveQuery, + _query: SealedLiveQuery, options?: RawLiveQueryOptions, ) => { if (options?.materialize !== false) { @@ -190,7 +191,7 @@ class NeonProvider implements ControlledProvider { } refreshQuery = () => - new Promise>((resolve) => { + new Promise>((resolve) => { this.pendingRefresh = resolve }) @@ -269,11 +270,14 @@ export function neonRealtimeDriver(): EagerLiveSourceDriver { // Expires within Neon's 10 s refresh lead, so a renewal is due at once. query: sealed(10_000), refreshQuery: provider.refreshQuery, - getKey: (row: ConformanceRow) => row.id, + getKey: (row: NeonRow) => row.id, }), ) return { - collection: collection as unknown as ObservedCollection, + collection: observedView(collection, { + row: fromSqlRow, + key: (key) => key as number, + }), provider, awaitTxId: (txid, timeout) => collection.utils.awaitTxId(txid, timeout), } diff --git a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts index c1a1fae116..7968b3cad5 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts @@ -53,9 +53,9 @@ const property = `adapter-conformance.eager-live-source.reference` const driver = referenceDriver() const historyArb = eagerLiveSourceHistoryArb(driver.capabilities) // The reference adapter declares every capability, so its grammar spreads -// over the most step kinds. Across 20 seeds, 400 histories reached every -// declared state at least 11 times. -const runs = 400 +// over the most step kinds. Across 20 seeds, 600 histories reached every +// declared state at least 15 times. +const runs = 600 // Timed waits sleep past their timeout, so a campaign takes several seconds. const campaignTimeoutMs = 120_000 @@ -116,6 +116,8 @@ describe(`eager live-source conformance: reference adapter`, () => { [`replays-per-batch`, `PUB-5`], [`ready-before-buffer`, `READY-10 (rows when ready)`], [`shows-buffered`, `READY-10/PUB-9`], + [`merges-updates`, `FID-1`], + [`relabels-inserts`, `PUB-2`], ] oracleReplayTest(it).each(mutants)( `rejects the %s mutant with %s`, @@ -278,6 +280,17 @@ const witnesses: Array<[string, EagerLiveSourceHistory]> = [ { kind: `snapshot`, rows: [{ id: 1, title: `a` }], seen: 0 }, ], ], + [ + // FID-1: an update without the optional field removes it. + `an update that omits a field removes it`, + [ + { kind: `snapshot`, rows: [{ id: 1, title: `a`, note: `x` }] }, + { + kind: `changes`, + changes: [{ type: `upsert`, row: { id: 1, title: `b` } }], + }, + ], + ], [ // lifecycle-oracle.property.test.ts, restart fencing `TrailBase: a restart is a fresh sync run`, @@ -291,8 +304,11 @@ const witnesses: Array<[string, EagerLiveSourceHistory]> = [ /** Whether `history` lies in the generator's domain. */ function withinGrammar(history: EagerLiveSourceHistory): boolean { - const row = (r: { id: number; title: string }) => - r.id >= 1 && r.id <= 5 && [`a`, `b`, `c`].includes(r.title) + const row = (r: { id: number; title: string; note?: string }) => + r.id >= 1 && + r.id <= 5 && + [`a`, `b`, `c`].includes(r.title) && + (!Object.hasOwn(r, `note`) || [`x`, `y`].includes(r.note!)) return ( history.length >= 1 && history.length <= 15 && @@ -423,6 +439,50 @@ describe(`eager live-source grammar controls`, () => { campaignTimeoutMs, ) + oracleReplayTest(it)( + `without notes, histories never reach field-removed`, + async () => { + const reach = createOracleReach( + `${property}.ablation`, + reachStatesFor(driver.capabilities), + { replaying: false }, + ) + const plain = (row: { id: number; title: string }) => ({ + id: row.id, + title: row.title, + }) + await fc.assert( + fc.asyncProperty( + historyArb.map((history) => + history.map((step): EagerLiveSourceStep => { + if (step.kind === `snapshot`) { + return { ...step, rows: step.rows.map(plain) } + } + if (step.kind !== `changes`) return step + return { + kind: `changes`, + changes: step.changes.map((change) => + change.type === `upsert` + ? { type: `upsert`, row: plain(change.row) } + : change, + ), + } + }), + ), + async (history) => { + reach.beginHistory() + await checkEagerLiveSourceHistory(driver, history, { + reach: reach.reach, + }) + }, + ), + { seed: 2118, numRuns: 100 }, + ) + expect(reach.counts().get(`field-removed`)).toBe(0) + }, + campaignTimeoutMs, + ) + // Ablation of the read point: when every fetch reads after all buffered // batches, no history reaches a cut whose result missed one. oracleReplayTest(it)( diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index 301cff5d24..3bfcc2a6e7 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -12,6 +12,7 @@ * renewing the same subscription. */ import { createCollection } from '@tanstack/db' +import { observedView } from '../src' import type { ConformanceRow, ControlledProvider, @@ -36,7 +37,7 @@ export type ReferenceMutant = | `transient-fails-loading` /** PUB-1: one provider batch is committed as one transaction per change. */ | `splits-batch` - /** PUB-3: a batch's deletes are not written. */ + /** PUB-3: a live batch's deletes are not written. */ | `drops-deletes` /** PUB-7: a later complete delivery merges into the rows. */ | `snapshot-merges` @@ -72,6 +73,13 @@ export type ReferenceMutant = | `ready-before-buffer` /** READY-10, PUB-9: a batch during the initial fetch shows before ready. */ | `shows-buffered` + /** FID-1: an update merges over the held row, so a removed field stays. */ + | `merges-updates` + /** + * PUB-2, checker calibration: the observed view relabels insert events as + * updates. The adapter is unchanged; the comparison must see the events. + */ + | `relabels-inserts` type ProviderEvent = | { type: `snapshot`; rows: ReadonlyArray } @@ -181,6 +189,8 @@ function referenceCollection( getKey: (row) => row.id, syncMode: `eager`, sync: { + // The provider delivers complete rows, so an update replaces the row. + rowUpdateMode: mutant === `merges-updates` ? `partial` : `full`, sync: ({ begin, write, @@ -226,7 +236,7 @@ function referenceCollection( value: change.row, }) keys.add(change.row.id) - } else if (mutant !== `drops-deletes`) { + } else if (mutant !== `drops-deletes` || !ready) { write({ type: `delete`, key: change.id }) keys.delete(change.id) } @@ -429,7 +439,35 @@ export function referenceDriver( mutant, perChange, ) - return { collection, provider, awaitTxId } + if (mutant !== `relabels-inserts`) { + return { collection, provider, awaitTxId } + } + const relabeled = observedView(collection, { + row: (row) => row, + key: (key) => key as number, + }) + const subscribeChanges: typeof relabeled.subscribeChanges = ( + callback, + options, + ) => + relabeled.subscribeChanges( + (changes) => + callback( + changes.map((change) => + change.type === `insert` + ? { ...change, type: `update` as const } + : change, + ), + ), + options, + ) + // The view's getters stay live on the derived object. + const view = Object.create(relabeled) as typeof relabeled + return { + collection: Object.assign(view, { subscribeChanges }), + provider, + awaitTxId, + } }, } } diff --git a/packages/trailbase-db-collection/tests/ORACLE.md b/packages/trailbase-db-collection/tests/ORACLE.md index 0415c6057e..608c35d5c0 100644 --- a/packages/trailbase-db-collection/tests/ORACLE.md +++ b/packages/trailbase-db-collection/tests/ORACLE.md @@ -256,7 +256,7 @@ No service-backed E2E run is claimed. `eager-live-source-conformance-oracle.property.test.ts` runs the shared eager live-source oracle from `packages/db-adapter-conformance` through `conformance-driver.ts`, with the same controlled RecordApi. It checks -READY-2/3/4, HEALTH-1, PUB-1 (per-change commits), PUB-2/3, FID-4, +READY-2/3/4, HEALTH-1, PUB-1 (per-change commits), PUB-2/3, FID-1/4, LIFE-2/5/10, and HEALTH-5 in eager mode. TrailBase declares no optional capability, so it is not checked for PUB-7, waits, retried failures, reconnect, or renewal. This file's diff --git a/packages/trailbase-db-collection/tests/conformance-driver.ts b/packages/trailbase-db-collection/tests/conformance-driver.ts index 0ca4c94734..ba1b4d3c7e 100644 --- a/packages/trailbase-db-collection/tests/conformance-driver.ts +++ b/packages/trailbase-db-collection/tests/conformance-driver.ts @@ -23,10 +23,17 @@ * `cancel`, so for it only the lock release is seen. * * The provider's own rows decide `Insert` versus `Update`, as a real service - * would. The driver never reads the Collection. + * would. A record is a SQL row with every column, so an absent `note` is + * `NULL` (`toSqlRow`), and the observed view maps `NULL` back to absent + * (`fromSqlRow`). The driver never reads the Collection. */ import { createCollection } from '@tanstack/db' import { trailBaseCollectionOptions } from '../src/trailbase' +import { + fromSqlRow, + observedView, + toSqlRow, +} from '../../db-adapter-conformance/src' import { MockRecordApi } from './mock-record-api' import type { Event, ListResponse } from 'trailbase' import type { @@ -34,23 +41,25 @@ import type { ControlledProvider, EagerLiveSourceDriver, EagerLiveSourceFixture, - ObservedCollection, ProviderChange, } from '../../db-adapter-conformance/src' +/** A TrailBase record: a SQL row, so an absent `note` is `NULL`. */ +type ConformanceRecord = ReturnType + interface ProviderSession { stream: ReadableStream controller: ReadableStreamDefaultController cancels: number locked: boolean list?: { - resolve: (response: ListResponse) => void + resolve: (response: ListResponse) => void reject: (error: unknown) => void } } class TrailBaseProvider implements ControlledProvider { - readonly api = new MockRecordApi() + readonly api = new MockRecordApi() readonly sessions: Array = [] private rows = new Map() @@ -75,7 +84,7 @@ class TrailBaseProvider implements ControlledProvider { }) this.api.list.mockImplementation( () => - new Promise>((resolve, reject) => { + new Promise>((resolve, reject) => { this.latest().list = { resolve, reject } }), ) @@ -106,7 +115,7 @@ class TrailBaseProvider implements ControlledProvider { deliverSnapshot = (rows: ReadonlyArray) => { this.rows = new Map(rows.map((row) => [row.id, row])) // One short page drains TrailBase's paged list. - this.pendingList().resolve({ records: [...rows] }) + this.pendingList().resolve({ records: rows.map(toSqlRow) }) } deliverChanges = (changes: ReadonlyArray) => { @@ -115,14 +124,13 @@ class TrailBaseProvider implements ControlledProvider { if (change.type === `upsert`) { const held = this.rows.has(change.row.id) this.rows.set(change.row.id, change.row) - controller.enqueue( - held ? { Update: { ...change.row } } : { Insert: { ...change.row } }, - ) + const record = toSqlRow(change.row) + controller.enqueue(held ? { Update: record } : { Insert: record }) } else { const row = this.rows.get(change.id) if (!row) throw new Error(`provider does not hold row ${change.id}`) this.rows.delete(change.id) - controller.enqueue({ Delete: { ...row } }) + controller.enqueue({ Delete: toSqlRow(row) }) } } } @@ -147,14 +155,17 @@ export function trailBaseConformanceDriver(): EagerLiveSourceDriver { const collection = createCollection( trailBaseCollectionOptions({ recordApi: provider.api, - getKey: (row: ConformanceRow) => row.id, + getKey: (row: ConformanceRecord) => row.id, syncMode: `eager`, parse: {}, serialize: {}, }), ) return { - collection: collection as unknown as ObservedCollection, + collection: observedView(collection, { + row: fromSqlRow, + key: (key) => key as number, + }), provider, } }, diff --git a/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts index d225befe77..8a506f2b42 100644 --- a/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts +++ b/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -35,7 +35,10 @@ import { trailBaseConformanceDriver } from './conformance-driver' const property = `adapter-conformance.eager-live-source.trailbase` const driver = trailBaseConformanceDriver() const historyArb = eagerLiveSourceHistoryArb(driver.capabilities) -const runs = 200 +// `field-removed` needs a held row with a note and a later upsert without +// one; at 400 histories every seed tried reached it at least 11 times. +const runs = 400 +const campaignTimeoutMs = 120_000 async function campaign(options: fc.Parameters<[unknown]>) { const reach = createOracleReach( @@ -50,10 +53,14 @@ async function campaign(options: fc.Parameters<[unknown]>) { } describe(`eager live-source conformance: TrailBase`, () => { - oracleReplayTest(it)(`conforms on a fixed-seed campaign`, async () => { - const reach = await campaign({ seed: 2118, numRuns: oracleRuns(runs) }) - reach.assertReached() - }) + oracleReplayTest(it)( + `conforms on a fixed-seed campaign`, + async () => { + const reach = await campaign({ seed: 2118, numRuns: oracleRuns(runs) }) + reach.assertReached() + }, + campaignTimeoutMs, + ) oracleReplayTest(it, property)( `conforms on a random campaign or a replayed seed`, @@ -61,6 +68,7 @@ describe(`eager live-source conformance: TrailBase`, () => { const reach = await campaign(oraclePropertyOptions(runs, property)) reach.assertReached() }, + campaignTimeoutMs, ) }) From 7aa1946bd9d7ed96f5625f433eddc2fe32105793 Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 29 Oct 2026 13:14:23 -0600 Subject: [PATCH 06/64] fix(rxdb): replace the synced row on a document update RxDB change events carry the whole document, but the adapter wrote them as sync updates under core's default `partial` row mode, which merges an update over the held row. A field that a document update removed stayed in the Collection. The sync config now declares `rowUpdateMode: 'full'`. It also drops `getSyncMetadata: undefined`, which core reads only through an optional call. The RxDB conformance driver moves to the optional `note` field and the shared `observedView`. - Law: FID-1 (values round-trip; null versus absent) from the adapter law set v18. - Reach: both RxDB campaigns (400 histories) assert field-removed and every other declared state at least five times; four repeats of the random campaign pass. - RED: on the parent, both campaigns failed FID-1 (fields), observed [[5,"a/x"]] where [[5,"a"]] is owed. The memory-storage witness `removes a field that a document update removes` received { id: '1', name: 'First', note: 'x' }. - GREEN: RxDB suite 23/23. - Mutant: removing `rowUpdateMode: 'full'` fails FID-1 in both campaigns. - Size: rxdb-db-collection unchanged against the parent (gzip), -1 byte against main. - Limits: FID-1 covers one optional string field; the claim covers the controlled RxCollection and one memory-storage witness. Co-authored-by: Isaac --- .changeset/rxdb-full-row-updates.md | 5 ++ packages/db-adapter-conformance/MIGRATION.md | 11 +++ packages/rxdb-db-collection/src/rxdb.ts | 5 +- .../tests/conformance-driver.ts | 75 +++++-------------- ...source-conformance-oracle.property.test.ts | 4 +- .../rxdb-db-collection/tests/rxdb.test.ts | 51 +++++++++++++ 6 files changed, 93 insertions(+), 58 deletions(-) create mode 100644 .changeset/rxdb-full-row-updates.md diff --git a/.changeset/rxdb-full-row-updates.md b/.changeset/rxdb-full-row-updates.md new file mode 100644 index 0000000000..d5c4d393c4 --- /dev/null +++ b/.changeset/rxdb-full-row-updates.md @@ -0,0 +1,5 @@ +--- +'@tanstack/rxdb-db-collection': patch +--- + +Remove a field from a synced row when an RxDB document update removes it. RxDB change events carry the whole document, so the adapter now replaces the row instead of merging the update into it. diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index 598224707a..e6d37e38dd 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -151,3 +151,14 @@ a buffered insert and update`) failed the same way. - **Neon:** still RED on READY-4 and HEALTH-1 only; a probe without failure steps passed 300 histories with notes. - **RxDB:** RED on FID-1; see the next entry. +- **2026-10-29, RxDB on field removal: a second RxDB defect.** The driver + stores an absent `note` as an absent document field and uses the shared + `observedView`. + - **RED:** both campaigns failed FID-1: a removed `note` stayed (`a/x` + where `a` is owed). RxDB wrote whole documents as sync `update`s, and + core's default `partial` row mode merged them over the held row. A + memory-storage witness (`rxdb.test.ts`, `removes a field that a +document update removes`) failed the same way. + - **Fix:** the RxDB sync config declares `rowUpdateMode: 'full'`. + - **GREEN:** RxDB suite 23/23; four repeats of the random campaign pass. + - **Mutant:** removing `rowUpdateMode` fails FID-1 in both campaigns. diff --git a/packages/rxdb-db-collection/src/rxdb.ts b/packages/rxdb-db-collection/src/rxdb.ts index 064cf51198..e9d36b1fe1 100644 --- a/packages/rxdb-db-collection/src/rxdb.ts +++ b/packages/rxdb-db-collection/src/rxdb.ts @@ -295,8 +295,9 @@ export function rxdbCollectionOptions( return stopOngoingFetch }, - // Expose the getSyncMetadata function - getSyncMetadata: undefined, + // An RxDB event carries the whole document, so an update replaces the + // row and a field the document no longer has goes. + rowUpdateMode: `full`, } const collectionConfig: CollectionConfig = { diff --git a/packages/rxdb-db-collection/tests/conformance-driver.ts b/packages/rxdb-db-collection/tests/conformance-driver.ts index 9ce35b0a72..8fb454d238 100644 --- a/packages/rxdb-db-collection/tests/conformance-driver.ts +++ b/packages/rxdb-db-collection/tests/conformance-driver.ts @@ -35,9 +35,9 @@ * found by the cursor's primary-key tie-break. * * RxDB primary keys are strings. The driver stores row `n` as document id - * `"n"`, and its observed view maps each key and row id back with - * `conformanceId`, which rejects anything that is not such a key. The view - * passes status, health, `preload`, and `cleanup` through unchanged. + * `"n"`, and the shared `observedView` maps each key and row id back with + * `conformanceKey`, which rejects anything that is not such a key. A JSON + * document omits an absent `note`, so rows otherwise pass through unchanged. */ import { createCollection } from '@tanstack/db' import { @@ -48,7 +48,7 @@ import { } from 'rxdb/plugins/core' import { Observable } from 'rxjs' import { rxdbCollectionOptions } from '../src/rxdb' -import type { ChangeMessage } from '@tanstack/db' +import { conformanceKey, observedView } from '../../db-adapter-conformance/src' import type { PreparedQuery, RxChangeEvent, @@ -61,11 +61,10 @@ import type { ControlledProvider, EagerLiveSourceDriver, EagerLiveSourceFixture, - ObservedCollection, ProviderChange, } from '../../db-adapter-conformance/src' -type Doc = { id: string; title: string } +type Doc = { id: string; title: string; note?: string } const schema = createRxSchema( { @@ -75,6 +74,7 @@ const schema = createRxSchema( properties: { id: { type: `string`, maxLength: 10 }, title: { type: `string` }, + note: { type: `string` }, }, required: [`id`, `title`], }, @@ -160,6 +160,8 @@ class RxdbProvider implements ControlledProvider { return { id: String(row.id), title: row.title, + // A JSON document omits an absent field. + ...(row.note === undefined ? {} : { note: row.note }), _deleted: deleted, _rev: `${lwt}-conformance`, _attachments: {}, @@ -206,7 +208,9 @@ class RxdbProvider implements ControlledProvider { const current = this.docs.get(id) return [ id, - current?.title === row.title ? current : this.document(row, lwt), + current && sameDocument(current, row) + ? current + : this.document(row, lwt), ] }), ) @@ -229,11 +233,7 @@ class RxdbProvider implements ControlledProvider { const previous = this.docs.get(id) if (!previous) throw new Error(`provider does not hold row ${id}`) this.docs.delete(id) - const doc = this.document( - { id: change.id, title: previous.title }, - lwt, - true, - ) + const doc = this.document(conformanceRow(previous), lwt, true) this.emit(`DELETE`, doc, previous) } } @@ -250,51 +250,13 @@ class RxdbProvider implements ControlledProvider { } } -/** Maps an RxDB key back to its conformance row id; rejects anything else. */ -function conformanceId(key: unknown): number { - const id = Number(key) - if (typeof key !== `string` || !Number.isInteger(id) || String(id) !== key) { - throw new Error(`RxDB key ${JSON.stringify(key)} is not a conformance id`) - } - return id -} - +/** An RxDB document as a conformance row: key `"n"` is row id `n`. */ function conformanceRow(doc: Doc): ConformanceRow { - return { ...doc, id: conformanceId(doc.id) } + return { ...doc, id: conformanceKey(doc.id) } } -/** The adapter's Collection, seen with conformance row ids. */ -function observedView( - collection: ReturnType, -): ObservedCollection { - return { - get status() { - return collection.status - }, - get health() { - return collection.health - }, - *values() { - for (const doc of collection.values()) yield conformanceRow(doc) - }, - preload: () => collection.preload(), - cleanup: () => collection.cleanup(), - on: (event, callback) => collection.on(event, callback), - subscribeChanges: (callback, options) => - collection.subscribeChanges( - (changes) => - callback( - changes.map((change): ChangeMessage => ({ - ...change, - key: conformanceId(change.key), - value: conformanceRow(change.value), - previousValue: - change.previousValue && conformanceRow(change.previousValue), - })), - ), - options, - ), - } +function sameDocument(doc: Doc, row: ConformanceRow): boolean { + return doc.title === row.title && doc.note === row.note } function createRxdbCollection(provider: RxdbProvider) { @@ -315,7 +277,10 @@ export function rxdbConformanceDriver(): EagerLiveSourceDriver { create(): EagerLiveSourceFixture { const provider = new RxdbProvider() return { - collection: observedView(createRxdbCollection(provider)), + collection: observedView(createRxdbCollection(provider), { + row: conformanceRow, + key: conformanceKey, + }), provider, } }, diff --git a/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts index 94cd33b069..762bfe7b27 100644 --- a/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts +++ b/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -37,7 +37,9 @@ import { rxdbConformanceDriver } from './conformance-driver' const property = `adapter-conformance.eager-live-source.rxdb` const driver = rxdbConformanceDriver() const historyArb = eagerLiveSourceHistoryArb(driver.capabilities) -const runs = 200 +// `field-removed` needs a held row with a note and a later upsert without +// one; at 400 histories every seed tried reached it at least 11 times. +const runs = 400 // Startup bursts lengthen histories, so a campaign takes several seconds. const campaignTimeoutMs = 120_000 diff --git a/packages/rxdb-db-collection/tests/rxdb.test.ts b/packages/rxdb-db-collection/tests/rxdb.test.ts index 7498018175..53314f4542 100644 --- a/packages/rxdb-db-collection/tests/rxdb.test.ts +++ b/packages/rxdb-db-collection/tests/rxdb.test.ts @@ -437,6 +437,57 @@ describe(`RxDB Integration`, () => { } }) + // A receiving witness for the conformance oracle's FID-1 lane on real + // memory storage: a document update that removes an optional field. + it(`removes a field that a document update removes`, async () => { + type NotedDoc = { id: string; name: string; note?: string } + const db = await createRxDatabase< + { noted: RxCollection }, + unknown, + unknown, + unknown + >({ + name: `my-rxdb-noted-` + dbNameId++, + ignoreDuplicate: true, + storage: wrappedValidateAjvStorage({ storage: getRxStorageMemory() }), + }) + const { noted } = await db.addCollections({ + noted: { + schema: { + version: 0, + type: `object`, + primaryKey: `id`, + properties: { + id: { type: `string`, maxLength: 100 }, + name: { type: `string` }, + note: { type: `string` }, + }, + required: [`id`, `name`], + }, + }, + }) + const doc = await noted.insert({ id: `1`, name: `First`, note: `x` }) + const collection = createCollection( + rxdbCollectionOptions({ rxCollection: noted }), + ) + + try { + await collection.preload() + await doc.incrementalModify( + ({ note: _note, ...rest }: NotedDoc) => rest, + ) + await flushPromises() + + expect(stripVirtualProps(collection.get(`1`))).toEqual({ + id: `1`, + name: `First`, + }) + } finally { + await collection.cleanup() + await db.remove() + } + }) + it(`does not let later live traffic extend the startup readiness boundary`, async () => { const db = await getDatababase() const rxCollection: RxCollection = db.test From 38122166aeff9d34cab68bf91a858e5a820d545e Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 29 Oct 2026 15:22:40 -0600 Subject: [PATCH 07/64] test(db): name replay owners for the adapter conformance properties The four registered `adapter-conformance.eager-live-source.*` properties had no named owner, so the replay manifest test failed. Each now names its owner file and random-lane test name. The Neon owner runs only under `vitest.neon.config.ts`, so the manifest gains a per-owner config map for owners inside packages/ that their default config excludes. - GREEN: tests/oracle-replay.test.ts 41/41. Co-authored-by: Isaac --- packages/db/tests/oracle-replay-manifest.ts | 41 ++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/packages/db/tests/oracle-replay-manifest.ts b/packages/db/tests/oracle-replay-manifest.ts index 5de3ae76b1..4357f06816 100644 --- a/packages/db/tests/oracle-replay-manifest.ts +++ b/packages/db/tests/oracle-replay-manifest.ts @@ -638,6 +638,26 @@ const ownerGroups: ReadonlyArray< `sync-metadata-session`, `persisted-flat persisted-nested persisted-seeds persisted-conflicts persisted-cached persisted-nested-seeds persisted-reads persisted-reads-seeds persisted-peers persisted-peers-full persisted-peers-on-demand`, ], + [ + `db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts`, + `adapter-conformance.eager-live-source`, + `reference`, + ], + [ + `db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts`, + `adapter-conformance.eager-live-source`, + `neon-realtime`, + ], + [ + `trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts`, + `adapter-conformance.eager-live-source`, + `trailbase`, + ], + [ + `rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts`, + `adapter-conformance.eager-live-source`, + `rxdb`, + ], ] const owners = new Map() @@ -724,6 +744,13 @@ export type OracleReplayManifestEntry = { // Pagination cells carry their property name in the test title so the guard // can select one cell without running the rest of the matrix. const replayTestNames = new Map([ + ...[`reference`, `neon-realtime`, `trailbase`, `rxdb`].map( + (driver) => + [ + `adapter-conformance.eager-live-source.${driver}`, + `conforms on a random campaign or a replayed seed`, + ] as const, + ), [ `source-health.required-history`, `required histories refine the model.*random`, @@ -869,6 +896,18 @@ const outsideReplayConfigs = new Map([ [`scripts`, `oracle-scripts.vitest.config.mjs`], ]) +/** + * An owner inside packages/ that its package's default config excludes, with + * the config that runs it. The Neon receiving driver is expected RED, so it + * runs only under its own config. + */ +const ownerReplayConfigs = new Map([ + [ + `packages/db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts`, + `vitest.neon.config.ts`, + ], +]) + export const oracleReplayManifest: ReadonlyArray = [ ...registeredOracleProperties, ].map((property) => { @@ -909,7 +948,7 @@ export const oracleReplayManifest: ReadonlyArray = [ ? `pnpm exec vitest run ${file.split(`/`).slice(2).join(`/`)}` : outsidePackages ? `node --import tsx ${`../`.repeat(cwd!.split(`/`).length)}packages/db/tests/oracle-replay.ts ${outsideConfig === undefined ? `` : `--config ${outsideConfig} `}${file.split(`/`).pop()}` - : `node --import tsx ${cwd === `packages/db` ? `tests` : `../db/tests`}/oracle-replay.ts ${file.split(`/`).slice(2).join(`/`)}`, + : `node --import tsx ${cwd === `packages/db` ? `tests` : `../db/tests`}/oracle-replay.ts ${ownerReplayConfigs.has(file) ? `--config ${ownerReplayConfigs.get(file)} ` : ``}${file.split(`/`).slice(2).join(`/`)}`, environment: status === `statistics-only` ? [ From e528597c7bfecb93f73579651db9a109886353b1 Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 29 Oct 2026 15:22:44 -0600 Subject: [PATCH 08/64] fix(db): publish a synced update that changes only -0 to 0 Generate typed values in the eager live-source conformance oracle (FID-1). A driver declares `valueKinds`, the kinds of its `valueDomain`; rows then carry a `value` drawn from edge cases of each kind and are compared by an independent strict description (`describeValue`): same type and prototype, `Object.is` for numbers, Dates by time, arrays and plain objects element by element with no extra own keys. Reach is declared per kind and per path (initial load, live change). A `retouch` change re-delivers a held row with every zero's sign flipped. A random campaign found a core defect: a synced update that changed a field only from -0 to 0 stored the new row but published no change event, so subscribers kept -0. Core compared rows with `deepEquals`, which treats -0 as 0. Maintainer decision (2026-10-29): -0 and 0 are distinct row values. The synced commit comparison now uses `sameRowValue`, deep equality with signed zeros; `deepEquals` keeps its documented semantics elsewhere. - Law: FID-1 (values round-trip, parameter `valueDomain`) and PUB-2 (change events fold to the rows) from the adapter law set v18. - Reach: reference fixed campaign (seed 2118, 600 histories) reaches all 20 value states and zero-sign-flipped (74); the lowest count of any state across 12 seeds is 12. RxDB and TrailBase reach their 14 JSON value states. - RED: on the parent core, random seed 95028 failed PUB-2 (change events): expected [[5,"a/y#n0"]], observed [[5,"a/y#n-0"]]. The replayed witness `FID-1, PUB-2: an update from -0 to 0 publishes a change` failed the same way. - GREEN: reference oracle 65/65; core suite 348 files, 23,927 tests; RxDB 23/23; TrailBase 151/151; Neon keeps its READY-4 and HEALTH-1 RED and passes 300 failure-free histories with values. - Mutant: value mutants dates-as-strings (live only), bigints-as-numbers (load only), drops-nulls, loses-negative-zero, each FID-1 (values). A core fix that tells -0 from 0 only at the top of a value survived until the retouch dimension, then failed PUB-2 in both campaigns. - Size: @tanstack/db +26 bytes gzip (app scenario +23). Folding the three equality flags into one bitmask was tried and measured +36, so the flag is threaded as a boolean. - Limits: no Temporal, Map, Set, typed-array, or class-instance values; no FID-3/PUB-4 rejection lane; core's optimistic and subscription reconciliation comparisons still use deepEquals (coverage map entry, witness needed). Co-authored-by: Isaac --- .changeset/sync-negative-zero-change.md | 5 + docs/contributing/oracle-coverage.md | 17 +- packages/db-adapter-conformance/MIGRATION.md | 23 + packages/db-adapter-conformance/README.md | 120 +++-- .../src/eager-live-source-oracle.ts | 454 +++++++++++++++--- packages/db-adapter-conformance/src/index.ts | 5 + .../src/observed-view.ts | 8 +- .../neon-realtime-oracle.property.test.ts | 12 +- .../tests/neon/neon-realtime-driver.ts | 3 +- .../reference-adapter-oracle.property.test.ts | 152 +++++- .../tests/reference-adapter.ts | 61 ++- packages/db/src/collection/state.ts | 4 +- packages/db/src/utils.ts | 34 +- .../tests/conformance-driver.ts | 19 +- ...source-conformance-oracle.property.test.ts | 4 +- .../tests/conformance-driver.ts | 18 +- ...source-conformance-oracle.property.test.ts | 9 +- 17 files changed, 790 insertions(+), 158 deletions(-) create mode 100644 .changeset/sync-negative-zero-change.md diff --git a/.changeset/sync-negative-zero-change.md b/.changeset/sync-negative-zero-change.md new file mode 100644 index 0000000000..591c96b907 --- /dev/null +++ b/.changeset/sync-negative-zero-change.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Publish a change when a synced update changes a field only from `-0` to `0`, or the reverse. Before, the Collection stored the new value but emitted no change event, so subscribers and live queries kept the old one. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index b2ce11637d..60ce312fb7 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -5974,8 +5974,16 @@ READY-2/3/4, HEALTH-1, PUB-1/2/3/7, FID-4, LIFE-2/5/10, HEALTH-5, EVID-1/2/3 (`await-evidence`), HEALTH-2 and RETRY-1 for retried failures (`retry-policy`), RECON-1 (`transport-reconnect`), RENEW-1 (`capability-renewal`), READY-8/10 with PUB-5/9 for batches committed -during the initial fetch (`listen-before-fetch`), and FID-1 removal of an -optional field. Every driver is also +during the initial fetch (`listen-before-fetch`), FID-1 removal of an +optional field, and FID-1 typed values per declared `valueKinds` on the +initial load and live changes. Its reference driver is also the generated +witness for core's synced commit change detection with `-0` and `0` +(maintainer decision 2026-10-29: distinct row values; `sameRowValue`). +Open in core, owner `packages/db/tests/change-event-history-oracle.test.ts` +and the subscription lifecycle publication oracle, witness needed: an +optimistic or local-only update and a subscription reconciliation that +change only `-0` to `0` (`collectOptimisticChanges` and `subscription.ts` +still compare with `deepEquals`). Every driver is also checked for the rows at the moment status becomes `ready`. Its model is an independent row Map and sync-run phase; drivers supply only a controlled provider and declare whether they commit per batch or per change. PUB-1 checks every change publication during @@ -5991,8 +5999,9 @@ READY-4 and HEALTH-1 until that adapter moves to `fail()`; probes show LIFE-10 (subscription kept open after an error state) behind them. Limits: no on-demand loading, mutation handlers, or reload; publications are observed through `subscribeChanges` only; EVID-3 does not distinguish a timed wait -that rejects before its timeout; FID-1 covers one optional string field, not -value domains; the Neon and TrailBase claims cover +that rejects before its timeout; FID-1 covers ten value kinds, not Temporal, +Map, Set, typed arrays, class instances, or field identity; the FID-3 and +PUB-4 rejection lane is not built; the Neon and TrailBase claims cover controlled providers, not real clients. Owner: `packages/db-adapter-conformance`. Evidence: the package README. Migration goal (maintainer, 2026-10-29): every collection adapter gets a diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index e6d37e38dd..ca69e13136 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -162,3 +162,26 @@ document update removes`) failed the same way. - **Fix:** the RxDB sync config declares `rowUpdateMode: 'full'`. - **GREEN:** RxDB suite 23/23; four repeats of the random campaign pass. - **Mutant:** removing `rowUpdateMode` fails FID-1 in both campaigns. +- **2026-10-29, eager live source, typed values (FID-1).** A driver declares + `valueKinds`, the kinds of its `valueDomain`; rows then carry a `value` + drawn from edge cases of each kind, compared by an independent strict + description (`describeValue`). Reach is declared per kind and per path + (initial load, live change). A `retouch` change re-delivers a held row + with every zero's sign flipped (`zero-sign-flipped`), or equal when the + domain has no `-0`. + - **Calibration:** `dates-as-strings` (live changes), `bigints-as-numbers` + (initial load), `drops-nulls`, `loses-negative-zero`, each FID-1 + (values). A path ablation shows the change-path states need their own + deliveries. + - **Core defect:** a synced update from `-0` to `0` published no change + event (PUB-2), because core compared rows with `deepEquals`. Maintainer + decision: `-0` and `0` are distinct row values. The synced commit path + now uses `sameRowValue`. A fix applied only at a value's top level + survived until the `retouch` dimension, then failed PUB-2. + - **Drivers:** RxDB, TrailBase, and Neon declare `jsonValueKinds`. RxDB and + TrailBase are GREEN; Neon passes 300 failure-free histories with values + and keeps its READY-4 and HEALTH-1 RED. TrailBase now runs 600 histories + (`field-removed` fell to 7 at 400). + - **Open:** the rejection lane (FID-3, PUB-4); the optimistic and + subscription-reconciliation paths in core still compare with + `deepEquals` (recorded in the coverage map). diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md index 85d4d385c3..b5df687c17 100644 --- a/packages/db-adapter-conformance/README.md +++ b/packages/db-adapter-conformance/README.md @@ -13,24 +13,24 @@ The package is private while its first oracle settles. pushes a complete result and then committed change batches. Its opening comment states the laws and their limits. In short: -| Law | Checked behavior | -| -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| READY-2, READY-3 | `loading` until the first complete delivery; then `ready` with exactly its rows | -| READY-4 | A failure before ready: status `error`, `preload()` rejects with the value, `health.error.cause` is the value | -| HEALTH-1 | A failure after ready: status stays `ready`, rows kept, `health.error.cause` is the value | -| PUB-1 | Every change publication during a step is one whole committed state, in commit order; per batch or per change, as the driver declares | -| PUB-2, PUB-3, PUB-7, FID-4 | Change events fold to the rows, with no insert of a held key and no update or delete of a missing one; batches fold onto rows; a complete delivery replaces (PUB-7 with `complete-snapshot`) | -| FID-1 | A delivered row is the whole backend row: a row without the optional `note` removes a `note` the Collection held | -| LIFE-2, LIFE-5 | Cleanup releases the provider subscription; a restart is a fresh, empty sync run | -| LIFE-10 | A failed provider's subscription is released at the failure, and no subscription is released twice | -| HEALTH-5 | No unhandled rejection and no undeclared `console.error` or `console.warn` | -| EVID-1, EVID-2, EVID-3 | `awaitTxId` fulfills after its batch applies, stays pending before, rejects after its timeout, and rejects at retirement | -| HEALTH-2, RETRY-1 | A retried failure stays hidden before ready, joins `failureReason` after it, and clears on the next delivery (`retry-policy`) | -| RECON-1 | Disconnect and reconnect keep status and rows | -| RENEW-1 | Capability renewal reuses the provider subscription and changes nothing visible | -| READY-8, READY-10 | With `listen-before-fetch`: no fetch before the listener is live; batches committed during the fetch apply after its result, before ready | -| PUB-5, PUB-9 | With `listen-before-fetch`: the cut publishes only backend states at or after the read point, in order; no replay regresses a row | -| READY-3, READY-10 | When status becomes `ready`, the rows already equal the initial cut | +| Law | Checked behavior | +| -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| READY-2, READY-3 | `loading` until the first complete delivery; then `ready` with exactly its rows | +| READY-4 | A failure before ready: status `error`, `preload()` rejects with the value, `health.error.cause` is the value | +| HEALTH-1 | A failure after ready: status stays `ready`, rows kept, `health.error.cause` is the value | +| PUB-1 | Every change publication during a step is one whole committed state, in commit order; per batch or per change, as the driver declares | +| PUB-2, PUB-3, PUB-7, FID-4 | Change events fold to the rows, with no insert of a held key and no update or delete of a missing one; batches fold onto rows; a complete delivery replaces (PUB-7 with `complete-snapshot`) | +| FID-1 | A delivered row is the whole backend row: a row without the optional `note` removes a `note` the Collection held. With declared `valueKinds`, each `value` strictly equals the delivered one on the initial load and on a live change: same type and prototype, `Object.is` for numbers (`-0` is not `0`), Dates by time, nested arrays and plain objects element by element | +| LIFE-2, LIFE-5 | Cleanup releases the provider subscription; a restart is a fresh, empty sync run | +| LIFE-10 | A failed provider's subscription is released at the failure, and no subscription is released twice | +| HEALTH-5 | No unhandled rejection and no undeclared `console.error` or `console.warn` | +| EVID-1, EVID-2, EVID-3 | `awaitTxId` fulfills after its batch applies, stays pending before, rejects after its timeout, and rejects at retirement | +| HEALTH-2, RETRY-1 | A retried failure stays hidden before ready, joins `failureReason` after it, and clears on the next delivery (`retry-policy`) | +| RECON-1 | Disconnect and reconnect keep status and rows | +| RENEW-1 | Capability renewal reuses the provider subscription and changes nothing visible | +| READY-8, READY-10 | With `listen-before-fetch`: no fetch before the listener is live; batches committed during the fetch apply after its result, before ready | +| PUB-5, PUB-9 | With `listen-before-fetch`: the cut publishes only backend states at or after the read point, in order; no replay regresses a row | +| READY-3, READY-10 | When status becomes `ready`, the rows already equal the initial cut | The law IDs come from the adapter conformance law set, version 18. @@ -44,6 +44,7 @@ Implement `EagerLiveSourceDriver`: - **`changeCommits`:** `per-batch` if your adapter commits a provider batch as one transaction, `per-change` if it commits each change alone. - **`declaredDiagnostic`:** optional; accepts console diagnostics your adapter documents, so HEALTH-5 does not reject them. - **`seed` and `fetchesBeforeListening`:** with `listen-before-fetch`, the provider holds backend rows set by `seed`, treats a batch before the first ready as a backend commit during the fetch, and counts fetches issued while its listener was not live (READY-8). +- **`valueKinds`:** the kinds of FID-1's `valueDomain` your backend represents; `jsonValueKinds` for a JSON backend. The driver delivers each value in the backend's own form and the Collection must show it decoded. - **`observedView`:** if your rows or keys differ from the conformance row, wrap the Collection with `observedView` and a one-to-one mapping. `conformanceKey` maps string keys; `toSqlRow` and `fromSqlRow` map an absent `note` to and from SQL `NULL`. Declare only the capabilities your adapter supports; the grammar generates @@ -83,30 +84,51 @@ reach counter, so the run fails when histories miss a state the laws need. | Declared state | Histories | | ------------------------------ | --------- | - | first-ready | 230 | - | replacing-snapshot | 74 | - | change-batch | 105 | - | multi-change-batch | 317 | - | failure-before-ready | 132 | - | failure-after-ready | 41 | - | restart | 136 | - | wait-before-evidence-fulfilled | 54 | - | wait-timed-out | 91 | - | wait-rejected-at-retirement | 199 | - | transient-failure-reported | 64 | - | transient-failure-recovered | 31 | - | reconnect | 65 | - | renewal | 154 | + | first-ready | 248 | + | replacing-snapshot | 77 | + | change-batch | 97 | + | multi-change-batch | 327 | + | failure-before-ready | 124 | + | failure-after-ready | 64 | + | restart | 116 | + | wait-before-evidence-fulfilled | 48 | + | wait-timed-out | 75 | + | wait-rejected-at-retirement | 188 | + | transient-failure-reported | 67 | + | transient-failure-recovered | 33 | + | reconnect | 62 | + | renewal | 141 | | change-before-ready | 514 | - | read-missed-change | 116 | - | seen-change-superseded | 31 | - | field-removed | 28 | + | read-missed-change | 101 | + | seen-change-superseded | 26 | + | field-removed | 27 | + | value:string:load | 66 | + | value:string:change | 95 | + | value:integer:load | 85 | + | value:integer:change | 130 | + | value:float:load | 49 | + | value:float:change | 61 | + | value:negative-zero:load | 54 | + | value:negative-zero:change | 103 | + | value:boolean:load | 64 | + | value:boolean:change | 104 | + | value:null:load | 62 | + | value:null:change | 95 | + | value:date:load | 51 | + | value:date:change | 93 | + | value:bigint:load | 74 | + | value:bigint:change | 91 | + | value:array:load | 71 | + | value:array:change | 99 | + | value:object:load | 56 | + | value:object:change | 99 | + | zero-sign-flipped | 74 | Each campaign fails when a state is reached fewer than five times. Across - 20 other seeds, the lowest count for any state was 15 (`field-removed`). + 20 other seeds, the lowest count for any state was 12 (`field-removed`), across 12 seeds. - **GREEN:** `pnpm exec vitest run tests/reference-adapter-oracle.property.test.ts`: - 58 tests pass (both campaigns, the per-change pair, 26 calibrations, and the + 65 tests pass (both campaigns, the per-change pair, 30 calibrations, and the grammar controls). TrailBase: its full suite, 151 tests, passes. - **Mutants:** the fixed campaign rejects each with the law it breaks: @@ -138,6 +160,10 @@ reach counter, so the run fails when histories miss a state the laws need. | `ready-before-buffer` | READY-10 (rows when ready) | | `merges-updates` | FID-1 | | `relabels-inserts` (checker calibration) | PUB-2 | + | `dates-as-strings` (live changes only) | FID-1 (values) | + | `bigints-as-numbers` (initial load only) | FID-1 (values) | + | `drops-nulls` | FID-1 (values) | + | `loses-negative-zero` | FID-1 (values) | | `shows-buffered` | READY-10/PUB-9 | `reverses-batch` reverses only batches of distinct keys, so its final rows @@ -147,10 +173,18 @@ reach counter, so the run fails when histories miss a state the laws need. - **Shrinking keeps the first violation (ORC-010):** `eagerLiveSourceProperty` treats a smaller history that fails a different law as passing. A shrunk counterexample therefore violates the law the campaign found. A cleanup error after a violation is kept with it in an `AggregateError`. - **Grammar controls (ORC-004):** these run in the reference oracle file. - - **Reconstruction:** eleven witnesses from the Neon, TrailBase, and RxDB suites and the retry contract are rebuilt as histories. Each is accepted with no skipped step and passes. + - **Reconstruction:** thirteen witnesses (eleven without values, run on the reference driver without value kinds, and two with values) from the Neon, TrailBase, and RxDB suites and the retry contract are rebuilt as histories. Each is accepted with no skipped step and passes. - **Exclusion:** six nearby invalid histories are each skipped by exactly one step; a batch before the first ready is invalid only without `listen-before-fetch`. - - **Ablation:** removing each step kind loses the declared state that only it reaches, reading after every buffered batch loses `read-missed-change`, and rows without notes lose `field-removed`. + - **Ablation:** removing each step kind loses the declared state that only it reaches, reading after every buffered batch loses `read-missed-change`, rows without notes lose `field-removed`, and live changes without values or retouches lose `value:date:change` while `value:date:load` stays reached. - **Replay:** during a path replay, fixed campaigns and calibrations stand aside (`oracleReplayTest`). +- **Core finding (typed values):** a random campaign (seed 95028) found + that a synced update changing only `-0` to `0` published no change + event: the Collection held `0` while every subscriber kept `-0` (PUB-2). + Core compared the rows with `deepEquals`, which treats `-0` as `0`. By + maintainer decision (2026-10-29), `-0` and `0` are distinct row values; + the synced commit path now compares with `sameRowValue`. A fix that tells + `-0` from `0` only at the top of a value survives without the `retouch` + dimension and is rejected with it. - **RED (Neon):** `pnpm test:neon`. Both campaigns fail. The fixed campaign shrinks to `[fail, wait]`: READY-4, status `loading` where `error` is owed. Random seeds find READY-4 or, after ready, HEALTH-1: `health.error` is @@ -172,8 +206,12 @@ reach counter, so the run fails when histories miss a state the laws need. timed wait settles within its own step, so no evidence arrives during it. An adapter that rejects a timed wait at once, before its timeout, is not distinguished from one that waits. - - FID-1 covers one optional string field. Value domains (Date, bigint, - nested objects) are not generated. + - FID-1 generates ten value kinds (string, integer, float, `-0`, boolean, + `null`, Date, bigint, array, plain object). Temporal values, Map, Set, + typed arrays, and class instances are not generated, and a stored + field's reference identity is not checked. + - The rejection side, FID-3 and PUB-4 (an out-of-domain value fails its + delivery and writes no row), has no lane yet. - READY-8's fetch order is checked only by a provider that implements `fetchesBeforeListening`. Commit receipts resolve at once in every driver, so a live commit whose receipt is still pending at the cut diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 6cb6be11b8..49060f89fd 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -29,7 +29,13 @@ * - **FID-1.** A delivered row is the complete backend row: a row without * the optional `note` has no `note` in the Collection, also when the row * it replaces had one. Fields compare by own property, so an own - * `undefined` differs from an absent field. + * `undefined` differs from an absent field. With declared `valueKinds`, + * each row's `value` strictly equals the delivered value, on the initial + * load and on a live change: same type and prototype, `Object.is` for + * numbers, Dates by time, and arrays and plain objects element by element + * with no extra own keys. The law set parameterizes FID-1 by each + * adapter's `valueDomain`; a driver declares the kinds of it this oracle + * generates. * - **PUB-7** (capability `complete-snapshot`). A later complete delivery * replaces the rows: keys it omits are gone. * - **LIFE-2, LIFE-5.** Cleanup releases the provider subscription. A restart @@ -105,8 +111,36 @@ export interface ConformanceRow { readonly id: number readonly title: string readonly note?: string + /** A value of a declared kind (FID-1); present only with `valueKinds`. */ + readonly value?: unknown } +/** + * Value kinds from FID-1's `valueDomain`. `jsonValueKinds` are those a JSON + * backend represents: `-0`, Dates, and bigints are not among them. + */ +export type ValueKind = + | `string` + | `integer` + | `float` + | `negative-zero` + | `boolean` + | `null` + | `date` + | `bigint` + | `array` + | `object` + +export const jsonValueKinds: ReadonlyArray = [ + `string`, + `integer`, + `float`, + `boolean`, + `null`, + `array`, + `object`, +] + /** One change in a provider's committed batch. */ export type ProviderChange = | { readonly type: `upsert`; readonly row: ConformanceRow } @@ -203,6 +237,12 @@ export interface EagerLiveSourceDriver { * change on its own. */ readonly changeCommits: `per-batch` | `per-change` + /** + * The value kinds the backend represents (FID-1). Rows then carry a + * `value` of one of these kinds; the driver delivers it in the backend's + * own form, and the Collection must show it decoded. + */ + readonly valueKinds?: ReadonlyArray /** * Console diagnostics the adapter declares (HEALTH-5). Any other * `console.error` or `console.warn` call is a violation. @@ -257,7 +297,7 @@ export type EagerLiveSourceStep = } | { readonly kind: `changes` - readonly changes: ReadonlyArray + readonly changes: ReadonlyArray } | { readonly kind: `wait` @@ -271,26 +311,104 @@ export type EagerLiveSourceStep = | { readonly kind: `renew` } | { readonly kind: `restart` } +/** + * A change in a generated batch. `retouch` re-delivers a row the source + * holds: the held key at position `id`, counted modulo the held keys in key + * order, so it rarely misses. With `negative-zero` in the value kinds, every zero in its + * value changes sign, so the row differs only by -0 versus 0. Without it, + * the re-delivery is equal. The model resolves it to an upsert. + */ +export type HistoryChange = + ProviderChange | { readonly type: `retouch`; readonly id: number } + export type EagerLiveSourceHistory = ReadonlyArray -const rowArb: fc.Arbitrary = fc.record( - { +/** + * Edge cases for each value kind, as factories so no two rows share an + * object. They cover FID-1's named cases: explicit falsy values, embedded + * newlines, integers past 2^31 and at 2^53 - 1, bigints beyond 2^53, Dates + * with milliseconds and before the epoch, and nested objects and arrays. + */ +const valueExamples: Record unknown>> = { + string: [() => ``, () => `plain`, () => `two\nlines`, () => `ünïcødé ✓`], + integer: [() => 0, () => -7, () => 2 ** 31, () => Number.MAX_SAFE_INTEGER], + float: [() => 0.5, () => -1.25, () => 1e-7, () => Number.MAX_VALUE], + // Nested too, so an equality that tells -0 from 0 only at the root fails. + 'negative-zero': [() => -0, () => [-0], () => ({ z: -0 })], + boolean: [() => true, () => false], + null: [() => null], + date: [ + () => new Date(0), + () => new Date(Date.UTC(2026, 9, 29, 12, 34, 56, 789)), + () => new Date(-86_400_000), + ], + bigint: [() => 9_007_199_254_740_993n, () => -(2n ** 63n), () => 0n], + // Zeros inside containers give a retouch a sign to flip. + array: [() => [], () => [0, `a`, null], () => [[1], [true]]], + object: [ + () => ({}), + () => ({ a: 0, b: `x` }), + () => ({ nested: { deep: [1, { k: null }] } }), + () => ({ '': false }), + ], +} + +function valueArb(kinds: ReadonlyArray): fc.Arbitrary { + return fc.oneof( + ...kinds.map((kind) => + fc + .integer({ min: 0, max: valueExamples[kind].length - 1 }) + .map((index) => valueExamples[kind][index]!()), + ), + ) +} + +function rowArb(kinds: ReadonlyArray): fc.Arbitrary { + const fields = { id: fc.integer({ min: 1, max: 5 }), title: fc.constantFrom(`a`, `b`, `c`), note: fc.constantFrom(`x`, `y`), - }, - { requiredKeys: [`id`, `title`] }, -) + } + if (kinds.length === 0) { + return fc.record(fields, { requiredKeys: [`id`, `title`] }) + } + return fc.record( + { ...fields, value: valueArb(kinds) }, + { requiredKeys: [`id`, `title`, `value`] }, + ) +} + +function changeArb( + kinds: ReadonlyArray, +): fc.Arbitrary { + const changes: Array> = [ + rowArb(kinds).map((row) => ({ type: `upsert` as const, row })), + fc + .integer({ min: 1, max: 5 }) + .map((id) => ({ type: `delete` as const, id })), + ] + if (kinds.length > 0) { + changes.push( + fc + .integer({ min: 1, max: 5 }) + .map((id) => ({ type: `retouch` as const, id })), + ) + } + return fc.oneof(...changes) +} -const changeArb: fc.Arbitrary = fc.oneof( - rowArb.map((row) => ({ type: `upsert` as const, row })), - fc.integer({ min: 1, max: 5 }).map((id) => ({ type: `delete` as const, id })), -) +/** What the grammar needs from a driver: its capabilities and value kinds. */ +export type GrammarDomain = Pick< + EagerLiveSourceDriver, + `capabilities` | `valueKinds` +> -/** Generates histories whose steps use only `capabilities`. */ +/** Generates histories whose steps use only the driver's declarations. */ export function eagerLiveSourceHistoryArb( - capabilities: ReadonlySet, + domain: GrammarDomain, ): fc.Arbitrary { + const { capabilities } = domain + const kinds = domain.valueKinds ?? [] const steps: Array<{ weight: number arbitrary: fc.Arbitrary @@ -299,7 +417,7 @@ export function eagerLiveSourceHistoryArb( weight: 4, arbitrary: fc .record({ - rows: fc.uniqueArray(rowArb, { + rows: fc.uniqueArray(rowArb(kinds), { selector: (row) => row.id, maxLength: 4, }), @@ -314,7 +432,7 @@ export function eagerLiveSourceHistoryArb( { weight: 6, arbitrary: fc - .array(changeArb, { maxLength: 3 }) + .array(changeArb(kinds), { maxLength: 3 }) .map((changes) => ({ kind: `changes` as const, changes })), }, { weight: 2, arbitrary: fc.constant({ kind: `fail` as const }) }, @@ -363,7 +481,7 @@ export function eagerLiveSourceHistoryArb( // outstanding. Without it, two buffered batches that touch one key before // the read point are too rare for PUB-5 to be checked. const startup = fc - .array(changeArb, { minLength: 1, maxLength: 3 }) + .array(changeArb(kinds), { minLength: 1, maxLength: 3 }) .map((changes) => ({ kind: `changes` as const, changes })) return fc .tuple(fc.array(startup, { maxLength: 3 }), history) @@ -439,22 +557,108 @@ export interface ExpectedObservation { } /** - * A row's value as the model compares it: its title, then its note when the - * row has its own `note` property. An own `undefined` shows as `undefined`. + * FID-1's strict equality as a canonical description, written here rather + * than taken from any codec or equality helper in production. Two values + * describe alike exactly when they have the same type and prototype, equal + * numbers under `Object.is`, equal Date times, and the same elements or own + * enumerable keys, whatever their key order. A value no kind produces, such + * as a class instance or an own `undefined`, describes as itself so that it + * differs from every generated value. */ -function rowValue(row: { readonly title: string; readonly note?: unknown }) { - return Object.hasOwn(row, `note`) +export function describeValue(value: unknown): string { + switch (typeof value) { + case `string`: + return `s${JSON.stringify(value)}` + case `number`: + return Object.is(value, -0) ? `n-0` : `n${String(value)}` + case `boolean`: + return `b${String(value)}` + case `bigint`: + return `i${String(value)}` + case `undefined`: + return `undefined` + case `object`: { + if (value === null) return `null` + const prototype = Object.getPrototypeOf(value) as unknown + if (value instanceof Date && prototype === Date.prototype) { + const time = value.getTime() + return Number.isNaN(time) ? `d?` : `d${time}` + } + if (Array.isArray(value) && prototype === Array.prototype) { + return `[${value.map(describeValue).join(`,`)}]` + } + if (prototype === Object.prototype || prototype === null) { + const fields = Object.keys(value) + .sort() + .map( + (key) => + `${JSON.stringify(key)}:${describeValue((value as Record)[key])}`, + ) + return `{${fields.join(`,`)}}` + } + return `?${Object.prototype.toString.call(value)}` + } + default: + return `?${typeof value}` + } +} + +/** The kind of a generated value, for reach. */ +function kindOf(value: unknown): ValueKind { + if (value === null) return `null` + if (describeValue(value).includes(`n-0`)) return `negative-zero` + if (typeof value === `number`) { + return Number.isInteger(value) ? `integer` : `float` + } + if (value instanceof Date) return `date` + if (Array.isArray(value)) return `array` + return typeof value === `object` ? `object` : (typeof value as ValueKind) +} + +type RowFields = Omit + +/** + * A row as the model compares it: its title, then `/note` when the row has + * its own `note`, then `#` and the described `value` when it has its own + * `value`. An own `undefined` note shows as `undefined`. + */ +function rowValue(row: RowFields): string { + const head = Object.hasOwn(row, `note`) ? `${row.title}/${String(row.note)}` : row.title + return Object.hasOwn(row, `value`) + ? `${head}#${describeValue(row.value)}` + : head +} + +/** Title and note, without the value. */ +function rowHead(value: string): string { + const mark = value.indexOf(`#`) + return mark === -1 ? value : value.slice(0, mark) +} + +/** + * Rows the model generated, by description, so that a computed state can be + * delivered as rows again (the initial cut's read). Only generated rows are + * remembered, never observed ones. + */ +const generatedRows = new Map() + +function modelValue(row: ConformanceRow): string { + const { id: _id, ...fields } = row + const value = rowValue(fields) + generatedRows.set(value, fields) + return value } function rowFromValue(id: number, value: string): ConformanceRow { - const [title = ``, note] = value.split(`/`) - return note === undefined ? { id, title } : { id, title, note } + const fields = generatedRows.get(value) + if (!fields) throw new Error(`the model lost row ${value}`) + return { id, ...fields } } function rowMap(rows: ReadonlyArray): Map { - return new Map(rows.map((row) => [row.id, rowValue(row)])) + return new Map(rows.map((row) => [row.id, modelValue(row)])) } /** The rows a Collection holds, as the model compares them. */ @@ -502,29 +706,66 @@ function rowState(rows: ReadonlyMap): RowState { return [...rows].sort(([a], [b]) => a - b) } +/** A value with the sign of every zero in it changed, nested ones too. */ +function flipZeros(value: unknown): unknown { + if (typeof value === `number`) { + return value === 0 ? (Object.is(value, -0) ? 0 : -0) : value + } + if (Array.isArray(value)) return value.map(flipZeros) + if ( + typeof value === `object` && + value !== null && + Object.getPrototypeOf(value) === Object.prototype + ) { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + key, + flipZeros(item), + ]), + ) + } + return value +} + /** * Folds one batch onto `rows` in order and returns the changes a provider - * delivers: a delete of a key it does not hold is dropped. `onChange` sees - * the rows after each delivered change. + * delivers: a delete or retouch of a key it does not hold is dropped, and a + * retouch delivers the held row again, its zeros flipped when `flipZeros`. + * `onChange` sees the rows after each delivered change; `flipped` counts + * retouches that changed a value. */ function foldBatch( rows: Map, - changes: ReadonlyArray, - onChange: () => void = () => {}, -): Array { + changes: ReadonlyArray, + options: { onChange?: () => void; flipZeros?: boolean } = {}, +): { delivered: Array; flipped: number } { const delivered: Array = [] + let flipped = 0 for (const change of changes) { if (change.type === `upsert`) { - rows.set(change.row.id, rowValue(change.row)) - } else if (rows.has(change.id)) { + rows.set(change.row.id, modelValue(change.row)) + delivered.push(change) + } else if (change.type === `delete`) { + if (!rows.has(change.id)) continue rows.delete(change.id) + delivered.push(change) } else { - continue + const keys = [...rows.keys()].sort((a, b) => a - b) + if (keys.length === 0) continue + const id = keys[change.id % keys.length]! + const held = rowFromValue(id, rows.get(id)!) + const row = + options.flipZeros && Object.hasOwn(held, `value`) + ? { ...held, value: flipZeros(held.value) } + : held + const value = modelValue(row) + if (value !== rows.get(id)) flipped++ + rows.set(id, value) + delivered.push({ type: `upsert`, row }) } - delivered.push(change) - onChange() + options.onChange?.() } - return delivered + return { delivered, flipped } } /** Whether a key held in both lost the `note` it had in `before`. */ @@ -534,7 +775,9 @@ function lostField( ): boolean { return [...before].some( ([id, value]) => - value.includes(`/`) && after.get(id)?.includes(`/`) === false, + rowHead(value).includes(`/`) && + after.has(id) && + !rowHead(after.get(id)!).includes(`/`), ) } @@ -569,6 +812,8 @@ type AcceptedStep = beforeReady: boolean /** Whether an upsert removed a field the row it replaced had. */ removesField: boolean + /** Whether a retouch changed a value only by the sign of a zero. */ + flipped: boolean } | { kind: `wait`; txid: number; timeout: boolean } | { kind: `fail`; failure: number } @@ -587,6 +832,8 @@ type AcceptedStep = interface ModelOptions { changeCommits: EagerLiveSourceDriver[`changeCommits`] listenBeforeFetch: boolean + /** The value kinds include `negative-zero`, so a retouch flips zeros. */ + flipZeros: boolean } /** EVID-1: a wait for `txid` fulfills once its batch applied. */ @@ -626,7 +873,9 @@ function initialCut( publishable.push(read) } const prefixes: Array = [] - foldBatch(backend, batch.changes, () => prefixes.push(rowState(backend))) + foldBatch(backend, batch.changes, { + onChange: () => prefixes.push(rowState(backend)), + }) if (index < seen) return if (changeCommits === `per-change`) publishable.push(...prefixes) else publishable.push(rowState(backend)) @@ -698,7 +947,9 @@ function stepModel( if (state.phase === `loading` && options.listenBeforeFetch) { // READY-8: the listener is live, so a batch committed during the // fetch reaches the adapter. It shows nothing until the cut. - const delivered = foldBatch(state.backend, step.changes) + const { delivered, flipped } = foldBatch(state.backend, step.changes, { + flipZeros: options.flipZeros, + }) state.lastTxid += 1 state.buffered.push({ changes: delivered, txid: state.lastTxid }) return { @@ -708,6 +959,7 @@ function stepModel( txid: state.lastTxid, beforeReady: true, removesField: false, + flipped: flipped > 0, }, publishable: [], } @@ -716,9 +968,10 @@ function stepModel( // PUB-3: fold the batch in order. PUB-1: each prefix is a committed // state when the provider commits each change alone. const prefixes: Array = [] - const delivered = foldBatch(state.rows, step.changes, () => - prefixes.push(rowState(state.rows)), - ) + const { delivered, flipped } = foldBatch(state.rows, step.changes, { + onChange: () => prefixes.push(rowState(state.rows)), + flipZeros: options.flipZeros, + }) // Reach: an upsert without `note` replaced a row that had one, before // or within this batch. const held = new Map(state.backend) @@ -730,11 +983,11 @@ function stepModel( } if ( !Object.hasOwn(change.row, `note`) && - held.get(change.row.id)?.includes(`/`) === true + rowHead(held.get(change.row.id) ?? ``).includes(`/`) ) { removesField = true } - held.set(change.row.id, rowValue(change.row)) + held.set(change.row.id, modelValue(change.row)) } foldBatch(state.backend, delivered) state.lastTxid += 1 @@ -747,6 +1000,7 @@ function stepModel( txid: state.lastTxid, beforeReady: false, removesField, + flipped: flipped > 0, }, publishable: options.changeCommits === `per-change` @@ -888,11 +1142,19 @@ export const eagerLiveSourceReachStates = [ `field-removed`, ] as const +/** + * Per value kind, a value delivered on the initial load and one delivered by + * a live change: a decoder can differ between the two paths. + */ +export type ValueReachState = `value:${ValueKind}:${`load` | `change`}` + export type EagerLiveSourceReachState = - (typeof eagerLiveSourceReachStates)[number] + | (typeof eagerLiveSourceReachStates)[number] + | ValueReachState + | `zero-sign-flipped` const capabilityOfState: Partial< - Record + Record<(typeof eagerLiveSourceReachStates)[number], EagerLiveSourceCapability> > = { 'replacing-snapshot': `complete-snapshot`, 'wait-before-evidence-fulfilled': `await-evidence`, @@ -907,14 +1169,38 @@ const capabilityOfState: Partial< 'seen-change-superseded': `listen-before-fetch`, } -/** The declared states a driver's capabilities let histories reach. */ +/** The declared states a driver's declarations let histories reach. */ export function reachStatesFor( - capabilities: ReadonlySet, + domain: GrammarDomain, ): Array { - return eagerLiveSourceReachStates.filter((state) => { - const capability = capabilityOfState[state] - return capability === undefined || capabilities.has(capability) - }) + const fixed: Array = + eagerLiveSourceReachStates.filter((state) => { + const capability = capabilityOfState[state] + return capability === undefined || domain.capabilities.has(capability) + }) + const values = (domain.valueKinds ?? []).flatMap( + (kind): Array => [ + `value:${kind}:load`, + `value:${kind}:change`, + ], + ) + const zeros: Array = domain.valueKinds?.includes( + `negative-zero`, + ) + ? [`zero-sign-flipped`] + : [] + return [...fixed, ...values, ...zeros] +} + +/** JSON for a message; a bigint shows as `n`, `-0` as `-0`. */ +function show(value: unknown): string { + return JSON.stringify(value, (_key, item: unknown) => + typeof item === `bigint` + ? `${String(item)}n` + : Object.is(item, -0) + ? `-0` + : item, + ) } /** A law violation at one checkpoint. */ @@ -927,8 +1213,8 @@ export class ConformanceViolation extends Error { readonly history: EagerLiveSourceHistory, ) { super( - `${law} violated at step ${step} of ${JSON.stringify(history)}: ` + - `expected ${JSON.stringify(expected)}, observed ${JSON.stringify(actual)}`, + `${law} violated at step ${step} of ${show(history)}: ` + + `expected ${show(expected)}, observed ${show(actual)}`, ) this.name = `ConformanceViolation` } @@ -968,7 +1254,11 @@ const timeoutSettleMs = 30 function lawFor(done: AcceptedStep, wasReady: boolean): string { switch (done.kind) { case `snapshot`: - return wasReady ? `PUB-7` : done.buffered > 0 ? `READY-10` : `READY-3` + return wasReady + ? `PUB-7` + : done.buffered > 0 + ? `READY-10/PUB-9` + : `READY-3` case `changes`: return done.beforeReady ? `READY-10/PUB-9` : `PUB-3/FID-4` case `wait`: @@ -1037,6 +1327,7 @@ export async function checkEagerLiveSourceHistory( const modelOptions: ModelOptions = { changeCommits: driver.changeCommits, listenBeforeFetch, + flipZeros: driver.valueKinds?.includes(`negative-zero`) === true, } // With `listen-before-fetch`, a sync run's backend starts with the rows of // its first complete delivery, committed before the adapter subscribes. @@ -1055,7 +1346,7 @@ export async function checkEagerLiveSourceHistory( const failures: Array = [] const failureCounter = { next: 0 } const waits: Array<{ current: Settlement }> = [] - if (listenBeforeFetch) provider.seed?.(seedFrom(0)) + if (listenBeforeFetch) provider.seed?.(structuredClone(seedFrom(0))) // READY-3, READY-10: the rows at the moment status becomes `ready`. let readyRows: RowState | undefined const watchReady = () => @@ -1151,16 +1442,28 @@ export async function checkEagerLiveSourceHistory( // FID-1: rows that differ only in a field, with the right keys and // titles, name the field law. if (!sameRows(rows, expected.rows) && sameTitles(rows, expected.rows)) { - throw violation(`FID-1 (fields)`, stepIndex, expected.rows, rows) + const valuesOnly = rows.every( + ([, value], index) => + rowHead(value) === rowHead(expected.rows[index]![1]), + ) + throw violation( + valuesOnly ? `FID-1 (values)` : `FID-1 (fields)`, + stepIndex, + expected.rows, + rows, + ) } check(`rows`, sameRows(rows, expected.rows), expected.rows, rows) if (readyRows !== undefined) { - check( - `rows when ready`, - sameRows(readyRows, expected.rows), - expected.rows, - readyRows, - ) + // READY-10 when the cut folded buffered batches; READY-3 otherwise. + if (!sameRows(readyRows, expected.rows)) { + throw violation( + `${law.includes(`READY-10`) ? `READY-10` : `READY-3`} (rows when ready)`, + stepIndex, + expected.rows, + readyRows, + ) + } readyRows = undefined } if (!sameRows(events.rows(), rows)) { @@ -1296,7 +1599,12 @@ export async function checkEagerLiveSourceHistory( const { done, publishable } = result switch (done.kind) { case `snapshot`: - provider.deliverSnapshot(done.rows) + provider.deliverSnapshot(structuredClone(done.rows)) + for (const row of done.rows) { + if (Object.hasOwn(row, `value`)) { + reach(`value:${kindOf(row.value)}:load`) + } + } reach(wasReady ? `replacing-snapshot` : `first-ready`) if (done.missed > 0) reach(`read-missed-change`) if (done.superseded) reach(`seen-change-superseded`) @@ -1304,9 +1612,21 @@ export async function checkEagerLiveSourceHistory( if (transientBefore > 0) reach(`transient-failure-recovered`) break case `changes`: - provider.deliverChanges(done.changes, String(done.txid)) + provider.deliverChanges( + structuredClone(done.changes), + String(done.txid), + ) + for (const change of done.changes) { + if ( + change.type === `upsert` && + Object.hasOwn(change.row, `value`) + ) { + reach(`value:${kindOf(change.row.value)}:change`) + } + } reach(done.beforeReady ? `change-before-ready` : `change-batch`) if (done.removesField) reach(`field-removed`) + if (done.flipped) reach(`zero-sign-flipped`) if (done.changes.length > 1) reach(`multi-change-batch`) if (transientBefore > 0) reach(`transient-failure-recovered`) if ( @@ -1367,7 +1687,9 @@ export async function checkEagerLiveSourceHistory( events.stop() await collection.cleanup() // The backend may change while no sync run listens. - if (listenBeforeFetch) provider.seed?.(seedFrom(index + 1)) + if (listenBeforeFetch) { + provider.seed?.(structuredClone(seedFrom(index + 1))) + } readyRows = undefined watchReady() preload = recordSettlement(collection.preload()) @@ -1510,7 +1832,7 @@ function supports( /** Same keys and titles, whatever the other fields. */ function sameTitles(a: RowState, b: RowState): boolean { - const title = (value: string) => value.split(`/`)[0] + const title = (value: string) => rowHead(value).split(`/`)[0] return ( a.length === b.length && a.every( @@ -1591,7 +1913,7 @@ export function eagerLiveSourceProperty( ) { let target: string | undefined return fc.asyncProperty( - eagerLiveSourceHistoryArb(driver.capabilities), + eagerLiveSourceHistoryArb(driver), async (history) => { reach?.beginHistory() try { diff --git a/packages/db-adapter-conformance/src/index.ts b/packages/db-adapter-conformance/src/index.ts index 5ef40bcaaa..1d1b6a2c74 100644 --- a/packages/db-adapter-conformance/src/index.ts +++ b/packages/db-adapter-conformance/src/index.ts @@ -1,10 +1,12 @@ export { checkEagerLiveSourceHistory, + describeValue, ConformanceViolation, eagerLiveSourceHistoryArb, eagerLiveSourceModel, eagerLiveSourceProperty, eagerLiveSourceReachStates, + jsonValueKinds, reachStatesFor, } from './eager-live-source-oracle' export type { @@ -18,9 +20,12 @@ export type { EagerLiveSourceReachState, EagerLiveSourceStep, ExpectedObservation, + GrammarDomain, ObservedCollection, ProviderChange, ReachCounter, + ValueKind, + ValueReachState, } from './eager-live-source-oracle' export { conformanceKey, diff --git a/packages/db-adapter-conformance/src/observed-view.ts b/packages/db-adapter-conformance/src/observed-view.ts index 54412e9912..6b6e046e6e 100644 --- a/packages/db-adapter-conformance/src/observed-view.ts +++ b/packages/db-adapter-conformance/src/observed-view.ts @@ -78,7 +78,13 @@ export function conformanceKey(key: unknown): number { /** A conformance row as a SQL row: an absent `note` is `NULL`. */ export function toSqlRow(row: ConformanceRow) { - return { id: row.id, title: row.title, note: row.note ?? null } + return { + id: row.id, + title: row.title, + note: row.note ?? null, + // A JSON column; present only when the driver declares value kinds. + ...(Object.hasOwn(row, `value`) ? { value: row.value } : {}), + } } /** diff --git a/packages/db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts b/packages/db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts index 4b234a5802..343ec1422e 100644 --- a/packages/db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/neon-realtime-oracle.property.test.ts @@ -35,7 +35,7 @@ import { neonRealtimeDriver } from './neon/neon-realtime-driver' const property = `adapter-conformance.eager-live-source.neon-realtime` const driver = neonRealtimeDriver() -const historyArb = eagerLiveSourceHistoryArb(driver.capabilities) +const historyArb = eagerLiveSourceHistoryArb(driver) // `field-removed` needs a held row with a note and a later upsert without // one; at 400 histories every seed tried reached it at least 11 times. const runs = 400 @@ -43,13 +43,9 @@ const runs = 400 const campaignTimeoutMs = 120_000 async function campaign(options: fc.Parameters<[unknown]>) { - const reach = createOracleReach( - property, - reachStatesFor(driver.capabilities), - { - minimum: 5, - }, - ) + const reach = createOracleReach(property, reachStatesFor(driver), { + minimum: 5, + }) await fc.assert(eagerLiveSourceProperty(driver, reach), options) return reach } diff --git a/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts b/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts index b3fcf47f50..68adc905d2 100644 --- a/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts +++ b/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts @@ -26,7 +26,7 @@ * judges the Collection's public state after such an exception. */ import { createCollection } from '@tanstack/db' -import { fromSqlRow, observedView, toSqlRow } from '../../src' +import { fromSqlRow, jsonValueKinds, observedView, toSqlRow } from '../../src' import { realtimeCollectionOptions } from './vendor/collection-options' import type { LiveQueryChange, @@ -255,6 +255,7 @@ export function neonRealtimeDriver(): EagerLiveSourceDriver { return { adapter: `@neon/realtime-tanstack 0.1.1`, changeCommits: `per-batch`, + valueKinds: jsonValueKinds, capabilities: new Set([ `complete-snapshot`, `await-evidence`, diff --git a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts index 7968b3cad5..b1fcd4ba5b 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts @@ -17,9 +17,11 @@ import { } from '../src' import { referenceDriver } from './reference-adapter' import type { + ConformanceRow, EagerLiveSourceHistory, EagerLiveSourceReachState, EagerLiveSourceStep, + GrammarDomain, } from '../src' import type { ReferenceMutant } from './reference-adapter' @@ -51,7 +53,7 @@ import type { ReferenceMutant } from './reference-adapter' const property = `adapter-conformance.eager-live-source.reference` const driver = referenceDriver() -const historyArb = eagerLiveSourceHistoryArb(driver.capabilities) +const historyArb = eagerLiveSourceHistoryArb(driver) // The reference adapter declares every capability, so its grammar spreads // over the most step kinds. Across 20 seeds, 600 histories reached every // declared state at least 15 times. @@ -60,13 +62,9 @@ const runs = 600 const campaignTimeoutMs = 120_000 async function campaign(options: fc.Parameters<[unknown]>) { - const reach = createOracleReach( - property, - reachStatesFor(driver.capabilities), - { - minimum: 5, - }, - ) + const reach = createOracleReach(property, reachStatesFor(driver), { + minimum: 5, + }) await fc.assert(eagerLiveSourceProperty(driver, reach), options) return reach } @@ -112,11 +110,15 @@ describe(`eager live-source conformance: reference adapter`, () => { [`disconnect-fails`, `RECON-1`], [`renew-resubscribes`, `RENEW-1`], [`drops-buffered`, `READY-10`], - [`buffer-before-rows`, `READY-10`], + [`buffer-before-rows`, `PUB-9`], [`replays-per-batch`, `PUB-5`], [`ready-before-buffer`, `READY-10 (rows when ready)`], [`shows-buffered`, `READY-10/PUB-9`], [`merges-updates`, `FID-1`], + [`dates-as-strings`, `FID-1 (values)`], + [`bigints-as-numbers`, `FID-1 (values)`], + [`drops-nulls`, `FID-1 (values)`], + [`loses-negative-zero`, `FID-1 (values)`], [`relabels-inserts`, `PUB-2`], ] oracleReplayTest(it).each(mutants)( @@ -303,12 +305,18 @@ const witnesses: Array<[string, EagerLiveSourceHistory]> = [ ] /** Whether `history` lies in the generator's domain. */ -function withinGrammar(history: EagerLiveSourceHistory): boolean { - const row = (r: { id: number; title: string; note?: string }) => +function withinGrammar( + history: EagerLiveSourceHistory, + domain: GrammarDomain, +): boolean { + // With value kinds every row carries a value; without them none does. + const valued = (domain.valueKinds ?? []).length > 0 + const row = (r: ConformanceRow) => r.id >= 1 && r.id <= 5 && [`a`, `b`, `c`].includes(r.title) && - (!Object.hasOwn(r, `note`) || [`x`, `y`].includes(r.note!)) + (!Object.hasOwn(r, `note`) || [`x`, `y`].includes(r.note!)) && + Object.hasOwn(r, `value`) === valued return ( history.length >= 1 && history.length <= 15 && @@ -339,11 +347,73 @@ function withinGrammar(history: EagerLiveSourceHistory): boolean { ) } +// The witnesses above carry no values, so they reconstruct in the grammar +// of the same driver without value kinds. Value witnesses use the full one. +const withoutValues = { ...driver, valueKinds: undefined } + +const valueWitnesses: Array<[string, EagerLiveSourceHistory]> = [ + [ + // A random campaign (seed 95028) found this: a synced update that only + // changes -0 to 0 published no change event, so the event fold kept -0 + // while the Collection held 0. Maintainer decision (2026-10-29): -0 and + // 0 are distinct row values. + `FID-1, PUB-2: an update from -0 to 0 publishes a change`, + [ + { + kind: `snapshot`, + rows: [{ id: 5, title: `a`, value: -0 }], + seen: 0, + }, + { + kind: `changes`, + changes: [{ type: `upsert`, row: { id: 5, title: `a`, value: 0 } }], + }, + ], + ], + [ + // FID-1's named cases: a Date and a bigint beyond 2^53 on the initial + // load, then an explicit falsy value and a nested object in a change. + `FID-1: typed values on the load and in a change`, + [ + { + kind: `snapshot`, + rows: [ + { id: 1, title: `a`, value: new Date(Date.UTC(2026, 9, 29)) }, + { id: 2, title: `a`, value: 9_007_199_254_740_993n }, + ], + seen: 0, + }, + { + kind: `changes`, + changes: [ + { type: `upsert`, row: { id: 1, title: `a`, value: false } }, + { + type: `upsert`, + row: { id: 3, title: `b`, value: { nested: { deep: [1] } } }, + }, + ], + }, + ], + ], +] + describe(`eager live-source grammar controls`, () => { oracleReplayTest(it).each(witnesses)( `reconstructs %s`, async (_name, history) => { - expect(withinGrammar(history)).toBe(true) + expect(withinGrammar(history, withoutValues)).toBe(true) + const { skipped } = await checkEagerLiveSourceHistory( + withoutValues, + history, + ) + expect(skipped).toBe(0) + }, + ) + + oracleReplayTest(it).each(valueWitnesses)( + `reconstructs %s`, + async (_name, history) => { + expect(withinGrammar(history, driver)).toBe(true) const { skipped } = await checkEagerLiveSourceHistory(driver, history) expect(skipped).toBe(0) }, @@ -393,7 +463,10 @@ describe(`eager live-source grammar controls`, () => { ], ] oracleReplayTest(it).each(invalid)(`excludes %s`, async (_name, history) => { - const { skipped } = await checkEagerLiveSourceHistory(driver, history) + const { skipped } = await checkEagerLiveSourceHistory( + withoutValues, + history, + ) expect(skipped).toBe(1) }) @@ -416,7 +489,7 @@ describe(`eager live-source grammar controls`, () => { oracleReplayTest(it).each(ablations)( `without %s steps, histories never reach %s`, async (kind, lost) => { - const states = reachStatesFor(driver.capabilities) + const states = reachStatesFor(driver) const reach = createOracleReach(`${property}.ablation`, states, { replaying: false, }) @@ -444,7 +517,7 @@ describe(`eager live-source grammar controls`, () => { async () => { const reach = createOracleReach( `${property}.ablation`, - reachStatesFor(driver.capabilities), + reachStatesFor(driver), { replaying: false }, ) const plain = (row: { id: number; title: string }) => ({ @@ -483,6 +556,51 @@ describe(`eager live-source grammar controls`, () => { campaignTimeoutMs, ) + // Path ablation: when live changes carry no value and no retouch, no + // history reaches a + // value kind on the change path, so the load and change states each need + // their own deliveries. + oracleReplayTest(it)( + `without values in live changes, histories never reach value:date:change`, + async () => { + const reach = createOracleReach( + `${property}.ablation`, + reachStatesFor(driver), + { replaying: false }, + ) + await fc.assert( + fc.asyncProperty( + historyArb.map((history) => + history.map((step): EagerLiveSourceStep => { + if (step.kind !== `changes`) return step + return { + kind: `changes`, + // A retouch re-delivers a held value, so it goes too. + changes: step.changes + .filter((change) => change.type !== `retouch`) + .map((change) => { + if (change.type !== `upsert`) return change + const { value: _value, ...row } = change.row + return { type: `upsert`, row } + }), + } + }), + ), + async (history) => { + reach.beginHistory() + await checkEagerLiveSourceHistory(driver, history, { + reach: reach.reach, + }) + }, + ), + { seed: 2118, numRuns: 100 }, + ) + expect(reach.counts().get(`value:date:change`)).toBe(0) + expect(reach.counts().get(`value:date:load`)).toBeGreaterThan(0) + }, + campaignTimeoutMs, + ) + // Ablation of the read point: when every fetch reads after all buffered // batches, no history reaches a cut whose result missed one. oracleReplayTest(it)( @@ -490,7 +608,7 @@ describe(`eager live-source grammar controls`, () => { async () => { const reach = createOracleReach( `${property}.ablation`, - reachStatesFor(driver.capabilities), + reachStatesFor(driver), { replaying: false }, ) await fc.assert( diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index 3bfcc2a6e7..c7fb4e1139 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -75,6 +75,14 @@ export type ReferenceMutant = | `shows-buffered` /** FID-1: an update merges over the held row, so a removed field stays. */ | `merges-updates` + /** FID-1: a live change stores a Date as its ISO string. */ + | `dates-as-strings` + /** FID-1: the initial load stores a bigint as a number. */ + | `bigints-as-numbers` + /** FID-1: a `null` value is stored as an absent field. */ + | `drops-nulls` + /** FID-1: `-0` is stored as `0`. */ + | `loses-negative-zero` /** * PUB-2, checker calibration: the observed view relabels insert events as * updates. The adapter is unchanged; the comparison must see the events. @@ -162,6 +170,40 @@ class ControlledPushProvider implements ControlledProvider { type Waiter = { resolve: () => void; reject: (error: Error) => void } +/** + * The row the adapter stores. A conforming adapter stores the delivered row + * itself; each value mutant changes one kind on one path. + */ +function storedRow( + row: ConformanceRow, + path: `load` | `change`, + mutant: ReferenceMutant | undefined, +): ConformanceRow { + const { value } = row + if ( + mutant === `dates-as-strings` && + path === `change` && + value instanceof Date + ) { + return { ...row, value: value.toISOString() } + } + if ( + mutant === `bigints-as-numbers` && + path === `load` && + typeof value === `bigint` + ) { + return { ...row, value: Number(value) } + } + if (mutant === `drops-nulls` && value === null) { + const { value: _value, ...rest } = row + return rest + } + if (mutant === `loses-negative-zero` && Object.is(value, -0)) { + return { ...row, value: 0 } + } + return row +} + const changeKey = (change: ProviderChange) => change.type === `upsert` ? change.row.id : change.id @@ -233,7 +275,7 @@ function referenceCollection( if (change.type === `upsert`) { write({ type: keys.has(change.row.id) ? `update` : `insert`, - value: change.row, + value: storedRow(change.row, `change`, mutant), }) keys.add(change.row.id) } else if (mutant !== `drops-deletes` || !ready) { @@ -290,7 +332,10 @@ function referenceCollection( } else { begin({ snapshot: true }) for (const row of event.rows) - write({ type: `insert`, value: row }) + write({ + type: `insert`, + value: storedRow(row, `load`, mutant), + }) void commit() keys = new Set(event.rows.map((row) => row.id)) } @@ -431,6 +476,18 @@ export function referenceDriver( `capability-renewal`, `listen-before-fetch`, ]), + valueKinds: [ + `string`, + `integer`, + `float`, + `negative-zero`, + `boolean`, + `null`, + `date`, + `bigint`, + `array`, + `object`, + ], changeCommits: perChange ? `per-change` : `per-batch`, create(): EagerLiveSourceFixture { const provider = new ControlledPushProvider() diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index cc728023b1..663a4e804e 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -1,4 +1,4 @@ -import { deepEquals } from '../utils' +import { deepEquals, sameRowValue } from '../utils' import { SortedMap } from '../SortedMap' import { enrichRowWithVirtualProps } from '../virtual-props.js' import { @@ -1453,7 +1453,7 @@ export class CollectionStateManager< previousVirtualProps.$hasPendingWrites !== nextVirtualProps.$hasPendingWrites || previousVirtualProps.$origin !== nextVirtualProps.$origin || - !deepEquals(previousVisibleValue, newVisibleValue) + !sameRowValue(previousVisibleValue, newVisibleValue) ) { events.push({ type: `update`, diff --git a/packages/db/src/utils.ts b/packages/db/src/utils.ts index 5c973f7627..e71f028714 100644 --- a/packages/db/src/utils.ts +++ b/packages/db/src/utils.ts @@ -45,6 +45,14 @@ export function deepEquals(a: any, b: any): boolean { return deepEqualsInternal(a, b, undefined) } +/** + * @internal Whether a synced row changed observably: deep equality that + * tells -0 from 0, as row values do (FID-1; maintainer decision 2026-10-29). + */ +export function sameRowValue(a: unknown, b: unknown): boolean { + return deepEqualsInternal(a, b, undefined, false, false, true) +} + /** @internal Compare persisted public and durable value shapes before skipping a reload. */ export function equalPersistedSnapshotValues(a: unknown, b: unknown): boolean { return deepEqualsInternal(a, b, undefined, true, true) @@ -78,9 +86,11 @@ export function deepEqualsInternal( visited: Map | undefined, draft = false, persistedSnapshot = false, + // Row change detection tells -0 from 0; general equality does not. + signedZero = false, ): boolean { // Handle strict equality (primitives, same reference) - if (a === b || Object.is(a, b)) return true + if (signedZero ? Object.is(a, b) : a === b || Object.is(a, b)) return true // Handle null/undefined if (a == null || b == null) return false @@ -141,6 +151,7 @@ export function deepEqualsInternal( visited, true, persistedSnapshot, + signedZero, ) : b.has(key) && deepEqualsInternal( @@ -149,6 +160,7 @@ export function deepEqualsInternal( visited, draft, persistedSnapshot, + signedZero, ), ) @@ -182,6 +194,7 @@ export function deepEqualsInternal( visited, draft, persistedSnapshot, + signedZero, ), ) visited.delete(a) @@ -212,6 +225,7 @@ export function deepEqualsInternal( candidateVisited, draft, persistedSnapshot, + signedZero, ) && matchValues( index + 1, @@ -311,7 +325,14 @@ export function deepEqualsInternal( visited.set(a, b) const result = a.every((item, index) => - deepEqualsInternal(item, b[index], visited, draft, persistedSnapshot), + deepEqualsInternal( + item, + b[index], + visited, + draft, + persistedSnapshot, + signedZero, + ), ) visited.delete(a) return result @@ -361,7 +382,14 @@ export function deepEqualsInternal( } if ( !Object.prototype.propertyIsEnumerable.call(b, key) || - !deepEqualsInternal(value, b[key], visited, draft, persistedSnapshot) + !deepEqualsInternal( + value, + b[key], + visited, + draft, + persistedSnapshot, + signedZero, + ) ) { result = false break diff --git a/packages/rxdb-db-collection/tests/conformance-driver.ts b/packages/rxdb-db-collection/tests/conformance-driver.ts index 8fb454d238..2e6eb8c21b 100644 --- a/packages/rxdb-db-collection/tests/conformance-driver.ts +++ b/packages/rxdb-db-collection/tests/conformance-driver.ts @@ -48,7 +48,12 @@ import { } from 'rxdb/plugins/core' import { Observable } from 'rxjs' import { rxdbCollectionOptions } from '../src/rxdb' -import { conformanceKey, observedView } from '../../db-adapter-conformance/src' +import { + conformanceKey, + describeValue, + jsonValueKinds, + observedView, +} from '../../db-adapter-conformance/src' import type { PreparedQuery, RxChangeEvent, @@ -64,7 +69,7 @@ import type { ProviderChange, } from '../../db-adapter-conformance/src' -type Doc = { id: string; title: string; note?: string } +type Doc = { id: string; title: string; note?: string; value?: unknown } const schema = createRxSchema( { @@ -75,6 +80,8 @@ const schema = createRxSchema( id: { type: `string`, maxLength: 10 }, title: { type: `string` }, note: { type: `string` }, + // Any JSON value (FID-1); RxDB documents are JSON. + value: {}, }, required: [`id`, `title`], }, @@ -162,6 +169,7 @@ class RxdbProvider implements ControlledProvider { title: row.title, // A JSON document omits an absent field. ...(row.note === undefined ? {} : { note: row.note }), + ...(Object.hasOwn(row, `value`) ? { value: row.value } : {}), _deleted: deleted, _rev: `${lwt}-conformance`, _attachments: {}, @@ -256,7 +264,11 @@ function conformanceRow(doc: Doc): ConformanceRow { } function sameDocument(doc: Doc, row: ConformanceRow): boolean { - return doc.title === row.title && doc.note === row.note + return ( + doc.title === row.title && + doc.note === row.note && + describeValue(doc.value) === describeValue(row.value) + ) } function createRxdbCollection(provider: RxdbProvider) { @@ -274,6 +286,7 @@ export function rxdbConformanceDriver(): EagerLiveSourceDriver { adapter: `@tanstack/rxdb-db-collection`, capabilities: new Set([`listen-before-fetch`]), changeCommits: `per-change`, + valueKinds: jsonValueKinds, create(): EagerLiveSourceFixture { const provider = new RxdbProvider() return { diff --git a/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts index 762bfe7b27..54eede25fe 100644 --- a/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts +++ b/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -36,7 +36,7 @@ import { rxdbConformanceDriver } from './conformance-driver' const property = `adapter-conformance.eager-live-source.rxdb` const driver = rxdbConformanceDriver() -const historyArb = eagerLiveSourceHistoryArb(driver.capabilities) +const historyArb = eagerLiveSourceHistoryArb(driver) // `field-removed` needs a held row with a note and a later upsert without // one; at 400 histories every seed tried reached it at least 11 times. const runs = 400 @@ -46,7 +46,7 @@ const campaignTimeoutMs = 120_000 async function campaign(options: fc.Parameters<[unknown]>) { const reach = createOracleReach( property, - reachStatesFor(driver.capabilities), + reachStatesFor(driver), { minimum: 5, }, diff --git a/packages/trailbase-db-collection/tests/conformance-driver.ts b/packages/trailbase-db-collection/tests/conformance-driver.ts index ba1b4d3c7e..46cc5ca6c0 100644 --- a/packages/trailbase-db-collection/tests/conformance-driver.ts +++ b/packages/trailbase-db-collection/tests/conformance-driver.ts @@ -31,6 +31,7 @@ import { createCollection } from '@tanstack/db' import { trailBaseCollectionOptions } from '../src/trailbase' import { fromSqlRow, + jsonValueKinds, observedView, toSqlRow, } from '../../db-adapter-conformance/src' @@ -45,7 +46,15 @@ import type { } from '../../db-adapter-conformance/src' /** A TrailBase record: a SQL row, so an absent `note` is `NULL`. */ -type ConformanceRecord = ReturnType +type ConformanceRecord = { + id: number + title: string + note: string | null + // Every row carries a value, since the driver declares value kinds. + value: unknown +} + +const toRecord = (row: ConformanceRow) => toSqlRow(row) as ConformanceRecord interface ProviderSession { stream: ReadableStream @@ -115,7 +124,7 @@ class TrailBaseProvider implements ControlledProvider { deliverSnapshot = (rows: ReadonlyArray) => { this.rows = new Map(rows.map((row) => [row.id, row])) // One short page drains TrailBase's paged list. - this.pendingList().resolve({ records: rows.map(toSqlRow) }) + this.pendingList().resolve({ records: rows.map(toRecord) }) } deliverChanges = (changes: ReadonlyArray) => { @@ -124,13 +133,13 @@ class TrailBaseProvider implements ControlledProvider { if (change.type === `upsert`) { const held = this.rows.has(change.row.id) this.rows.set(change.row.id, change.row) - const record = toSqlRow(change.row) + const record = toRecord(change.row) controller.enqueue(held ? { Update: record } : { Insert: record }) } else { const row = this.rows.get(change.id) if (!row) throw new Error(`provider does not hold row ${change.id}`) this.rows.delete(change.id) - controller.enqueue({ Delete: toSqlRow(row) }) + controller.enqueue({ Delete: toRecord(row) }) } } } @@ -150,6 +159,7 @@ export function trailBaseConformanceDriver(): EagerLiveSourceDriver { adapter: `@tanstack/trailbase-db-collection (eager)`, capabilities: new Set(), changeCommits: `per-change`, + valueKinds: jsonValueKinds, create(): EagerLiveSourceFixture { const provider = new TrailBaseProvider() const collection = createCollection( diff --git a/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts index 8a506f2b42..50fdc47519 100644 --- a/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts +++ b/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -34,16 +34,17 @@ import { trailBaseConformanceDriver } from './conformance-driver' const property = `adapter-conformance.eager-live-source.trailbase` const driver = trailBaseConformanceDriver() -const historyArb = eagerLiveSourceHistoryArb(driver.capabilities) +const historyArb = eagerLiveSourceHistoryArb(driver) // `field-removed` needs a held row with a note and a later upsert without -// one; at 400 histories every seed tried reached it at least 11 times. -const runs = 400 +// one. With value kinds, 400 histories reached it as few as 7 times across +// eight seeds, so TrailBase runs 600. +const runs = 600 const campaignTimeoutMs = 120_000 async function campaign(options: fc.Parameters<[unknown]>) { const reach = createOracleReach( property, - reachStatesFor(driver.capabilities), + reachStatesFor(driver), { minimum: 5, }, From 443bd61a1b5e1fdcd554ea85c612abc64cfb5063 Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 29 Oct 2026 21:44:17 -0600 Subject: [PATCH 09/64] fix(db): treat -0 and 0 as distinct in writes and change detection Maintainer decision (2026-10-29): -0 and 0 are distinct row values, for synced rows and for user writes. Beyond the synced commit path fixed earlier, four core paths treated them as equal: - A draft write of 0 over -0 recorded no mutation, so the write was lost (draft proxy and flat change tracking). - Two stacked pending optimistic writes, -0 then 0, stored 0 while subscribers kept -0 (`collectOptimisticChanges`). - A subscription replay replacing -0 with 0 published nothing (`createStateDiff`). - A source write of 0 over a row a failed replay retained as -0 published nothing (`reconcileStalePublishedChanges`). Each now compares with signed zeros (`Object.is`, `sameRowValue`). - Law: FID-1 (written and delivered values round-trip) and PUB-2 (change events fold to the rows), with the decision above. The flat change-tracking law "-0 written over 0 is not a change" is reversed. - Reach: proxy-oracle `zero-sign-write` (declared, minimum 5); optimistic-history `zeroSignEditsOverPending` via a `flipZeros` edit; subscription replay `zeroSignReplacements`; lifecycle publication `zeroSignWrites` via a flip source mutation. Retained-row reconcile is reached only by a pinned replay (coverage map). - RED: on the parent, proxy-oracle failed (values [-0,0], forEach delta 0: draft kept -0); flat change tracking failed its fixed and random campaigns and the pinned `-0 written over 0 is a change`; the retention oracle failed its event replica after a flipZeros edit; the replay oracle failed a -0 to 0 replacement; the pinned retained -0 history failed. - GREEN: core 348 files, 23,931 tests; persistence-core 2,753; query-db 1,204; Electric 839; adapter conformance 65; RxDB 23; TrailBase 151. - Mutant: each fixed site reverted alone fails its owner: draft equality without signed zeros; the flat tracker's `===` shortcut; `collectOptimisticChanges` with deepEquals; `createStateDiff` with deepEquals; `reconcileStalePublishedChanges` with deepEquals (pinned replay only). - Model changes with that authority: the proxy oracle's changed test, the flat oracle's sameValue and normalization, the optimistic model's authored-change, no-op, and unchanged-settlement rules, and the replay and publication models' diffs. - Size: @tanstack/db +11 bytes gzip against main (was +26 after the synced fix; two comparisons got simpler). - Limits: typed arrays still compare -0 as 0 (documented deepEquals law); generated publication histories do not reach retained rows. Co-authored-by: Isaac --- .changeset/sync-negative-zero-change.md | 2 +- docs/contributing/oracle-coverage.md | 19 ++- packages/db-adapter-conformance/MIGRATION.md | 24 +++- packages/db/src/collection/state.ts | 2 +- packages/db/src/collection/subscription.ts | 6 +- packages/db/src/proxy.ts | 6 +- ...on-state-retention-oracle.property.test.ts | 13 +++ ...ecycle-publication-oracle.property.test.ts | 108 ++++++++++++++++-- ...ubscription-replay-oracle.property.test.ts | 36 +++++- ...at-change-tracking-oracle.property.test.ts | 29 ++--- .../db/tests/optimistic-history-oracle.ts | 105 ++++++++++++++--- packages/db/tests/proxy-oracle.test.ts | 52 +++++++-- 12 files changed, 328 insertions(+), 74 deletions(-) diff --git a/.changeset/sync-negative-zero-change.md b/.changeset/sync-negative-zero-change.md index 591c96b907..306e2ca42f 100644 --- a/.changeset/sync-negative-zero-change.md +++ b/.changeset/sync-negative-zero-change.md @@ -2,4 +2,4 @@ '@tanstack/db': patch --- -Publish a change when a synced update changes a field only from `-0` to `0`, or the reverse. Before, the Collection stored the new value but emitted no change event, so subscribers and live queries kept the old one. +Treat `-0` and `0` as distinct row values. A synced update, a replay that replaces a subscription's rows, and an optimistic update that change a field only from `-0` to `0` (or back) now publish a change; before, the Collection could store the new value while subscribers kept the old one. A draft write of `0` over `-0` is now recorded as a change instead of being dropped. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 60ce312fb7..1319ed9152 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -1809,7 +1809,7 @@ generic visitor behavior outside the named collectors. | Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. Peers may also order their rows by `id` or by a `g` that can be null, with explicit `nulls`; order comes from the reference. A comparator that ignores direction or `nulls`, an order left out of the partition key, and key order instead of the query's order each fail the pinned reorder history and both campaigns. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition while mounted and releases every source subscription after the final framework unmount. The [resource-lifetime oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc-oracle.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, and with no `gcTime`, where both use the live-query Collection default, checks that a partition waits for its views' longest `gcTime`, checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does, including a view with a residual conjunct, and checks that a partition that subscribes again still serves new mounts from one source subscription. A generated cleanup-admission lane (`pooled-live-query.cleanup-admission`) crosses 2–3 views of one partition, observed or not, read by a subscribed live query or not, and view cleanups with or without `dependents`; it checks the model's rejection rule and rows and status against live-query Collections. A view that gives up its partition share when its cleanup rejects fails it. A view whose own cleanup fulfilled must show `cleaned-up` with no rows (law 14, maintainer decision), checked against the model and the live-query Collection; a view that keeps its source's status or its group's rows fails it. Further focused witnesses check that a partition keeps groups only for values that have rows or watchers, that a detached reader never sees a dropped group's revision reused, keep a pooled query's public Collection live while the view is observed, accept it as a query source, and check that a pooled query turns terminal when source cleanup starts while the adapter's cleanup is still pending, as a live-query Collection does. A detached wholesale reader of every view, kept after its peer unmounts, must match the live-query Collection at every checkpoint, and generated histories cross the dependents cascade with readers on watched and unwatched groups that hold rows (declared reach); a cleanup that empties groups without advancing their revisions, or advances only watched groups, fails it. Every view and its live-query Collection also carry an app status listener that throws; each listener gets its own call and the throw is reported uncaught, so a view that lets a listener's throw stop the others or reject the source's cleanup fails it. Generated callback-boundary histories also compare ordinary compiled and pooled core observers across return/throw, Error/undefined, zero through three healthy peers, callback position before/after peers, and `gcTime` 1–3. Failed granular bootstrap retains no unreturned ownership after grace; a held cascade cleanup (`cleanup({ dependents: true })`) gives every admitted wholesale observer one terminal `cleaned-up` snapshot/callback despite a sibling throw, and an ended partition stops following its source's health, and preserves the exact failure at settlement. Original production and independent rollback-only or all-peer-only repairs fail the respective ownership or cleanup-start comparisons. These controlled core receiving paths do not establish framework schedules, arbitrary callback programs or provider shutdown. A pinned oracle witness checks that an `eq` path whose getter throws excludes the row on both paths; a pooled path that rethrows fails it. The pooled oracle's grammar never releases a partition, so release and resubscribe histories rely on these fixed witnesses; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. Removing the partition's final-listener release survives the mounted row checks but fails the shared framework unmount checkpoint and core release-timing owner. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. The shared adapter conformance suite checks the opt-in boundary in every framework: queries without `pool` keep a source subscription each in the function, config, and parameterized forms, and a pooled parameterized query follows its parameter between pool groups beside an unchanged peer. Pooling every query by default, a pooled identity that ignores literal values (React then reuses the old query), and an Angular `params` form that drops `pool` each fail it. `pooled-param-change` also reads each driver's publication log: every value published after a switch holds only the new group's rows. Each driver also logs its keyed `state` map at the same points, and the law applies to it. A pooled view that serves the previous view's rows until a later task replaces them passes every final-state check and fails only the log, in all six drivers. A view that sends the previous rows as its first change set fails the `state` log in Vue; Solid rebuilds its map from a snapshot within the same turn, and Angular builds its map from `entries()`, so that mutant cannot reach them. The `invalid-pool` scenario requires `InvalidPoolOptionError` for `true` and for `false` (the value adapters pass to say a hook cannot pool), alone, beside `gcTime`, beside `queryKey`, on a parameterized query, and on a disabled config or parameterized query, in all six drivers; a check in only one of the three readers of `pool` fails it in four of them, and Angular's `params` form failed the `false` and disabled cases until it checked its pool first. The shared `grouping-operator-switch` scenario re-renders a grouped query without `groupKeys()`, and `compile-error-after-rerender` re-renders to a select that mixes a source field with an aggregate; both require the error and no rows after it. A binding that keeps its last compiled query when compilation throws fails both in Solid, Svelte, and Angular; one that throws once and then serves that query fails the no-rows check in Solid and Svelte, and cannot occur in Angular, whose `computed` rethrows a cached error until an input changes. In `grouping-operator-switch`, React and Preact kept the grouped query before query identity held operator code, and Vue served the previous rows before its compile error stopped reaching readers through a throwing computed (also on `main` at `5649a07d4`). | | Live-query deferred acquisition | [deferred acquisition oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/live-query-deferred-acquisition-oracle.test.ts), [React Suspense warm-cache witness](https://github.com/TanStack/db/blob/main/packages/react-db/tests/suspense-warm-cache.test.tsx) | An independent model of the law that a live-query Collection starts no provider work before a subscriber or a preload in its current sync run crosses seven source states (eager or on-demand, idle, already started or render-deferred, and an on-demand source that already holds the rows), an unordered query and an ordered window, one or two live-query levels, and eleven histories: start and read, subscribe or preload after a start, subscribe or preload without one, deferred subscription and resumption with or without a start, cleanup before a subscriber, peer start, source truncate, and restart after a run that had one. After every command it checks source sync-run starts, acquisition attempts, readiness when the call returns (the synchronous observation cut), settled readiness, and rows. A pooled block checks that building a pooled view starts no idle eager source until a view subscribes or preloads, and eight DbClient histories check that a preload answered by a pending or settled stream still resumes acquisition across eager/on-demand idle and render-deferred sources. The React streaming witness receives the on-demand render-deferral premise, starts local acquisition before the stream settles, and retains server rows until browser acquisition completes. Mutants that never defer, keep the subscriber-or-preload fact across cleanup, ignore preload, never resume a builder's subscriptions, let a deferring subscriber start a base source, build a pooled partition without deferral, never resume it, mark a preload one microtask late, or drop the stream's preload mark each fail it. The React witness checks that `useLiveSuspenseQuery` over a source that already holds the rows renders without the fallback, for eager and on-demand sources and both shapes; creating the Suspense collection started, or not preloading it in render, fails it. A later review added `peer` and `truncate` commands: a peer that starts an idle on-demand source, or a truncate on a running one, made the live query acquire before any subscriber, and both guards are killed separately. Pinned blocks check that a subscriber surviving cleanup is served after a restart while a surviving deferring subscriber is not, and that `toArrayWhenReady()` and `stateWhenReady()` count as a preload. A pinned block checks that a deferring live query is not counted in its source's `subscriberCount` or `subscribers:change` until its first subscriber, while still keeping the source from garbage collection; a Query Collection test checks that it causes no refetch. Another pinned block checks that a preload or subscriber requested by an inner live query's status handler, while the outer query creates its subscription to it, still resumes acquisition. Solid's Suspense receiving witnesses check that a resource whose preload settles first retains its live-query Collection while its sibling remains pending past the unsubscribed floor, then releases that temporary owner on both commit and abandonment, and that a preload settling after its owner unmounts cannot publish an error through the abandoned result. Open: source failure and asynchronous loads before resumption, which the subscription lifecycle oracle owns once acquisition resumes; a resume that throws, which leaves later sources unstarted only for a live query that is already in error; asynchronous sources; and arbitrary Suspense schedules with a `DbClient` beyond the bounded abandoned-render and streaming handoffs recorded above. | | Live-query notification algebra | [real-query and checker calibration](https://github.com/TanStack/db/blob/main/packages/db/tests/notification-laws-oracle.test.ts), [independent Map checker](https://github.com/TanStack/db/blob/main/packages/db-collection-e2e/src/utils/notification-laws-oracle.ts) | One-key synchronous source histories check exact live-query callback reconstruction and unsubscribe silence. Synthetic multi-key histories calibrate the checker; direct source Collection events remain with the change-event history owner. | -| Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows, frozen or not and with or without a non-enumerable field (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields; data and accessor `defineProperty`; stored drafts; throws), run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal, or throw the same error and leave the rows unchanged. Defining a field acts as assigning it, and a non-enumerable field is row data only once written; the proxy broke three of those laws on `main` and the flat tracker one. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history and both campaigns fail without that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows, frozen or not and with or without a non-enumerable field (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields; data and accessor `defineProperty`; stored drafts; throws), run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets, with `0` and `-0` distinct (maintainer decision, 2026-10-29: writing one over the other is a change), or throw the same error and leave the rows unchanged. Defining a field acts as assigning it, and a non-enumerable field is row data only once written; the proxy broke three of those laws on `main` and the flat tracker one. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history and both campaigns fail without that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Virtual props cache | [cache laws](https://github.com/TanStack/db/blob/main/packages/db/tests/virtual-props-cache.test.ts) | Focused laws for the per-key cache of rows read with virtual props: a change's published value is the row later `get` and `toArray` reads return, for every subscriber, after sync and local writes, and a key that is deleted or rolled back leaves the cache. Enriching the value before the previous value fails the first two; skipping the drop on delete events fails the third. Rows read with virtual props through other paths, such as joins, are outside these laws. | | LocalStorage Collection | [mutation order](../../packages/db/tests/local-storage-order-oracle.test.ts), [peer snapshots and ownership](../../packages/db/tests/local-storage-peer-history.test.ts) (fixed tests; no oracle owner (tracked in #2264)) | Finite two-write histories compare authored same/disjoint-key updates, both handler completion orders, and all four application-decision pairs against public rows, durable rows, caller settlement, and fresh restore. A nested fire-and-forget admission checks that the outer write reaches storage first; awaiting the nested receipt inside its earlier handler is an excluded dependency cycle. Handler-free direct insert, update, and delete are checked at the method-return boundary for synchronous Storage effects. A later handler failure is checked while its earlier handler remains held: its receipt rejects and its optimistic row rolls back promptly, while the durable predecessor remains pending. Fixed tests in `packages/db/tests/local-storage-order.test.ts` (no oracle owner (tracked in #2264)) cover the next two histories. An automatic insert followed by manual acceptance crosses same-key update versus disjoint insert and both earlier-handler decisions; manual settlement waits behind that handler, and public, durable, and fresh-restore rows match the authored fold. Un-awaited manual acceptance crosses direct and held-predecessor paths with success or Storage failure: both commit and persistence receipts wait for every registered write, report the exact fault on failure, and agree with durable and public rows at settlement. A two-Collection witness checks that one manual receipt waits for both un-awaited acceptances. Peer witnesses check serialized disjoint typed-key writes before delayed event delivery, equal-ID manual Collection ownership, construction-time rejection of direct or shallow-copy reuse before or after first preload with the first Collection still writable, a peer insert during pending local insert, listener release across cleanup/restart, malformed startup snapshots, transient startup read failures and restart, the null-versus-empty storage sentinel, local and peer clear publication with subsequent public/durable agreement and a pending optimistic mutation across clear, and storage/parser read failures during writes and event delivery. Reload witnesses let another tab (a second Storage object over the same bytes) update or delete a kept row and insert a row, or clear storage, while no event reaches the first tab, with and without a prior local edit of the kept row: `reload()` must resolve with the Collection `ready` and its public rows equal to the durable rows and a fresh restore, and the reloaded run must confirm a later write. The adapter that re-inserted stored rows in each run crashed on the update histories and kept the deleted row; a snapshot only for non-empty storage fails the clear histories. The [same-tab audit](oracle-reviews/2026-10-08-localstorage-same-tab.md) records witnesses that withhold events through each persistence receipt: disjoint typed keys, a manually accepted same-key update followed by automatic update and delete, clear, a rejected Storage write, a pending same-key optimistic insert, cleanup/restart, and a write started by a ready-status listener compare public snapshots with the authored durable fold. A default jsdom Storage witness receives a same-tab write. The old event-only adapter fails seven no-event checkpoints. The original adapter also fails the second-handler-first, delayed-peer, equal-ID, listener-release, read-failure, synchronous direct-write, prompt-failure, mixed manual/automatic order, un-awaited manual receipt, reused-options admission, malformed-startup, and clear-publication witnesses. A failed write read rejects without replacing durable rows; a failed event read retains the public snapshot until a later valid event. These checks do not establish simultaneous cross-tab read-modify-write atomicity, atomic acceptance across distinct storage keys, manual acceptance without a mutation Collection owner, arbitrary long histories, native browser scheduling, or synchronization across distinct custom Storage wrapper objects. Existing LocalStorage failure and CRUD tests remain supporting examples. | | LocalStorage value fidelity | [JSON value model](../../packages/db/tests/local-storage-value-fidelity-oracle.property.test.ts) | With the default JSON parser, an insert, update, manual insert, or manual update is accepted only if a fresh restore can return the row. A recursive value grammar labels each generated value, while it is built, with its expected JSON form or as lost. Lost: functions, symbols, enumerable symbol keys, `NaN` and infinities (also boxed), `undefined` and missing array entries, arrays with extra or symbol properties, a `toJSON` that returns `undefined` in an array, and Map, Set, WeakMap, Promise, RegExp, Uint8Array, and SharedArrayBuffer values. Normalized: `toJSON` values (Date, a plain object, and a function with its own `toJSON`), `undefined` properties and `toJSON` results, `-0`, boxed finite numbers, and non-enumerable symbol keys. Histories cross the four entry paths with and without an automatic handler. At `when('settled')` settlement, an accepted row must keep the authored writer value, store the model's form durably, and restore it in a fresh Collection; a lost row must reject with `SerializationError`, leave the writer's predecessor and storage bytes unchanged, and never call the handler. Pinned witnesses cover every lost kind and normalization on each path, with and without a handler; a fixed-seed and a random campaign of 300 histories run under `local-storage.value-fidelity`. A pinned witness per entry path uses a `toJSON` that changes after its first call: storage must hold the JSON the check accepted, and serializing the row again during the key check or the snapshot write each fails it. The stringify-only check on `main` fails 162 of 230 cells. The first walker in this PR failed 66 cells of the expanded grammar. Mutants that reject `undefined` properties, accept `undefined` array entries, skip boxed-number unwrapping, treat non-enumerable symbol keys as lost, skip the array-property check, or shrink the opaque-type list each fail at least six cells. Open: custom parsers, including a wrapper around `JSON` (accepted by design, documented); class instances (accepted by design; the guide states they restore as plain objects); cross-realm natives; getters; non-enumerable string data; update payloads whose shape the draft copy changes (array extra properties and symbol keys are dropped, and a hidden symbol key becomes enumerable, so an update of an object with a hidden symbol key rejects although its insert succeeds), which the proxy detachment owner must decide; peer or event delivery of a rejected write, which never reaches storage; and peer delivery of normalized values: the peer owner checks only `Date` at that boundary, so a peer receiving witness for the generated value grammar remains with the peer owner. IndexedDB clone rejection is owned by the IndexedDB settlement and values oracles, which require `SerializationError`; local-only rows keep values by reference, pinned in `local-only.test.ts`. | @@ -5979,11 +5979,18 @@ optional field, and FID-1 typed values per declared `valueKinds` on the initial load and live changes. Its reference driver is also the generated witness for core's synced commit change detection with `-0` and `0` (maintainer decision 2026-10-29: distinct row values; `sameRowValue`). -Open in core, owner `packages/db/tests/change-event-history-oracle.test.ts` -and the subscription lifecycle publication oracle, witness needed: an -optimistic or local-only update and a subscription reconciliation that -change only `-0` to `0` (`collectOptimisticChanges` and `subscription.ts` -still compare with `deepEquals`). Every driver is also +The decision covers user writes too (2026-10-29): the draft proxy +(`proxy-oracle`, `flat-change-tracking-oracle`), optimistic change +detection (`optimistic-history-oracle`, with a `flipZeros` edit and +`zeroSignEditsOverPending` reach), subscription truncate replay +(`collection-subscription-replay-oracle`), and stale-row reconciliation +(`collection-subscription-lifecycle-publication-oracle`) all compare row +values with signed zeros. Open: generated publication histories never write +over a row a failed replay retained (the random grammar inserts source +writes into green lifecycle histories), so that reconcile path has only +pinned replays, including a `-0` one (owner: the lifecycle publication +oracle; witness needed: a generated failed-replay history with later source +writes). Every driver is also checked for the rows at the moment status becomes `ready`. Its model is an independent row Map and sync-run phase; drivers supply only a controlled provider and declare whether they commit per batch or per change. PUB-1 checks every change publication during diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index ca69e13136..be78f0cf88 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -182,6 +182,24 @@ document update removes`) failed the same way. TrailBase are GREEN; Neon passes 300 failure-free histories with values and keeps its READY-4 and HEALTH-1 RED. TrailBase now runs 600 histories (`field-removed` fell to 7 at 400). - - **Open:** the rejection lane (FID-3, PUB-4); the optimistic and - subscription-reconciliation paths in core still compare with - `deepEquals` (recorded in the coverage map). + - **Open then:** the rejection lane (FID-3, PUB-4), and core's other + `-0` paths; see the next entry. +- **2026-10-29, core `-0` paths beyond sync (from the typed-value lane).** + Maintainer decision: `-0` and `0` are distinct for user writes too. Each + owner was extended first and went RED on the parent: + - **Draft proxy:** a write of `0` over `-0` recorded no mutation. + `proxy-oracle` gained `-0` values and deltas (`zero-sign-write` reach); + the flat change-tracking law "-0 written over 0 is not a change" was + reversed. Both trackers now compare with `Object.is`. + - **Optimistic path:** two stacked pending writes, `-0` then `0`, left + subscribers at `-0`. `optimistic-history-oracle` gained `-0`, a + `flipZeros` edit, and `zeroSignEditsOverPending` reach; two model rules + that used `===` (`author`'s no-op test and the unchanged-settlement + check) now tell the zeros apart. `collectOptimisticChanges` uses + `sameRowValue`. + - **Subscription replay and stale reconcile:** a replay replacing `-0` + with `0` published nothing. The replay and lifecycle publication + oracles gained `-0` and zero-sign reach, and their models use + `Object.is`. `createStateDiff` is rejected by generated histories; + `reconcileStalePublishedChanges` only by a pinned replay, since + generated histories never reach retained rows (coverage map). diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 663a4e804e..83fbf4c8ca 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -787,7 +787,7 @@ export class CollectionStateManager< } else if ( previousValue !== undefined && currentValue !== undefined && - (!deepEquals(previousValue, currentValue) || + (!sameRowValue(previousValue, currentValue) || previousVirtualProps.$origin !== nextVirtualProps.$origin || previousVirtualProps.$hasPendingWrites !== nextVirtualProps.$hasPendingWrites) diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index de491b3325..f88a7ea218 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -4,7 +4,7 @@ import { PropRef, Value, getPropRefPropertyPath } from '../query/ir.js' import { EventEmitter } from '../event-emitter.js' import { compileExpression } from '../query/compiler/evaluators.js' import { buildCursor, buildCursorCurrent } from '../utils/cursor.js' -import { deepEquals } from '../utils.js' +import { sameRowValue } from '../utils.js' import { normalizeError } from '../utils/error.js' import { reportUncaught, runAllCallbacks } from '../utils/callbacks.js' import { createDeferred } from '../deferred.js' @@ -834,7 +834,7 @@ export class CollectionSubscription key, value: previousValue, }) - } else if (!deepEquals(value, previousValue)) { + } else if (!sameRowValue(value, previousValue)) { replacement.push({ type: `update`, key, @@ -1906,7 +1906,7 @@ export class CollectionSubscription value: previous, previousValue: undefined, }) - } else if (!deepEquals(previous, change.value)) { + } else if (!sameRowValue(previous, change.value)) { reconciled.push({ ...change, type: `update`, diff --git a/packages/db/src/proxy.ts b/packages/db/src/proxy.ts index f576990a2c..ec5116f476 100644 --- a/packages/db/src/proxy.ts +++ b/packages/db/src/proxy.ts @@ -357,13 +357,14 @@ function deepClone( } // Generic value equality intentionally ignores some native state. Draft -// assignments must also preserve Set contents/order and RegExp match position. +// assignments must also preserve Set contents/order and RegExp match position, +// and tell -0 from 0, which are distinct row values. function draftValuesEqual( left: unknown, right: unknown, paired?: Map, ): boolean { - return deepEqualsInternal(left, right, paired, true) + return deepEqualsInternal(left, right, paired, true, false, true) } /** @@ -927,7 +928,6 @@ export function withFlatChangeTracking( if ( !Object.hasOwn(original, key) || !( - value === before || Object.is(value, before) || (typeof before === `object` && before !== null && diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 4130e98fb6..05b4f26d20 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -742,6 +742,19 @@ it(`writes distinguishing partial updates in the fixed partial campaign`, async expect(distinguishing).toBeGreaterThan(0) }) +// Reach: an edit over an in-flight optimistic write that changes the +// visible row only by a zero's sign. Neither the pending-writes flag nor +// deep equality without signed zeros sees that change. +it(`writes zero-sign edits over in-flight writes in the fixed campaign`, async () => { + const histories = fc.sample(optimisticHistory, { seed: 86103, numRuns: 40 }) + let zeroSignEdits = 0 + for (const { initial, steps } of histories) { + const counts = await runOptimisticHistory(initial, steps) + zeroSignEdits += counts.zeroSignEditsOverPending + } + expect(zeroSignEdits).toBeGreaterThan(0) +}) + // Pinned replays for a source delete of a key the source never held. The // backend accepted an optimistic insert, then deleted the row before the // source streamed it. The model's drain applies the delete to the applied diff --git a/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts b/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts index 882042c34f..2af7158204 100644 --- a/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts @@ -99,8 +99,26 @@ type SourceMutation = { key: RowKey action: `upsert` | `delete` value: number + // Upsert the visible row's zero with the other sign; a key whose visible + // value is not a zero upserts `value`. + flip?: boolean } +/** Resolves a `flip` mutation against the model's visible rows. */ +function withFlippedZero( + command: PublicationCommand, + publication: PublicationModel, +): PublicationCommand { + if (command.type !== `source` || !command.flip) return command + const visible = publication.visible.get(command.key)?.value + return visible === 0 + ? { ...command, value: Object.is(visible, -0) ? 0 : -0 } + : command +} + +// Reach: source writes that change a public row only by the sign of a zero. +let zeroSignWrites = 0 + type PublicationCommand = | Exclude | { type: `truncate`; replacement?: Row } @@ -190,9 +208,14 @@ function recordSourceWrite(publication: PublicationModel, row: Row): void { // Retained public rows take precedence when reconciling a stale snapshot. const previous = publication.visible.get(row.id) ?? publication.source.get(row.id) + if (previous?.value === row.value && !Object.is(previous.value, row.value)) { + zeroSignWrites++ + } publication.source.set(row.id, cloneRow(row)) publication.retainedKeys.delete(row.id) - if (previous?.value === row.value) return + // The same value publishes nothing; -0 and 0 are distinct row values + // (maintainer decision, 2026-10-29). + if (previous !== undefined && Object.is(previous.value, row.value)) return publication.visible.set(row.id, cloneRow(row)) publication.sentKeys.add(row.id) publication.batches.push([ @@ -212,7 +235,9 @@ const mapsEqual = ( right: ReadonlyMap, ): boolean => left.size === right.size && - [...left].every(([id, row]) => right.get(id)?.value === row.value) + [...left].every( + ([id, row]) => right.has(id) && Object.is(right.get(id)!.value, row.value), + ) function publicationPhase( publication: PublicationModel, @@ -243,7 +268,7 @@ function publicationDiff( const value = next.get(key) if (!value) { changes.push({ type: `delete`, key, value: cloneRow(previousValue) }) - } else if (value.value !== previousValue.value) { + } else if (!Object.is(value.value, previousValue.value)) { changes.push({ type: `update`, key, @@ -511,12 +536,28 @@ function projectPublication( } } -const sourceMutationArbitrary: fc.Arbitrary = fc.record({ - type: fc.constant(`source` as const), - key: fc.constantFrom(`a` as const, `b` as const, `c` as const, `d` as const), - action: fc.constantFrom(`upsert` as const, `delete` as const), - value: fc.integer({ min: 0, max: 5 }), -}) +const sourceMutationArbitrary: fc.Arbitrary = fc.oneof( + fc.record({ + type: fc.constant(`source` as const), + key: fc.constantFrom( + `a` as const, + `b` as const, + `c` as const, + `d` as const, + ), + action: fc.constantFrom(`upsert` as const, `delete` as const), + // -0 is a distinct row value from 0, so a write can change only a sign. + value: fc.oneof(fc.integer({ min: 0, max: 5 }), fc.constant(-0)), + }), + // Writes 0 over a non-zero row, and flips the sign of a zero one. + fc.record({ + type: fc.constant(`source` as const), + key: fc.constantFrom(`a` as const, `b` as const), + action: fc.constant(`upsert` as const), + value: fc.constant(0), + flip: fc.constant(true), + }), +) // Generated publication histories contain 1–20 lifecycle commands from the // shared grammar plus 1–5 atomic source mutations inserted at any boundary, @@ -778,7 +819,8 @@ async function runPublicationHistory( const failures: Array = [] try { - for (const [index, command] of history.entries()) { + for (const [index, authored] of history.entries()) { + const command = withFlippedZero(authored, publication) const priorPublicationCount = lifecycle.publications const phaseBefore = publicationPhase(publication, lifecycle) const pendingAttemptsBefore = [...attempts.values()].filter( @@ -2158,6 +2200,39 @@ describe(`CollectionSubscription lifecycle publication oracle`, () => { { continueAfterMismatch: true }, ) }) + // The history above with zeros: the failed replay retains b as -0, and the + // later source write of 0 must reconcile as an update, since -0 and 0 are + // distinct row values (maintainer decision, 2026-10-29). + it(`reconciles a retained -0 row with a later source write of 0`, async () => { + await runPublicationHistory( + [ + { type: `source`, key: `b`, action: `upsert`, value: -0 }, + { type: `request`, demand: `a` }, + { + type: `settle`, + demand: `a`, + scope: `current`, + age: `oldest`, + outcome: `resolve`, + }, + { type: `truncate` }, + { type: `source`, key: `b`, action: `upsert`, value: 51 }, + { + type: `settle`, + demand: `a`, + scope: `current`, + age: `oldest`, + outcome: `reject`, + }, + { type: `release`, demand: `a` }, + { type: `source`, key: `b`, action: `upsert`, value: 0 }, + { type: `cleanup` }, + { type: `restart` }, + { type: `unsubscribe` }, + ], + { continueAfterMismatch: true }, + ) + }) it(`does not delete an independent public row when releasing a restarted demand`, async () => { await runPublicationHistory( [ @@ -2389,6 +2464,19 @@ describe(`CollectionSubscription lifecycle publication oracle`, () => { }) const { multiplier, ...replay } = readOracleRunConfig() const runs = 60 * multiplier + // Generated histories never write over a retained row (a failed replay's + // carried row): the random grammar inserts source writes into green + // lifecycle histories. The pinned zero-sign history below covers that path. + it(`reaches a zero-sign source write in the fixed campaign`, async () => { + zeroSignWrites = 0 + for (const history of fc.sample(publicationCommandHistoryArbitrary, { + seed: 1_657_005, + numRuns: runs, + })) { + await runPublicationHistory(history) + } + expect(zeroSignWrites).toBeGreaterThan(0) + }, 120_000) fcTest.prop([publicationCommandHistoryArbitrary], { numRuns: runs, seed: 1_657_005, diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index 63ef5d389d..91c59838c6 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -139,9 +139,14 @@ type PendingReplay = { settled: boolean } +// Values include -0, a distinct row value from 0 (maintainer decision, +// 2026-10-29), so a replay can replace a row that differs only by a zero's +// sign. +const replayValue = fc.oneof(fc.integer({ min: -2, max: 2 }), fc.constant(-0)) + const rowArbitrary: fc.Arbitrary = fc.record({ id: fc.constantFrom(`one` as const, `two` as const), - value: fc.integer({ min: -2, max: 2 }), + value: replayValue, }) const rowsArbitrary = fc.uniqueArray(rowArbitrary, { @@ -156,7 +161,7 @@ function replayLoadArbitrary( return fc.record({ demandId: fc.constant(demandId), rows: fc - .option(fc.integer({ min: -2, max: 2 }), { nil: undefined }) + .option(replayValue, { nil: undefined }) .map((value) => (value === undefined ? [] : [{ id: demandId, value }])), outcome: fc.constantFrom(`resolve` as const, `reject` as const), writeBeforeSettlement: fc.boolean(), @@ -168,7 +173,7 @@ function sourceActionArbitrary( ): fc.Arbitrary { return fc.oneof( fc - .tuple(fc.constantFrom(...demandIds), fc.integer({ min: -2, max: 2 })) + .tuple(fc.constantFrom(...demandIds), replayValue) .map(([id, value]) => ({ type: `put` as const, row: { id, value } })), fc .constantFrom(...demandIds) @@ -307,6 +312,9 @@ function rowsById( return new Map(rows.map((row) => [row.id, { ...row }])) } +// Reach: replacements in which a row changed only by the sign of a zero. +let zeroSignReplacements = 0 + function publicationDiff( baseline: ReadonlyMap, finalRows: ReadonlyMap, @@ -320,7 +328,9 @@ function publicationDiff( key, value: { ...previousValue }, }) - } else if (value.value !== previousValue.value) { + } else if (!Object.is(value.value, previousValue.value)) { + // -0 and 0 are distinct row values. + if (value.value === previousValue.value) zeroSignReplacements++ changes.push({ type: `update`, key, @@ -526,7 +536,9 @@ async function runReplayScenario(scenario: ReplayScenario): Promise { } const previous = sourceRows.get(action.row.id) - if (previous?.value === action.row.value) return false + // A put of the value the source holds writes nothing; -0 is not 0. + if (previous !== undefined && Object.is(previous.value, action.row.value)) + return false applyRows([action.row], { type: `ordinary` }) return true } @@ -2049,6 +2061,20 @@ describe(`CollectionSubscription replay oracle`, () => { seed: 1756, })(`matches replay and ownership laws for a fixed seed`, runReplayScenario) + // The fixed campaign's histories reach a replacement that changes a row + // only by a zero's sign, where a diff without signed zeros publishes + // nothing. + it(`reaches a zero-sign replacement in the fixed campaign`, async () => { + zeroSignReplacements = 0 + for (const scenario of fc.sample(replayScenarioArbitrary, { + seed: 1756, + numRuns: generatedRuns, + })) { + await runReplayScenario(scenario) + } + expect(zeroSignReplacements).toBeGreaterThan(0) + }) + fcTest.prop( [replayScenarioArbitrary], oracleRandomParameters( diff --git a/packages/db/tests/flat-change-tracking-oracle.property.test.ts b/packages/db/tests/flat-change-tracking-oracle.property.test.ts index 85f693db6a..e81a44f039 100644 --- a/packages/db/tests/flat-change-tracking-oracle.property.test.ts +++ b/packages/db/tests/flat-change-tracking-oracle.property.test.ts @@ -20,7 +20,7 @@ * * Model: `expectedChanges` folds every row's operations, in callback order, * over plain copies of the rows. It keeps enumerable fields whose final value - * differs under `===` or `Object.is` from the row's own field, or, for the + * differs under `Object.is` from the row's own field, or, for the * object a non-enumerable field holds, by contents; plus deleted enumerable * fields. A copy holds no non-enumerable field, so reading one gives * `undefined`, deleting one does nothing, and writing one is a change unless @@ -50,14 +50,16 @@ * `withArrayChangeTracking` and `withChangeTracking` run the same operations. * * Refinement check: the flat result, the proxy result, and the model are - * strictly equal, including present `undefined` fields and the cycles a - * stored draft creates, except that `0` and `-0` compare equal and - * prototypes are ignored: collection equality does not distinguish them, and - * the proxy skips writing a value equal to the current one. Mutating an - * assigned object after the callback leaves both results unchanged. + * strictly equal, including present `undefined` fields, zero signs, and the + * cycles a stored draft creates, except that prototypes are ignored: + * collection equality does not distinguish them. `-0` and `0` are distinct + * row values (maintainer decision, 2026-10-29), so writing one over the + * other is a change, as `NaN` over `NaN` is not. Mutating an assigned object + * after the callback leaves both results unchanged. * - * Calibration: a flat diff with `!==` reports unchanged `NaN` fields, one with - * `Object.is` alone reports `-0` written over `0`, one that skips deletions + * Calibration: a flat diff with `!==` reports unchanged `NaN` fields, one + * that also accepts `===` drops `-0` written over `0` (the behavior before + * the 2026-10-29 decision, in both trackers), one that skips deletions * loses removed fields, one that ignores whether the row owns a field drops * added `undefined` fields, and one that edits rows in place breaks the throw * and stored-draft histories; each fails a pinned history and the fixed @@ -137,8 +139,10 @@ type History = { // Model // --------------------------------------------------------------------------- +// The same value: `NaN` is itself, and `-0` is not `0` (maintainer +// decision, 2026-10-29: they are distinct row values). function sameValue(a: unknown, b: unknown): boolean { - return a === b || Object.is(a, b) + return Object.is(a, b) } // Only a non-enumerable field holds an object, always `{ nested: 1 }`; an @@ -242,10 +246,9 @@ function applyOperation( } } -// Collection equality does not distinguish -0 from 0 or prototypes. A -// stored draft can make a change set cyclic, so the copy keeps cycles. +// Collection equality does not distinguish prototypes. A stored draft can +// make a change set cyclic, so the copy keeps cycles. function normalize(value: unknown, copies = new Map()): unknown { - if (value === 0) return 0 if (value === null || typeof value !== `object`) return value const existing = copies.get(value) if (existing) return existing @@ -413,7 +416,7 @@ const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ }, }, { - name: `-0 written over 0 is not a change`, + name: `-0 written over 0 is a change`, history: { rows: [plainRow([0, 1, MISSING])], operations: [[{ kind: `set`, field: `a`, value: -0 }]], diff --git a/packages/db/tests/optimistic-history-oracle.ts b/packages/db/tests/optimistic-history-oracle.ts index 68c2413633..90e21f885e 100644 --- a/packages/db/tests/optimistic-history-oracle.ts +++ b/packages/db/tests/optimistic-history-oracle.ts @@ -180,6 +180,9 @@ export type OptimisticStep = optimistic: boolean pending?: boolean inHandler?: HandlerBatch | undefined + // Replace `fields` with the visible row's zero fields, signs flipped. + // A missing row, or one with no zero, keeps `fields`. + flipZeros?: boolean } | { type: `delete` @@ -322,11 +325,13 @@ class HistoryModel { ): number | undefined { const row = this.visible().get(step.key) if (step.type === `delete` && !row) return + // An edit that writes only the values the row holds is no transaction. + // -0 and 0 are distinct row values (maintainer decision, 2026-10-29). if ( step.type === `edit` && row && - Object.entries(step.fields).every( - ([key, value]) => row[key as keyof Fields] === value, + Object.entries(step.fields).every(([key, value]) => + Object.is(row[key as keyof Fields], value), ) ) return @@ -645,12 +650,36 @@ export async function withHistoryCleanup( return result as T } -// The history grammar shared by every production path. +// The history grammar shared by every production path. Field values include +// -0, a distinct row value from 0 (maintainer decision, 2026-10-29), so a +// write can change a field only by the sign of a zero. +const fieldValue = fc.oneof(fc.integer({ min: -2, max: 2 }), fc.constant(-0)) +const fieldNames = [`a`, `b`, `c`] as const + +/** + * Resolves a `flipZeros` edit against the model's visible rows: it writes + * each zero field of the edited row with the other sign. Other steps, and an + * edit of a row without a zero, are unchanged. + */ +function withFlippedZeros( + step: OptimisticStep, + visible: ReadonlyMap, +): OptimisticStep { + if (step.type !== `edit` || !step.flipZeros) return step + const row = visible.get(step.key) + if (row === undefined) return step + const fields = Object.fromEntries( + fieldNames + .filter((field) => row[field] === 0) + .map((field) => [field, Object.is(row[field], -0) ? 0 : -0]), + ) + return Object.keys(fields).length === 0 ? step : { ...step, fields } +} export const historyRow = fc.record({ id: fc.integer({ min: 1, max: 3 }), - a: fc.integer({ min: -2, max: 2 }), - b: fc.integer({ min: -2, max: 2 }), - c: fc.integer({ min: -2, max: 2 }), + a: fieldValue, + b: fieldValue, + c: fieldValue, }) const sourceBatch = fc.record({ type: fc.constant(`sync` as const), @@ -700,11 +729,7 @@ export const optimisticStep: fc.Arbitrary = fc.oneof( key: fc.integer({ min: 1, max: 3 }), fields: fc .record( - { - a: fc.integer({ min: -2, max: 2 }), - b: fc.integer({ min: -2, max: 2 }), - c: fc.integer({ min: -2, max: 2 }), - }, + { a: fieldValue, b: fieldValue, c: fieldValue }, { requiredKeys: [] }, ) .filter((fields) => Object.keys(fields).length > 0), @@ -712,6 +737,22 @@ export const optimisticStep: fc.Arbitrary = fc.oneof( inHandler: handlerSourceBatch, }), }, + { + // An edit that changes only the sign of the row's zeros (-0 and 0 are + // distinct row values). Random fields rarely do it by chance. It + // matters over a write still in flight, so it needs a second flip edit on + // the same key before the first settles; the fixed campaigns reach it. + weight: 2, + arbitrary: fc.record({ + type: fc.constant(`edit` as const), + key: fc.integer({ min: 1, max: 3 }), + // With no zero to flip, the edit writes zeros, so a later one can. + fields: fc.constant({ a: 0, b: 0, c: 0 }), + optimistic: fc.boolean(), + inHandler: fc.constant(undefined), + flipZeros: fc.constant(true), + }), + }, { weight: 4, arbitrary: fc.record({ @@ -1004,6 +1045,10 @@ export async function runOptimisticHistory( // Partial updates whose omitted `c` differs from the source's held row, // so only a merge keeps the held value. distinguishingPartialUpdates: 0, + // Edits over a row that already had an optimistic write in flight whose + // visible row changed only by the sign of a zero: no pending-writes flag + // changes, so only value equality can publish the change. + zeroSignEditsOverPending: 0, // Drains that kept a settled row held for a deferred transaction. deferredHolds: 0, abortedDeferred: 0, @@ -1388,16 +1433,34 @@ export async function runOptimisticHistory( const initialFrame = publications.at(-1) check(`initial`) initialPublication = false - for (const [position, step] of steps.entries()) { + for (const [position, authored] of steps.entries()) { + const step = withFlippedZeros(authored, model.visible()) settling = step.type === `settle` const before = sorted(model.visible().values()) const deliveredBefore = deliveries if (step.type === `edit` || step.type === `delete`) { if (step.type === `edit` && step.pending && step.inHandler) throw new Error(`A pending edit cannot write from its handler yet`) + const inFlight = model.transactions.some( + (transaction) => + transaction.key === step.key && + (transaction.state === `pending` || + transaction.state === `persisting`), + ) const index = model.author(step) if (index === undefined) continue const intent = model.transactions[index]! + const was = before.find((row) => row.id === step.key) + const now = model.visible().get(step.key) + if ( + inFlight && + was !== undefined && + now !== undefined && + fieldNames.every((field) => was[field] === now[field]) && + fieldNames.some((field) => !Object.is(was[field], now[field])) + ) { + counts.zeroSignEditsOverPending++ + } cuts = [sorted(model.visible().values())] if (step.inHandler && !open) { const batch: HandlerBatch = resolveSourceBatch(step.inHandler) @@ -1479,12 +1542,18 @@ export async function runOptimisticHistory( b: intent.row.b, } : plain(intent.row) - : Object.fromEntries( + : // An authored field is a change unless it is the same + // value; -0 and 0 are distinct row values (maintainer + // decision, 2026-10-29). + Object.fromEntries( Object.entries(step.fields).filter( ([key, value]) => - before.find((row) => row.id === intent.key)?.[ - key as keyof Fields - ] !== value, + !Object.is( + before.find((row) => row.id === intent.key)?.[ + key as keyof Fields + ], + value, + ), ), ), } @@ -1646,10 +1715,10 @@ export async function runOptimisticHistory( check(`${position}: ${JSON.stringify(step)}`) // Count events as well as final values; value-only oracles miss redundant // publications when a mutation moves into completed retention. + // Strict equality, so a row whose zero changed sign counts as changed. if ( step.type === `settle` && - JSON.stringify(before) === - JSON.stringify(sorted(model.visible().values())) + isDeepStrictEqual(before, sorted(model.visible().values())) ) { expect(deliveries, `unchanged settlement ${position}`).toBe( deliveredBefore, diff --git a/packages/db/tests/proxy-oracle.test.ts b/packages/db/tests/proxy-oracle.test.ts index 27d4f27593..9fc76a6570 100644 --- a/packages/db/tests/proxy-oracle.test.ts +++ b/packages/db/tests/proxy-oracle.test.ts @@ -9,6 +9,7 @@ import { oracleRuns, } from './oracle-config' import { declareOracle } from './oracle-declaration' +import { createOracleReach } from './oracle-reach' import type { FrozenRow, StoredRow } from './proxy-native-rows' /** @@ -121,8 +122,10 @@ function observeArrayCallbacks( const expectedCall = runArrayCallback(native, step) const actualCall = runArrayCallback(proxy, step) const expectedRow = copyCallbackRow(native) + // A value is changed unless it is the same value. -0 and 0 are distinct + // row values (maintainer decision, 2026-10-29), so `Object.is` decides. const changed = native.items.some( - (item, index) => item.value !== values[index], + (item, index) => !Object.is(item.value, values[index]), ) const changes: Partial = getChanges() return { @@ -269,36 +272,63 @@ describe(`native array callback oracle`, () => { }, ) + // -0 joins the values and deltas, so `-0 += 0` writes 0: a change of a + // zero's sign alone, which a draft must record like any other write. + const signedInteger = (min: number, max: number) => + fc.oneof(fc.integer({ min, max }), fc.constant(-0)) const step = fc.record({ method: fc.constantFrom(...callbackMethods), target: fc.integer({ min: 0, max: 5 }), - delta: fc.integer({ min: -5, max: 5 }), + delta: signedInteger(-5, 5), }) // A fixed and a random campaign. A path replay of // `proxy.callback-history` runs only the random campaign, from its path. const callbackHistory = fc.record({ - values: fc.array(fc.integer({ min: -10, max: 10 }), { + values: fc.array(signedInteger(-10, 10), { minLength: 1, maxLength: 6, }), steps: fc.tuple(step, step), }) - const callbackProperty = fc.property(callbackHistory, ({ values, steps }) => { - assertArrayCallbacks(observeArrayCallbacks(values, steps)) - }) + const callbackCampaign = (options: fc.Parameters<[unknown]>) => { + const reach = createOracleReach( + `proxy.callback-history`, + [`zero-sign-write`], + { minimum: 5 }, + ) + fc.assert( + fc.property(callbackHistory, ({ values, steps }) => { + reach.beginHistory() + const cuts = observeArrayCallbacks(values, steps) + // Reach: a step left an item whose value differs from its start only + // by the sign of a zero. + if ( + cuts.some((cut) => + cut.expectedRow.items.some( + (item, index) => + item.value === values[index] && + !Object.is(item.value, values[index]), + ), + ) + ) { + reach.reach(`zero-sign-write`) + } + assertArrayCallbacks(cuts) + }), + options as fc.Parameters<[{ values: Array; steps: unknown }]>, + ) + reach.assertReached() + } oracleReplayTest(it)( `matches native two-step callback histories (2026103)`, () => { - fc.assert(callbackProperty, { seed: 2026103, numRuns: oracleRuns(200) }) + callbackCampaign({ seed: 2026103, numRuns: oracleRuns(200) }) }, ) oracleReplayTest(it, `proxy.callback-history`)( `matches native two-step callback histories (random)`, () => { - fc.assert( - callbackProperty, - oraclePropertyOptions(200, `proxy.callback-history`), - ) + callbackCampaign(oraclePropertyOptions(200, `proxy.callback-history`)) }, ) From 9856fa510cb1a27d7bdf13ac3be0b9b88e7d1e41 Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 29 Oct 2026 21:58:48 -0600 Subject: [PATCH 10/64] test(conformance): generate complete deliveries with two rows for one key Add a `malformed` step to the eager live-source oracle: a complete delivery that holds two rows with one key, for drivers that declare `duplicate-key-delivery` and the law set's `duplicateInsert` policy. Under `error` the model expects PUB-4: no row written, nothing published, rows accepted earlier kept, and a sync failure carrying the adapter's own error (before ready, status `error` and `preload()` rejects; after it, a health `error`). Under `replace` it expects PUB-3: the later row wins. - Law: PUB-4 (a rejected delivery changes nothing) and PUB-3 (parameter `duplicateInsert`) from the adapter law set v18. - Reach: malformed-before-ready and malformed-after-ready (the latter with `complete-snapshot`), each declared with minimum 5. - RED: the first version assumed `error` for every driver and failed TrailBase (status `ready` where `error` was owed); the law set declares `duplicateInsert: replace` for TrailBase, so the model now takes the policy from the driver. - GREEN: reference 67/67; TrailBase 151/151; RxDB 23/23 (capability does not apply); Neon keeps its READY-4 and HEALTH-1 RED (no declared policy, so no capability). - Mutant: reference commits-malformed-prefix and ignores-malformed fail PUB-4. A first-row-wins edit of TrailBase's eager load failed PUB-3 (duplicateInsert: replace) in both campaigns, then was reverted. - Limits: one malformed shape; no malformed keys, undecodable rows, or FID-3 out-of-domain values; provider release at a rejection is not checked. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 6 +- packages/db-adapter-conformance/MIGRATION.md | 17 ++ packages/db-adapter-conformance/README.md | 13 +- .../src/eager-live-source-oracle.ts | 161 ++++++++++++++++-- .../reference-adapter-oracle.property.test.ts | 2 + .../tests/reference-adapter.ts | 30 +++- .../tests/conformance-driver.ts | 7 +- 7 files changed, 212 insertions(+), 24 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 1319ed9152..4132d6e0b4 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6007,8 +6007,10 @@ READY-4 and HEALTH-1 until that adapter moves to `fail()`; probes show LIFE-10 on-demand loading, mutation handlers, or reload; publications are observed through `subscribeChanges` only; EVID-3 does not distinguish a timed wait that rejects before its timeout; FID-1 covers ten value kinds, not Temporal, -Map, Set, typed arrays, class instances, or field identity; the FID-3 and -PUB-4 rejection lane is not built; the Neon and TrailBase claims cover +Map, Set, typed arrays, class instances, or field identity; PUB-4 covers a +complete delivery with two rows for one key, under each driver's +`duplicateInsert` (TrailBase `replace`, reference `error`), not malformed +keys, undecodable rows, or FID-3 out-of-domain values; the Neon and TrailBase claims cover controlled providers, not real clients. Owner: `packages/db-adapter-conformance`. Evidence: the package README. Migration goal (maintainer, 2026-10-29): every collection adapter gets a diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index be78f0cf88..23b4a2d976 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -203,3 +203,20 @@ document update removes`) failed the same way. `Object.is`. `createStateDiff` is rejected by generated histories; `reconcileStalePublishedChanges` only by a pinned replay, since generated histories never reach retained rows (coverage map). +- **2026-10-29, eager live source, rejected deliveries (PUB-4).** A driver + whose provider can deliver two rows with one key declares + `duplicate-key-delivery` and the law set's `duplicateInsert` policy. A + `malformed` step delivers such a complete result. Under `error` the + model expects PUB-4: no row written, nothing published, earlier rows + kept, and a sync failure with the adapter's own error. Under `replace` it + expects PUB-3: the later row wins. + - **Reference:** `error`; mutants `commits-malformed-prefix` and + `ignores-malformed` fail PUB-4. + - **TrailBase:** `replace` (law set); GREEN. A first-row-wins edit of its + eager load was rejected with PUB-3 (duplicateInsert: replace) by both + campaigns, then reverted. The first run assumed `error` for every + driver and failed TrailBase; the law set's policy parameter settled it. + - **Neon:** the law set declares no `duplicateInsert` for Neon, so its + driver does not declare the capability. + - **RxDB:** its storage cannot hold two documents with one key, so the + capability does not apply. diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md index b5df687c17..58060fa85f 100644 --- a/packages/db-adapter-conformance/README.md +++ b/packages/db-adapter-conformance/README.md @@ -31,6 +31,7 @@ comment states the laws and their limits. In short: | READY-8, READY-10 | With `listen-before-fetch`: no fetch before the listener is live; batches committed during the fetch apply after its result, before ready | | PUB-5, PUB-9 | With `listen-before-fetch`: the cut publishes only backend states at or after the read point, in order; no replay regresses a row | | READY-3, READY-10 | When status becomes `ready`, the rows already equal the initial cut | +| PUB-4, PUB-3 | With `duplicate-key-delivery`: a complete delivery holding two rows with one key follows the driver's `duplicateInsert` policy. Under `error` it is rejected: no row written, nothing published, earlier rows kept, a sync failure reported. Under `replace` the later row wins | The law IDs come from the adapter conformance law set, version 18. @@ -44,6 +45,7 @@ Implement `EagerLiveSourceDriver`: - **`changeCommits`:** `per-batch` if your adapter commits a provider batch as one transaction, `per-change` if it commits each change alone. - **`declaredDiagnostic`:** optional; accepts console diagnostics your adapter documents, so HEALTH-5 does not reject them. - **`seed` and `fetchesBeforeListening`:** with `listen-before-fetch`, the provider holds backend rows set by `seed`, treats a batch before the first ready as a backend commit during the fetch, and counts fetches issued while its listener was not live (READY-8). +- **`duplicateInsert`:** with `duplicate-key-delivery`, the law set's policy for an insert of a held key, `error` or `replace`. - **`valueKinds`:** the kinds of FID-1's `valueDomain` your backend represents; `jsonValueKinds` for a JSON backend. The driver delivers each value in the backend's own form and the Collection must show it decoded. - **`observedView`:** if your rows or keys differ from the conformance row, wrap the Collection with `observedView` and a one-to-one mapping. `conformanceKey` maps string keys; `toSqlRow` and `fromSqlRow` map an absent `note` to and from SQL `NULL`. @@ -128,7 +130,7 @@ reach counter, so the run fails when histories miss a state the laws need. 20 other seeds, the lowest count for any state was 12 (`field-removed`), across 12 seeds. - **GREEN:** `pnpm exec vitest run tests/reference-adapter-oracle.property.test.ts`: - 65 tests pass (both campaigns, the per-change pair, 30 calibrations, and the + 67 tests pass (both campaigns, the per-change pair, 32 calibrations, and the grammar controls). TrailBase: its full suite, 151 tests, passes. - **Mutants:** the fixed campaign rejects each with the law it breaks: @@ -164,6 +166,8 @@ reach counter, so the run fails when histories miss a state the laws need. | `bigints-as-numbers` (initial load only) | FID-1 (values) | | `drops-nulls` | FID-1 (values) | | `loses-negative-zero` | FID-1 (values) | + | `commits-malformed-prefix` | PUB-4 | + | `ignores-malformed` | PUB-4 | | `shows-buffered` | READY-10/PUB-9 | `reverses-batch` reverses only batches of distinct keys, so its final rows @@ -210,8 +214,11 @@ reach counter, so the run fails when histories miss a state the laws need. `null`, Date, bigint, array, plain object). Temporal values, Map, Set, typed arrays, and class instances are not generated, and a stored field's reference identity is not checked. - - The rejection side, FID-3 and PUB-4 (an out-of-domain value fails its - delivery and writes no row), has no lane yet. + - PUB-4 covers one malformed shape, two rows with one key in a complete + delivery. A malformed key, a row the adapter cannot decode, and an + out-of-domain value (FID-3) are not generated: a driver needs a backend + form for each. Whether an adapter releases its provider subscription + when it rejects a delivery is not checked; no law fixes it. - READY-8's fetch order is checked only by a provider that implements `fetchesBeforeListening`. Commit receipts resolve at once in every driver, so a live commit whose receipt is still pending at the cut diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 49060f89fd..23f5404db3 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -74,6 +74,17 @@ * publication is a backend state at or after the read point, in order. * - **READY-3, READY-10.** When status becomes `ready`, the rows already * equal the initial cut. A `status:ready` listener never sees fewer rows. + * - **PUB-4, PUB-3** (capability `duplicate-key-delivery`, parameter + * `duplicateInsert`). A complete delivery can hold two rows with one key. + * Under `replace`, the later row wins (PUB-3) and the delivery otherwise + * applies as a complete delivery. Under `error`, the adapter rejects it + * (PUB-4): no row of it is written and nothing publishes; rows accepted + * earlier stay. The failure + * is reported as a sync failure: before the first ready, status `error` + * and `preload()` rejects; after it, status stays `ready` with a health + * `error`. The error is the adapter's own, so its value is not fixed. + * Whether the adapter releases its provider subscription at that point is + * not fixed either; cleanup still releases each one exactly once. * * Authority: the sync callback contract in `packages/db/src/types.ts` * (`begin`, `commit`, `markReady`, `fail`, `reportError`, `reportHealthy`), @@ -154,6 +165,7 @@ export type EagerLiveSourceCapability = | `capability-renewal` | `retry-policy` | `listen-before-fetch` + | `duplicate-key-delivery` /** * The public Collection surface the oracle reads. A driver passes the @@ -243,6 +255,11 @@ export interface EagerLiveSourceDriver { * own form, and the Collection must show it decoded. */ readonly valueKinds?: ReadonlyArray + /** + * The law set's `duplicateInsert` policy for an insert of a key the run + * holds; required with `duplicate-key-delivery`. + */ + readonly duplicateInsert?: `error` | `replace` /** * Console diagnostics the adapter declares (HEALTH-5). Any other * `console.error` or `console.warn` call is a violation. @@ -276,6 +293,8 @@ export interface EagerLiveSourceDriver { * an applied batch (`ahead` 0) or a future one. With `timeout`, the wait * carries a short timeout. * - `fail`: the provider stops. + * - `malformed`: a complete delivery holding two rows with one key, legal + * where a complete delivery is (PUB-4). * - `transient-failure`: a failure the adapter keeps retrying. * - `disconnect`, `reconnect`: transport loss and return. * - `renew`: the adapter's renewal of an expiring capability completes. Legal @@ -305,6 +324,16 @@ export type EagerLiveSourceStep = readonly timeout?: boolean } | { readonly kind: `fail` } + | { + /** + * A complete delivery of `rows` plus a second row for the key of + * `rows[duplicate % rows.length]` with another title (capability + * `duplicate-key-delivery`). + */ + readonly kind: `malformed` + readonly rows: ReadonlyArray + readonly duplicate: number + } | { readonly kind: `transient-failure` } | { readonly kind: `disconnect` } | { readonly kind: `reconnect` } @@ -438,6 +467,25 @@ export function eagerLiveSourceHistoryArb( { weight: 2, arbitrary: fc.constant({ kind: `fail` as const }) }, { weight: 1, arbitrary: fc.constant({ kind: `restart` as const }) }, ] + if (capabilities.has(`duplicate-key-delivery`)) { + steps.push({ + weight: 1, + arbitrary: fc + .record({ + rows: fc.uniqueArray(rowArb(kinds), { + selector: (row) => row.id, + minLength: 1, + maxLength: 3, + }), + duplicate: fc.nat(2), + }) + .map(({ rows, duplicate }) => ({ + kind: `malformed` as const, + rows, + duplicate, + })), + }) + } if (capabilities.has(`await-evidence`)) { // One wait in four is timed. A timed wait settles at its own step, so // only untimed waits can be fulfilled by a later batch. @@ -534,7 +582,8 @@ interface ModelState { lastTxid: number committed: Set renewed: boolean - failure: number | undefined + /** A provider failure's index, or `adapter` for the adapter's own error. */ + failure: number | `adapter` | undefined transient: Array waits: Array<{ run: number; txid: number; expected: WaitExpectation }> } @@ -544,13 +593,14 @@ export interface ExpectedObservation { status: `loading` | `ready` | `error` rows: RowState /** Index of the failure the health node's `error` must carry, or none. */ - healthError: number | null + healthError: number | `adapter` | null /** Indices of the failures `failureReason.errors` must list, in order. */ failureReason: ReadonlyArray - preload: `pending` | `fulfilled` | { rejectedWith: number } + preload: `pending` | `fulfilled` | { rejectedWith: number | `adapter` } waits: ReadonlyArray /** Unreleased provider subscriptions: one while the provider runs. */ - openSubscriptions: number + /** Undefined when no law fixes it: after the adapter rejected a delivery. */ + openSubscriptions: number | undefined /** Subscriptions opened in total: one per sync run, renewal included. */ openedSubscriptions: number renewalsThisRun: number @@ -718,10 +768,7 @@ function flipZeros(value: unknown): unknown { Object.getPrototypeOf(value) === Object.prototype ) { return Object.fromEntries( - Object.entries(value).map(([key, item]) => [ - key, - flipZeros(item), - ]), + Object.entries(value).map(([key, item]) => [key, flipZeros(item)]), ) } return value @@ -817,6 +864,13 @@ type AcceptedStep = } | { kind: `wait`; txid: number; timeout: boolean } | { kind: `fail`; failure: number } + | { + kind: `malformed` + rows: ReadonlyArray + afterReady: boolean + /** `duplicateInsert: replace` applied the later row. */ + replaced: boolean + } | { kind: `transient-failure`; failure: number } | { kind: `disconnect` } | { kind: `reconnect` } @@ -834,6 +888,7 @@ interface ModelOptions { listenBeforeFetch: boolean /** The value kinds include `negative-zero`, so a retouch flips zeros. */ flipZeros: boolean + duplicateInsert: `error` | `replace` } /** EVID-1: a wait for `txid` fulfills once its batch applied. */ @@ -1042,6 +1097,42 @@ function stepModel( } return { done: { kind: `fail`, failure }, publishable: [] } } + case `malformed`: { + if (!canDeliver(state)) return undefined + // PUB-4: the adapter rejects the delivery. No row of it is written, + // so the rows stay; the sync run fails with the adapter's own error. + const base = step.rows[step.duplicate % step.rows.length]! + const titles = [`a`, `b`, `c`].filter((title) => title !== base.title) + const rows = [...step.rows, { ...base, title: titles[0]! }] + const afterReady = state.phase === `ready` + if (options.duplicateInsert === `replace`) { + // PUB-3: the later row wins; the delivery applies as a complete one. + // Before the first ready of a listen-before-fetch run, the cut's + // read point would need its own rule, so the step is not legal there. + if (options.listenBeforeFetch && !afterReady) return undefined + state.rows = rowMap(rows) + state.backend = rowMap(rows) + state.phase = `ready` + state.transient = [] + return { + done: { kind: `malformed`, rows, afterReady, replaced: true }, + publishable: [rowState(state.rows)], + } + } + state.failure = `adapter` + state.phase = afterReady ? `stopped` : `loading-error` + state.transient = [] + state.buffered = [] + for (const wait of state.waits) { + if (wait.run === state.run && wait.expected === `pending`) { + wait.expected = `unchecked` + } + } + return { + done: { kind: `malformed`, rows, afterReady, replaced: false }, + publishable: [], + } + } case `transient-failure`: { if (!canDeliver(state)) return undefined const failure = failures.next++ @@ -1110,7 +1201,11 @@ function expectedObservation(state: ModelState): ExpectedObservation { : `fulfilled`, waits: state.waits.map((wait) => wait.expected), // LIFE-10: a failed provider's source is released at its end. - openSubscriptions: running ? 1 : 0, + openSubscriptions: running + ? 1 + : state.failure === `adapter` + ? undefined + : 0, openedSubscriptions: state.run, renewalsThisRun: state.renewed ? 1 : 0, } @@ -1140,6 +1235,8 @@ export const eagerLiveSourceReachStates = [ `read-missed-change`, `seen-change-superseded`, `field-removed`, + `malformed-before-ready`, + `malformed-after-ready`, ] as const /** @@ -1167,6 +1264,8 @@ const capabilityOfState: Partial< 'change-before-ready': `listen-before-fetch`, 'read-missed-change': `listen-before-fetch`, 'seen-change-superseded': `listen-before-fetch`, + 'malformed-before-ready': `duplicate-key-delivery`, + 'malformed-after-ready': `duplicate-key-delivery`, } /** The declared states a driver's declarations let histories reach. */ @@ -1178,6 +1277,12 @@ export function reachStatesFor( const capability = capabilityOfState[state] return capability === undefined || domain.capabilities.has(capability) }) + // A malformed delivery after ready also needs `complete-snapshot`. + const reachable = fixed.filter( + (state) => + state !== `malformed-after-ready` || + domain.capabilities.has(`complete-snapshot`), + ) const values = (domain.valueKinds ?? []).flatMap( (kind): Array => [ `value:${kind}:load`, @@ -1189,7 +1294,7 @@ export function reachStatesFor( ) ? [`zero-sign-flipped`] : [] - return [...fixed, ...values, ...zeros] + return [...reachable, ...values, ...zeros] } /** JSON for a message; a bigint shows as `n`, `-0` as `-0`. */ @@ -1265,6 +1370,8 @@ function lawFor(done: AcceptedStep, wasReady: boolean): string { return done.timeout ? `EVID-3` : `EVID-1` case `fail`: return wasReady ? `HEALTH-1` : `READY-4` + case `malformed`: + return done.replaced ? `PUB-3 (duplicateInsert: replace)` : `PUB-4` case `transient-failure`: return wasReady ? `HEALTH-1` : `RETRY-1` case `disconnect`: @@ -1328,6 +1435,13 @@ export async function checkEagerLiveSourceHistory( changeCommits: driver.changeCommits, listenBeforeFetch, flipZeros: driver.valueKinds?.includes(`negative-zero`) === true, + duplicateInsert: driver.duplicateInsert ?? `error`, + } + if ( + driver.capabilities.has(`duplicate-key-delivery`) && + driver.duplicateInsert === undefined + ) { + throw new Error(`duplicate-key-delivery needs a duplicateInsert policy`) } // With `listen-before-fetch`, a sync run's backend starts with the rows of // its first complete delivery, committed before the adapter subscribes. @@ -1472,6 +1586,8 @@ export async function checkEagerLiveSourceHistory( const health = collection.health if (expected.healthError === null) { check(`health error`, health.error === null, null, health.error?.message) + } else if (expected.healthError === `adapter`) { + check(`health error`, health.error !== null, `an error`, null) } else { check( `health error cause`, @@ -1508,7 +1624,8 @@ export async function checkEagerLiveSourceHistory( typeof want === `string` ? got === want : typeof got === `object` && - got.rejected === failure(want.rejectedWith), + (want.rejectedWith === `adapter` || + got.rejected === failure(want.rejectedWith)), want, typeof got === `object` ? { rejected: String(got.rejected) } : got, ) @@ -1542,7 +1659,10 @@ export async function checkEagerLiveSourceHistory( const releases = provider.releases() const open = releases.filter((count) => count === 0).length const failed = state.phase === `loading-error` || state.phase === `stopped` - if (open !== expected.openSubscriptions) { + if ( + expected.openSubscriptions !== undefined && + open !== expected.openSubscriptions + ) { throw violation( `${failed ? `LIFE-10` : law} (open provider subscriptions)`, stepIndex, @@ -1662,6 +1782,14 @@ export async function checkEagerLiveSourceHistory( reach(wasReady ? `failure-after-ready` : `failure-before-ready`) break } + case `malformed`: + provider.deliverSnapshot(structuredClone(done.rows)) + reach( + done.afterReady + ? `malformed-after-ready` + : `malformed-before-ready`, + ) + break case `transient-failure`: { const error = new Error( `conformance transient failure ${done.failure}`, @@ -1713,7 +1841,9 @@ export async function checkEagerLiveSourceHistory( ? `PUB-5/PUB-9` : done.kind === `changes` && done.beforeReady ? `READY-10/PUB-9` - : `PUB-1`, + : done.kind === `malformed` && !done.replaced + ? `PUB-4` + : `PUB-1`, ) } else { events.take() @@ -1813,6 +1943,11 @@ function supports( switch (step.kind) { case `snapshot`: return !ready || driver.capabilities.has(`complete-snapshot`) + case `malformed`: + return ( + driver.capabilities.has(`duplicate-key-delivery`) && + (!ready || driver.capabilities.has(`complete-snapshot`)) + ) case `wait`: return driver.capabilities.has(`await-evidence`) && !!fixture.awaitTxId case `transient-failure`: diff --git a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts index b1fcd4ba5b..8651690ca1 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts @@ -119,6 +119,8 @@ describe(`eager live-source conformance: reference adapter`, () => { [`bigints-as-numbers`, `FID-1 (values)`], [`drops-nulls`, `FID-1 (values)`], [`loses-negative-zero`, `FID-1 (values)`], + [`commits-malformed-prefix`, `PUB-4`], + [`ignores-malformed`, `PUB-4`], [`relabels-inserts`, `PUB-2`], ] oracleReplayTest(it).each(mutants)( diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index c7fb4e1139..b84bf7385f 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -83,6 +83,10 @@ export type ReferenceMutant = | `drops-nulls` /** FID-1: `-0` is stored as `0`. */ | `loses-negative-zero` + /** PUB-4: a rejected delivery commits the rows written before the failure. */ + | `commits-malformed-prefix` + /** PUB-4: a duplicate key is skipped and the delivery applies. */ + | `ignores-malformed` /** * PUB-2, checker calibration: the observed view relabels insert events as * updates. The adapter is unchanged; the comparison must see the events. @@ -330,12 +334,25 @@ function referenceCollection( } void commit() } else { + // PUB-4: a delivery that cannot apply, such as one holding two + // rows with one key, writes nothing and fails the sync run. begin({ snapshot: true }) - for (const row of event.rows) - write({ - type: `insert`, - value: storedRow(row, `load`, mutant), - }) + const written = new Set() + try { + for (const row of event.rows) { + if (mutant === `ignores-malformed` && written.has(row.id)) + continue + write({ + type: `insert`, + value: storedRow(row, `load`, mutant), + }) + written.add(row.id) + } + } catch (error) { + if (mutant === `commits-malformed-prefix`) void commit() + stop(subscription, error as Error) + return + } void commit() keys = new Set(event.rows.map((row) => row.id)) } @@ -475,6 +492,7 @@ export function referenceDriver( `transport-reconnect`, `capability-renewal`, `listen-before-fetch`, + `duplicate-key-delivery`, ]), valueKinds: [ `string`, @@ -489,6 +507,8 @@ export function referenceDriver( `object`, ], changeCommits: perChange ? `per-change` : `per-batch`, + // A delivery holding two rows with one key fails the run (PUB-4). + duplicateInsert: `error`, create(): EagerLiveSourceFixture { const provider = new ControlledPushProvider() const { collection, awaitTxId } = referenceCollection( diff --git a/packages/trailbase-db-collection/tests/conformance-driver.ts b/packages/trailbase-db-collection/tests/conformance-driver.ts index 46cc5ca6c0..df4154a3e5 100644 --- a/packages/trailbase-db-collection/tests/conformance-driver.ts +++ b/packages/trailbase-db-collection/tests/conformance-driver.ts @@ -157,8 +157,13 @@ class TrailBaseProvider implements ControlledProvider { export function trailBaseConformanceDriver(): EagerLiveSourceDriver { return { adapter: `@tanstack/trailbase-db-collection (eager)`, - capabilities: new Set(), + // A paged list can return one record twice, for example when a write + // moves it between pages (PUB-4). + capabilities: new Set([`duplicate-key-delivery`]), changeCommits: `per-change`, + // The law set declares `duplicateInsert: replace` for TrailBase: page + // rows and stream events are complete rows, and the later one wins. + duplicateInsert: `replace`, valueKinds: jsonValueKinds, create(): EagerLiveSourceFixture { const provider = new TrailBaseProvider() From 9f0841af80f25eddcd0f9bbd4e8042bb7a4a3223 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 07:29:17 -0600 Subject: [PATCH 11/64] test(db): generate failed-replay histories in the publication oracle The lifecycle publication oracle's random histories insert source writes into green lifecycle histories, so none wrote over a row a failed replay retained, and `reconcileStalePublishedChanges` had only a pinned replay. Add a generated failed-replay history, drawn one time in four: source writes, a load, a truncate, a rejected replay, the final owner's release, and more source writes, each part generated. Declare reach for writes over retained rows (at least five) and for zero-sign ones. The new histories reached one model rule no history had: the model published a write equal to the installed source row to a subscriber still showing a retained older value. Production publishes nothing. PUB-6 (equal values do not churn) and this oracle's own law ("a later request delivers its installed row, so a row that the subscriber still shows at an older value is updated then") support production, so the model now publishes nothing for such a write and keeps the key retained. - Law: PUB-2 and PUB-6 for retained rows after a failed replay, with -0 and 0 distinct (maintainer decision, 2026-10-29). - Reach: the fixed campaign writes over retained rows at least five times and over a retained zero with the other sign at least once. - RED: with `reconcileStalePublishedChanges` comparing by deepEquals, the fixed and random campaigns, the reach test, and the pinned -0 replay fail; before, only the pinned replay failed. - GREEN: the publication oracle 140/140 on three runs; subscription suites and the replay manifest 777/777. - Limits: no mutant shows the corrected equal-write rule rejecting a production that publishes such a write; core suppresses the write before the subscription sees it. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 13 ++-- packages/db-adapter-conformance/MIGRATION.md | 10 +++ ...ecycle-publication-oracle.property.test.ts | 75 +++++++++++++++++-- 3 files changed, 84 insertions(+), 14 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 4132d6e0b4..398ec177f4 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -5985,12 +5985,13 @@ detection (`optimistic-history-oracle`, with a `flipZeros` edit and `zeroSignEditsOverPending` reach), subscription truncate replay (`collection-subscription-replay-oracle`), and stale-row reconciliation (`collection-subscription-lifecycle-publication-oracle`) all compare row -values with signed zeros. Open: generated publication histories never write -over a row a failed replay retained (the random grammar inserts source -writes into green lifecycle histories), so that reconcile path has only -pinned replays, including a `-0` one (owner: the lifecycle publication -oracle; witness needed: a generated failed-replay history with later source -writes). Every driver is also +values with signed zeros. A generated failed-replay +history in the lifecycle publication oracle (source writes, a load, a +truncate, a rejected replay, the final owner's release, more source writes) +reaches source writes over retained rows; both campaigns reject a stale +reconcile without signed zeros. Under PUB-6 and that oracle's law, a write +equal to the installed source row publishes nothing, even to a subscriber +that still shows a retained older value. Every driver is also checked for the rows at the moment status becomes `ready`. Its model is an independent row Map and sync-run phase; drivers supply only a controlled provider and declare whether they commit per batch or per change. PUB-1 checks every change publication during diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index 23b4a2d976..444b691964 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -220,3 +220,13 @@ document update removes`) failed the same way. driver does not declare the capability. - **RxDB:** its storage cannot hold two documents with one key, so the capability does not apply. +- **2026-10-30, retained rows in core's publication oracle.** The lifecycle + publication oracle gained a generated failed-replay history, drawn one + time in four, and declared reach for source writes over retained rows (at + least five) and zero-sign ones. Its generated campaigns now reject + `reconcileStalePublishedChanges` with deepEquals, which only a pinned + replay rejected before. The new histories exposed one model rule no + history had reached: the model published a write equal to the installed + source row to a subscriber that still showed a retained older value. + Production publishes nothing, as PUB-6 and the oracle's own law (a later + request updates that row) require; the model now agrees. diff --git a/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts b/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts index 2af7158204..d38b5480f6 100644 --- a/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts @@ -116,8 +116,11 @@ function withFlippedZero( : command } -// Reach: source writes that change a public row only by the sign of a zero. +// Reach: source writes that change a public row only by the sign of a zero, +// and source writes over a row a failed replay or cleanup retained. let zeroSignWrites = 0 +let retainedWrites = 0 +let zeroSignRetainedWrites = 0 type PublicationCommand = | Exclude @@ -208,9 +211,19 @@ function recordSourceWrite(publication: PublicationModel, row: Row): void { // Retained public rows take precedence when reconciling a stale snapshot. const previous = publication.visible.get(row.id) ?? publication.source.get(row.id) - if (previous?.value === row.value && !Object.is(previous.value, row.value)) { - zeroSignWrites++ + const zeroSign = + previous?.value === row.value && !Object.is(previous.value, row.value) + if (zeroSign) zeroSignWrites++ + if (publication.retainedKeys.has(row.id)) { + retainedWrites++ + if (zeroSign) zeroSignRetainedWrites++ } + // A write of the value the source already holds changes nothing in the + // Collection, so it publishes nothing (PUB-6), even to a subscriber that + // still shows a retained older value. That subscriber is updated by a + // later change or request, as this file's law states. + const installed = publication.source.get(row.id) + if (installed !== undefined && Object.is(installed.value, row.value)) return publication.source.set(row.id, cloneRow(row)) publication.retainedKeys.delete(row.id) // The same value publishes nothing; -0 and 0 are distinct row values @@ -576,7 +589,7 @@ const sourceMutationArbitrary: fc.Arbitrary = fc.oneof( // Truncate-with-replacement rows are outside this random grammar and have // separate fixed cases. A malformed sync transaction with an unmatched begin // is excluded: each generated source mutation commits as one transaction. -const publicationCommandHistoryArbitrary: fc.Arbitrary< +const randomPublicationHistoryArbitrary: fc.Arbitrary< Array > = greenLifecycleHistoryArbitrary.chain((history) => fc @@ -598,6 +611,51 @@ const publicationCommandHistoryArbitrary: fc.Arbitrary< }), ) +/** + * A failed replay that leaves public rows retained, then source writes over + * them: rows the source wrote, a demand that loads, a truncate, a rejected + * replay, and the final owner's release, with generated source writes + * before, during, and after. A random lifecycle history rarely puts these + * in this order, and only this order reaches the stale reconcile of a + * retained row. + */ +const failedReplayHistoryArbitrary: fc.Arbitrary> = fc + .record({ + before: fc.array(sourceMutationArbitrary, { minLength: 1, maxLength: 3 }), + during: fc.array(sourceMutationArbitrary, { maxLength: 2 }), + after: fc.array(sourceMutationArbitrary, { minLength: 1, maxLength: 3 }), + demand: fc.constantFrom(`a`, `b`), + }) + .map(({ before, during, after, demand }): Array => [ + ...before, + { type: `request`, demand }, + { + type: `settle`, + demand, + scope: `current`, + age: `oldest`, + outcome: `resolve`, + }, + { type: `truncate` }, + ...during, + { + type: `settle`, + demand, + scope: `current`, + age: `oldest`, + outcome: `reject`, + }, + { type: `release`, demand }, + ...after, + ]) + +const publicationCommandHistoryArbitrary: fc.Arbitrary< + Array +> = fc.oneof( + { weight: 3, arbitrary: randomPublicationHistoryArbitrary }, + { weight: 1, arbitrary: failedReplayHistoryArbitrary }, +) + async function runPublicationHistory( history: ReadonlyArray, runOptions: PublicationRunOptions = {}, @@ -2464,11 +2522,10 @@ describe(`CollectionSubscription lifecycle publication oracle`, () => { }) const { multiplier, ...replay } = readOracleRunConfig() const runs = 60 * multiplier - // Generated histories never write over a retained row (a failed replay's - // carried row): the random grammar inserts source writes into green - // lifecycle histories. The pinned zero-sign history below covers that path. - it(`reaches a zero-sign source write in the fixed campaign`, async () => { + it(`reaches zero-sign and retained-row source writes in the fixed campaign`, async () => { zeroSignWrites = 0 + retainedWrites = 0 + zeroSignRetainedWrites = 0 for (const history of fc.sample(publicationCommandHistoryArbitrary, { seed: 1_657_005, numRuns: runs, @@ -2476,6 +2533,8 @@ describe(`CollectionSubscription lifecycle publication oracle`, () => { await runPublicationHistory(history) } expect(zeroSignWrites).toBeGreaterThan(0) + expect(retainedWrites).toBeGreaterThanOrEqual(5) + expect(zeroSignRetainedWrites).toBeGreaterThan(0) }, 120_000) fcTest.prop([publicationCommandHistoryArbitrary], { numRuns: runs, From 7958b8ee883448fce5b827ef0c82326774cd8e16 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 09:52:40 -0600 Subject: [PATCH 12/64] test(db): hold back the failed-replay publication generator Commit 9f0841af8 added a generated failed-replay history to the lifecycle publication oracle and, to make it pass, a model rule that let a subscriber keep a stale retained row after an equal source write. That rule contradicts the law that a subscriber that applied every change message equals a direct read. Remove the rule and, with it, the generator dimension and its reach counters: without the rule, the generator finds the counterexample on some seeds against today's production, and fixing production first needs the replay-convergence law settled (ARCHITECTURE.md law 9 states the opposite retention rule, and #2294 depends on the same rule). The stale reconcile keeps its pinned replays, including the -0 one, as before 9f0841af8. The coverage map records the open counterexample, its shrunk trace, and its owner. - GREEN: the publication oracle 140/140 on three runs. - Limits: generated publication histories do not reach retained rows. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 19 +++-- packages/db-adapter-conformance/MIGRATION.md | 21 +++--- ...ecycle-publication-oracle.property.test.ts | 75 ++----------------- 3 files changed, 31 insertions(+), 84 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 398ec177f4..c7f24ed5af 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -5985,13 +5985,18 @@ detection (`optimistic-history-oracle`, with a `flipZeros` edit and `zeroSignEditsOverPending` reach), subscription truncate replay (`collection-subscription-replay-oracle`), and stale-row reconciliation (`collection-subscription-lifecycle-publication-oracle`) all compare row -values with signed zeros. A generated failed-replay -history in the lifecycle publication oracle (source writes, a load, a -truncate, a rejected replay, the final owner's release, more source writes) -reaches source writes over retained rows; both campaigns reject a stale -reconcile without signed zeros. Under PUB-6 and that oracle's law, a write -equal to the installed source row publishes nothing, even to a subscriber -that still shows a retained older value. Every driver is also +values with signed zeros. Open: generated publication histories never write +over a row a failed replay retained (the random grammar inserts source +writes into green lifecycle histories), so the stale reconcile is reached +only by pinned replays, including a `-0` one. A generated failed-replay +history reached an open counterexample: after a failed, released replay, a +write equal to the committed row leaves the subscriber at its older retained +value while a direct read returns the new one, against the law that a +subscriber equals a direct read. That contradicts ARCHITECTURE.md law 9's +retention rule, so the generator is held back until the replay-convergence +law is settled with #2294 (owner: the lifecycle publication oracle; witness +needed: a generated failed-replay history; shrunk trace: source `a`=0, request, +settle resolve, truncate, source `a`=5, settle reject, release, source `a`=5). Every driver is also checked for the rows at the moment status becomes `ready`. Its model is an independent row Map and sync-run phase; drivers supply only a controlled provider and declare whether they commit per batch or per change. PUB-1 checks every change publication during diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index 444b691964..5e8af2bebe 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -220,13 +220,14 @@ document update removes`) failed the same way. driver does not declare the capability. - **RxDB:** its storage cannot hold two documents with one key, so the capability does not apply. -- **2026-10-30, retained rows in core's publication oracle.** The lifecycle - publication oracle gained a generated failed-replay history, drawn one - time in four, and declared reach for source writes over retained rows (at - least five) and zero-sign ones. Its generated campaigns now reject - `reconcileStalePublishedChanges` with deepEquals, which only a pinned - replay rejected before. The new histories exposed one model rule no - history had reached: the model published a write equal to the installed - source row to a subscriber that still showed a retained older value. - Production publishes nothing, as PUB-6 and the oracle's own law (a later - request updates that row) require; the model now agrees. +- **2026-10-30, retained rows in core's publication oracle: held back.** + A generated failed-replay history (one time in four) reached source writes + over retained rows. It exposed an open counterexample: after a failed, + released replay, a write equal to the committed row leaves the subscriber + stale while a direct read returns the new value. A model change that + accepted this was wrong under the law that a subscriber equals a direct + read, and was reverted. Fixing it needs the replay-convergence law settled + first (ARCHITECTURE.md law 9 says the opposite, and #2294 depends on the + same rule), so the generator dimension is removed until then and the + stale reconcile keeps its pinned replays. The shrunk trace and owner are + in the coverage map. diff --git a/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts b/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts index d38b5480f6..2af7158204 100644 --- a/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts @@ -116,11 +116,8 @@ function withFlippedZero( : command } -// Reach: source writes that change a public row only by the sign of a zero, -// and source writes over a row a failed replay or cleanup retained. +// Reach: source writes that change a public row only by the sign of a zero. let zeroSignWrites = 0 -let retainedWrites = 0 -let zeroSignRetainedWrites = 0 type PublicationCommand = | Exclude @@ -211,19 +208,9 @@ function recordSourceWrite(publication: PublicationModel, row: Row): void { // Retained public rows take precedence when reconciling a stale snapshot. const previous = publication.visible.get(row.id) ?? publication.source.get(row.id) - const zeroSign = - previous?.value === row.value && !Object.is(previous.value, row.value) - if (zeroSign) zeroSignWrites++ - if (publication.retainedKeys.has(row.id)) { - retainedWrites++ - if (zeroSign) zeroSignRetainedWrites++ + if (previous?.value === row.value && !Object.is(previous.value, row.value)) { + zeroSignWrites++ } - // A write of the value the source already holds changes nothing in the - // Collection, so it publishes nothing (PUB-6), even to a subscriber that - // still shows a retained older value. That subscriber is updated by a - // later change or request, as this file's law states. - const installed = publication.source.get(row.id) - if (installed !== undefined && Object.is(installed.value, row.value)) return publication.source.set(row.id, cloneRow(row)) publication.retainedKeys.delete(row.id) // The same value publishes nothing; -0 and 0 are distinct row values @@ -589,7 +576,7 @@ const sourceMutationArbitrary: fc.Arbitrary = fc.oneof( // Truncate-with-replacement rows are outside this random grammar and have // separate fixed cases. A malformed sync transaction with an unmatched begin // is excluded: each generated source mutation commits as one transaction. -const randomPublicationHistoryArbitrary: fc.Arbitrary< +const publicationCommandHistoryArbitrary: fc.Arbitrary< Array > = greenLifecycleHistoryArbitrary.chain((history) => fc @@ -611,51 +598,6 @@ const randomPublicationHistoryArbitrary: fc.Arbitrary< }), ) -/** - * A failed replay that leaves public rows retained, then source writes over - * them: rows the source wrote, a demand that loads, a truncate, a rejected - * replay, and the final owner's release, with generated source writes - * before, during, and after. A random lifecycle history rarely puts these - * in this order, and only this order reaches the stale reconcile of a - * retained row. - */ -const failedReplayHistoryArbitrary: fc.Arbitrary> = fc - .record({ - before: fc.array(sourceMutationArbitrary, { minLength: 1, maxLength: 3 }), - during: fc.array(sourceMutationArbitrary, { maxLength: 2 }), - after: fc.array(sourceMutationArbitrary, { minLength: 1, maxLength: 3 }), - demand: fc.constantFrom(`a`, `b`), - }) - .map(({ before, during, after, demand }): Array => [ - ...before, - { type: `request`, demand }, - { - type: `settle`, - demand, - scope: `current`, - age: `oldest`, - outcome: `resolve`, - }, - { type: `truncate` }, - ...during, - { - type: `settle`, - demand, - scope: `current`, - age: `oldest`, - outcome: `reject`, - }, - { type: `release`, demand }, - ...after, - ]) - -const publicationCommandHistoryArbitrary: fc.Arbitrary< - Array -> = fc.oneof( - { weight: 3, arbitrary: randomPublicationHistoryArbitrary }, - { weight: 1, arbitrary: failedReplayHistoryArbitrary }, -) - async function runPublicationHistory( history: ReadonlyArray, runOptions: PublicationRunOptions = {}, @@ -2522,10 +2464,11 @@ describe(`CollectionSubscription lifecycle publication oracle`, () => { }) const { multiplier, ...replay } = readOracleRunConfig() const runs = 60 * multiplier - it(`reaches zero-sign and retained-row source writes in the fixed campaign`, async () => { + // Generated histories never write over a retained row (a failed replay's + // carried row): the random grammar inserts source writes into green + // lifecycle histories. The pinned zero-sign history below covers that path. + it(`reaches a zero-sign source write in the fixed campaign`, async () => { zeroSignWrites = 0 - retainedWrites = 0 - zeroSignRetainedWrites = 0 for (const history of fc.sample(publicationCommandHistoryArbitrary, { seed: 1_657_005, numRuns: runs, @@ -2533,8 +2476,6 @@ describe(`CollectionSubscription lifecycle publication oracle`, () => { await runPublicationHistory(history) } expect(zeroSignWrites).toBeGreaterThan(0) - expect(retainedWrites).toBeGreaterThanOrEqual(5) - expect(zeroSignRetainedWrites).toBeGreaterThan(0) }, 120_000) fcTest.prop([publicationCommandHistoryArbitrary], { numRuns: runs, From dad3408581532c6b914e08f401ab056b813b1e55 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 11:12:42 -0600 Subject: [PATCH 13/64] test(conformance): allow asynchronous providers and a propagation step Extend the eager live-source oracle's driver contract for adapters whose provider is a real local database: - Provider steps may return a promise; the check awaits them. - A fixture may supply `settle()`, its law-set `propagationStep`, which the check awaits before every comparison, after the usual event-loop turns. A change watcher with a throttle needs it. - A post-ready `fail` step is now gated on `source-failure-after-ready`: a provider that can fail terminally after the first ready (the law set's `terminal-source-end`, or an `errorMapping` that reports a terminal failure). Without it, `fail` is generated only before ready and `failure-after-ready` is not a required reach state. The reference, TrailBase, RxDB, and Neon drivers declare it. - GREEN: reference 67/67; RxDB 23/23; TrailBase 151/151; Neon keeps its READY-4 and HEALTH-1 RED. - Limits: existing drivers neither return promises nor declare `settle`, so the new paths are exercised by the PowerSync driver in the next commit. Co-authored-by: Isaac --- .../src/eager-live-source-oracle.ts | 56 +++++++++++++------ .../tests/neon/neon-realtime-driver.ts | 2 + .../tests/reference-adapter.ts | 1 + .../tests/conformance-driver.ts | 6 +- .../tests/conformance-driver.ts | 6 +- 5 files changed, 51 insertions(+), 20 deletions(-) diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 23f5404db3..8d39721172 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -166,6 +166,7 @@ export type EagerLiveSourceCapability = | `retry-policy` | `listen-before-fetch` | `duplicate-key-delivery` + | `source-failure-after-ready` /** * The public Collection surface the oracle reads. A driver passes the @@ -198,17 +199,20 @@ export interface ControlledProvider { */ releases: () => ReadonlyArray /** Delivers a complete result: every row the source holds. */ - deliverSnapshot: (rows: ReadonlyArray) => void + deliverSnapshot: (rows: ReadonlyArray) => void | Promise /** Delivers one committed change batch that carries `txid`. */ - deliverChanges: (changes: ReadonlyArray, txid: string) => void + deliverChanges: ( + changes: ReadonlyArray, + txid: string, + ) => void | Promise /** The provider stops with `error` and delivers nothing more. */ - fail: (error: Error) => void + fail: (error: Error) => void | Promise /** A failure the adapter keeps retrying (capability `retry-policy`). */ - transientFailure?: (error: Error) => void + transientFailure?: (error: Error) => void | Promise /** Drops the transport (capability `transport-reconnect`). */ - disconnect?: () => void + disconnect?: () => void | Promise /** Restores the transport (capability `transport-reconnect`). */ - reconnect?: () => void + reconnect?: () => void | Promise /** * Supplies the replacement capability that the adapter requested and * resolves after the adapter renewed its provider subscription with it @@ -224,7 +228,7 @@ export interface ControlledProvider { * is a backend commit on top of them; `deliverSnapshot` then delivers the * fetch result the oracle computed. */ - seed?: (rows: ReadonlyArray) => void + seed?: (rows: ReadonlyArray) => void | Promise /** Fetches the adapter issued while its change listener was not live. */ fetchesBeforeListening?: () => number } @@ -237,6 +241,13 @@ export interface EagerLiveSourceFixture { awaitTxId?: (txid: string, timeout?: number) => Promise /** Releases fixture resources after the history's final cleanup. */ dispose?: () => void | Promise + /** + * The adapter's propagation step (the law set's `propagationStep`), when a + * delivery reaches the Collection later than a few event-loop turns, as + * through a throttled change watcher. The check awaits it before every + * comparison. + */ + settle?: () => Promise } /** A production driver: it builds a fresh fixture for each history. */ @@ -292,7 +303,11 @@ export interface EagerLiveSourceDriver { * - `wait`: `awaitTxId` for the last txid plus `ahead`, so a wait can target * an applied batch (`ahead` 0) or a future one. With `timeout`, the wait * carries a short timeout. - * - `fail`: the provider stops. + * - `fail`: the provider stops. After the first ready only with + * `source-failure-after-ready`: a provider that can end terminally then, as + * the law set's `terminal-source-end` or an `errorMapping` that reports a + * terminal failure describes. Without it, the adapter's provider has no + * failure after ready to deliver. * - `malformed`: a complete delivery holding two rows with one key, legal * where a complete delivery is (PUB-4). * - `transient-failure`: a failure the adapter keeps retrying. @@ -1265,6 +1280,7 @@ const capabilityOfState: Partial< 'read-missed-change': `listen-before-fetch`, 'seen-change-superseded': `listen-before-fetch`, 'malformed-before-ready': `duplicate-key-delivery`, + 'failure-after-ready': `source-failure-after-ready`, 'malformed-after-ready': `duplicate-key-delivery`, } @@ -1344,7 +1360,7 @@ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) /** * The checkpoint premise: the adapters in scope apply a delivery within a few - * event-loop turns. Three macrotask turns cover microtask receipts and one + * event-loop turns, or within the propagation step their fixture declares. Three macrotask turns cover microtask receipts and one * timer hop. A slower adapter shows as a mismatch at this cut, not as a pass. */ async function settle(): Promise { @@ -1460,7 +1476,7 @@ export async function checkEagerLiveSourceHistory( const failures: Array = [] const failureCounter = { next: 0 } const waits: Array<{ current: Settlement }> = [] - if (listenBeforeFetch) provider.seed?.(structuredClone(seedFrom(0))) + if (listenBeforeFetch) await provider.seed?.(structuredClone(seedFrom(0))) // READY-3, READY-10: the rows at the moment status becomes `ready`. let readyRows: RowState | undefined const watchReady = () => @@ -1533,6 +1549,7 @@ export async function checkEagerLiveSourceHistory( delivery: boolean = false, ) => { await settle() + await fixture.settle?.() const expected = expectedObservation(state) const rows = observedRows(collection) const check = (name: string, ok: boolean, want: unknown, got: unknown) => { @@ -1719,7 +1736,7 @@ export async function checkEagerLiveSourceHistory( const { done, publishable } = result switch (done.kind) { case `snapshot`: - provider.deliverSnapshot(structuredClone(done.rows)) + await provider.deliverSnapshot(structuredClone(done.rows)) for (const row of done.rows) { if (Object.hasOwn(row, `value`)) { reach(`value:${kindOf(row.value)}:load`) @@ -1732,7 +1749,7 @@ export async function checkEagerLiveSourceHistory( if (transientBefore > 0) reach(`transient-failure-recovered`) break case `changes`: - provider.deliverChanges( + await provider.deliverChanges( structuredClone(done.changes), String(done.txid), ) @@ -1778,12 +1795,12 @@ export async function checkEagerLiveSourceHistory( `conformance provider failure ${done.failure}`, ) failures[done.failure] = error - provider.fail(error) + await provider.fail(error) reach(wasReady ? `failure-after-ready` : `failure-before-ready`) break } case `malformed`: - provider.deliverSnapshot(structuredClone(done.rows)) + await provider.deliverSnapshot(structuredClone(done.rows)) reach( done.afterReady ? `malformed-after-ready` @@ -1795,15 +1812,15 @@ export async function checkEagerLiveSourceHistory( `conformance transient failure ${done.failure}`, ) failures[done.failure] = error - provider.transientFailure!(error) + await provider.transientFailure!(error) if (wasReady) reach(`transient-failure-reported`) break } case `disconnect`: - provider.disconnect!() + await provider.disconnect!() break case `reconnect`: - provider.reconnect!() + await provider.reconnect!() reach(`reconnect`) break case `renew`: @@ -1816,7 +1833,7 @@ export async function checkEagerLiveSourceHistory( await collection.cleanup() // The backend may change while no sync run listens. if (listenBeforeFetch) { - provider.seed?.(structuredClone(seedFrom(index + 1))) + await provider.seed?.(structuredClone(seedFrom(index + 1))) } readyRows = undefined watchReady() @@ -1867,6 +1884,7 @@ export async function checkEagerLiveSourceHistory( events.stop() await collection.cleanup() await settle() + await fixture.settle?.() if (primary === undefined) { retireWaits(state) primary = retiredWaitViolation(state, waits, history) @@ -1943,6 +1961,8 @@ function supports( switch (step.kind) { case `snapshot`: return !ready || driver.capabilities.has(`complete-snapshot`) + case `fail`: + return !ready || driver.capabilities.has(`source-failure-after-ready`) case `malformed`: return ( driver.capabilities.has(`duplicate-key-delivery`) && diff --git a/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts b/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts index 68adc905d2..8fb961a7fa 100644 --- a/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts +++ b/packages/db-adapter-conformance/tests/neon/neon-realtime-driver.ts @@ -261,6 +261,8 @@ export function neonRealtimeDriver(): EagerLiveSourceDriver { `await-evidence`, `transport-reconnect`, `capability-renewal`, + // Its live query's `error` state ends the subscription (markError). + `source-failure-after-ready`, ]), create(): EagerLiveSourceFixture { const provider = new NeonProvider() diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index b84bf7385f..8ef0cf7fa2 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -493,6 +493,7 @@ export function referenceDriver( `capability-renewal`, `listen-before-fetch`, `duplicate-key-delivery`, + `source-failure-after-ready`, ]), valueKinds: [ `string`, diff --git a/packages/rxdb-db-collection/tests/conformance-driver.ts b/packages/rxdb-db-collection/tests/conformance-driver.ts index 2e6eb8c21b..8868b3f6ce 100644 --- a/packages/rxdb-db-collection/tests/conformance-driver.ts +++ b/packages/rxdb-db-collection/tests/conformance-driver.ts @@ -284,7 +284,11 @@ function createRxdbCollection(provider: RxdbProvider) { export function rxdbConformanceDriver(): EagerLiveSourceDriver { return { adapter: `@tanstack/rxdb-db-collection`, - capabilities: new Set([`listen-before-fetch`]), + // `$` can error after ready (terminal-source-end). + capabilities: new Set([ + `listen-before-fetch`, + `source-failure-after-ready`, + ]), changeCommits: `per-change`, valueKinds: jsonValueKinds, create(): EagerLiveSourceFixture { diff --git a/packages/trailbase-db-collection/tests/conformance-driver.ts b/packages/trailbase-db-collection/tests/conformance-driver.ts index df4154a3e5..c3ad1e71ee 100644 --- a/packages/trailbase-db-collection/tests/conformance-driver.ts +++ b/packages/trailbase-db-collection/tests/conformance-driver.ts @@ -159,7 +159,11 @@ export function trailBaseConformanceDriver(): EagerLiveSourceDriver { adapter: `@tanstack/trailbase-db-collection (eager)`, // A paged list can return one record twice, for example when a write // moves it between pages (PUB-4). - capabilities: new Set([`duplicate-key-delivery`]), + // The change stream can error after ready (terminal-source-end). + capabilities: new Set([ + `duplicate-key-delivery`, + `source-failure-after-ready`, + ]), changeCommits: `per-change`, // The law set declares `duplicateInsert: replace` for TrailBase: page // rows and stream events are complete rows, and the later one wins. From 3cbd0ffe3412305d0e8306bbf9b75eaa749c2b48 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 11:12:50 -0600 Subject: [PATCH 14/64] fix(powersync): release the change watcher when the first load fails Run PowerSync on the eager live-source conformance oracle, in eager mode over a real local PowerSync database (SQLite through @powersync/node). A complete delivery writes the table and resolves the public `onLoad` hook the eager load awaits; a batch is one SQLite write transaction; a failure before ready rejects `onLoad`. The fixture's `settle` waits on the database side: for the initial load's diff trigger after a complete delivery, and for the watcher's callback after a batch. The oracle found that a failed first load kept the sync run's change watcher registered until cleanup. A failed first load ends its sync run, so the adapter now aborts the run's watcher and disposes any diff trigger it created. - Law: LIFE-10 (sources are released at their own end, CAP:live-changes) from the adapter law set v18, with law 16 (a failed first load is final for its sync run). - Reach: 400 histories per campaign; the lowest count of any declared state across four seeds is 16 (`field-removed`). - RED: on the parent, both campaigns failed LIFE-10 (open provider subscriptions), expected 0, observed 1, shrunk to `[fail]`. - GREEN: PowerSync suite 195/195 on four full-suite runs; two concurrent campaigns pass. - Mutant: the original defect (LIFE-10), dropped deletes in the diff drain (PUB-3/FID-4), one commit per diff operation (PUB-1). Ready before awaiting the initial receipts survives and is equivalent here: every receipt resolves at once without pending optimistic transactions. - Size: powersync-db-collection +6 bytes gzip. - Limits: local SQLite writes only, not the PowerSync sync service; no value kinds yet (typed columns need a schema); on-demand startup failures, cleanup waiting on an unsettled `onLoad`, and post-ready change-handler errors are logged only and recorded for other families. Co-authored-by: Isaac --- .changeset/powersync-failed-load-release.md | 5 + docs/contributing/oracle-coverage.md | 8 +- packages/db-adapter-conformance/MIGRATION.md | 66 +++- packages/db-adapter-conformance/README.md | 6 + packages/db/tests/oracle-config.ts | 1 + packages/db/tests/oracle-replay-manifest.ts | 7 +- .../powersync-db-collection/src/powersync.ts | 8 +- .../tests/conformance-driver.ts | 314 ++++++++++++++++++ ...source-conformance-oracle.property.test.ts | 78 +++++ 9 files changed, 476 insertions(+), 17 deletions(-) create mode 100644 .changeset/powersync-failed-load-release.md create mode 100644 packages/powersync-db-collection/tests/conformance-driver.ts create mode 100644 packages/powersync-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts diff --git a/.changeset/powersync-failed-load-release.md b/.changeset/powersync-failed-load-release.md new file mode 100644 index 0000000000..af4ed773fc --- /dev/null +++ b/.changeset/powersync-failed-load-release.md @@ -0,0 +1,5 @@ +--- +'@tanstack/powersync-db-collection': patch +--- + +Release the change watcher and any diff trigger when an eager Collection's first load fails, instead of keeping them until cleanup. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index c7f24ed5af..77c9fc821f 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6031,4 +6031,10 @@ two owners until this entry moves them. RxDB controlled RxCollection with RxDB's own query matcher and sort comparator, `listen-before-fetch`) is GREEN after a fix for a buffered insert that the initial read already reflects; `rxdb.test.ts` holds a receiving witness for -it on memory storage. +it on memory storage. PowerSync (`adapter-conformance.eager-live-source.powersync`, +eager mode over a real local PowerSync database, per-batch commits, a +database-side propagation step) is GREEN after a fix: a failed first load now +releases the run's change watcher (LIFE-10). Its startup failures in +on-demand mode, cleanup waiting on an unsettled `onLoad`, and post-ready +change-handler errors are logged only; they belong to the on-demand and +lifecycle families, which are not built. diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index 5e8af2bebe..fa8f0a71aa 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -27,23 +27,23 @@ reviewed change. ## Adapters -| Adapter | Families that apply (section A) | Driver status | -| ----------------- | ------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Neon Realtime | eager live source, mutations | Eager: RED (READY-4, HEALTH-1, and LIFE-10 behind them); adapter update pending | -| Electric | eager live source, mutations, on-demand, source shapes, reload and retry, persistence | Not started | -| PowerSync | eager live source, mutations, on-demand | Not started | -| RxDB | eager live source, mutations | Eager: GREEN (2026-10-29) after a fix, per-change commits, `listen-before-fetch`. Driver `rxdb-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.rxdb`. Mutations: not started | -| TrailBase | eager live source, mutations, on-demand | Eager: GREEN (2026-10-29), per-change commits. Driver `trailbase-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.trailbase`; declares no optional capability. Mutations, on-demand: not started | -| Convex | eager live source, mutations, on-demand, source shapes | Not started | -| query-db | client cache and refetch, on-demand, mutations, reload and retry, persistence | Not started; not a push source, so the eager family does not apply | -| local-only | local and shared stores, mutations | Not started | -| localStorage | local and shared stores, mutations | Not started | -| Persisted wrapper | persistence wrapper | Not started | -| Storage hosts (8) | storage hosts | Not started | +| Adapter | Families that apply (section A) | Driver status | +| ----------------- | ------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Neon Realtime | eager live source, mutations | Eager: RED (READY-4, HEALTH-1, and LIFE-10 behind them); adapter update pending | +| Electric | eager live source, mutations, on-demand, source shapes, reload and retry, persistence | Not started | +| PowerSync | eager live source, mutations, on-demand | Eager: GREEN (2026-10-30) after a fix, per-batch commits, over real local SQLite. Driver `powersync-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.powersync`. Mutations, on-demand: not started | +| RxDB | eager live source, mutations | Eager: GREEN (2026-10-29) after a fix, per-change commits, `listen-before-fetch`. Driver `rxdb-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.rxdb`. Mutations: not started | +| TrailBase | eager live source, mutations, on-demand | Eager: GREEN (2026-10-29), per-change commits. Driver `trailbase-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.trailbase`; declares no optional capability. Mutations, on-demand: not started | +| Convex | eager live source, mutations, on-demand, source shapes | Not started | +| query-db | client cache and refetch, on-demand, mutations, reload and retry, persistence | Not started; not a push source, so the eager family does not apply | +| local-only | local and shared stores, mutations | Not started | +| localStorage | local and shared stores, mutations | Not started | +| Persisted wrapper | persistence wrapper | Not started | +| Storage hosts (8) | storage hosts | Not started | ## Suggested order -1. **Push adapters on the existing family:** RxDB (done), TrailBase (done), PowerSync, Electric (eager mode), and Convex. Each needs only a controlled-provider driver. A RED result is a finding against that adapter. +1. **Push adapters on the existing family:** RxDB (done), TrailBase (done), PowerSync (done), Electric (eager mode), and Convex. Each needs only a controlled-provider driver. A RED result is a finding against that adapter. 2. **The Neon update, run against its driver.** This shows an adapter moving from RED to GREEN. 3. **The mutations family.** It applies to every adapter except the storage hosts. 4. **The other families,** one at a time. Each family's new driver work goes to the adapters that declare it. @@ -231,3 +231,41 @@ document update removes`) failed the same way. same rule), so the generator dimension is removed until then and the stale reconcile keeps its pinned replays. The shrunk trace and owner are in the coverage map. +- **2026-10-30, PowerSync, eager live source: one PowerSync defect.** The + driver runs the real adapter over a real local PowerSync database + (SQLite through `@powersync/node`). A complete delivery writes the table + and then resolves the public `onLoad` hook the eager load awaits; a batch + is one SQLite write transaction; a failure before ready rejects `onLoad`. + A release is the abort of the change-watcher registration's signal. + - **Oracle changes this exposed:** provider steps may be asynchronous, and + a fixture may declare its propagation step (`settle`, the law set's + `propagationStep`): PowerSync's changes reach the Collection through a + throttled watcher. `settle` waits, on the database side, for the initial + load's diff trigger after a complete delivery and for the watcher's + callback after a batch. Post-ready `fail` is now a capability + (`source-failure-after-ready`): the law set gives PowerSync no + `terminal-source-end`. Without it, histories still fail before ready. + - **RED:** both campaigns failed LIFE-10 (open provider subscriptions): + after a failed first load, the run's change watcher stayed registered + until cleanup. Shrunk: `[fail]`. + - **Fix:** a failed first load aborts the run's watcher and disposes any + diff trigger it created. + - **GREEN:** PowerSync suite 195/195 on four full-suite runs; two + concurrent campaigns pass. 400 histories; the lowest reach across four + seeds is 16 (`field-removed`). + - **Mutants of `src/powersync.ts`:** the original defect (LIFE-10), dropped + deletes in the diff drain (PUB-3/FID-4), one commit per diff operation + (PUB-1). Ready before awaiting the initial receipts survives; it is + equivalent here, since every receipt resolves at once without pending + optimistic transactions. + - **Driver lessons:** an initial version waited out a timeout for each + batch that rewrote a row with its own values (no diff, no callback), and + hung when cleanup met a still-gated `onLoad`; the driver now expects a + drain only for a changing write and opens the gate when the adapter + aborts the run. Under full-suite load, a check ran before a restarted + run's initial load finished; `settle` now waits for its trigger. + - **Observed, not yet tested** (for the on-demand and lifecycle families): + an on-demand startup failure is only logged, with no `fail()` and no + release; `onLoad` takes no abort signal and cleanup waits for it, so a + hook that never settles leaves cleanup pending; after ready, an error in + the eager change handler is only logged. diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md index 58060fa85f..6a067ca2d3 100644 --- a/packages/db-adapter-conformance/README.md +++ b/packages/db-adapter-conformance/README.md @@ -47,6 +47,8 @@ Implement `EagerLiveSourceDriver`: - **`seed` and `fetchesBeforeListening`:** with `listen-before-fetch`, the provider holds backend rows set by `seed`, treats a batch before the first ready as a backend commit during the fetch, and counts fetches issued while its listener was not live (READY-8). - **`duplicateInsert`:** with `duplicate-key-delivery`, the law set's policy for an insert of a held key, `error` or `replace`. - **`valueKinds`:** the kinds of FID-1's `valueDomain` your backend represents; `jsonValueKinds` for a JSON backend. The driver delivers each value in the backend's own form and the Collection must show it decoded. +- **Asynchronous steps and `settle`:** provider methods may return a promise, which the check awaits. A fixture whose deliveries reach the Collection later than a few event-loop turns supplies `settle()`, its law-set `propagationStep`, which the check awaits before every comparison. +- **`source-failure-after-ready`:** declare it when your provider can fail terminally after the first ready (the law set's `terminal-source-end`, or an `errorMapping` that reports a terminal failure); otherwise `fail` is generated only before ready. - **`observedView`:** if your rows or keys differ from the conformance row, wrap the Collection with `observedView` and a one-to-one mapping. `conformanceKey` maps string keys; `toSqlRow` and `fromSqlRow` map an absent `note` to and from SQL `NULL`. Declare only the capabilities your adapter supports; the grammar generates @@ -71,6 +73,10 @@ reach counter, so the run fails when histories miss a state the laws need. until the adapter is updated, so it has its own config: `pnpm --filter @tanstack/db-adapter-conformance test:neon`. +- **PowerSync** (`packages/powersync-db-collection/tests/conformance-driver.ts`): + the published adapter in eager mode over a real local PowerSync database, + with per-batch commits and a declared propagation step (`settle`). It runs + in that package's default `pnpm test`. - **TrailBase** (`packages/trailbase-db-collection/tests/conformance-driver.ts`): the published adapter in eager mode over a controlled `RecordApi`, with per-change commits and no optional capability. It runs in that package's diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index a85dbb43c5..644a24b6a4 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -146,6 +146,7 @@ const staticOracleProperties = [ `adapter-conformance.eager-live-source.neon-realtime`, `adapter-conformance.eager-live-source.trailbase`, `adapter-conformance.eager-live-source.rxdb`, + `adapter-conformance.eager-live-source.powersync`, `trailbase.lifecycle`, `source-health.history`, `source-health.required-history`, diff --git a/packages/db/tests/oracle-replay-manifest.ts b/packages/db/tests/oracle-replay-manifest.ts index 4357f06816..32717e5091 100644 --- a/packages/db/tests/oracle-replay-manifest.ts +++ b/packages/db/tests/oracle-replay-manifest.ts @@ -658,6 +658,11 @@ const ownerGroups: ReadonlyArray< `adapter-conformance.eager-live-source`, `rxdb`, ], + [ + `powersync-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts`, + `adapter-conformance.eager-live-source`, + `powersync`, + ], ] const owners = new Map() @@ -744,7 +749,7 @@ export type OracleReplayManifestEntry = { // Pagination cells carry their property name in the test title so the guard // can select one cell without running the rest of the matrix. const replayTestNames = new Map([ - ...[`reference`, `neon-realtime`, `trailbase`, `rxdb`].map( + ...[`reference`, `neon-realtime`, `trailbase`, `rxdb`, `powersync`].map( (driver) => [ `adapter-conformance.eager-live-source.${driver}`, diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index 63b87c7c32..dfb52c15f4 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -656,7 +656,13 @@ function createPowerSyncCollectionConfig< message: `Could not start syncing process for ${viewName} into ${trackedTableName}`, error, }) - if (!ready) fail(error) + if (!ready) { + fail(error) + // A failed first load ends this sync run, so its change watcher + // and any diff trigger it set up are released now, not at cleanup. + abortController.abort() + void disposeTrackingAfterAbort() + } }) const invokeOnUnload = (): void | Promise => { diff --git a/packages/powersync-db-collection/tests/conformance-driver.ts b/packages/powersync-db-collection/tests/conformance-driver.ts new file mode 100644 index 0000000000..1dbd27b8ae --- /dev/null +++ b/packages/powersync-db-collection/tests/conformance-driver.ts @@ -0,0 +1,314 @@ +/** + * Production driver for the shared eager live-source conformance oracle + * (`packages/db-adapter-conformance`). It runs the real PowerSync adapter in + * eager mode over a real local PowerSync database (SQLite through + * `@powersync/node`), as the other tests in this package do. The provider is + * that database: the driver writes rows to it with SQL, and the adapter reads + * them through its own diff triggers and change watcher. + * + * The oracle's steps map onto PowerSync like this: + * + * - a complete delivery is the initial load. The adapter awaits its public + * `onLoad` hook before it establishes tracking and reads the table, so the + * driver writes the rows in one write transaction and then resolves the + * pending `onLoad`. PowerSync reads its table once per sync run, so the + * driver does not declare `complete-snapshot`; + * - a change batch is one SQLite write transaction: `INSERT` for a key the + * provider does not hold, `UPDATE` for one it does, `DELETE` otherwise. The + * adapter applies one diff drain as one commit, so the driver declares + * `per-batch` commits; + * - a failure before ready rejects the pending `onLoad`. PowerSync has no + * provider failure after ready (the law set gives it no + * `terminal-source-end`), so the driver does not declare + * `source-failure-after-ready`; + * - a provider subscription is a change-watcher registration + * (`onChangeWithCallback`); its release is the abort of the signal the + * adapter passed (LIFE-2, LIFE-10); + * - the propagation step (`propagationStep`: the diff-trigger watch) is + * `settle()`. After a complete delivery it waits until the initial load + * created its diff trigger, which it does once its read applied; after a + * batch, until the adapter's change callback has run and returned. + * + * PowerSync keys are strings, so row `n` is id `"n"`, and the shared + * `observedView` maps each key back with `conformanceKey`. `note` is a + * nullable text column, so an absent `note` is SQL `NULL` (`toSqlRow`, + * `fromSqlRow`). The driver never reads the Collection. + */ +import { randomUUID } from 'node:crypto' +import { rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { PowerSyncDatabase, Schema, Table, column } from '@powersync/node' +import { createCollection } from '@tanstack/db' +import { powerSyncCollectionOptions } from '../src' +import { + conformanceKey, + fromSqlRow, + observedView, + toSqlRow, +} from '../../db-adapter-conformance/src' +import { TEST_DATABASE_IMPLEMENTATION } from './test-db-implementation' +import type { + ConformanceRow, + ControlledProvider, + EagerLiveSourceDriver, + EagerLiveSourceFixture, + ProviderChange, +} from '../../db-adapter-conformance/src' + +const SCHEMA = new Schema({ + rows: new Table({ title: column.text, note: column.text }), +}) + +type Registration = { + signal: AbortSignal | undefined + running: number + drained: number +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** + * How long the driver waits for an adapter step (an `onLoad` call, an initial + * load, a drain) before letting the comparison decide. + */ +const drainTimeoutMs = 2_000 + +class PowerSyncProvider implements ControlledProvider { + readonly database: PowerSyncDatabase + readonly file: string + private readonly registrations: Array = [] + private pendingLoad: + { resolve: () => void; reject: (error: unknown) => void } | undefined + private rows = new Map() + // The drain count the latest write must see before the adapter applied it. + private drainTarget: number | undefined + // Diff triggers created so far, and the count an initial load must reach: + // a trigger is created once its initial read applied (`beforeCreate`). + private triggersCreated = 0 + private triggerTarget: number | undefined + + constructor() { + if (!TEST_DATABASE_IMPLEMENTATION) { + throw new Error(`no SQLite implementation is available`) + } + this.file = `conformance-${randomUUID()}.sqlite` + this.database = new PowerSyncDatabase({ + database: { + dbFilename: this.file, + dbLocation: tmpdir(), + implementation: TEST_DATABASE_IMPLEMENTATION, + }, + schema: SCHEMA, + }) + // Count the diff triggers the adapter creates. + const createDiffTrigger = this.database.triggers.createDiffTrigger.bind( + this.database.triggers, + ) + this.database.triggers.createDiffTrigger = async (options) => { + const dispose = await createDiffTrigger(options) + this.triggersCreated++ + return dispose + } + // Record each change-watcher registration the adapter makes, its release + // signal, and when its change callbacks run. + const register = this.database.onChangeWithCallback.bind(this.database) + this.database.onChangeWithCallback = (handler, options) => { + const registration: Registration = { + signal: options?.signal, + running: 0, + drained: 0, + } + this.registrations.push(registration) + // A real initial read finishes on its own. When the adapter abandons a + // run whose load is still waiting for the oracle, the gate opens, so + // cleanup does not wait on a hook no provider would hold. + options?.signal?.addEventListener( + `abort`, + () => { + const pending = this.pendingLoad + this.pendingLoad = undefined + pending?.resolve() + }, + { once: true }, + ) + return register( + { + ...handler, + onChange: async (event) => { + registration.running++ + try { + await handler?.onChange(event) + } finally { + registration.running-- + registration.drained++ + } + }, + }, + options, + ) + } + } + + /** The `onLoad` hook: the initial load waits for the oracle's delivery. */ + load = () => + new Promise((resolve, reject) => { + this.pendingLoad = { resolve, reject } + }) + + // The adapter calls `onLoad` asynchronously after the sync run starts. + private async takePendingLoad() { + const deadline = Date.now() + drainTimeoutMs + while (!this.pendingLoad && Date.now() < deadline) await sleep(5) + const pending = this.pendingLoad + if (!pending) throw new Error(`the adapter has not called onLoad`) + this.pendingLoad = undefined + return pending + } + + releases = () => + this.registrations.map((registration) => + registration.signal?.aborted === true ? 1 : 0, + ) + + deliverSnapshot = async (rows: ReadonlyArray) => { + const pending = await this.takePendingLoad() + await this.database.writeTransaction(async (tx) => { + await tx.execute(`DELETE FROM rows`) + for (const row of rows) { + const sql = toSqlRow(row) + await tx.execute( + `INSERT INTO rows (id, title, note) VALUES (?, ?, ?)`, + [String(sql.id), sql.title, sql.note], + ) + } + }) + this.rows = new Map(rows.map((row) => [row.id, row])) + this.triggerTarget = this.triggersCreated + 1 + pending.resolve() + } + + deliverChanges = async (changes: ReadonlyArray) => { + if (changes.length === 0) return + const registration = this.registrations.at(-1) + // A drain follows only a write that changes a row: rewriting a row with + // the values it holds records no diff. + const before = new Map(this.rows) + await this.database.writeTransaction(async (tx) => { + for (const change of changes) { + if (change.type === `upsert`) { + const sql = toSqlRow(change.row) + if (this.rows.has(change.row.id)) { + await tx.execute( + `UPDATE rows SET title = ?, note = ? WHERE id = ?`, + [sql.title, sql.note, String(sql.id)], + ) + } else { + await tx.execute( + `INSERT INTO rows (id, title, note) VALUES (?, ?, ?)`, + [String(sql.id), sql.title, sql.note], + ) + } + this.rows.set(change.row.id, change.row) + } else { + if (!this.rows.has(change.id)) { + throw new Error(`provider does not hold row ${change.id}`) + } + await tx.execute(`DELETE FROM rows WHERE id = ?`, [String(change.id)]) + this.rows.delete(change.id) + } + } + }) + const changed = + before.size !== this.rows.size || + [...this.rows].some(([id, row]) => { + const held = before.get(id) + return held?.title !== row.title || held.note !== row.note + }) + if (registration && changed) this.drainTarget = registration.drained + 1 + } + + fail = async (error: Error) => { + const pending = await this.takePendingLoad() + pending.reject(error) + } + + /** + * The propagation step: after a complete delivery, the initial load that + * reads it has created its diff trigger; after a batch, the change callback + * that follows the write ran. + */ + settle = async () => { + const registration = this.registrations.at(-1) + const loadTarget = this.triggerTarget + if (loadTarget !== undefined) { + this.triggerTarget = undefined + const deadline = Date.now() + drainTimeoutMs + while ( + this.triggersCreated < loadTarget && + registration?.signal?.aborted !== true && + Date.now() < deadline + ) { + await sleep(5) + } + // Readiness follows the trigger by a few microtasks. + for (let turn = 0; turn < 3; turn++) await sleep(0) + } + const target = this.drainTarget + if (target === undefined) return + this.drainTarget = undefined + if (!registration) return + const deadline = Date.now() + drainTimeoutMs + while ( + (registration.drained < target || registration.running > 0) && + registration.signal?.aborted !== true && + Date.now() < deadline + ) { + await sleep(5) + } + } + + dispose = async () => { + await this.database.disconnectAndClear() + await this.database.close() + await rm(join(tmpdir(), this.file), { force: true }) + } +} + +type PowerSyncRow = { + id: string + title?: string | null + note?: string | null +} + +export function powerSyncConformanceDriver(): EagerLiveSourceDriver { + return { + adapter: `@tanstack/powersync-db-collection (eager)`, + capabilities: new Set(), + changeCommits: `per-batch`, + // PowerSync reports a failed start through its logger (law set + // capability `logger`). + declaredDiagnostic: (_level, args) => + args.some((arg) => String(arg).includes(`Could not start syncing`)), + create(): EagerLiveSourceFixture { + const provider = new PowerSyncProvider() + const collection = createCollection( + powerSyncCollectionOptions({ + database: provider.database, + table: SCHEMA.props.rows, + onLoad: provider.load, + }), + ) + return { + collection: observedView(collection, { + row: (row: PowerSyncRow) => + fromSqlRow({ ...row, id: conformanceKey(row.id) }), + key: conformanceKey, + }), + provider, + settle: provider.settle, + dispose: provider.dispose, + } + }, + } +} diff --git a/packages/powersync-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/powersync-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts new file mode 100644 index 0000000000..b24358385d --- /dev/null +++ b/packages/powersync-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -0,0 +1,78 @@ +import { fc } from '@fast-check/vitest' +import { describe, it } from 'vitest' +import { + oraclePropertyOptions, + oracleReplayTest, + oracleRuns, +} from '../../db/tests/oracle-config' +import { declareOracle } from '../../db/tests/oracle-declaration' +import { createOracleReach } from '../../db/tests/oracle-reach' +import { + checkEagerLiveSourceHistory, + eagerLiveSourceHistoryArb, + eagerLiveSourceModel, + eagerLiveSourceProperty, + reachStatesFor, +} from '../../db-adapter-conformance/src' +import { powerSyncConformanceDriver } from './conformance-driver' + +/** + * # PowerSync on the shared eager live-source conformance oracle + * + * A receiving driver for `packages/db-adapter-conformance`. The shared + * module states the laws, the model, the history grammar, and the check; + * this file supplies only the PowerSync production driver in + * `conformance-driver.ts` and the campaigns. It states no expected results. + * + * PowerSync runs here over a real local SQLite database. It declares none of + * the oracle's optional capabilities: it reads its table once per sync run, + * its changes carry no txid, and it has no provider failure after ready. Its + * propagation step is the diff-trigger watch, which the fixture's `settle` + * awaits. The claim is limited to local SQLite writes; a claim about the + * PowerSync sync service needs a receiving witness against it. + */ + +const property = `adapter-conformance.eager-live-source.powersync` +const driver = powerSyncConformanceDriver() +const historyArb = eagerLiveSourceHistoryArb(driver) +// `field-removed` needs a held row with a note and a later upsert without +// one: 100 histories reached it only four times. Each history runs on a +// fresh SQLite database and waits for the change watcher after every batch, +// so 400 histories take about a minute. +const runs = 400 +// Startup bursts lengthen histories, so a campaign takes several seconds. +const campaignTimeoutMs = 600_000 + +async function campaign(options: fc.Parameters<[unknown]>) { + const reach = createOracleReach(property, reachStatesFor(driver), { + minimum: 5, + }) + await fc.assert(eagerLiveSourceProperty(driver, reach), options) + return reach +} + +describe(`eager live-source conformance: PowerSync`, () => { + oracleReplayTest(it)( + `conforms on a fixed-seed campaign`, + async () => { + const reach = await campaign({ seed: 2118, numRuns: oracleRuns(runs) }) + reach.assertReached() + }, + campaignTimeoutMs, + ) + + oracleReplayTest(it, property)( + `conforms on a random campaign or a replayed seed`, + async () => { + const reach = await campaign(oraclePropertyOptions(runs, property)) + reach.assertReached() + }, + campaignTimeoutMs, + ) +}) + +declareOracle({ + check: [checkEagerLiveSourceHistory, eagerLiveSourceProperty], + model: [eagerLiveSourceModel.step, eagerLiveSourceModel.expected], + grammar: [historyArb], +}) From 283045411dbc7e1c1808113bf415b64531b590e9 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 11:44:24 -0600 Subject: [PATCH 15/64] test(conformance): generate typed fields and run them through PowerSync Add typed fields to the eager live-source oracle (FID-1): a driver declares `fieldKinds`, the kinds its backend stores in typed columns, and each row then carries `typed`, one field per kind, compared by the same strict description as `value`, with reach per kind on the load and change paths. A driver may declare `excludesValue` for values its backend itself cannot store as given; each use is recorded in the coverage map. PowerSync declares string, integer, float, boolean (INTEGER 0/1), and Date (ISO TEXT) columns, decoded by a user `deserializationSchema` and validated by `schema`, as an app writes them. The driver now rejects its `onLoad` gate when the database refuses a write, rather than strand it. - Law: FID-1 (values round-trip; parameter `valueDomain`) from the adapter law set v18; PowerSync's domain is SQLite text, integer, and real, with booleans as 0/1 and Dates as ISO text by default. - Reach: reference 20 typed-field states; PowerSync 10, with every campaign asserting each at least five times. - RED: mutants that skip `deserializeSyncRow` on PowerSync's diff path and on its initial load fail FID-1 (values) at a change and at the load (a boolean shown as 1 and a Date as its ISO string). The reference mutant typed-booleans-as-numbers fails FID-1 (values). - GREEN: reference 68/68; PowerSync 195/195; RxDB 23/23; TrailBase 151/151; Neon keeps its READY-4 and HEALTH-1 RED. - Limits: PowerSync's storage keeps numbers at 15 significant digits (9007199254740991 reads back as 9007199254740990, 0.30000000000000004 as 0.3) and rejects Number.MAX_VALUE; the driver excludes such numbers. This is a platform limit, not an adapter defect. Typed fields are non-null; nullability is covered by `note`. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 9 +- packages/db-adapter-conformance/MIGRATION.md | 18 ++ packages/db-adapter-conformance/README.md | 112 ++++++------- .../src/eager-live-source-oracle.ts | 154 ++++++++++++++---- .../reference-adapter-oracle.property.test.ts | 20 ++- .../tests/reference-adapter.ts | 25 +++ .../tests/conformance-driver.ts | 143 ++++++++++++---- 7 files changed, 358 insertions(+), 123 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 77c9fc821f..4dcd64c4a5 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6037,4 +6037,11 @@ database-side propagation step) is GREEN after a fix: a failed first load now releases the run's change watcher (LIFE-10). Its startup failures in on-demand mode, cleanup waiting on an unsettled `onLoad`, and post-ready change-handler errors are logged only; they belong to the on-demand and -lifecycle families, which are not built. +lifecycle families, which are not built. PowerSync also runs typed fields (string, integer, float, +boolean as INTEGER 0/1, Date as ISO TEXT) through user deserialization and +output schemas. Excluded by `excludesValue` (platform limit, not adapter +behavior): numbers PowerSync's storage does not keep at full precision. Its +table views keep 15 significant digits (9007199254740991 reads back as +9007199254740990, 0.30000000000000004 as 0.3) and reject Number.MAX_VALUE +(`powersync_diff: number out of range`); owner: the PowerSync driver; +evidence: a direct write and read through `@powersync/node` 0.20.0. diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index fa8f0a71aa..d9a308dba0 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -269,3 +269,21 @@ document update removes`) failed the same way. release; `onLoad` takes no abort signal and cleanup waits for it, so a hook that never settles leaves cleanup pending; after ready, an error in the eager change handler is only logged. +- **2026-10-30, eager live source, typed fields (FID-1).** A driver declares + `fieldKinds`, the kinds its backend stores in typed columns; each row then + carries `typed`, one field per kind, with reach per kind on the load and + change paths. A driver may declare `excludesValue` for values its backend + itself cannot store as given. Reference mutant: `typed-booleans-as-numbers` + (FID-1 values). + - **PowerSync:** typed columns for string, integer, float, boolean (INTEGER + 0/1), and Date (ISO TEXT), decoded by user `deserializationSchema` and + validated by `schema`, as an app writes them. GREEN; mutants that skip + `deserializeSyncRow` on the diff path or on the initial load fail FID-1 + (values) at a change and at the load. No adapter defect. + - **PowerSync platform limit (excluded, recorded):** its storage keeps a + number at 15 significant digits. Through its table views + 9007199254740991 reads back as 9007199254740990 and 0.30000000000000004 + as 0.3, and Number.MAX_VALUE is rejected (`powersync_diff: number out of +range`). The Collection shows what the database stored, so this is not + the adapter's FID-1; the driver excludes such numbers. Worth reporting + to PowerSync. diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md index 6a067ca2d3..ff2cb7ea41 100644 --- a/packages/db-adapter-conformance/README.md +++ b/packages/db-adapter-conformance/README.md @@ -13,25 +13,25 @@ The package is private while its first oracle settles. pushes a complete result and then committed change batches. Its opening comment states the laws and their limits. In short: -| Law | Checked behavior | -| -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| READY-2, READY-3 | `loading` until the first complete delivery; then `ready` with exactly its rows | -| READY-4 | A failure before ready: status `error`, `preload()` rejects with the value, `health.error.cause` is the value | -| HEALTH-1 | A failure after ready: status stays `ready`, rows kept, `health.error.cause` is the value | -| PUB-1 | Every change publication during a step is one whole committed state, in commit order; per batch or per change, as the driver declares | -| PUB-2, PUB-3, PUB-7, FID-4 | Change events fold to the rows, with no insert of a held key and no update or delete of a missing one; batches fold onto rows; a complete delivery replaces (PUB-7 with `complete-snapshot`) | -| FID-1 | A delivered row is the whole backend row: a row without the optional `note` removes a `note` the Collection held. With declared `valueKinds`, each `value` strictly equals the delivered one on the initial load and on a live change: same type and prototype, `Object.is` for numbers (`-0` is not `0`), Dates by time, nested arrays and plain objects element by element | -| LIFE-2, LIFE-5 | Cleanup releases the provider subscription; a restart is a fresh, empty sync run | -| LIFE-10 | A failed provider's subscription is released at the failure, and no subscription is released twice | -| HEALTH-5 | No unhandled rejection and no undeclared `console.error` or `console.warn` | -| EVID-1, EVID-2, EVID-3 | `awaitTxId` fulfills after its batch applies, stays pending before, rejects after its timeout, and rejects at retirement | -| HEALTH-2, RETRY-1 | A retried failure stays hidden before ready, joins `failureReason` after it, and clears on the next delivery (`retry-policy`) | -| RECON-1 | Disconnect and reconnect keep status and rows | -| RENEW-1 | Capability renewal reuses the provider subscription and changes nothing visible | -| READY-8, READY-10 | With `listen-before-fetch`: no fetch before the listener is live; batches committed during the fetch apply after its result, before ready | -| PUB-5, PUB-9 | With `listen-before-fetch`: the cut publishes only backend states at or after the read point, in order; no replay regresses a row | -| READY-3, READY-10 | When status becomes `ready`, the rows already equal the initial cut | -| PUB-4, PUB-3 | With `duplicate-key-delivery`: a complete delivery holding two rows with one key follows the driver's `duplicateInsert` policy. Under `error` it is rejected: no row written, nothing published, earlier rows kept, a sync failure reported. Under `replace` the later row wins | +| Law | Checked behavior | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| READY-2, READY-3 | `loading` until the first complete delivery; then `ready` with exactly its rows | +| READY-4 | A failure before ready: status `error`, `preload()` rejects with the value, `health.error.cause` is the value | +| HEALTH-1 | A failure after ready: status stays `ready`, rows kept, `health.error.cause` is the value | +| PUB-1 | Every change publication during a step is one whole committed state, in commit order; per batch or per change, as the driver declares | +| PUB-2, PUB-3, PUB-7, FID-4 | Change events fold to the rows, with no insert of a held key and no update or delete of a missing one; batches fold onto rows; a complete delivery replaces (PUB-7 with `complete-snapshot`) | +| FID-1 | A delivered row is the whole backend row: a row without the optional `note` removes a `note` the Collection held. With declared `valueKinds`, each `value` strictly equals the delivered one on the initial load and on a live change: same type and prototype, `Object.is` for numbers (`-0` is not `0`), Dates by time, nested arrays and plain objects element by element. With declared `fieldKinds`, each row's `typed` record holds one field per kind, as typed columns do, checked the same way | +| LIFE-2, LIFE-5 | Cleanup releases the provider subscription; a restart is a fresh, empty sync run | +| LIFE-10 | A failed provider's subscription is released at the failure, and no subscription is released twice | +| HEALTH-5 | No unhandled rejection and no undeclared `console.error` or `console.warn` | +| EVID-1, EVID-2, EVID-3 | `awaitTxId` fulfills after its batch applies, stays pending before, rejects after its timeout, and rejects at retirement | +| HEALTH-2, RETRY-1 | A retried failure stays hidden before ready, joins `failureReason` after it, and clears on the next delivery (`retry-policy`) | +| RECON-1 | Disconnect and reconnect keep status and rows | +| RENEW-1 | Capability renewal reuses the provider subscription and changes nothing visible | +| READY-8, READY-10 | With `listen-before-fetch`: no fetch before the listener is live; batches committed during the fetch apply after its result, before ready | +| PUB-5, PUB-9 | With `listen-before-fetch`: the cut publishes only backend states at or after the read point, in order; no replay regresses a row | +| READY-3, READY-10 | When status becomes `ready`, the rows already equal the initial cut | +| PUB-4, PUB-3 | With `duplicate-key-delivery`: a complete delivery holding two rows with one key follows the driver's `duplicateInsert` policy. Under `error` it is rejected: no row written, nothing published, earlier rows kept, a sync failure reported. Under `replace` the later row wins | The law IDs come from the adapter conformance law set, version 18. @@ -45,6 +45,7 @@ Implement `EagerLiveSourceDriver`: - **`changeCommits`:** `per-batch` if your adapter commits a provider batch as one transaction, `per-change` if it commits each change alone. - **`declaredDiagnostic`:** optional; accepts console diagnostics your adapter documents, so HEALTH-5 does not reject them. - **`seed` and `fetchesBeforeListening`:** with `listen-before-fetch`, the provider holds backend rows set by `seed`, treats a batch before the first ready as a backend commit during the fetch, and counts fetches issued while its listener was not live (READY-8). +- **`fieldKinds` and `excludesValue`:** `fieldKinds` are the kinds your backend stores in typed columns; rows then carry `typed`, one field per kind, which the driver writes in the column's form. `excludesValue` names generated values the backend itself cannot store as given (a platform limit, not adapter behavior); record each use in the coverage map. - **`duplicateInsert`:** with `duplicate-key-delivery`, the law set's policy for an insert of a held key, `error` or `replace`. - **`valueKinds`:** the kinds of FID-1's `valueDomain` your backend represents; `jsonValueKinds` for a JSON backend. The driver delivers each value in the backend's own form and the Collection must show it decoded. - **Asynchronous steps and `settle`:** provider methods may return a promise, which the check awaits. A fixture whose deliveries reach the Collection later than a few event-loop turns supplies `settle()`, its law-set `propagationStep`, which the check awaits before every comparison. @@ -136,45 +137,46 @@ reach counter, so the run fails when histories miss a state the laws need. 20 other seeds, the lowest count for any state was 12 (`field-removed`), across 12 seeds. - **GREEN:** `pnpm exec vitest run tests/reference-adapter-oracle.property.test.ts`: - 67 tests pass (both campaigns, the per-change pair, 32 calibrations, and the + 68 tests pass (both campaigns, the per-change pair, 32 calibrations, and the grammar controls). TrailBase: its full suite, 151 tests, passes. - **Mutants:** the fixed campaign rejects each with the law it breaks: - | Mutant | Law | - | ---------------------------------------- | -------------------------- | - | `ready-before-delivery` | READY-2 | - | `wraps-loading-failure` | READY-4 | - | `failure-clears-rows` | HEALTH-1 | - | `transient-is-fatal` | HEALTH-1 | - | `never-recovers` | HEALTH-2 | - | `transient-fails-loading` | RETRY-1 | - | `splits-batch` | PUB-1 | - | `reverses-batch` (per-change) | PUB-1 | - | `drops-deletes` | PUB-3 | - | `snapshot-merges` | PUB-7 | - | `leaks-subscription` | LIFE-2 | - | `restart-replays-rows` | LIFE-5 | - | `keeps-failed-source` | LIFE-10 | - | `double-release` | LIFE-10 | - | `detached-rejection` | HEALTH-5 | - | `logs-failure` | HEALTH-5 | - | `wait-off-by-one` | EVID-1 | - | `wait-ignores-timeout` | EVID-3 | - | `disconnect-fails` | RECON-1 | - | `renew-resubscribes` | RENEW-1 | - | `drops-buffered` | READY-10 | - | `buffer-before-rows` | READY-10 | - | `replays-per-batch` | PUB-5 | - | `ready-before-buffer` | READY-10 (rows when ready) | - | `merges-updates` | FID-1 | - | `relabels-inserts` (checker calibration) | PUB-2 | - | `dates-as-strings` (live changes only) | FID-1 (values) | - | `bigints-as-numbers` (initial load only) | FID-1 (values) | - | `drops-nulls` | FID-1 (values) | - | `loses-negative-zero` | FID-1 (values) | - | `commits-malformed-prefix` | PUB-4 | - | `ignores-malformed` | PUB-4 | - | `shows-buffered` | READY-10/PUB-9 | + | Mutant | Law | + | ----------------------------------------------- | -------------------------- | + | `ready-before-delivery` | READY-2 | + | `wraps-loading-failure` | READY-4 | + | `failure-clears-rows` | HEALTH-1 | + | `transient-is-fatal` | HEALTH-1 | + | `never-recovers` | HEALTH-2 | + | `transient-fails-loading` | RETRY-1 | + | `splits-batch` | PUB-1 | + | `reverses-batch` (per-change) | PUB-1 | + | `drops-deletes` | PUB-3 | + | `snapshot-merges` | PUB-7 | + | `leaks-subscription` | LIFE-2 | + | `restart-replays-rows` | LIFE-5 | + | `keeps-failed-source` | LIFE-10 | + | `double-release` | LIFE-10 | + | `detached-rejection` | HEALTH-5 | + | `logs-failure` | HEALTH-5 | + | `wait-off-by-one` | EVID-1 | + | `wait-ignores-timeout` | EVID-3 | + | `disconnect-fails` | RECON-1 | + | `renew-resubscribes` | RENEW-1 | + | `drops-buffered` | READY-10 | + | `buffer-before-rows` | READY-10 | + | `replays-per-batch` | PUB-5 | + | `ready-before-buffer` | READY-10 (rows when ready) | + | `merges-updates` | FID-1 | + | `relabels-inserts` (checker calibration) | PUB-2 | + | `dates-as-strings` (live changes only) | FID-1 (values) | + | `bigints-as-numbers` (initial load only) | FID-1 (values) | + | `drops-nulls` | FID-1 (values) | + | `loses-negative-zero` | FID-1 (values) | + | `typed-booleans-as-numbers` (live changes only) | FID-1 (values) | + | `commits-malformed-prefix` | PUB-4 | + | `ignores-malformed` | PUB-4 | + | `shows-buffered` | READY-10/PUB-9 | `reverses-batch` reverses only batches of distinct keys, so its final rows are right and only the intermediate publications are wrong. An earlier diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 8d39721172..73db3e7194 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -33,7 +33,9 @@ * each row's `value` strictly equals the delivered value, on the initial * load and on a live change: same type and prototype, `Object.is` for * numbers, Dates by time, and arrays and plain objects element by element - * with no extra own keys. The law set parameterizes FID-1 by each + * with no extra own keys. With declared `fieldKinds`, each row's `typed` + * record holds one field per kind, as typed columns do, and each field + * must round-trip the same way on both paths. The law set parameterizes FID-1 by each * adapter's `valueDomain`; a driver declares the kinds of it this oracle * generates. * - **PUB-7** (capability `complete-snapshot`). A later complete delivery @@ -124,6 +126,11 @@ export interface ConformanceRow { readonly note?: string /** A value of a declared kind (FID-1); present only with `valueKinds`. */ readonly value?: unknown + /** + * One field per declared `fieldKinds` kind, each holding a value of that + * kind, as a typed column does (FID-1); present only with `fieldKinds`. + */ + readonly typed?: Readonly>> } /** @@ -266,6 +273,19 @@ export interface EagerLiveSourceDriver { * own form, and the Collection must show it decoded. */ readonly valueKinds?: ReadonlyArray + /** + * Kinds the backend stores in typed columns (FID-1). Rows then carry + * `typed`, one field per kind; the driver stores each in the backend's form + * for its column, and the Collection must show it decoded. + */ + readonly fieldKinds?: ReadonlyArray + /** + * Generated values the backend itself cannot store as given, such as a + * number its storage rounds: a platform limit, not adapter behavior. The + * grammar avoids them. Every use needs a coverage-map entry naming the + * limit and its evidence. + */ + readonly excludesValue?: (kind: ValueKind, value: unknown) => boolean /** * The law set's `duplicateInsert` policy for an insert of a key the run * holds; required with `duplicate-key-delivery`. @@ -397,41 +417,73 @@ const valueExamples: Record unknown>> = { ], } -function valueArb(kinds: ReadonlyArray): fc.Arbitrary { +/** A value the driver's backend cannot store as given; see `excludesValue`. */ +type ValueExclusion = (kind: ValueKind, value: unknown) => boolean + +function valueArb( + kinds: ReadonlyArray, + excludes: ValueExclusion = () => false, +): fc.Arbitrary { return fc.oneof( - ...kinds.map((kind) => - fc - .integer({ min: 0, max: valueExamples[kind].length - 1 }) - .map((index) => valueExamples[kind][index]!()), - ), + ...kinds.map((kind) => { + const examples = valueExamples[kind].filter( + (example) => !excludes(kind, example()), + ) + if (examples.length === 0) { + throw new Error(`every ${kind} example is excluded`) + } + return fc + .integer({ min: 0, max: examples.length - 1 }) + .map((index) => examples[index]!()) + }), ) } -function rowArb(kinds: ReadonlyArray): fc.Arbitrary { +function rowArb( + kinds: ReadonlyArray, + fieldKinds: ReadonlyArray = [], + excludes?: ValueExclusion, +): fc.Arbitrary { const fields = { id: fc.integer({ min: 1, max: 5 }), title: fc.constantFrom(`a`, `b`, `c`), note: fc.constantFrom(`x`, `y`), + ...(kinds.length > 0 ? { value: valueArb(kinds, excludes) } : {}), + ...(fieldKinds.length > 0 + ? { + typed: fc.record( + Object.fromEntries( + fieldKinds.map((kind) => [kind, valueArb([kind], excludes)]), + ) as Record>, + ), + } + : {}), } - if (kinds.length === 0) { - return fc.record(fields, { requiredKeys: [`id`, `title`] }) - } - return fc.record( - { ...fields, value: valueArb(kinds) }, - { requiredKeys: [`id`, `title`, `value`] }, - ) + return fc.record(fields, { + requiredKeys: [ + `id`, + `title`, + ...(kinds.length > 0 ? [`value` as const] : []), + ...(fieldKinds.length > 0 ? [`typed` as const] : []), + ], + }) as fc.Arbitrary } function changeArb( kinds: ReadonlyArray, + fieldKinds: ReadonlyArray = [], + excludes?: ValueExclusion, ): fc.Arbitrary { const changes: Array> = [ - rowArb(kinds).map((row) => ({ type: `upsert` as const, row })), + rowArb(kinds, fieldKinds, excludes).map((row) => ({ + type: `upsert` as const, + row, + })), fc .integer({ min: 1, max: 5 }) .map((id) => ({ type: `delete` as const, id })), ] - if (kinds.length > 0) { + if (kinds.length > 0 || fieldKinds.length > 0) { changes.push( fc .integer({ min: 1, max: 5 }) @@ -444,7 +496,7 @@ function changeArb( /** What the grammar needs from a driver: its capabilities and value kinds. */ export type GrammarDomain = Pick< EagerLiveSourceDriver, - `capabilities` | `valueKinds` + `capabilities` | `valueKinds` | `fieldKinds` | `excludesValue` > /** Generates histories whose steps use only the driver's declarations. */ @@ -453,6 +505,7 @@ export function eagerLiveSourceHistoryArb( ): fc.Arbitrary { const { capabilities } = domain const kinds = domain.valueKinds ?? [] + const fieldKinds = domain.fieldKinds ?? [] const steps: Array<{ weight: number arbitrary: fc.Arbitrary @@ -461,10 +514,13 @@ export function eagerLiveSourceHistoryArb( weight: 4, arbitrary: fc .record({ - rows: fc.uniqueArray(rowArb(kinds), { - selector: (row) => row.id, - maxLength: 4, - }), + rows: fc.uniqueArray( + rowArb(kinds, fieldKinds, domain.excludesValue), + { + selector: (row) => row.id, + maxLength: 4, + }, + ), seen: fc.integer({ min: 0, max: 3 }), }) .map(({ rows, seen }) => @@ -476,7 +532,9 @@ export function eagerLiveSourceHistoryArb( { weight: 6, arbitrary: fc - .array(changeArb(kinds), { maxLength: 3 }) + .array(changeArb(kinds, fieldKinds, domain.excludesValue), { + maxLength: 3, + }) .map((changes) => ({ kind: `changes` as const, changes })), }, { weight: 2, arbitrary: fc.constant({ kind: `fail` as const }) }, @@ -487,11 +545,14 @@ export function eagerLiveSourceHistoryArb( weight: 1, arbitrary: fc .record({ - rows: fc.uniqueArray(rowArb(kinds), { - selector: (row) => row.id, - minLength: 1, - maxLength: 3, - }), + rows: fc.uniqueArray( + rowArb(kinds, fieldKinds, domain.excludesValue), + { + selector: (row) => row.id, + minLength: 1, + maxLength: 3, + }, + ), duplicate: fc.nat(2), }) .map(({ rows, duplicate }) => ({ @@ -544,7 +605,10 @@ export function eagerLiveSourceHistoryArb( // outstanding. Without it, two buffered batches that touch one key before // the read point are too rare for PUB-5 to be checked. const startup = fc - .array(changeArb(kinds), { minLength: 1, maxLength: 3 }) + .array(changeArb(kinds, fieldKinds, domain.excludesValue), { + minLength: 1, + maxLength: 3, + }) .map((changes) => ({ kind: `changes` as const, changes })) return fc .tuple(fc.array(startup, { maxLength: 3 }), history) @@ -691,14 +755,18 @@ function rowValue(row: RowFields): string { const head = Object.hasOwn(row, `note`) ? `${row.title}/${String(row.note)}` : row.title - return Object.hasOwn(row, `value`) - ? `${head}#${describeValue(row.value)}` - : head + const value = Object.hasOwn(row, `value`) + ? `#${describeValue(row.value)}` + : `` + const typed = Object.hasOwn(row, `typed`) + ? `$${describeValue(row.typed)}` + : `` + return `${head}${value}${typed}` } /** Title and note, without the value. */ function rowHead(value: string): string { - const mark = value.indexOf(`#`) + const mark = value.search(/[#$]/) return mark === -1 ? value : value.slice(0, mark) } @@ -1258,7 +1326,9 @@ export const eagerLiveSourceReachStates = [ * Per value kind, a value delivered on the initial load and one delivered by * a live change: a decoder can differ between the two paths. */ -export type ValueReachState = `value:${ValueKind}:${`load` | `change`}` +export type ValueReachState = + | `value:${ValueKind}:${`load` | `change`}` + | `field:${ValueKind}:${`load` | `change`}` export type EagerLiveSourceReachState = | (typeof eagerLiveSourceReachStates)[number] @@ -1305,12 +1375,18 @@ export function reachStatesFor( `value:${kind}:change`, ], ) + const fields = (domain.fieldKinds ?? []).flatMap( + (kind): Array => [ + `field:${kind}:load`, + `field:${kind}:change`, + ], + ) const zeros: Array = domain.valueKinds?.includes( `negative-zero`, ) ? [`zero-sign-flipped`] : [] - return [...reachable, ...values, ...zeros] + return [...reachable, ...values, ...fields, ...zeros] } /** JSON for a message; a bigint shows as `n`, `-0` as `-0`. */ @@ -1741,6 +1817,9 @@ export async function checkEagerLiveSourceHistory( if (Object.hasOwn(row, `value`)) { reach(`value:${kindOf(row.value)}:load`) } + for (const kind of Object.keys(row.typed ?? {})) { + reach(`field:${kind as ValueKind}:load`) + } } reach(wasReady ? `replacing-snapshot` : `first-ready`) if (done.missed > 0) reach(`read-missed-change`) @@ -1760,6 +1839,11 @@ export async function checkEagerLiveSourceHistory( ) { reach(`value:${kindOf(change.row.value)}:change`) } + if (change.type === `upsert`) { + for (const kind of Object.keys(change.row.typed ?? {})) { + reach(`field:${kind as ValueKind}:change`) + } + } } reach(done.beforeReady ? `change-before-ready` : `change-batch`) if (done.removesField) reach(`field-removed`) diff --git a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts index 8651690ca1..d27799147f 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts @@ -119,6 +119,7 @@ describe(`eager live-source conformance: reference adapter`, () => { [`bigints-as-numbers`, `FID-1 (values)`], [`drops-nulls`, `FID-1 (values)`], [`loses-negative-zero`, `FID-1 (values)`], + [`typed-booleans-as-numbers`, `FID-1 (values)`], [`commits-malformed-prefix`, `PUB-4`], [`ignores-malformed`, `PUB-4`], [`relabels-inserts`, `PUB-2`], @@ -313,12 +314,14 @@ function withinGrammar( ): boolean { // With value kinds every row carries a value; without them none does. const valued = (domain.valueKinds ?? []).length > 0 + const typed = (domain.fieldKinds ?? []).length > 0 const row = (r: ConformanceRow) => r.id >= 1 && r.id <= 5 && [`a`, `b`, `c`].includes(r.title) && (!Object.hasOwn(r, `note`) || [`x`, `y`].includes(r.note!)) && - Object.hasOwn(r, `value`) === valued + Object.hasOwn(r, `value`) === valued && + Object.hasOwn(r, `typed`) === typed return ( history.length >= 1 && history.length <= 15 && @@ -351,7 +354,13 @@ function withinGrammar( // The witnesses above carry no values, so they reconstruct in the grammar // of the same driver without value kinds. Value witnesses use the full one. -const withoutValues = { ...driver, valueKinds: undefined } +const withoutValues = { + ...driver, + valueKinds: undefined, + fieldKinds: undefined, +} +// The value witnesses carry values but no typed fields. +const withoutFields = { ...driver, fieldKinds: undefined } const valueWitnesses: Array<[string, EagerLiveSourceHistory]> = [ [ @@ -415,8 +424,11 @@ describe(`eager live-source grammar controls`, () => { oracleReplayTest(it).each(valueWitnesses)( `reconstructs %s`, async (_name, history) => { - expect(withinGrammar(history, driver)).toBe(true) - const { skipped } = await checkEagerLiveSourceHistory(driver, history) + expect(withinGrammar(history, withoutFields)).toBe(true) + const { skipped } = await checkEagerLiveSourceHistory( + withoutFields, + history, + ) expect(skipped).toBe(0) }, ) diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index 8ef0cf7fa2..7415a2352f 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -83,6 +83,8 @@ export type ReferenceMutant = | `drops-nulls` /** FID-1: `-0` is stored as `0`. */ | `loses-negative-zero` + /** FID-1: a live change stores a typed boolean field as a number. */ + | `typed-booleans-as-numbers` /** PUB-4: a rejected delivery commits the rows written before the failure. */ | `commits-malformed-prefix` /** PUB-4: a duplicate key is skipped and the delivery applies. */ @@ -205,6 +207,16 @@ function storedRow( if (mutant === `loses-negative-zero` && Object.is(value, -0)) { return { ...row, value: 0 } } + if ( + mutant === `typed-booleans-as-numbers` && + path === `change` && + typeof row.typed?.boolean === `boolean` + ) { + return { + ...row, + typed: { ...row.typed, boolean: Number(row.typed.boolean) }, + } + } return row } @@ -495,6 +507,19 @@ export function referenceDriver( `duplicate-key-delivery`, `source-failure-after-ready`, ]), + // The same ten kinds as typed fields, one per row field. + fieldKinds: [ + `string`, + `integer`, + `float`, + `negative-zero`, + `boolean`, + `null`, + `date`, + `bigint`, + `array`, + `object`, + ], valueKinds: [ `string`, `integer`, diff --git a/packages/powersync-db-collection/tests/conformance-driver.ts b/packages/powersync-db-collection/tests/conformance-driver.ts index 1dbd27b8ae..99c0799b44 100644 --- a/packages/powersync-db-collection/tests/conformance-driver.ts +++ b/packages/powersync-db-collection/tests/conformance-driver.ts @@ -32,7 +32,10 @@ * PowerSync keys are strings, so row `n` is id `"n"`, and the shared * `observedView` maps each key back with `conformanceKey`. `note` is a * nullable text column, so an absent `note` is SQL `NULL` (`toSqlRow`, - * `fromSqlRow`). The driver never reads the Collection. + * `fromSqlRow`). Typed fields (FID-1) are typed columns: the driver writes + * a boolean as 0 or 1 and a Date as ISO text, the user schemas an app would + * write decode them, and the view only renames the columns into `typed`. + * The driver never reads the Collection. */ import { randomUUID } from 'node:crypto' import { rm } from 'node:fs/promises' @@ -40,6 +43,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { PowerSyncDatabase, Schema, Table, column } from '@powersync/node' import { createCollection } from '@tanstack/db' +import { z } from 'zod' import { powerSyncCollectionOptions } from '../src' import { conformanceKey, @@ -56,10 +60,68 @@ import type { ProviderChange, } from '../../db-adapter-conformance/src' +// Typed fields (FID-1) live in typed columns: SQLite stores a boolean as +// INTEGER 0 or 1 and a Date as ISO-8601 TEXT, as PowerSync does by default. const SCHEMA = new Schema({ - rows: new Table({ title: column.text, note: column.text }), + rows: new Table({ + title: column.text, + note: column.text, + t_string: column.text, + t_integer: column.integer, + t_float: column.real, + t_boolean: column.integer, + t_date: column.text, + }), }) +/** The kinds PowerSync's typed columns hold; bigint, -0, and JSON do not fit. */ +const FIELD_KINDS = [`string`, `integer`, `float`, `boolean`, `date`] as const + +/** + * The user schemas an app writes for this table: `deserializationSchema` + * turns SQLite values into the Collection's types, and `schema` validates + * them. The adapter must apply them on the initial load and every change. + */ +const outputSchema = z.object({ + id: z.string(), + title: z.string().nullable(), + note: z.string().nullable(), + t_string: z.string(), + t_integer: z.number(), + t_float: z.number(), + t_boolean: z.boolean(), + t_date: z.date(), +}) +const deserializationSchema = z.object({ + id: z.string(), + title: z.string().nullable(), + note: z.string().nullable(), + t_string: z.string(), + t_integer: z.number(), + t_float: z.number(), + t_boolean: z.number().transform((value) => value === 1), + t_date: z.string().transform((value) => new Date(value)), +}) + +/** A conformance row as the SQL columns the provider writes. */ +function sqlColumns(row: ConformanceRow): Array { + const sql = toSqlRow(row) + const typed = row.typed ?? {} + return [ + String(sql.id), + sql.title, + sql.note, + typed.string as string, + typed.integer as number, + typed.float as number, + typed.boolean === true ? 1 : 0, + (typed.date as Date).toISOString(), + ] +} + +const insertRow = `INSERT INTO rows (id, title, note, t_string, t_integer, t_float, t_boolean, t_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` +const updateRow = `UPDATE rows SET title = ?, note = ?, t_string = ?, t_integer = ?, t_float = ?, t_boolean = ?, t_date = ? WHERE id = ?` + type Registration = { signal: AbortSignal | undefined running: number @@ -173,16 +235,16 @@ class PowerSyncProvider implements ControlledProvider { deliverSnapshot = async (rows: ReadonlyArray) => { const pending = await this.takePendingLoad() - await this.database.writeTransaction(async (tx) => { - await tx.execute(`DELETE FROM rows`) - for (const row of rows) { - const sql = toSqlRow(row) - await tx.execute( - `INSERT INTO rows (id, title, note) VALUES (?, ?, ?)`, - [String(sql.id), sql.title, sql.note], - ) - } - }) + try { + await this.database.writeTransaction(async (tx) => { + await tx.execute(`DELETE FROM rows`) + for (const row of rows) await tx.execute(insertRow, sqlColumns(row)) + }) + } catch (error) { + // The database refused the rows: the load fails rather than wait. + pending.reject(error) + throw error + } this.rows = new Map(rows.map((row) => [row.id, row])) this.triggerTarget = this.triggersCreated + 1 pending.resolve() @@ -197,17 +259,11 @@ class PowerSyncProvider implements ControlledProvider { await this.database.writeTransaction(async (tx) => { for (const change of changes) { if (change.type === `upsert`) { - const sql = toSqlRow(change.row) + const [id, ...columns] = sqlColumns(change.row) if (this.rows.has(change.row.id)) { - await tx.execute( - `UPDATE rows SET title = ?, note = ? WHERE id = ?`, - [sql.title, sql.note, String(sql.id)], - ) + await tx.execute(updateRow, [...columns, id!]) } else { - await tx.execute( - `INSERT INTO rows (id, title, note) VALUES (?, ?, ?)`, - [String(sql.id), sql.title, sql.note], - ) + await tx.execute(insertRow, [id!, ...columns]) } this.rows.set(change.row.id, change.row) } else { @@ -223,7 +279,10 @@ class PowerSyncProvider implements ControlledProvider { before.size !== this.rows.size || [...this.rows].some(([id, row]) => { const held = before.get(id) - return held?.title !== row.title || held.note !== row.note + return ( + held === undefined || + JSON.stringify(sqlColumns(held)) !== JSON.stringify(sqlColumns(row)) + ) }) if (registration && changed) this.drainTarget = registration.drained + 1 } @@ -275,10 +334,28 @@ class PowerSyncProvider implements ControlledProvider { } } -type PowerSyncRow = { - id: string - title?: string | null - note?: string | null +type PowerSyncRow = z.infer + +/** A Collection row as a conformance row: typed columns become `typed`. */ +function conformanceRow({ + t_string, + t_integer, + t_float, + t_boolean, + t_date, + ...row +}: PowerSyncRow): ConformanceRow { + return fromSqlRow({ + ...row, + id: conformanceKey(row.id), + typed: { + string: t_string, + integer: t_integer, + float: t_float, + boolean: t_boolean, + date: t_date, + }, + }) } export function powerSyncConformanceDriver(): EagerLiveSourceDriver { @@ -286,6 +363,14 @@ export function powerSyncConformanceDriver(): EagerLiveSourceDriver { adapter: `@tanstack/powersync-db-collection (eager)`, capabilities: new Set(), changeCommits: `per-batch`, + fieldKinds: FIELD_KINDS, + // PowerSync's storage keeps a number at 15 significant digits: through + // its table views, 9007199254740991 reads back as 9007199254740990 and + // 0.30000000000000004 as 0.3, and Number.MAX_VALUE is rejected + // ("powersync_diff: number out of range"). A platform limit, not the + // adapter's: the Collection shows what the database stored. + excludesValue: (_kind, value) => + typeof value === `number` && Number(value.toPrecision(15)) !== value, // PowerSync reports a failed start through its logger (law set // capability `logger`). declaredDiagnostic: (_level, args) => @@ -296,13 +381,15 @@ export function powerSyncConformanceDriver(): EagerLiveSourceDriver { powerSyncCollectionOptions({ database: provider.database, table: SCHEMA.props.rows, + schema: outputSchema, + deserializationSchema, + onDeserializationError: () => {}, onLoad: provider.load, }), ) return { collection: observedView(collection, { - row: (row: PowerSyncRow) => - fromSqlRow({ ...row, id: conformanceKey(row.id) }), + row: conformanceRow, key: conformanceKey, }), provider, From 39294b6e2e55969f3985f3ff5bfca584c62219f4 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 13:45:03 -0600 Subject: [PATCH 16/64] test(conformance): client-created failures, change-carried txids, retired evidence, retry thresholds Extend the eager live-source oracle for adapters whose client library creates failure values and carries txids on row messages, and for retrying sources that report only an extended outage. - failureValue: any. A failure's reported value is the client's: the check owes some error. For retried failures it owes one to one listed failure per attempt, cause the newest, and the outage's first failure kept until recovery (RETRY-1, its own violation name). - txidEvidence: change. A batch with no delivered change carries no txid. - retryReportAfter: k. A retried failure shows from the kth consecutive failed attempt after ready; fewer show nothing. Authority: maintainer decision 2026-10-30 (a retrying source must not flap its health; only an extended outage reports). Default 1 keeps every existing driver. - EVID-2. The model remembers txids committed in retired runs and names EVID-2 for a wait they fulfill. Law set v18 EVID-2: evidence from a retired run or from before a restart does not resolve a wait. - Scenario prefixes, one history in six each: a batch, a restart, and a wait for its txid (await-evidence); an outage one attempt longer than retryReportAfter (retry-policy). New reach states: wait-on-retired-evidence, transient-failure-repeated, transient-failure-below-threshold (only when retryReportAfter > 1). - The wait-off-by-one mutant now shrinks to a wait for a buffered batch, so that label names EVID-1/READY-10/PUB-9. Evidence: - Law: HEALTH-1, HEALTH-2, RETRY-1, EVID-2 (law set v18) with the retryReportAfter maintainer decision above. - Reach: reference fixed campaign, 600 histories, every declared state at least 5 (asserted); Electric's driver (next commit) reaches the new states 19 to 57 times across six seeds at 400 histories. - RED: new mutant keeps-retired-evidence (the reference adapter keeps its applied txids across sync runs) is rejected with EVID-2; Electric's unchanged adapter is RED on EVID-2, HEALTH-1, and the threshold (next commit). - GREEN: reference suite 69/69; TrailBase, RxDB, PowerSync campaigns pass. - Mutant: keeps-retired-evidence -> EVID-2. - Limits: failureValue any does not check failure identity; a held commit receipt is not generated. Co-authored-by: Isaac --- .../src/eager-live-source-oracle.ts | 256 +++++++++++++++--- .../reference-adapter-oracle.property.test.ts | 1 + .../tests/reference-adapter.ts | 4 +- 3 files changed, 224 insertions(+), 37 deletions(-) diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 73db3e7194..9c3deb621e 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -53,12 +53,20 @@ * applied, also when the batch came first, and stays pending before. A * wait with a timeout and no evidence rejects after the timeout, and later * evidence does not change that. A wait still pending when its sync run - * retires rejects. + * retires rejects. Evidence from a retired sync run does not fulfill a + * wait in a later one (EVID-2). * - **HEALTH-1, HEALTH-2, RETRY-1** (capability `retry-policy`). A transient * failure that the adapter keeps retrying shows nothing before the first * ready. After it, the failure joins the node's `failureReason`, oldest * first, with `cause` the newest and no `error`. The next successful - * delivery clears it. + * delivery clears it. With `retryReportAfter: k` (maintainer decision + * 2026-10-30: a retrying source must not flap its health), only an + * extended outage shows: an attempt joins `failureReason` from the `k`th + * consecutive failed attempt after ready on, and fewer show nothing. Under + * `failureValue: any` the reported values are + * the client library's, so the node lists at least one failure and at + * most one per retried attempt, with `cause` its newest; a later attempt + * does not replace the failure first listed for the outage (RETRY-1). * - **RECON-1** (capability `transport-reconnect`). A transport disconnect and * reconnect keep status and rows, and later deliveries still apply. * - **RENEW-1** (capability `capability-renewal`). While the source's @@ -291,6 +299,27 @@ export interface EagerLiveSourceDriver { * holds; required with `duplicate-key-delivery`. */ readonly duplicateInsert?: `error` | `replace` + /** + * `any` when the Collection reports a provider failure with a value the + * adapter's client library creates, such as an HTTP client's error for a + * failed response, rather than the provider's own value. The check then + * owes some error, not that value, also for a retried failure's + * `failureReason`. Default `provider`. + */ + readonly failureValue?: `provider` | `any` + /** + * What carries a batch's txid (`await-evidence`): the batch itself + * (`batch`, the default), or each change in it (`change`), as Electric's + * row messages do. Under `change`, a batch with no delivered change + * carries no txid, so it is no evidence. + */ + readonly txidEvidence?: `batch` | `change` + /** + * The consecutive failed attempts after ready from which a retried + * failure shows in `failureReason` (`retry-policy`); fewer show nothing. + * Default 1: every retried failure shows. + */ + readonly retryReportAfter?: number /** * Console diagnostics the adapter declares (HEALTH-5). Any other * `console.error` or `console.warn` call is a violation. @@ -496,7 +525,11 @@ function changeArb( /** What the grammar needs from a driver: its capabilities and value kinds. */ export type GrammarDomain = Pick< EagerLiveSourceDriver, - `capabilities` | `valueKinds` | `fieldKinds` | `excludesValue` + | `capabilities` + | `valueKinds` + | `fieldKinds` + | `excludesValue` + | `retryReportAfter` > /** Generates histories whose steps use only the driver's declarations. */ @@ -599,7 +632,60 @@ export function eagerLiveSourceHistoryArb( arbitrary: fc.constant({ kind: `renew` as const }), }) } - const history = fc.array(fc.oneof(...steps), { minLength: 1, maxLength: 12 }) + const generated = fc.array(fc.oneof(...steps), { + minLength: 1, + maxLength: 12, + }) + // Some laws need a short sequence that random steps rarely produce, so a + // history can start with one; one in six does per declared scenario. + const snapshot = steps[0]!.arbitrary + const scenarios: Array>> = [] + if (capabilities.has(`await-evidence`)) { + // EVID-2: a txid that a retired run committed and the current run has + // not: a batch with a change, a restart, and a wait for it. + scenarios.push( + fc + .tuple( + snapshot, + fc.array(changeArb(kinds, fieldKinds, domain.excludesValue), { + minLength: 1, + maxLength: 3, + }), + fc.boolean(), + ) + .map(([first, changes, timeout]): Array => [ + first, + { kind: `changes`, changes }, + { kind: `restart` }, + { kind: `wait`, ahead: 0, timeout }, + ]), + ) + } + if (capabilities.has(`retry-policy`)) { + // RETRY-1: an outage after ready one attempt longer than + // `retryReportAfter`, so its first report and a later attempt show. + const attempts = (domain.retryReportAfter ?? 1) + 1 + scenarios.push( + snapshot.map((first): Array => [ + first, + ...Array.from({ length: attempts }, () => ({ + kind: `transient-failure` as const, + })), + ]), + ) + } + const history = + scenarios.length === 0 + ? generated + : fc + .tuple( + fc.oneof( + { weight: 5, arbitrary: fc.constant([]) }, + ...scenarios.map((arbitrary) => ({ weight: 1, arbitrary })), + ), + generated, + ) + .map(([prefix, rest]) => [...prefix, ...rest]) if (!capabilities.has(`listen-before-fetch`)) return history // A startup burst: batches the backend commits while the first fetch is // outstanding. Without it, two buffered batches that touch one key before @@ -660,10 +746,17 @@ interface ModelState { connected: boolean lastTxid: number committed: Set + /** Txids committed in retired sync runs; they name EVID-2. */ + retiredEvidence: Set renewed: boolean - /** A provider failure's index, or `adapter` for the adapter's own error. */ - failure: number | `adapter` | undefined + /** + * A provider failure's index; `any` for one whose reported value the + * client library creates; `adapter` for the adapter's own error. + */ + failure: number | `any` | `adapter` | undefined transient: Array + /** Consecutive failed attempts after ready in the current outage. */ + outage: number waits: Array<{ run: number; txid: number; expected: WaitExpectation }> } @@ -812,9 +905,11 @@ function initialModel(seed: ReadonlyArray = []): ModelState { connected: true, lastTxid: 0, committed: new Set(), + retiredEvidence: new Set(), renewed: false, failure: undefined, transient: [], + outage: 0, waits: [], } } @@ -827,6 +922,12 @@ function canDeliver(state: ModelState): boolean { } /** Retiring a sync run rejects its pending waits (EVID-2). */ +/** HEALTH-2: a delivery, failure, or restart ends the outage. */ +function endOutage(state: ModelState): void { + state.transient = [] + state.outage = 0 +} + function retireWaits(state: ModelState): void { for (const wait of state.waits) { if (wait.run === state.run && wait.expected === `pending`) { @@ -972,6 +1073,9 @@ interface ModelOptions { /** The value kinds include `negative-zero`, so a retouch flips zeros. */ flipZeros: boolean duplicateInsert: `error` | `replace` + failureValue: `provider` | `any` + txidEvidence: `batch` | `change` + retryReportAfter: number } /** EVID-1: a wait for `txid` fulfills once its batch applied. */ @@ -1040,7 +1144,7 @@ function stepModel( case `snapshot`: { if (!canDeliver(state)) return undefined // HEALTH-2: a successful delivery clears the open transient failures. - state.transient = [] + endOutage(state) if (options.listenBeforeFetch && state.phase === `loading`) { const buffered = state.buffered.length const seen = Math.min(step.seen ?? buffered, buffered) @@ -1048,7 +1152,11 @@ function stepModel( state.rows = cut.backend state.backend = new Map(cut.backend) state.phase = `ready` - for (const batch of state.buffered) applyEvidence(state, batch.txid) + for (const batch of state.buffered) { + if (options.txidEvidence === `batch` || batch.changes.length > 0) { + applyEvidence(state, batch.txid) + } + } state.buffered = [] return { done: { @@ -1129,8 +1237,11 @@ function stepModel( } foldBatch(state.backend, delivered) state.lastTxid += 1 - state.transient = [] - applyEvidence(state, state.lastTxid) + endOutage(state) + // EVID-1: under `change`, only a batch with a change carries its txid. + if (options.txidEvidence === `batch` || delivered.length > 0) { + applyEvidence(state, state.lastTxid) + } return { done: { kind: `changes`, @@ -1167,11 +1278,11 @@ function stepModel( case `fail`: { if (!canDeliver(state)) return undefined const failure = failures.next++ - state.failure = failure + state.failure = options.failureValue === `any` ? `any` : failure // READY-4 before the first ready; HEALTH-1 after it. LIFE-10: the // source ends here. state.phase = state.phase === `loading` ? `loading-error` : `stopped` - state.transient = [] + endOutage(state) // No law fixes what a pending txid wait does after its source failed. for (const wait of state.waits) { if (wait.run === state.run && wait.expected === `pending`) { @@ -1196,7 +1307,7 @@ function stepModel( state.rows = rowMap(rows) state.backend = rowMap(rows) state.phase = `ready` - state.transient = [] + endOutage(state) return { done: { kind: `malformed`, rows, afterReady, replaced: true }, publishable: [rowState(state.rows)], @@ -1204,7 +1315,7 @@ function stepModel( } state.failure = `adapter` state.phase = afterReady ? `stopped` : `loading-error` - state.transient = [] + endOutage(state) state.buffered = [] for (const wait of state.waits) { if (wait.run === state.run && wait.expected === `pending`) { @@ -1220,8 +1331,14 @@ function stepModel( if (!canDeliver(state)) return undefined const failure = failures.next++ // RETRY-1: before the first ready a retried failure shows nothing. - // HEALTH-1: after it, the failure joins `failureReason`. - if (state.phase === `ready`) state.transient.push(failure) + // HEALTH-1: after it, the failure joins `failureReason` once the + // outage reached `retryReportAfter` attempts. + if (state.phase === `ready`) { + state.outage += 1 + if (state.outage >= options.retryReportAfter) { + state.transient.push(failure) + } + } return { done: { kind: `transient-failure`, failure }, publishable: [] } } case `disconnect`: { @@ -1254,10 +1371,12 @@ function stepModel( state.buffered = [] state.connected = true state.lastTxid = 0 + // EVID-2: the retired run's evidence does not fulfill a later wait. + for (const txid of state.committed) state.retiredEvidence.add(txid) state.committed = new Set() state.renewed = false state.failure = undefined - state.transient = [] + endOutage(state) return { done: { kind: `restart` }, publishable: [] } } } @@ -1274,13 +1393,20 @@ function expectedObservation(state: ModelState): ExpectedObservation { ? `error` : `ready`, rows: rowState(state.rows), - healthError: running ? null : state.failure!, + healthError: running + ? null + : state.failure === `any` + ? `adapter` + : state.failure!, failureReason: state.transient, preload: state.phase === `loading` ? `pending` : state.phase === `loading-error` - ? { rejectedWith: state.failure! } + ? { + rejectedWith: + state.failure === `any` ? `adapter` : state.failure!, + } : `fulfilled`, waits: state.waits.map((wait) => wait.expected), // LIFE-10: a failed provider's source is released at its end. @@ -1310,7 +1436,10 @@ export const eagerLiveSourceReachStates = [ `wait-before-evidence-fulfilled`, `wait-timed-out`, `wait-rejected-at-retirement`, + `wait-on-retired-evidence`, `transient-failure-reported`, + `transient-failure-below-threshold`, + `transient-failure-repeated`, `transient-failure-recovered`, `reconnect`, `renewal`, @@ -1342,7 +1471,10 @@ const capabilityOfState: Partial< 'wait-before-evidence-fulfilled': `await-evidence`, 'wait-timed-out': `await-evidence`, 'wait-rejected-at-retirement': `await-evidence`, + 'wait-on-retired-evidence': `await-evidence`, 'transient-failure-reported': `retry-policy`, + 'transient-failure-below-threshold': `retry-policy`, + 'transient-failure-repeated': `retry-policy`, 'transient-failure-recovered': `retry-policy`, reconnect: `transport-reconnect`, renewal: `capability-renewal`, @@ -1363,11 +1495,14 @@ export function reachStatesFor( const capability = capabilityOfState[state] return capability === undefined || domain.capabilities.has(capability) }) - // A malformed delivery after ready also needs `complete-snapshot`. + // A malformed delivery after ready also needs `complete-snapshot`; an + // unreported attempt needs `retryReportAfter` above 1. const reachable = fixed.filter( (state) => - state !== `malformed-after-ready` || - domain.capabilities.has(`complete-snapshot`), + (state !== `malformed-after-ready` || + domain.capabilities.has(`complete-snapshot`)) && + (state !== `transient-failure-below-threshold` || + (domain.retryReportAfter ?? 1) > 1), ) const values = (domain.valueKinds ?? []).flatMap( (kind): Array => [ @@ -1528,6 +1663,9 @@ export async function checkEagerLiveSourceHistory( listenBeforeFetch, flipZeros: driver.valueKinds?.includes(`negative-zero`) === true, duplicateInsert: driver.duplicateInsert ?? `error`, + failureValue: driver.failureValue ?? `provider`, + txidEvidence: driver.txidEvidence ?? `batch`, + retryReportAfter: driver.retryReportAfter ?? 1, } if ( driver.capabilities.has(`duplicate-key-delivery`) && @@ -1616,9 +1754,13 @@ export async function checkEagerLiveSourceHistory( // `law` names the step's law. Observations whose law does not depend on // the step name their own: HEALTH-2 when a delivery should have cleared a // retried failure, EVID-1 for a wait that fulfilled without evidence - // (READY-10/PUB-9 when its batch should still be buffered), + // (also READY-10/PUB-9 when its batch should still be buffered; EVID-2 + // when a retired run committed it), // EVID-3 for a timed wait that did not reject, and LIFE-10 for a failed // provider's subscription that stays open or is released twice. + // Under `failureValue: any`, the model's first open transient failure + // names the outage, and `cause` is the failure first reported for it. + let firstReported: { outage: number; cause: unknown } | undefined const compare = async ( stepIndex: number, law: string, @@ -1702,14 +1844,43 @@ export async function checkEagerLiveSourceHistory( got, ) } - checkReason( - reasons.length === wantReasons.length && - reasons.every((cause, index) => cause === wantReasons[index]) && - (wantReasons.length === 0 || - health.failureReason?.cause === wantReasons.at(-1)), - wantReasons.map((error) => error.message), - reasons.map(String), - ) + if (modelOptions.failureValue === `provider`) { + checkReason( + reasons.length === wantReasons.length && + reasons.every((cause, index) => cause === wantReasons[index]) && + (wantReasons.length === 0 || + health.failureReason?.cause === wantReasons.at(-1)), + wantReasons.map((error) => error.message), + reasons.map(String), + ) + } else { + // The client library's values: one to one per attempt, newest as + // `cause`, and the outage's first failure kept until it recovers. + checkReason( + wantReasons.length === 0 + ? reasons.length === 0 + : reasons.length >= 1 && + reasons.length <= wantReasons.length && + health.failureReason?.cause === reasons.at(-1), + wantReasons.length === 0 ? [] : `1 to ${wantReasons.length} failures`, + reasons.map(String), + ) + const outage = expected.failureReason[0] + if ( + outage !== undefined && + outage === firstReported?.outage && + reasons[0] !== firstReported.cause + ) { + throw violation( + `RETRY-1 (health failureReason)`, + stepIndex, + `the outage's first failure kept`, + `a replaced failure`, + ) + } + firstReported = + outage === undefined ? undefined : { outage, cause: reasons[0] } + } const want = expected.preload const got = preload.current check( @@ -1727,8 +1898,8 @@ export async function checkEagerLiveSourceHistory( const gotWait = waits[index]!.current const gotKind = typeof gotWait === `object` ? `rejected` : gotWait if (gotKind === wantWait) return - // A wait for a batch still buffered before the cut fulfilled because - // the adapter applied that batch early. + // A wait for a batch still buffered before the cut fulfilled: the + // adapter applied that batch early, or fulfilled the wait without it. const buffered = state.buffered.some( (batch) => batch.txid === state.waits[index]!.txid && @@ -1737,8 +1908,10 @@ export async function checkEagerLiveSourceHistory( const waitLaw = gotKind === `fulfilled` ? buffered - ? `READY-10/PUB-9` - : `EVID-1` + ? `EVID-1/READY-10/PUB-9` + : state.retiredEvidence.has(state.waits[index]!.txid) + ? `EVID-2` + : `EVID-1` : wantWait === `rejected` && state.waits[index]!.run === state.run ? `EVID-3` : law @@ -1867,6 +2040,13 @@ export async function checkEagerLiveSourceHistory( done.timeout ? waitTimeoutMs : undefined, ) waits.push(recordSettlement(pending)) + // EVID-2: a retired run committed this txid; this run has not. + if ( + state.retiredEvidence.has(done.txid) && + !state.committed.has(done.txid) + ) { + reach(`wait-on-retired-evidence`) + } if (done.timeout) { await sleep(timeoutSettleMs) if (state.waits.at(-1)!.expected === `rejected`) @@ -1897,7 +2077,11 @@ export async function checkEagerLiveSourceHistory( ) failures[done.failure] = error await provider.transientFailure!(error) - if (wasReady) reach(`transient-failure-reported`) + if (state.transient.at(-1) === done.failure) { + reach(`transient-failure-reported`) + } else if (wasReady) reach(`transient-failure-below-threshold`) + // RETRY-1: a later attempt in the same outage. + if (transientBefore > 0) reach(`transient-failure-repeated`) break } case `disconnect`: diff --git a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts index d27799147f..703d2702fc 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts @@ -106,6 +106,7 @@ describe(`eager live-source conformance: reference adapter`, () => { [`detached-rejection`, `HEALTH-5`], [`logs-failure`, `HEALTH-5`], [`wait-off-by-one`, `EVID-1`], + [`keeps-retired-evidence`, `EVID-2`], [`wait-ignores-timeout`, `EVID-3`], [`disconnect-fails`, `RECON-1`], [`renew-resubscribes`, `RENEW-1`], diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index 7415a2352f..35f67b46d3 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -55,6 +55,8 @@ export type ReferenceMutant = | `logs-failure` /** EVID-1: a wait for the next txid fulfills before that batch arrives. */ | `wait-off-by-one` + /** EVID-2: a retired sync run's evidence fulfills a later wait. */ + | `keeps-retired-evidence` /** EVID-3: a wait ignores its timeout. */ | `wait-ignores-timeout` /** RECON-1: a transport drop is reported as a failure. */ @@ -267,7 +269,7 @@ function referenceCollection( }> = [] let keys = new Set() const opened: Array = [] - applied = new Set() + if (mutant !== `keeps-retired-evidence`) applied = new Set() waiters = new Map() highestTxid = 0 From 8158548154fb17217ffb65e798616ddaa37dc992 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 13:45:35 -0600 Subject: [PATCH 17/64] fix(electric): report extended outages, scope txid evidence to a run, give up on a throwing onError Run the Electric adapter (eager mode, installed @electric-sql/client, a controlled fetchClient that answers the HTTP shape protocol, typed Postgres columns) on the shared eager live-source conformance oracle, and fix the three defects it found. - After ready, a retried outage now shows in collection.health. The adapter wraps the client's fetchClient and counts attempts the client retries (network error, 429, 5xx) and onError retry decisions; the fifth in a row reports the outage once through reportError, and a commit point ends it, clearing its failure once that commit applies. Shorter outages show nothing (maintainer decision 2026-10-30: no flapping). The Electric docs said a retry reports nothing; they now state the threshold. - Retirement forgets the run's txid and snapshot evidence, as scoped recovery already did, so a restarted Collection's awaitTxId waits for the new stream's evidence (EVID-2; Electric README: transaction evidence belongs to one collection lifecycle). - An onError that throws gives up by returning nothing, so the client reports its error to the subscriber; rethrown, it rejected the client's own start unobserved. Evidence: - Law: HEALTH-1, HEALTH-2, RETRY-1 with retryReportAfter 5 (maintainer decision 2026-10-30), EVID-2, HEALTH-5 (law set v18; "handler throws" is a give-up under RETRY-1). - Reach: 400 histories per campaign; across six seeds the lowest declared state is field-removed at 19; transient-failure-reported 48+, transient-failure-recovered 32+, wait-on-retired-evidence 41+. - RED (unchanged adapter): HEALTH-1 at [snapshot [], transient-failure, changes []]; the same for a 400 that onError retries; EVID-2 at a wait after restart for the retired run's txid; HEALTH-5 at [fail, restart, fail] (throwing give-up). The first fix, reporting every attempt, went RED against the threshold decision. - GREEN: both campaigns pass; Electric suite 1242 passed (4 skipped); TrailBase, RxDB, PowerSync, and reference campaigns pass. - Mutants of src/electric.ts, each rejected: no retry reporting, no count for an onError retry, threshold 4, threshold 6, a count a commit does not reset (HEALTH-1); a new failure per attempt that replaces the first (RETRY-1); no recovery (HEALTH-2); evidence kept across retirement (EVID-2); a rethrown onError error (HEALTH-5). - Limits: eager mode only; no persisted resume, so a held commit receipt is not exercised; no real Electric server; failure identity unchecked under failureValue any. - Size: electric-db-collection +223 gzip bytes (+2.3%); the app scenario is unchanged. onFailedAttempt was ruled out: it carries no error and also fires on a must-refetch 409 and on aborts. Co-authored-by: Isaac --- .../electric-retry-health-and-evidence.md | 5 + docs/collections/electric-collection.md | 16 +- docs/contributing/oracle-coverage.md | 26 +- packages/db-adapter-conformance/MIGRATION.md | 93 +++- packages/db-adapter-conformance/README.md | 7 +- packages/db/tests/oracle-config.ts | 1 + packages/db/tests/oracle-replay-manifest.ts | 14 +- .../electric-db-collection/src/electric.ts | 97 +++- .../tests/conformance-driver.ts | 434 ++++++++++++++++++ ...source-conformance-oracle.property.test.ts | 77 ++++ 10 files changed, 729 insertions(+), 41 deletions(-) create mode 100644 .changeset/electric-retry-health-and-evidence.md create mode 100644 packages/electric-db-collection/tests/conformance-driver.ts create mode 100644 packages/electric-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts diff --git a/.changeset/electric-retry-health-and-evidence.md b/.changeset/electric-retry-health-and-evidence.md new file mode 100644 index 0000000000..d24cb1c58c --- /dev/null +++ b/.changeset/electric-retry-health-and-evidence.md @@ -0,0 +1,5 @@ +--- +'@tanstack/electric-db-collection': patch +--- + +Report an extended Electric outage in `collection.health`. After the Collection is ready, five failed attempts in a row, retried by the client or by an `onError` that returns a retry, show as the source node's `failureReason`, and the next applied up-to-date clears it; shorter outages leave health unchanged. A restarted Collection no longer resolves `awaitTxId` from a txid the previous sync run delivered. An `onError` handler that throws now gives up the stream without an unhandled rejection. diff --git a/docs/collections/electric-collection.md b/docs/collections/electric-collection.md index a206107d60..ecf4296a78 100644 --- a/docs/collections/electric-collection.md +++ b/docs/collections/electric-collection.md @@ -344,7 +344,9 @@ await todosCollection.utils.awaitMatch( Transaction evidence and pending `awaitTxId`/`awaitMatch` calls belong to one collection lifecycle. Explicit cleanup and automatic garbage collection reject pending waits with `StreamAbortedError`; callbacks from the retired stream cannot -settle waits in a restarted collection. Observe these promises even when the +settle waits in a restarted collection, and a restarted collection does not +treat a txid the retired stream delivered as synced: `awaitTxId` waits for the +new stream's evidence. Observe these promises even when the component or collection may be disposed before they settle. An on-demand Collection with managed SQLite persistence can also replace its @@ -429,10 +431,14 @@ source node shows an `error` in `collection.health` and in each live query's `health` tree. This does not silently retry the failed stream; `collection.reload()` refetches. -While the Electric stream retries a failed request, the Collection reports -nothing: it stays `loading` before its first up-to-date, and keeps its health tree -unchanged afterward. When the stream gives up, it calls `fail()`: a failed first load -before readiness, or the source node's `error` after it. Complete updates from an explicit `replica: 'full'` stream remain valid. +While the Electric stream retries a failed request, the Collection stays +`loading` before its first up-to-date. After it, a short outage leaves the +health tree unchanged, so a blip does not flap it. Once five attempts in a row +have failed, whether the client retried them or your `onError` returned a +retry, the source node shows the outage as its `failureReason`; the next +applied up-to-date clears it. When the stream gives up, including when +`onError` throws, it calls `fail()`: a failed first load before readiness, or +the source node's `error` after it. Complete updates from an explicit `replica: 'full'` stream remain valid. On-demand streams can observe updates outside their loaded subsets; unknown partial rows are ignored, while transaction acknowledgement evidence is retained. Outside scoped recovery, complete rows published by persistence reloads or diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 4dcd64c4a5..5325d71aa1 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6003,7 +6003,7 @@ commit per batch or per change. PUB-1 checks every change publication during a step, not only the settled checkpoint. The reference push adapter (`tests/reference-adapter-oracle.property.test.ts`, `adapter-conformance.eager-live-source.reference`) runs fixed and random -campaigns of 600 histories with declared reach, a per-change variant, and 27 +campaigns of 600 histories with declared reach, a per-change variant, and 35 law-named hostile mutants, with ORC-004 reconstruction, exclusion, and ablation controls. The Neon Realtime receiving driver (`adapter-conformance.eager-live-source.neon-realtime`, `test:neon`) runs @@ -6045,3 +6045,27 @@ table views keep 15 significant digits (9007199254740991 reads back as 9007199254740990, 0.30000000000000004 as 0.3) and reject Number.MAX_VALUE (`powersync_diff: number out of range`); owner: the PowerSync driver; evidence: a direct write and read through `@powersync/node` 0.20.0. +Electric (`adapter-conformance.eager-live-source.electric`, eager mode over +the installed `@electric-sql/client` and a controlled `fetchClient` that +answers the HTTP protocol, per-batch commits, typed Postgres columns for +nine kinds) is GREEN after three fixes. After ready, an outage of five +consecutive failed attempts that the client retries (network error, 429, +5xx) or the shape's `onError` retries is now reported in `failureReason`, +once, and the next applied commit point clears it; shorter outages show +nothing (HEALTH-1, HEALTH-2, RETRY-1, driver parameter +`retryReportAfter: 5`; maintainer decision 2026-10-30: a retrying source +must not flap its health, so only an extended outage reports). A restarted run no +longer settles `awaitTxId` from the retired run's txids (EVID-2). An +`onError` that throws now gives up without a detached rejection (HEALTH-5, +RETRY-1); the driver gives up alternately by returning nothing and by +throwing. Electric's +failures are values its client creates (`failureValue: any`), so the check +owes some error, at most one per attempt, and the outage's first failure +kept; it does not check identity. Its empty batches carry no txid +(`txidEvidence: change`). Excluded by `excludesValue` (column domains, not +adapter behavior): int4 values outside 32 bits, and a `-0` nested in an +array or object for the float8 column. Limits: no on-demand or progressive +mode, no persisted resume (so a held commit receipt that clears an outage +late is not exercised), and no real Electric server. Two generated scenario +prefixes reach EVID-2 (a batch, a restart, a wait for its txid) and the +second retried failure of one outage. diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index d9a308dba0..12b0de0317 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -27,23 +27,23 @@ reviewed change. ## Adapters -| Adapter | Families that apply (section A) | Driver status | -| ----------------- | ------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Neon Realtime | eager live source, mutations | Eager: RED (READY-4, HEALTH-1, and LIFE-10 behind them); adapter update pending | -| Electric | eager live source, mutations, on-demand, source shapes, reload and retry, persistence | Not started | -| PowerSync | eager live source, mutations, on-demand | Eager: GREEN (2026-10-30) after a fix, per-batch commits, over real local SQLite. Driver `powersync-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.powersync`. Mutations, on-demand: not started | -| RxDB | eager live source, mutations | Eager: GREEN (2026-10-29) after a fix, per-change commits, `listen-before-fetch`. Driver `rxdb-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.rxdb`. Mutations: not started | -| TrailBase | eager live source, mutations, on-demand | Eager: GREEN (2026-10-29), per-change commits. Driver `trailbase-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.trailbase`; declares no optional capability. Mutations, on-demand: not started | -| Convex | eager live source, mutations, on-demand, source shapes | Not started | -| query-db | client cache and refetch, on-demand, mutations, reload and retry, persistence | Not started; not a push source, so the eager family does not apply | -| local-only | local and shared stores, mutations | Not started | -| localStorage | local and shared stores, mutations | Not started | -| Persisted wrapper | persistence wrapper | Not started | -| Storage hosts (8) | storage hosts | Not started | +| Adapter | Families that apply (section A) | Driver status | +| ----------------- | ------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Neon Realtime | eager live source, mutations | Eager: RED (READY-4, HEALTH-1, and LIFE-10 behind them); adapter update pending | +| Electric | eager live source, mutations, on-demand, source shapes, reload and retry, persistence | Eager: GREEN (2026-10-30) after three fixes, per-batch commits, typed Postgres columns, over the installed client. Driver `electric-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.electric`. Other families: not started | +| PowerSync | eager live source, mutations, on-demand | Eager: GREEN (2026-10-30) after a fix, per-batch commits, over real local SQLite. Driver `powersync-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.powersync`. Mutations, on-demand: not started | +| RxDB | eager live source, mutations | Eager: GREEN (2026-10-29) after a fix, per-change commits, `listen-before-fetch`. Driver `rxdb-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.rxdb`. Mutations: not started | +| TrailBase | eager live source, mutations, on-demand | Eager: GREEN (2026-10-29), per-change commits. Driver `trailbase-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.trailbase`; declares no optional capability. Mutations, on-demand: not started | +| Convex | eager live source, mutations, on-demand, source shapes | Not started | +| query-db | client cache and refetch, on-demand, mutations, reload and retry, persistence | Not started; not a push source, so the eager family does not apply | +| local-only | local and shared stores, mutations | Not started | +| localStorage | local and shared stores, mutations | Not started | +| Persisted wrapper | persistence wrapper | Not started | +| Storage hosts (8) | storage hosts | Not started | ## Suggested order -1. **Push adapters on the existing family:** RxDB (done), TrailBase (done), PowerSync (done), Electric (eager mode), and Convex. Each needs only a controlled-provider driver. A RED result is a finding against that adapter. +1. **Push adapters on the existing family:** RxDB (done), TrailBase (done), PowerSync (done), Electric (eager mode, done), and Convex. Each needs only a controlled-provider driver. A RED result is a finding against that adapter. 2. **The Neon update, run against its driver.** This shows an adapter moving from RED to GREEN. 3. **The mutations family.** It applies to every adapter except the storage hosts. 4. **The other families,** one at a time. Each family's new driver work goes to the adapters that declare it. @@ -287,3 +287,68 @@ document update removes`) failed the same way. range`). The Collection shows what the database stored, so this is not the adapter's FID-1; the driver excludes such numbers. Worth reporting to PowerSync. +- **2026-10-30, Electric, eager live source: three Electric defects.** The + driver runs the real adapter over the installed `@electric-sql/client`, + whose `fetchClient` the driver answers with the HTTP shape protocol, as + `electric-sdk-delivery-oracle` does. A complete delivery answers the + initial request, or, after ready, a must-refetch (409) and the refetch; a + batch answers the live request with row messages that carry its txid; a + failure is an HTTP 400 that the shape's `onError` gives up on; a retried + failure is an HTTP 503 the client retries or an HTTP 400 that `onError` + retries. The propagation step waits until the client asks again, or until + `onError` gave up. + - **Oracle changes this exposed:** `failureValue: any` for failures whose + value the client creates. The retried-failure check then owes one to one + listed failure per attempt, `cause` its newest, and the outage's first + failure kept until recovery (RETRY-1). `retryReportAfter: k`: a retried + failure shows only from the `k`th consecutive failed attempt after ready + (maintainer decision 2026-10-30: no flapping on a blip; Electric + declares 5), with reach state `transient-failure-below-threshold`. + `txidEvidence: change`: Electric's + empty batches carry no txid. The model names EVID-2 for a wait fulfilled + by a retired run's txid; reference mutant `keeps-retired-evidence`. Two + generated scenario prefixes (one history in six each) reach EVID-2 and a + repeated retried failure, with reach states `wait-on-retired-evidence` + and `transient-failure-repeated`. The `wait-off-by-one` mutant now + shrinks to a wait for a buffered batch, so that label names + `EVID-1/READY-10/PUB-9`. + - **RED 1 (HEALTH-1):** after ready, a retried 503 left health empty. + Shrunk: `[snapshot [], transient-failure, changes []]`. The same for a + 400 that `onError` retries. After the threshold decision, the first + fix (report every attempt) went RED in turn: one 503 after ready showed + a failure where the law owes none. + - **RED 2 (EVID-2):** after a restart, `awaitTxId` for a txid the retired + run delivered fulfilled before the new run delivered it. Electric kept + `seenTxids` for the Collection's life, against its README ("Transaction + evidence ... belong to one collection lifecycle"). + - **RED 3 (HEALTH-5):** an `onError` that throws was rethrown into the + client, whose start then rejected unobserved. Shrunk: + `[fail, restart, fail]`. + - **Fixes:** the adapter wraps the client's `fetchClient` and counts each + retried attempt (network error, 429, 5xx) after ready, and each `onError` + retry decision; the fifth in a row reports the outage once through + `reportError`. A commit point ends the outage, and `reportHealthy` + clears its failure once that commit applies. Retirement forgets the run's + evidence, as scoped recovery already did. A throwing `onError` gives up + by returning nothing, so the client reports its error to the subscriber. + - **GREEN:** Electric suite 1242 passed (4 skipped); both campaigns pass + at 400 histories. Across six seeds the lowest reach is 19 + (`field-removed`). + - **Mutants of `src/electric.ts`:** no retry reporting (HEALTH-1), no + count for an `onError` retry (HEALTH-1), a threshold of 4 or 6 + (HEALTH-1), an attempt count that a commit does not reset (HEALTH-1), a + new failure per attempt that replaces the first (RETRY-1), no recovery + (HEALTH-2), evidence kept across retirement (EVID-2), a rethrown + `onError` error (HEALTH-5). + - **Size:** Electric +223 gzip bytes (+2.3%); the `app` scenario does not + include Electric. The cost is the `fetchClient` wrapper, which builds a + `FetchError` with the status and headers an app can read, and one + attempt counter shared by both retry paths. `onFailedAttempt` was ruled + out: it carries no error and also fires for a must-refetch 409 and for + aborts. + - **Driver lessons:** a retry is part of its run's request chain; only a + request after an abort or a give-up starts a new run. A float8 column + holds a scalar `-0` only, and int4 holds 32 bits; both are column + domains (`excludesValue`). Waiting out a deadline after each give-up + made histories take 1 s each; the driver now counts the give-ups its + own `onError` made. diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md index ff2cb7ea41..824925d14a 100644 --- a/packages/db-adapter-conformance/README.md +++ b/packages/db-adapter-conformance/README.md @@ -62,7 +62,7 @@ reach counter, so the run fails when histories miss a state the laws need. - **Reference adapter** (`tests/reference-adapter.ts`): a small push adapter written to conform. `tests/reference-adapter-oracle.property.test.ts` runs a fixed-seed and a random campaign with declared reach, a per-change variant, - and 20 hostile mutants of the adapter, each of which the check must reject + and 35 hostile mutants of the adapter, each of which the check must reject with its law. It runs in the default `pnpm test`. - **Neon Realtime** (`tests/neon/`): `@neon/realtime-tanstack` 0.1.1, vendored @@ -74,6 +74,11 @@ reach counter, so the run fails when histories miss a state the laws need. until the adapter is updated, so it has its own config: `pnpm --filter @tanstack/db-adapter-conformance test:neon`. +- **Electric** (`packages/electric-db-collection/tests/conformance-driver.ts`): + the published adapter in eager mode over the installed + `@electric-sql/client`, through a controlled `fetchClient` that answers + the HTTP shape protocol, with typed Postgres columns and per-batch + commits. It runs in that package's default `pnpm test`. - **PowerSync** (`packages/powersync-db-collection/tests/conformance-driver.ts`): the published adapter in eager mode over a real local PowerSync database, with per-batch commits and a declared propagation step (`settle`). It runs diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 644a24b6a4..65e425b8dd 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -147,6 +147,7 @@ const staticOracleProperties = [ `adapter-conformance.eager-live-source.trailbase`, `adapter-conformance.eager-live-source.rxdb`, `adapter-conformance.eager-live-source.powersync`, + `adapter-conformance.eager-live-source.electric`, `trailbase.lifecycle`, `source-health.history`, `source-health.required-history`, diff --git a/packages/db/tests/oracle-replay-manifest.ts b/packages/db/tests/oracle-replay-manifest.ts index 32717e5091..5c0c6de6ae 100644 --- a/packages/db/tests/oracle-replay-manifest.ts +++ b/packages/db/tests/oracle-replay-manifest.ts @@ -663,6 +663,11 @@ const ownerGroups: ReadonlyArray< `adapter-conformance.eager-live-source`, `powersync`, ], + [ + `electric-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts`, + `adapter-conformance.eager-live-source`, + `electric`, + ], ] const owners = new Map() @@ -749,7 +754,14 @@ export type OracleReplayManifestEntry = { // Pagination cells carry their property name in the test title so the guard // can select one cell without running the rest of the matrix. const replayTestNames = new Map([ - ...[`reference`, `neon-realtime`, `trailbase`, `rxdb`, `powersync`].map( + ...[ + `reference`, + `neon-realtime`, + `trailbase`, + `rxdb`, + `powersync`, + `electric`, + ].map( (driver) => [ `adapter-conformance.eager-live-source.${driver}`, diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 14ecda8745..669d74c4b1 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1,4 +1,5 @@ import { + FetchError, ShapeStream, isChangeMessage, isControlMessage, @@ -953,10 +954,11 @@ class ElectricLifecycle> { } this.pendingTxidWaits.clear() this.matchBuffer = { committedMessages: [], pendingMessages: [] } - this.evidence.hydratedResumeState = undefined + // A retired run's evidence does not settle a later run's waits (EVID-2). + this.forgetEvidence() } - forgetEvidenceAfterScopedRecovery(): void { + forgetEvidence(): void { this.evidence.seenTxids.clear() this.evidence.seenSnapshots.length = 0 this.evidence.hydratedResumeState = undefined @@ -1628,7 +1630,7 @@ function createElectricSync>( } = tagState if (freshScopedGeneration) clearTagTrackingState() const lifecycle = getLifecycle(params.collection) - if (freshScopedGeneration) lifecycle.forgetEvidenceAfterScopedRecovery() + if (freshScopedGeneration) lifecycle.forgetEvidence() const lifecycleEpoch = lifecycle.start() const isActiveLifecycle = () => lifecycle.isActive(lifecycleEpoch) Object.assign(params.collection.utils, lifecycle.utils) @@ -1650,6 +1652,32 @@ function createElectricSync>( ready = true params.markReady() } + // After the first ready, a retried outage shows only once it lasted + // five failed attempts in a row, so health does not flap on a blip + // (maintainer decision). It is reported once: later attempts keep its + // failure. A commit point ends the outage, and its failure clears once + // that commit applies (HEALTH-1, HEALTH-2, RETRY-1). A give-up after + // ready ends health reports; the other failures abort the run. + let failedAttempts = 0 + let retrying: unknown + let gaveUp = false + const reportsHealth = () => ready && !gaveUp && isActiveLifecycle() + const reportRetry = (cause: unknown) => { + if (!reportsHealth() || abortController.signal.aborted) return + if (++failedAttempts < 5) return + retrying ??= cause + params.reportError(retrying) + } + const endOutage = () => { + const recovered = retrying + failedAttempts = 0 + retrying = undefined + return () => { + if (recovered !== undefined && reportsHealth()) { + params.reportHealthy(recovered) + } + } + } // The stream can give up (fail()) from several places; the first one // stops this sync run. Core ignores a later fail() in the same run, so // the first error stays, and a later markReady(), such as a late @@ -1731,7 +1759,7 @@ function createElectricSync>( requiresFreshSourceEvidence && persistence?.managedCacheGeneration === true && persistence.startScopedRecovery !== undefined - if (scopedRecovery) lifecycle.forgetEvidenceAfterScopedRecovery() + if (scopedRecovery) lifecycle.forgetEvidence() const usesChangesOnlyLog = syncMode === `on-demand` && (!requiresFreshSourceEvidence || scopedRecovery) @@ -1896,6 +1924,31 @@ function createElectricSync>( shapeOptions.handle ?? (canUsePersistedResume ? persistedResumeState.handle : undefined), signal: abortController.signal, + // The client retries a network error, 429, and 5xx without calling + // onError; onError decides the other failures. + fetchClient: async (input, init) => { + try { + const response = await (shapeOptions.fetchClient ?? fetch)( + input, + init, + ) + if (response.status === 429 || response.status >= 500) { + reportRetry( + new FetchError( + response.status, + undefined, + undefined, + Object.fromEntries(response.headers), + String(input), + ), + ) + } + return response + } catch (error) { + if (!init?.signal?.aborted) reportRetry(error) + throw error + } + }, onError: async (errorParams) => { if (!isActiveLifecycle() || abortController.signal.aborted) return streamErrorVersion++ @@ -1919,10 +1972,14 @@ function createElectricSync>( try { retry = await shapeOptions.onError(errorParams) } catch (error) { + // A throwing handler gives up. Rethrown, its error would reject + // the client's own start unobserved; returning nothing stops the + // stream, and the client reports its error to the subscriber. failWaits(error) - throw error + return } if (!retry || typeof retry !== `object`) failWaits(errorParams) + else reportRetry(errorParams) return retry } else { const initialSyncFailed = !ready @@ -2554,19 +2611,18 @@ function createElectricSync>( () => undefined, ) } - if (applied !== true) { - void applied.then( - () => undefined, - (error: unknown) => { - if (!isActiveLifecycle() || abortController.signal.aborted) { - return - } - streamErrorVersion++ - stream.unsubscribeAll() - abortController.abort() - fail(error) - }, - ) + const recover = endOutage() + if (applied === true) recover() + else { + void applied.then(recover, (error: unknown) => { + if (!isActiveLifecycle() || abortController.signal.aborted) { + return + } + streamErrorVersion++ + stream.unsubscribeAll() + abortController.abort() + fail(error) + }) } if (finishesReset) { @@ -2623,7 +2679,10 @@ function createElectricSync>( rejectFullSnapshot(error) if (isActiveLifecycle() && !abortController.signal.aborted) { if (!ready) failInitialSync(error) - else fail(error) + else { + gaveUp = true + fail(error) + } } // A source that stops must end its reserved transaction. The initial // failure decides first: abandoning commits. diff --git a/packages/electric-db-collection/tests/conformance-driver.ts b/packages/electric-db-collection/tests/conformance-driver.ts new file mode 100644 index 0000000000..8edad5c7ea --- /dev/null +++ b/packages/electric-db-collection/tests/conformance-driver.ts @@ -0,0 +1,434 @@ +/** + * Production driver for the shared eager live-source conformance oracle + * (`packages/db-adapter-conformance`). It runs the real Electric adapter in + * eager mode over the installed `@electric-sql/client` `ShapeStream`, which + * fetches through a controlled `fetchClient`, as + * `electric-sdk-delivery-oracle` does. The driver answers the Electric HTTP + * protocol; the real client parses each value from its Postgres type in + * `electric-schema`, and the real adapter applies it. + * + * The oracle's steps map onto Electric like this: + * + * - a complete delivery answers the initial request (`offset=-1`) with an + * insert per row and `up-to-date`. A later one (`complete-snapshot`) + * answers the pending live request with `must-refetch` (HTTP 409), then + * the refetch with the new rows and `up-to-date`; + * - a change batch answers the pending live request with one message per + * change, each carrying the batch's txid (`await-evidence`), and + * `up-to-date`. `insert` for a key the provider does not hold, `update` + * for one it does, `delete` otherwise. The adapter commits at + * `up-to-date`, so the driver declares `per-batch` commits; + * - a failure answers with HTTP 400, which the client gives to the shape's + * `onError`. The handler gives up, alternately by returning nothing and by + * throwing, and the adapter reports the client's or the handler's error, + * so the driver declares `failureValue: any`. + * The law set gives Electric a give-up after retries (`retryBound`), so the + * driver declares `source-failure-after-ready`; + * - a retried failure (`retry-policy`) is one failed attempt; health shows + * an outage from its fifth consecutive attempt (`retryReportAfter`). It + * answers with HTTP 503, which the + * client retries (its backoff is set short), or, every other time, with + * HTTP 400, which the shape's `onError` retries; + * - a provider subscription is one sync run's request chain; its release is + * the abort of the chain's pending request (LIFE-2, LIFE-10); + * - the propagation step (`settle`) waits until the client asks again after + * a response, or, after a failure, until the shape's `onError` gave up. + * + * Typed fields (FID-1) are Postgres columns: text, int4, float8 (also for + * -0), bool, timestamptz, int8, and jsonb. The client's default parsers + * decode all but timestamptz, which an app parses with a `parser` entry; the + * driver supplies that entry. int4 holds a 32-bit integer, so larger + * integers are excluded (`excludesValue`), as Postgres would reject them. + * `note` is a nullable text column (`toSqlRow`, `fromSqlRow`). The driver + * never reads the Collection. + */ +import { createCollection } from '@tanstack/db' +import { electricCollectionOptions } from '../src/electric' +import { + fromSqlRow, + observedView, + toSqlRow, +} from '../../db-adapter-conformance/src' +import type { + ConformanceRow, + ControlledProvider, + EagerLiveSourceDriver, + EagerLiveSourceFixture, + ProviderChange, + ValueKind, +} from '../../db-adapter-conformance/src' + +/** Each typed field kind and the Postgres column type that holds it. */ +const COLUMNS = { + string: `text`, + integer: `int4`, + float: `float8`, + 'negative-zero': `float8`, + boolean: `bool`, + date: `timestamptz`, + bigint: `int8`, + array: `jsonb`, + object: `jsonb`, +} as const satisfies Partial> + +type FieldKind = keyof typeof COLUMNS +const FIELD_KINDS = Object.keys(COLUMNS) as Array +const column = (kind: FieldKind) => `t_${kind.replace(`-`, `_`)}` + +const SCHEMA = JSON.stringify({ + id: { type: `int4`, pk_index: 0 }, + title: { type: `text` }, + note: { type: `text` }, + ...Object.fromEntries( + FIELD_KINDS.map((kind) => [column(kind), { type: COLUMNS[kind] }]), + ), +}) + +/** A value in Postgres text form, as Electric sends it. */ +function wire(kind: FieldKind, value: unknown): string { + switch (COLUMNS[kind]) { + case `float8`: + return Object.is(value, -0) ? `-0` : String(value) + case `bool`: + return value === true ? `true` : `false` + case `timestamptz`: + return (value as Date).toISOString() + case `jsonb`: + return JSON.stringify(value) + default: + return String(value) + } +} + +/** A conformance row as an Electric row value: every column as text. */ +function rowValue(row: ConformanceRow): Record { + const sql = toSqlRow(row) + return { + id: String(sql.id), + title: sql.title, + note: sql.note, + ...Object.fromEntries( + FIELD_KINDS.map((kind) => [column(kind), wire(kind, row.typed?.[kind])]), + ), + } +} + +type Request = { + url: URL + signal: AbortSignal | undefined + respond: (response: Response) => void +} + +type Session = { + handle: string + latest: Request | undefined + // The latest response stopped the client, as a give-up does. + stopped: boolean +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** How long the driver waits for the client before letting the check decide. */ +const clientTimeoutMs = 2_000 + +class ElectricProvider implements ControlledProvider { + private readonly queued: Array = [] + private readonly sessions: Array = [] + private rows = new Map() + private offset = 0 + private answered: Request | undefined + // Give-ups so far, and the count the latest failure must reach. + private gaveUp = 0 + private giveUpTarget = 0 + private transientFailures = 0 + private retryNextError = false + + /** + * An app's `onError`. It retries the HTTP 400 of a retried failure by + * returning an empty retry. It gives up on any other error, alternately + * by returning nothing and by throwing; both stop the stream (RETRY-1), + * and the adapter fails. + */ + readonly onError = () => { + if (this.retryNextError) { + this.retryNextError = false + return {} + } + if (this.gaveUp++ % 2 === 1) throw new Error(`conformance give-up`) + return undefined + } + + readonly fetchClient: typeof fetch = (input, init) => + new Promise((resolve, reject) => { + const url = new URL(String(input)) + const signal = init?.signal ?? undefined + const request: Request = { + url, + signal, + respond: (response) => { + signal?.removeEventListener(`abort`, abort) + resolve(response) + }, + } + const abort = () => { + const index = this.queued.indexOf(request) + if (index >= 0) this.queued.splice(index, 1) + reject( + signal?.reason ?? new DOMException(`request aborted`, `AbortError`), + ) + } + if (signal?.aborted) { + abort() + return + } + signal?.addEventListener(`abort`, abort, { once: true }) + // A sync run's request chain continues while its latest request is + // live: a retry or a refetch belongs to it. A request after that chain + // was aborted, or after the client stopped, starts a new sync run's + // chain. + let session = this.sessions.at(-1) + if ( + !session || + session.stopped || + session.latest?.signal?.aborted === true + ) { + session = { + handle: `shape-${this.sessions.length + 1}`, + latest: undefined, + stopped: false, + } + this.sessions.push(session) + this.rows = new Map() + } + session.latest = request + session.stopped = false + this.queued.push(request) + }) + + private session(request: Request): Session { + const session = this.sessions.find((s) => s.latest === request) + if (!session) throw new Error(`the request belongs to no sync run`) + return session + } + + private async take(): Promise { + const deadline = Date.now() + clientTimeoutMs + while (this.queued.length === 0 && Date.now() < deadline) await sleep(2) + const request = this.queued.shift() + if (!request) throw new Error(`the client has not requested`) + this.answered = request + return request + } + + private headers(session: Session, upToDate = true) { + this.offset++ + return { + 'electric-handle': session.handle, + 'electric-offset': `${this.offset}_0`, + 'electric-cursor': String(this.offset), + 'electric-schema': SCHEMA, + ...(upToDate ? { 'electric-up-to-date': `true` } : {}), + } + } + + private respondRows(request: Request, messages: Array): void { + const session = this.session(request) + request.respond( + new Response( + JSON.stringify([ + ...messages, + { + headers: { + control: `up-to-date`, + global_last_seen_lsn: String(this.offset + 1), + }, + }, + ]), + { headers: this.headers(session) }, + ), + ) + } + + private inserts(rows: ReadonlyArray) { + return rows.map((row) => ({ + key: String(row.id), + value: rowValue(row), + headers: { operation: `insert` }, + })) + } + + // A run's chain is released when its pending request is aborted, or when + // the client stopped after a terminal response and holds no request: an + // HTTP long-poll that stopped holds nothing open. + releases = () => + this.sessions.map((session) => + session.latest?.signal?.aborted === true || session.stopped ? 1 : 0, + ) + + deliverSnapshot = async (rows: ReadonlyArray) => { + let request = await this.take() + if (request.url.searchParams.get(`offset`) !== `-1`) { + // A complete delivery after ready: must-refetch, then the refetch. + const session = this.session(request) + session.handle = `${session.handle}-r${this.offset}` + request.respond( + new Response( + JSON.stringify([{ headers: { control: `must-refetch` } }]), + { + status: 409, + headers: { 'electric-handle': session.handle }, + }, + ), + ) + request = await this.take() + } + this.rows = new Map(rows.map((row) => [row.id, row])) + this.respondRows(request, this.inserts(rows)) + } + + deliverChanges = async ( + changes: ReadonlyArray, + txid: string, + ) => { + const request = await this.take() + const messages = changes.map((change) => { + if (change.type === `upsert`) { + const held = this.rows.has(change.row.id) + this.rows.set(change.row.id, change.row) + return { + key: String(change.row.id), + value: rowValue(change.row), + headers: { + operation: held ? `update` : `insert`, + txids: [Number(txid)], + }, + } + } + const row = this.rows.get(change.id) + if (!row) throw new Error(`provider does not hold row ${change.id}`) + this.rows.delete(change.id) + return { + key: String(change.id), + value: { id: String(change.id) }, + headers: { operation: `delete`, txids: [Number(txid)] }, + } + }) + this.respondRows(request, messages) + } + + fail = async () => { + const request = await this.take() + this.session(request).stopped = true + this.giveUpTarget = this.gaveUp + 1 + request.respond(new Response(`conformance failure`, { status: 400 })) + } + + // Retried failures alternate: the client retries an HTTP 503 itself, and + // the app's `onError` retries an HTTP 400. + transientFailure = async () => { + const request = await this.take() + const byApp = this.transientFailures++ % 2 === 1 + this.retryNextError = byApp + request.respond( + new Response(`conformance transient failure`, { + status: byApp ? 400 : 503, + }), + ) + } + + /** + * The propagation step: the client asks again after a response, or, after + * a terminal one, gives up through `onError`. + */ + settle = async () => { + const answered = this.answered + if (!answered) return + this.answered = undefined + const stopped = this.sessions.find((s) => s.latest === answered)?.stopped + const deadline = Date.now() + clientTimeoutMs + while ( + (stopped ? this.gaveUp < this.giveUpTarget : this.queued.length === 0) && + answered.signal?.aborted !== true && + Date.now() < deadline + ) { + await sleep(2) + } + // The client's subscriber delivery follows its next request closely. + for (let turn = 0; turn < 3; turn++) await sleep(0) + } +} + +type ElectricRow = Record & { + id: number + title: string | null + note: string | null +} + +/** A Collection row as a conformance row: typed columns become `typed`. */ +function conformanceRow(row: ElectricRow): ConformanceRow { + const typed: Record = {} + const rest: Record = {} + for (const [key, value] of Object.entries(row)) { + const kind = FIELD_KINDS.find((k) => column(k) === key) + if (kind) typed[kind] = value + else rest[key] = value + } + return fromSqlRow({ ...rest, typed } as { note?: unknown }) +} + +let sequence = 0 + +export function electricConformanceDriver(): EagerLiveSourceDriver { + return { + adapter: `@tanstack/electric-db-collection (eager)`, + capabilities: new Set([ + `complete-snapshot`, + `await-evidence`, + `retry-policy`, + `source-failure-after-ready`, + ]), + changeCommits: `per-batch`, + // The Collection reports the client's error for the failed response. + failureValue: `any`, + // Electric carries txids on row messages; an empty batch sends none. + txidEvidence: `change`, + // Only an extended outage shows (maintainer decision 2026-10-30). + retryReportAfter: 5, + fieldKinds: FIELD_KINDS, + // Column domains: int4 holds a 32-bit integer, and Postgres rejects a + // larger one; a float8 column holds a scalar -0, not one nested in an + // array or object. + excludesValue: (kind, value) => + (kind === `integer` && + typeof value === `number` && + (value > 2_147_483_647 || value < -2_147_483_648)) || + (kind === `negative-zero` && typeof value !== `number`), + create(): EagerLiveSourceFixture { + const provider = new ElectricProvider() + const collection = createCollection( + electricCollectionOptions({ + id: `electric-conformance-${++sequence}`, + shapeOptions: { + url: `http://conformance/shape-${sequence}`, + params: { table: `rows` }, + fetchClient: provider.fetchClient, + parser: { timestamptz: (value: string) => new Date(value) }, + backoffOptions: { initialDelay: 1, maxDelay: 2, multiplier: 1 }, + onError: provider.onError, + }, + getKey: (row) => row.id, + }), + ) + return { + collection: observedView(collection, { + row: conformanceRow, + key: (key) => key as number, + }), + provider, + awaitTxId: (txid, timeout) => + collection.utils.awaitTxId(Number(txid), timeout), + settle: provider.settle, + } + }, + } +} diff --git a/packages/electric-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/electric-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts new file mode 100644 index 0000000000..3283ba1119 --- /dev/null +++ b/packages/electric-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -0,0 +1,77 @@ +import { fc } from '@fast-check/vitest' +import { describe, it } from 'vitest' +import { + oraclePropertyOptions, + oracleReplayTest, + oracleRuns, +} from '../../db/tests/oracle-config' +import { declareOracle } from '../../db/tests/oracle-declaration' +import { createOracleReach } from '../../db/tests/oracle-reach' +import { + checkEagerLiveSourceHistory, + eagerLiveSourceHistoryArb, + eagerLiveSourceModel, + eagerLiveSourceProperty, + reachStatesFor, +} from '../../db-adapter-conformance/src' +import { electricConformanceDriver } from './conformance-driver' + +/** + * # Electric on the shared eager live-source conformance oracle + * + * A receiving driver for `packages/db-adapter-conformance`. The shared + * module states the laws, the model, the history grammar, and the check; + * this file supplies only the Electric production driver in + * `conformance-driver.ts` and the campaigns. It states no expected results. + * + * Electric runs here in eager mode over the installed `@electric-sql/client` + * through a controlled `fetchClient`. It declares `complete-snapshot` + * (must-refetch), `await-evidence` (txids), `retry-policy` (an HTTP 503 the + * client retries, or an HTTP 400 the shape's `onError` retries), and + * `source-failure-after-ready` (a give-up through `onError`), and typed + * Postgres columns. The claim is limited to the controlled HTTP provider; a + * claim about an Electric server needs a receiving witness against it. + */ + +const property = `adapter-conformance.eager-live-source.electric` +const driver = electricConformanceDriver() +const historyArb = eagerLiveSourceHistoryArb(driver) +// The rarest state is `field-removed`: at 400 histories, six seeds reached +// it 19 to 31 times. +const runs = 400 +// A campaign takes about 20 seconds. +const campaignTimeoutMs = 120_000 + +async function campaign(options: fc.Parameters<[unknown]>) { + const reach = createOracleReach(property, reachStatesFor(driver), { + minimum: 5, + }) + await fc.assert(eagerLiveSourceProperty(driver, reach), options) + return reach +} + +describe(`eager live-source conformance: Electric`, () => { + oracleReplayTest(it)( + `conforms on a fixed-seed campaign`, + async () => { + const reach = await campaign({ seed: 2118, numRuns: oracleRuns(runs) }) + reach.assertReached() + }, + campaignTimeoutMs, + ) + + oracleReplayTest(it, property)( + `conforms on a random campaign or a replayed seed`, + async () => { + const reach = await campaign(oraclePropertyOptions(runs, property)) + reach.assertReached() + }, + campaignTimeoutMs, + ) +}) + +declareOracle({ + check: [checkEagerLiveSourceHistory, eagerLiveSourceProperty], + model: [eagerLiveSourceModel.step, eagerLiveSourceModel.expected], + grammar: [historyArb], +}) From 8da7afaf65797f54e4564d3434ff80400dca4fc1 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 16:28:01 -0600 Subject: [PATCH 18/64] test(conformance): outages anywhere, matched failure values, waits retire at failure Address the oracle review of 39294b6e2 and 815854815 (three independent oracle reviewers) and encode the maintainer decisions it prompted. - Outages are a generated history element: runs of failed attempts below and past retryReportAfter, and a recurring outage around a live batch, anywhere in a history. Attempt kinds are a generator dimension (driver transientVariants). New reach states: each kind, and an outage reported again in a run, after a live batch, and in a later run. The retry scenario prefix is gone. - failureValue: any is replaced by the driver's matchesFailure relation: the health error, the preload rejection, and failureReason compare through it, so a stale or replaced value is rejected. Under it the outage shows the attempt that crossed the threshold, and may list later attempts after it (maintainer decision 2026-10-30). - A sync run retires when it fails: its pending waits reject, and a wait registered after a failure rejects unless the shown rows reflect its evidence (law set v18 "retired ... when it fails"; maintainer decision 2026-10-30). Waits are now generated after a failure. - A retired run's evidence fulfills a wait once the current run is ready (maintainer decision 2026-10-30, revising v18 EVID-2's "never"). Txids number batches across runs, as a backend's do, so EVID-1 and EVID-2 labels are diagnostic. - A field-only publication mismatch names FID-1. - The check refuses retryReportAfter above 1 with transport-reconnect or capability-renewal: no law fixes whether those end an outage. - Reference adapter: implements the decisions; an outage-threshold variant (k=3, client-created values) conforms with declared reach. New mutants: retired-evidence-before-ready, forgets-retired-evidence, leaks-waits-after-fail (EVID-2); stale-outage-value (HEALTH-2), replaces-outage-value (RETRY-1), reports-outage-early and outage-count-survives-delivery (HEALTH-1) on the threshold variant. buffer-before-rows now also shows as PUB-4 (it writes before a rejected delivery), so its entry accepts either law. Evidence: - Law: HEALTH-1/2, RETRY-1, HEALTH-4, EVID-1/2/3 with the decisions above. - Reach: reference fixed campaign 600 histories, every declared state at least 5 (asserted), threshold variant likewise; Electric (next commit) at least 15 across four seeds. - RED: Electric's 815854815 adapter fails EVID-2 at [fail, wait, snapshot] (next commit); each new reference mutant is rejected with its law. - GREEN: reference suite 77/77; TrailBase, RxDB, PowerSync campaigns pass. - Limits: the grammar has no accepted response that commits nothing; awaitMatch and snapshot-visibility evidence are not generated. Co-authored-by: Isaac --- .../src/eager-live-source-oracle.ts | 398 ++++++++++++------ .../reference-adapter-oracle.property.test.ts | 59 ++- .../tests/reference-adapter.ts | 143 ++++++- 3 files changed, 452 insertions(+), 148 deletions(-) diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 9c3deb621e..feabab990d 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -14,7 +14,9 @@ * - **HEALTH-1.** A provider failure after the first ready keeps status * `ready` and the rows, and reports the value as the health node's `error` * cause with no `failureReason`. Before any failure, the node shows no - * problem. + * problem. A driver whose client library creates the reported value + * declares `matchesFailure`, the relation that names the value owed for + * each injected failure (HEALTH-4); the check then compares through it. * - **PUB-1.** Each change publication equals one whole committed state: the * rows after one provider commit, never part of a commit, never an earlier * state than one already published. A commit with no changes publishes @@ -52,9 +54,15 @@ * `awaitTxId(txid)` fulfills once a committed batch carrying that id has * applied, also when the batch came first, and stays pending before. A * wait with a timeout and no evidence rejects after the timeout, and later - * evidence does not change that. A wait still pending when its sync run - * retires rejects. Evidence from a retired sync run does not fulfill a - * wait in a later one (EVID-2). + * evidence does not change that. A sync run retires when it fails or is + * cleaned up, and a wait still pending then rejects; a wait registered + * after a failure rejects unless the shown rows already reflect its + * evidence (EVID-2; maintainer decision 2026-10-30, the law set's "retired + * ... when it fails"). A retired run's evidence fulfills a wait in a later + * run once that run is ready, because its rows are read after that + * evidence, and not before (EVID-2 as revised by maintainer decision + * 2026-10-30). Txids number batches across runs, as a backend's + * transaction ids do. * - **HEALTH-1, HEALTH-2, RETRY-1** (capability `retry-policy`). A transient * failure that the adapter keeps retrying shows nothing before the first * ready. After it, the failure joins the node's `failureReason`, oldest @@ -63,10 +71,10 @@ * 2026-10-30: a retrying source must not flap its health), only an * extended outage shows: an attempt joins `failureReason` from the `k`th * consecutive failed attempt after ready on, and fewer show nothing. Under - * `failureValue: any` the reported values are - * the client library's, so the node lists at least one failure and at - * most one per retried attempt, with `cause` its newest; a later attempt - * does not replace the failure first listed for the outage (RETRY-1). + * `matchesFailure` the node shows the attempt that crossed the threshold + * (maintainer decision 2026-10-30) and may list later attempts after it, + * in order; a later attempt does not replace it (RETRY-1). Attempts come + * in each of the driver's `transientVariants` kinds. * - **RECON-1** (capability `transport-reconnect`). A transport disconnect and * reconnect keep status and rows, and later deliveries still apply. * - **RENEW-1** (capability `capability-renewal`). While the source's @@ -222,8 +230,11 @@ export interface ControlledProvider { ) => void | Promise /** The provider stops with `error` and delivers nothing more. */ fail: (error: Error) => void | Promise - /** A failure the adapter keeps retrying (capability `retry-policy`). */ - transientFailure?: (error: Error) => void | Promise + /** + * One failed attempt the adapter keeps retrying (capability + * `retry-policy`), of the driver's `transientVariants` kind `variant`. + */ + transientFailure?: (error: Error, variant: number) => void | Promise /** Drops the transport (capability `transport-reconnect`). */ disconnect?: () => void | Promise /** Restores the transport (capability `transport-reconnect`). */ @@ -300,13 +311,21 @@ export interface EagerLiveSourceDriver { */ readonly duplicateInsert?: `error` | `replace` /** - * `any` when the Collection reports a provider failure with a value the - * adapter's client library creates, such as an HTTP client's error for a - * failed response, rather than the provider's own value. The check then - * owes some error, not that value, also for a retried failure's - * `failureReason`. Default `provider`. + * For an adapter whose client library creates the reported failure value, + * such as an HTTP client's error for a failed response: whether `reported` + * is the value the adapter owes for the injected failure `failure`. The + * driver carries the failure's identity through its transport (for + * example in a response header), so a stale or replaced value does not + * match. Without it, the check compares the provider's value by identity. */ - readonly failureValue?: `provider` | `any` + readonly matchesFailure?: (reported: unknown, failure: number) => boolean + /** + * Names of the kinds of failed attempt the driver's provider can make + * (`retry-policy`), such as an HTTP status the client retries or a + * network error. Each is generated and reached after ready. Default: one + * kind. + */ + readonly transientVariants?: ReadonlyArray /** * What carries a batch's txid (`await-evidence`): the batch itself * (`batch`, the default), or each change in it (`change`), as Electric's @@ -359,7 +378,11 @@ export interface EagerLiveSourceDriver { * failure after ready to deliver. * - `malformed`: a complete delivery holding two rows with one key, legal * where a complete delivery is (PUB-4). - * - `transient-failure`: a failure the adapter keeps retrying. + * - `transient-failure`: one failed attempt the adapter keeps retrying, of + * one of the driver's kinds. Attempts come in outages: runs of one to two + * more than `retryReportAfter` attempts, anywhere in a history, so an + * outage can cross the threshold after a live batch, a second time in one + * run, or in a later run. * - `disconnect`, `reconnect`: transport loss and return. * - `renew`: the adapter's renewal of an expiring capability completes. Legal * once per sync run. @@ -398,7 +421,11 @@ export type EagerLiveSourceStep = readonly rows: ReadonlyArray readonly duplicate: number } - | { readonly kind: `transient-failure` } + | { + /** One failed attempt of the driver's kind `variant`. */ + readonly kind: `transient-failure` + readonly variant?: number + } | { readonly kind: `disconnect` } | { readonly kind: `reconnect` } | { readonly kind: `renew` } @@ -530,6 +557,7 @@ export type GrammarDomain = Pick< | `fieldKinds` | `excludesValue` | `retryReportAfter` + | `transientVariants` > /** Generates histories whose steps use only the driver's declarations. */ @@ -612,11 +640,44 @@ export function eagerLiveSourceHistoryArb( })), }) } + // An outage is a run of failed attempts. It is one element of the + // history, so it lands anywhere; other steps are elements of one step. + const outages: Array>> = [] if (capabilities.has(`retry-policy`)) { - steps.push({ - weight: 3, - arbitrary: fc.constant({ kind: `transient-failure` as const }), + const variants = domain.transientVariants?.length ?? 1 + const threshold = domain.retryReportAfter ?? 1 + const attempt = fc + .nat(variants - 1) + .map((variant) => ({ kind: `transient-failure` as const, variant })) + // Half the outages stay below the threshold, half cross it by up to two + // attempts. A recurring outage crosses it, recovers on a live batch, and + // crosses it again, so a second report in one run is not rare. + const crossing = fc.array(attempt, { + minLength: threshold, + maxLength: threshold + 2, }) + outages.push( + fc.oneof( + fc.array(attempt, { + minLength: 1, + maxLength: Math.max(1, threshold - 1), + }), + crossing, + ), + fc + .tuple( + crossing, + fc.array(changeArb(kinds, fieldKinds, domain.excludesValue), { + maxLength: 3, + }), + crossing, + ) + .map(([first, changes, second]): Array => [ + ...first, + { kind: `changes`, changes }, + ...second, + ]), + ) } if (capabilities.has(`transport-reconnect`)) { steps.push( @@ -632,12 +693,23 @@ export function eagerLiveSourceHistoryArb( arbitrary: fc.constant({ kind: `renew` as const }), }) } - const generated = fc.array(fc.oneof(...steps), { - minLength: 1, - maxLength: 12, - }) - // Some laws need a short sequence that random steps rarely produce, so a - // history can start with one; one in six does per declared scenario. + const generated = fc + .array( + fc.oneof( + ...steps.map(({ weight, arbitrary }) => ({ + weight, + arbitrary: arbitrary.map((step) => [step]), + })), + ...outages.map((arbitrary, index) => ({ + weight: index === 0 ? 3 : 1, + arbitrary, + })), + ), + { minLength: 1, maxLength: 12 }, + ) + .map((elements) => elements.flat()) + // EVID-2 needs a sequence that random steps rarely produce, so one history + // in six starts with it. const snapshot = steps[0]!.arbitrary const scenarios: Array>> = [] if (capabilities.has(`await-evidence`)) { @@ -661,19 +733,6 @@ export function eagerLiveSourceHistoryArb( ]), ) } - if (capabilities.has(`retry-policy`)) { - // RETRY-1: an outage after ready one attempt longer than - // `retryReportAfter`, so its first report and a later attempt show. - const attempts = (domain.retryReportAfter ?? 1) + 1 - scenarios.push( - snapshot.map((first): Array => [ - first, - ...Array.from({ length: attempts }, () => ({ - kind: `transient-failure` as const, - })), - ]), - ) - } const history = scenarios.length === 0 ? generated @@ -721,7 +780,10 @@ export function eagerLiveSourceHistoryArb( * initial cut folds them onto the fetch result. * - `lastTxid` numbers the sync run's batches. A wait targets it. * - `transient` lists the open transient failures, oldest first. Only a - * failure after the first ready is listed (RETRY-1). + * failure after the first ready is listed (RETRY-1), from the + * `retryReportAfter`th attempt of the outage on. + * - `outage` counts the current outage's failed attempts after ready. + * - `retiredEvidence` holds the txids that retired runs committed (EVID-2). * - `waits` records every `awaitTxId` call with the settlement the model * expects, across sync runs, because a retired wait must reject. * @@ -731,8 +793,8 @@ export function eagerLiveSourceHistoryArb( */ type Phase = `loading` | `ready` | `loading-error` | `stopped` -/** A wait's expected settlement. `unchecked` after its sync run failed. */ -type WaitExpectation = `pending` | `fulfilled` | `rejected` | `unchecked` +/** A wait's expected settlement. */ +type WaitExpectation = `pending` | `fulfilled` | `rejected` type RowState = ReadonlyArray<[number, string]> @@ -746,18 +808,26 @@ interface ModelState { connected: boolean lastTxid: number committed: Set - /** Txids committed in retired sync runs; they name EVID-2. */ + /** + * Txids committed in retired sync runs (EVID-2). Txids number batches + * across runs, as a backend's transaction ids do. + */ retiredEvidence: Set renewed: boolean /** - * A provider failure's index; `any` for one whose reported value the - * client library creates; `adapter` for the adapter's own error. + * A provider failure's index; `adapter` for the adapter's own error. */ - failure: number | `any` | `adapter` | undefined + failure: number | `adapter` | undefined transient: Array /** Consecutive failed attempts after ready in the current outage. */ outage: number - waits: Array<{ run: number; txid: number; expected: WaitExpectation }> + /** Each wait, and the law that rejects it when it must reject. */ + waits: Array<{ + run: number + txid: number + expected: WaitExpectation + rejectLaw?: `EVID-2` | `EVID-3` + }> } /** What the model predicts at a checkpoint. */ @@ -921,17 +991,41 @@ function canDeliver(state: ModelState): boolean { ) } -/** Retiring a sync run rejects its pending waits (EVID-2). */ /** HEALTH-2: a delivery, failure, or restart ends the outage. */ function endOutage(state: ModelState): void { state.transient = [] state.outage = 0 } +/** + * EVID-2: a sync run retires when it fails or is cleaned up, and its pending + * waits reject then; no evidence can arrive for them. Authority: the law + * set's "retired ... when it fails", confirmed by maintainer decision + * 2026-10-30 (a pending wait rejects at the failure). + */ function retireWaits(state: ModelState): void { for (const wait of state.waits) { if (wait.run === state.run && wait.expected === `pending`) { wait.expected = `rejected` + wait.rejectLaw = `EVID-2` + } + } +} + +/** + * READY-3: the first complete delivery opens the gate. EVID-2: a retired + * run's evidence fulfills a wait once the current run is ready, because its + * rows are read after that evidence (maintainer decision 2026-10-30). + */ +function openGate(state: ModelState): void { + state.phase = `ready` + for (const wait of state.waits) { + if ( + wait.run === state.run && + wait.expected === `pending` && + state.retiredEvidence.has(wait.txid) + ) { + wait.expected = `fulfilled` } } } @@ -1055,7 +1149,7 @@ type AcceptedStep = /** `duplicateInsert: replace` applied the later row. */ replaced: boolean } - | { kind: `transient-failure`; failure: number } + | { kind: `transient-failure`; failure: number; variant: number } | { kind: `disconnect` } | { kind: `reconnect` } | { kind: `renew` } @@ -1073,7 +1167,6 @@ interface ModelOptions { /** The value kinds include `negative-zero`, so a retouch flips zeros. */ flipZeros: boolean duplicateInsert: `error` | `replace` - failureValue: `provider` | `any` txidEvidence: `batch` | `change` retryReportAfter: number } @@ -1151,7 +1244,7 @@ function stepModel( const cut = initialCut(state, seen, options.changeCommits) state.rows = cut.backend state.backend = new Map(cut.backend) - state.phase = `ready` + openGate(state) for (const batch of state.buffered) { if (options.txidEvidence === `batch` || batch.changes.length > 0) { applyEvidence(state, batch.txid) @@ -1175,7 +1268,7 @@ function stepModel( const removesField = lostField(state.rows, rowMap(step.rows)) state.rows = rowMap(step.rows) state.backend = rowMap(step.rows) - state.phase = `ready` + openGate(state) return { done: { kind: `snapshot`, @@ -1258,37 +1351,40 @@ function stepModel( } } case `wait`: { - if (state.phase !== `loading` && state.phase !== `ready`) return undefined const txid = Math.max(1, state.lastTxid + step.ahead) - // EVID-1: evidence already applied fulfills the wait. EVID-3: a wait - // with a timeout and no evidence rejects once the timeout passes, which - // is before the next checkpoint. + // EVID-1: evidence the shown rows reflect fulfills the wait: this + // run's, or a retired run's once the Collection is ready (EVID-2). + // EVID-3: a wait with a timeout and no evidence rejects once the + // timeout passes, which is before the next checkpoint. EVID-2: after + // a failure no evidence can arrive, so the wait rejects. const timeout = step.timeout === true + const running = state.phase === `loading` || state.phase === `ready` + const shown = + state.committed.has(txid) || + (state.phase !== `loading` && + state.phase !== `loading-error` && + state.retiredEvidence.has(txid)) state.waits.push({ run: state.run, txid, - expected: state.committed.has(txid) + expected: shown ? `fulfilled` - : timeout + : !running || timeout ? `rejected` : `pending`, + rejectLaw: running ? `EVID-3` : `EVID-2`, }) return { done: { kind: `wait`, txid, timeout }, publishable: [] } } case `fail`: { if (!canDeliver(state)) return undefined const failure = failures.next++ - state.failure = options.failureValue === `any` ? `any` : failure + state.failure = failure // READY-4 before the first ready; HEALTH-1 after it. LIFE-10: the // source ends here. state.phase = state.phase === `loading` ? `loading-error` : `stopped` endOutage(state) - // No law fixes what a pending txid wait does after its source failed. - for (const wait of state.waits) { - if (wait.run === state.run && wait.expected === `pending`) { - wait.expected = `unchecked` - } - } + retireWaits(state) return { done: { kind: `fail`, failure }, publishable: [] } } case `malformed`: { @@ -1306,7 +1402,7 @@ function stepModel( if (options.listenBeforeFetch && !afterReady) return undefined state.rows = rowMap(rows) state.backend = rowMap(rows) - state.phase = `ready` + openGate(state) endOutage(state) return { done: { kind: `malformed`, rows, afterReady, replaced: true }, @@ -1317,11 +1413,7 @@ function stepModel( state.phase = afterReady ? `stopped` : `loading-error` endOutage(state) state.buffered = [] - for (const wait of state.waits) { - if (wait.run === state.run && wait.expected === `pending`) { - wait.expected = `unchecked` - } - } + retireWaits(state) return { done: { kind: `malformed`, rows, afterReady, replaced: false }, publishable: [], @@ -1339,7 +1431,14 @@ function stepModel( state.transient.push(failure) } } - return { done: { kind: `transient-failure`, failure }, publishable: [] } + return { + done: { + kind: `transient-failure`, + failure, + variant: step.variant ?? 0, + }, + publishable: [], + } } case `disconnect`: { if (!canDeliver(state)) return undefined @@ -1370,8 +1469,8 @@ function stepModel( state.seed = rowMap(nextSeed) state.buffered = [] state.connected = true - state.lastTxid = 0 - // EVID-2: the retired run's evidence does not fulfill a later wait. + // EVID-2: the retired run's evidence fulfills a later wait only once + // this run is ready (`openGate`). for (const txid of state.committed) state.retiredEvidence.add(txid) state.committed = new Set() state.renewed = false @@ -1393,20 +1492,13 @@ function expectedObservation(state: ModelState): ExpectedObservation { ? `error` : `ready`, rows: rowState(state.rows), - healthError: running - ? null - : state.failure === `any` - ? `adapter` - : state.failure!, + healthError: running ? null : state.failure!, failureReason: state.transient, preload: state.phase === `loading` ? `pending` : state.phase === `loading-error` - ? { - rejectedWith: - state.failure === `any` ? `adapter` : state.failure!, - } + ? { rejectedWith: state.failure! } : `fulfilled`, waits: state.waits.map((wait) => wait.expected), // LIFE-10: a failed provider's source is released at its end. @@ -1440,6 +1532,9 @@ export const eagerLiveSourceReachStates = [ `transient-failure-reported`, `transient-failure-below-threshold`, `transient-failure-repeated`, + `outage-reported-again`, + `outage-reported-after-batch`, + `outage-reported-in-later-run`, `transient-failure-recovered`, `reconnect`, `renewal`, @@ -1463,6 +1558,8 @@ export type EagerLiveSourceReachState = | (typeof eagerLiveSourceReachStates)[number] | ValueReachState | `zero-sign-flipped` + /** A failed attempt of each declared kind after ready. */ + | `transient:${string}` const capabilityOfState: Partial< Record<(typeof eagerLiveSourceReachStates)[number], EagerLiveSourceCapability> @@ -1474,6 +1571,9 @@ const capabilityOfState: Partial< 'wait-on-retired-evidence': `await-evidence`, 'transient-failure-reported': `retry-policy`, 'transient-failure-below-threshold': `retry-policy`, + 'outage-reported-again': `retry-policy`, + 'outage-reported-after-batch': `retry-policy`, + 'outage-reported-in-later-run': `retry-policy`, 'transient-failure-repeated': `retry-policy`, 'transient-failure-recovered': `retry-policy`, reconnect: `transport-reconnect`, @@ -1521,7 +1621,14 @@ export function reachStatesFor( ) ? [`zero-sign-flipped`] : [] - return [...reachable, ...values, ...fields, ...zeros] + const variants: Array = domain.capabilities.has( + `retry-policy`, + ) + ? (domain.transientVariants ?? []).map( + (name): EagerLiveSourceReachState => `transient:${name}`, + ) + : [] + return [...reachable, ...values, ...fields, ...zeros, ...variants] } /** JSON for a message; a bigint shows as `n`, `-0` as `-0`. */ @@ -1663,7 +1770,6 @@ export async function checkEagerLiveSourceHistory( listenBeforeFetch, flipZeros: driver.valueKinds?.includes(`negative-zero`) === true, duplicateInsert: driver.duplicateInsert ?? `error`, - failureValue: driver.failureValue ?? `provider`, txidEvidence: driver.txidEvidence ?? `batch`, retryReportAfter: driver.retryReportAfter ?? 1, } @@ -1673,6 +1779,17 @@ export async function checkEagerLiveSourceHistory( ) { throw new Error(`duplicate-key-delivery needs a duplicateInsert policy`) } + // No law fixes whether a reconnect or a renewal ends an outage that a + // threshold is still counting; a connection-aware source reports at once. + if ( + (driver.retryReportAfter ?? 1) > 1 && + (driver.capabilities.has(`transport-reconnect`) || + driver.capabilities.has(`capability-renewal`)) + ) { + throw new Error( + `retryReportAfter above 1 has no law with transport-reconnect or capability-renewal`, + ) + } // With `listen-before-fetch`, a sync run's backend starts with the rows of // its first complete delivery, committed before the adapter subscribes. const seedFrom = (start: number): ReadonlyArray => { @@ -1740,8 +1857,18 @@ export async function checkEagerLiveSourceHistory( (rows, commit) => commit > last && sameRows(rows, publication), ) if (index === -1) { + // FID-1: a publication with a committed state's keys and titles + // that differs only in a field names the field law. + const near = publishable.find((rows) => sameTitles(rows, publication)) + const valuesOnly = near?.every( + ([, value], row) => rowHead(value) === rowHead(publication[row]![1]), + ) throw violation( - `${law} (publication)`, + near === undefined + ? `${law} (publication)` + : valuesOnly + ? `FID-1 (values)` + : `FID-1 (fields)`, stepIndex, publishable, publication, @@ -1758,9 +1885,12 @@ export async function checkEagerLiveSourceHistory( // when a retired run committed it), // EVID-3 for a timed wait that did not reject, and LIFE-10 for a failed // provider's subscription that stays open or is released twice. - // Under `failureValue: any`, the model's first open transient failure - // names the outage, and `cause` is the failure first reported for it. - let firstReported: { outage: number; cause: unknown } | undefined + // HEALTH-4: the reported value for an injected failure is the provider's + // value, or under `matchesFailure` the value the driver's relation names. + const matches = (reported: unknown, index: number) => + driver.matchesFailure + ? driver.matchesFailure(reported, index) + : reported === failure(index) const compare = async ( stepIndex: number, law: string, @@ -1826,7 +1956,7 @@ export async function checkEagerLiveSourceHistory( } else { check( `health error cause`, - health.error?.cause === failure(expected.healthError), + matches(health.error?.cause, expected.healthError), failure(expected.healthError).message, health.error?.message ?? null, ) @@ -1844,7 +1974,7 @@ export async function checkEagerLiveSourceHistory( got, ) } - if (modelOptions.failureValue === `provider`) { + if (!driver.matchesFailure) { checkReason( reasons.length === wantReasons.length && reasons.every((cause, index) => cause === wantReasons[index]) && @@ -1854,32 +1984,32 @@ export async function checkEagerLiveSourceHistory( reasons.map(String), ) } else { - // The client library's values: one to one per attempt, newest as - // `cause`, and the outage's first failure kept until it recovers. - checkReason( - wantReasons.length === 0 - ? reasons.length === 0 - : reasons.length >= 1 && - reasons.length <= wantReasons.length && - health.failureReason?.cause === reasons.at(-1), - wantReasons.length === 0 ? [] : `1 to ${wantReasons.length} failures`, - reasons.map(String), - ) - const outage = expected.failureReason[0] - if ( - outage !== undefined && - outage === firstReported?.outage && - reasons[0] !== firstReported.cause - ) { + // Client-created values: the outage shows the attempt that crossed + // the threshold (maintainer decision 2026-10-30), and may list later + // attempts after it, in order. RETRY-1: a later attempt does not + // replace that first failure. + const want = expected.failureReason + if (want.length > 0 && !matches(reasons[0], want[0]!)) { throw violation( - `RETRY-1 (health failureReason)`, + reasons.length > 0 && + want.slice(1).some((i) => matches(reasons[0], i)) + ? `RETRY-1 (health failureReason)` + : `${reasonLaw} (health failureReason)`, stepIndex, - `the outage's first failure kept`, - `a replaced failure`, + failure(want[0]!).message, + reasons.map(String), ) } - firstReported = - outage === undefined ? undefined : { outage, cause: reasons[0] } + let next = 0 + checkReason( + reasons.length <= want.length && + reasons.every((cause) => { + while (next < want.length && !matches(cause, want[next]!)) next++ + return next++ < want.length + }), + want.map((index) => failure(index).message), + reasons.map(String), + ) } const want = expected.preload const got = preload.current @@ -1889,12 +2019,11 @@ export async function checkEagerLiveSourceHistory( ? got === want : typeof got === `object` && (want.rejectedWith === `adapter` || - got.rejected === failure(want.rejectedWith)), + matches(got.rejected, want.rejectedWith)), want, typeof got === `object` ? { rejected: String(got.rejected) } : got, ) expected.waits.forEach((wantWait, index) => { - if (wantWait === `unchecked`) return const gotWait = waits[index]!.current const gotKind = typeof gotWait === `object` ? `rejected` : gotWait if (gotKind === wantWait) return @@ -1905,16 +2034,27 @@ export async function checkEagerLiveSourceHistory( batch.txid === state.waits[index]!.txid && state.waits[index]!.run === state.run, ) + const model = state.waits[index]! + const retired = state.retiredEvidence.has(model.txid) + const current = state.committed.has(model.txid) + // A fulfilled wait names the evidence it lacked; an unfulfilled one + // names why it should have settled. const waitLaw = gotKind === `fulfilled` ? buffered ? `EVID-1/READY-10/PUB-9` - : state.retiredEvidence.has(state.waits[index]!.txid) + : retired && !current + ? `EVID-2` + : !current + ? `EVID-1` + : (model.rejectLaw ?? law) + : wantWait === `fulfilled` + ? retired && !current ? `EVID-2` : `EVID-1` - : wantWait === `rejected` && state.waits[index]!.run === state.run - ? `EVID-3` - : law + : wantWait === `rejected` + ? (model.rejectLaw ?? law) + : law throw violation( `${waitLaw} (awaitTxId ${index})`, stepIndex, @@ -1963,6 +2103,9 @@ export async function checkEagerLiveSourceHistory( try { await compare(-1, `READY-2`) + // Reported outages and applied live batches in the current run. + let outagesThisRun = 0 + let batchesThisRun = 0 for (const [index, step] of history.entries()) { const wasReady = state.phase === `ready` const transientBefore = state.transient.length @@ -2019,6 +2162,7 @@ export async function checkEagerLiveSourceHistory( } } reach(done.beforeReady ? `change-before-ready` : `change-batch`) + if (!done.beforeReady) batchesThisRun++ if (done.removesField) reach(`field-removed`) if (done.flipped) reach(`zero-sign-flipped`) if (done.changes.length > 1) reach(`multi-change-batch`) @@ -2076,12 +2220,22 @@ export async function checkEagerLiveSourceHistory( `conformance transient failure ${done.failure}`, ) failures[done.failure] = error - await provider.transientFailure!(error) + await provider.transientFailure!(error, done.variant) + const kind = driver.transientVariants?.[done.variant] + if (wasReady && kind !== undefined) reach(`transient:${kind}`) if (state.transient.at(-1) === done.failure) { reach(`transient-failure-reported`) } else if (wasReady) reach(`transient-failure-below-threshold`) // RETRY-1: a later attempt in the same outage. if (transientBefore > 0) reach(`transient-failure-repeated`) + if (transientBefore === 0 && state.transient.length > 0) { + // An outage crossed the threshold: after an earlier reported + // outage of this run, after a live batch, or in a later run. + if (outagesThisRun > 0) reach(`outage-reported-again`) + if (batchesThisRun > 0) reach(`outage-reported-after-batch`) + if (state.run > 1) reach(`outage-reported-in-later-run`) + outagesThisRun++ + } break } case `disconnect`: @@ -2108,6 +2262,8 @@ export async function checkEagerLiveSourceHistory( preload = recordSettlement(collection.preload()) events = foldChanges(collection) reach(`restart`) + outagesThisRun = 0 + batchesThisRun = 0 if (pendingBefore) reach(`wait-rejected-at-retirement`) break } diff --git a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts index 703d2702fc..fc666d8791 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts @@ -89,7 +89,7 @@ describe(`eager live-source conformance: reference adapter`, () => { ) // Each mutant breaks one law; the check must report that law. - const mutants: Array<[ReferenceMutant, string]> = [ + const mutants: Array<[ReferenceMutant, string | RegExp]> = [ [`ready-before-delivery`, `READY-2`], [`wraps-loading-failure`, `READY-4`], [`failure-clears-rows`, `HEALTH-1`], @@ -106,12 +106,16 @@ describe(`eager live-source conformance: reference adapter`, () => { [`detached-rejection`, `HEALTH-5`], [`logs-failure`, `HEALTH-5`], [`wait-off-by-one`, `EVID-1`], - [`keeps-retired-evidence`, `EVID-2`], + [`retired-evidence-before-ready`, `EVID-2`], + [`forgets-retired-evidence`, `EVID-2`], + [`leaks-waits-after-fail`, `EVID-2`], [`wait-ignores-timeout`, `EVID-3`], [`disconnect-fails`, `RECON-1`], [`renew-resubscribes`, `RENEW-1`], [`drops-buffered`, `READY-10`], - [`buffer-before-rows`, `PUB-9`], + // It applies buffered batches before the rows, also before a delivery + // that must write nothing. + [`buffer-before-rows`, /PUB-9|PUB-4/], [`replays-per-batch`, `PUB-5`], [`ready-before-buffer`, `READY-10 (rows when ready)`], [`shows-buffered`, `READY-10/PUB-9`], @@ -137,7 +141,7 @@ describe(`eager live-source conformance: reference adapter`, () => { ) expect(result.failed).toBe(true) expect(result.errorInstance).toBeInstanceOf(ConformanceViolation) - expect((result.errorInstance as ConformanceViolation).law).toContain(law) + expect((result.errorInstance as ConformanceViolation).law).toMatch(law) }, campaignTimeoutMs, ) @@ -176,6 +180,53 @@ describe(`eager live-source conformance: per-change commits`, () => { ) }) +// An outage threshold with client-created values (`retryReportAfter`, +// `matchesFailure`), as Electric declares. The reference adapter with that +// policy must conform, and each mutant of its outage reporting must be +// rejected with its law. +describe(`eager live-source conformance: outage threshold, client values`, () => { + const outage = { reportAfter: 3, clientValues: true } + const thresholdDriver = referenceDriver(undefined, { outage }) + + oracleReplayTest(it)( + `the reference adapter with an outage threshold conforms`, + async () => { + const reach = createOracleReach( + `${property}.outage`, + reachStatesFor(thresholdDriver), + { minimum: 5 }, + ) + await fc.assert(eagerLiveSourceProperty(thresholdDriver, reach), { + seed: 2118, + numRuns: runs, + }) + reach.assertReached() + }, + campaignTimeoutMs, + ) + + const outageMutants: Array<[ReferenceMutant, string]> = [ + [`stale-outage-value`, `HEALTH-`], + [`replaces-outage-value`, `RETRY-1`], + [`reports-outage-early`, `HEALTH-1`], + [`outage-count-survives-delivery`, `HEALTH-1`], + [`wraps-loading-failure`, `READY-4`], + ] + oracleReplayTest(it).each(outageMutants)( + `rejects the %s mutant with %s`, + async (mutant, law) => { + const result = await fc.check( + eagerLiveSourceProperty(referenceDriver(mutant, { outage })), + { seed: 2118, numRuns: runs }, + ) + expect(result.failed).toBe(true) + expect(result.errorInstance).toBeInstanceOf(ConformanceViolation) + expect((result.errorInstance as ConformanceViolation).law).toContain(law) + }, + campaignTimeoutMs, + ) +}) + // --------------------------------------------------------------------------- // Grammar controls (ORC-004) // --------------------------------------------------------------------------- diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index 35f67b46d3..cbf72d0893 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -55,8 +55,20 @@ export type ReferenceMutant = | `logs-failure` /** EVID-1: a wait for the next txid fulfills before that batch arrives. */ | `wait-off-by-one` - /** EVID-2: a retired sync run's evidence fulfills a later wait. */ - | `keeps-retired-evidence` + /** EVID-2: a retired run's evidence fulfills a wait before the run is ready. */ + | `retired-evidence-before-ready` + /** EVID-2: a retired run's evidence never fulfills a later wait. */ + | `forgets-retired-evidence` + /** EVID-2: a failure leaves pending waits pending until cleanup. */ + | `leaks-waits-after-fail` + /** HEALTH-2, client values: a later outage reports the first outage's value. */ + | `stale-outage-value` + /** RETRY-1, client values: each attempt replaces the reported failure. */ + | `replaces-outage-value` + /** HEALTH-1, threshold: the outage shows one attempt early. */ + | `reports-outage-early` + /** HEALTH-1, threshold: a delivery does not reset the attempt count. */ + | `outage-count-survives-delivery` /** EVID-3: a wait ignores its timeout. */ | `wait-ignores-timeout` /** RECON-1: a transport drop is reported as a failure. */ @@ -225,15 +237,43 @@ function storedRow( const changeKey = (change: ProviderChange) => change.type === `upsert` ? change.row.id : change.id +/** How the reference adapter reports retried failures. */ +interface OutagePolicy { + /** The failed attempts in a row after ready from which an outage shows. */ + reportAfter: number + /** + * Reports its own error, with the provider's failure as `cause`, as a + * client library's error would carry an HTTP failure. + */ + clientValues: boolean +} + /** Builds the reference adapter's Collection over `provider`. */ function referenceCollection( provider: ControlledPushProvider, mutant: ReferenceMutant | undefined, perChange: boolean, + policy: OutagePolicy, ) { + // Evidence of the current sync run, and of retired runs, which counts + // once the current run is ready (EVID-2). let applied = new Set() + const retired = new Set() let waiters = new Map>() let highestTxid = 0 + let runReady = false + let runFailed = false + // The first outage value, kept by the `stale-outage-value` mutant. + let firstOutage: Error | undefined + const wrap = (error: Error) => + policy.clientValues + ? new Error(`reference failure`, { cause: error }) + : error + const shown = (txid: string) => + applied.has(txid) || + (retired.has(txid) && + (runReady || mutant === `retired-evidence-before-ready`) && + mutant !== `forgets-retired-evidence`) // Only the `restart-replays-rows` mutant reads this across sync runs. let previousRows: ReadonlyArray = [] @@ -269,9 +309,26 @@ function referenceCollection( }> = [] let keys = new Set() const opened: Array = [] - if (mutant !== `keeps-retired-evidence`) applied = new Set() + for (const txid of applied) retired.add(txid) + applied = new Set() waiters = new Map() highestTxid = 0 + runReady = false + runFailed = false + // Failed attempts after ready since the last delivery, and the value + // that reports the outage. + let attempts = 0 + let outage: Error | undefined + const becomeReady = () => { + ready = true + runReady = true + markReady() + for (const [txid, set] of waiters) { + if (!shown(txid)) continue + for (const waiter of set) waiter.resolve() + waiters.delete(txid) + } + } const release = (subscription: ProviderSubscription) => { if (subscription.releases === 0 || mutant === `double-release`) { @@ -284,10 +341,7 @@ function referenceCollection( void commit() keys = new Set(previousRows.map((row) => row.id)) } - if (mutant === `ready-before-delivery`) { - ready = true - markReady() - } + if (mutant === `ready-before-delivery`) becomeReady() const writeChange = (change: ProviderChange) => { if (change.type === `upsert`) { @@ -325,13 +379,23 @@ function referenceCollection( const delivered = () => { if (failing && mutant !== `never-recovers`) reportHealthy() failing = false + if (mutant !== `outage-count-survives-delivery`) attempts = 0 + outage = undefined } const stop = (subscription: ProviderSubscription, error: Error) => { if (mutant === `detached-rejection`) void Promise.reject(error) if (mutant === `logs-failure`) console.error(`source failed`, error) if (mutant !== `keeps-failed-source`) release(subscription) - fail(error) + // EVID-2: the failure retires the run, so its pending waits reject. + runFailed = true + if (mutant !== `leaks-waits-after-fail`) { + for (const set of waiters.values()) { + for (const waiter of set) waiter.reject(error) + } + waiters.clear() + } + fail(wrap(error)) } const subscribe = ( @@ -370,16 +434,10 @@ function referenceCollection( void commit() keys = new Set(event.rows.map((row) => row.id)) } - if (mutant === `ready-before-buffer` && !ready) { - ready = true - markReady() - } + if (mutant === `ready-before-buffer` && !ready) becomeReady() if (mutant !== `buffer-before-rows`) applyBuffered(cut) previousRows = event.rows - if (!ready) { - ready = true - markReady() - } + if (!ready) becomeReady() delivered() } else if (event.type === `changes`) { if (!ready && mutant !== `shows-buffered`) { @@ -424,8 +482,25 @@ function referenceCollection( stop(subscription, event.error) return } + // An outage shows from its `reportAfter`th attempt in a row. + attempts += 1 + const threshold = + policy.reportAfter - (mutant === `reports-outage-early` ? 1 : 0) + if (attempts < threshold) return failing = true - reportError(event.error) + if (!policy.clientValues) { + reportError(event.error) + } else { + if (mutant === `replaces-outage-value` && outage) { + reportHealthy(outage) + outage = undefined + } + outage ??= + mutant === `stale-outage-value` + ? (firstOutage ??= wrap(event.error)) + : wrap(event.error) + reportError(outage) + } } else if (!ready) { stop( subscription, @@ -467,7 +542,11 @@ function referenceCollection( }) const awaitTxId = (txid: string, timeout?: number): Promise => { - if (applied.has(txid)) return Promise.resolve(true) + if (shown(txid)) return Promise.resolve(true) + // EVID-2: after a failure no evidence can arrive. + if (runFailed && mutant !== `leaks-waits-after-fail`) { + return Promise.reject(new Error(`reference source failed`)) + } if (mutant === `wait-off-by-one` && Number(txid) <= highestTxid + 1) { return Promise.resolve(true) } @@ -491,20 +570,37 @@ function referenceCollection( /** * A production driver for the reference adapter, optionally mutated. With * `perChange`, the adapter commits each change of a batch on its own and - * declares `per-change` commits, as TrailBase does. + * declares `per-change` commits, as TrailBase does. With `outage`, it + * reports a retried outage from its `reportAfter`th attempt, with its own + * error values when `clientValues` is set, as Electric does. */ export function referenceDriver( mutant?: ReferenceMutant, - { perChange = false }: { perChange?: boolean } = {}, + { + perChange = false, + outage = { reportAfter: 1, clientValues: false }, + }: { perChange?: boolean; outage?: OutagePolicy } = {}, ): EagerLiveSourceDriver { + const variant = outage.reportAfter > 1 || outage.clientValues return { - adapter: `reference${perChange ? ` per-change` : ``}${mutant ? ` (${mutant})` : ``}`, + adapter: `reference${perChange ? ` per-change` : ``}${variant ? ` outage ${outage.reportAfter}` : ``}${mutant ? ` (${mutant})` : ``}`, + retryReportAfter: outage.reportAfter, + // The adapter's error carries the provider's failure as `cause`. + matchesFailure: outage.clientValues + ? (reported, failure) => + reported instanceof Error && + reported.cause instanceof Error && + reported.cause.message.endsWith(` failure ${failure}`) + : undefined, + // A threshold has no law across a reconnect or a renewal, so a driver + // with one declares neither. capabilities: new Set([ `complete-snapshot`, `await-evidence`, `retry-policy`, - `transport-reconnect`, - `capability-renewal`, + ...(outage.reportAfter > 1 + ? [] + : ([`transport-reconnect`, `capability-renewal`] as const)), `listen-before-fetch`, `duplicate-key-delivery`, `source-failure-after-ready`, @@ -543,6 +639,7 @@ export function referenceDriver( provider, mutant, perChange, + outage, ) if (mutant !== `relabels-inserts`) { return { collection, provider, awaitTxId } From e67c4bae9df7cfc986705d07a4febe06d1fdaa90 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 16:28:24 -0600 Subject: [PATCH 19/64] fix(electric): count only shape-log outages, keep retired evidence until ready, reject waits at failure Fix the Electric findings from the oracle review of 815854815 under the maintainer decisions of 2026-10-30. - Retried subset snapshot requests no longer count toward the source's outage (glossary: a subset load's failure belongs to its demand). They reach the fetchClient without an abort signal, GET or POST; shape-log requests always carry one. Only an applied up-to-date ends an outage; a subset's end does not. - "Five in a row": the fetchClient wrapper counts every failed shape-log attempt (network error, 429, 5xx, and a 4xx that onError then decides), and an accepted response restarts the count. onError no longer counts, so the last 5xx before an onError retry is not counted twice. - A throwing onError after ready fails the run with its own error, as it rejects preload before ready. - The lifecycle keeps a retired run's evidence and counts it once the current run is ready, instead of forgetting it: a reload between a txid's delivery and a handler's awaitTxId no longer times out. Exported sync metadata includes it. - A failure rejects pending awaitTxId and awaitMatch waits with the failure, and later unmet waits reject at once. - Driver: generated attempt kinds (503, 429 with Retry-After, a rejected fetch, a 400 onError retries); every failed response carries the injected failure's index in a header, and matchesFailure names the value owed, a thrown give-up by identity; a give-up throws for an odd failure index. - electric-sdk-delivery-oracle's on-demand subset lane: 0..7 retried 503s per subset request, up to three extra pausing loads, a live outage of 0..6 attempts after the first load, and a source-health check after every retried attempt and delivery. Evidence: - Law: HEALTH-1/2/4, RETRY-1, EVID-2 and the glossary's request error, with the 2026-10-30 decisions (threshold 5 restarting on an accepted response, the crossing attempt's value, retirement at failure, retired evidence after ready, the thrown error). - Reach: eager campaigns 400 histories; across four seeds every declared state at least 15 (outage-reported-in-later-run), each attempt kind at least 66. - RED on 815854815: EVID-2 at [fail, wait, snapshot]; on-demand lane source health showed a subset request's retried 503s (failures [0,5]). Mutants restoring each earlier behavior are rejected: retired txids forgotten (EVID-2), thrown error unreported (HEALTH-1 health error cause), a subset end ending a live outage (on-demand lane). - GREEN: Electric suite 1242 passed (4 skipped); SDK delivery oracle 34/34; eager campaigns pass; reference 77/77; TrailBase, RxDB, PowerSync pass. - Mutants rejected: waits kept after a failure, retired evidence before ready (EVID-2); 429 or network attempts uncounted, threshold 6 (HEALTH-1); subset attempts counted, aborted requests counted (on-demand lane). Survives: never restarting the count on an accepted response, since every accepted eager response is an up-to-date (recorded limit). - Limits: snapshot-end evidence across a restart and awaitMatch after a failure are not generated; no held commit receipt, persisted resume, or real Electric server; the on-demand lane has 12 histories and no declared reach. - Size: electric-db-collection +415 gzip bytes over the merge base (+4.2%), +192 over 815854815, for the retired-evidence set, failure-time wait rejection, and the merged export; the app scenario is unchanged. Co-authored-by: Isaac --- .../electric-retry-health-and-evidence.md | 2 +- docs/collections/electric-collection.md | 36 ++-- docs/contributing/glossary.md | 2 +- docs/contributing/oracle-coverage.md | 58 ++++--- packages/db-adapter-conformance/MIGRATION.md | 53 ++++++ packages/db-adapter-conformance/README.md | 2 +- .../electric-db-collection/src/electric.ts | 156 ++++++++++++------ .../tests/conformance-driver.ts | 118 ++++++++++--- ...source-conformance-oracle.property.test.ts | 6 +- ...ctric-sdk-delivery-oracle.property.test.ts | 84 +++++++++- 10 files changed, 398 insertions(+), 119 deletions(-) diff --git a/.changeset/electric-retry-health-and-evidence.md b/.changeset/electric-retry-health-and-evidence.md index d24cb1c58c..413d218d9d 100644 --- a/.changeset/electric-retry-health-and-evidence.md +++ b/.changeset/electric-retry-health-and-evidence.md @@ -2,4 +2,4 @@ '@tanstack/electric-db-collection': patch --- -Report an extended Electric outage in `collection.health`. After the Collection is ready, five failed attempts in a row, retried by the client or by an `onError` that returns a retry, show as the source node's `failureReason`, and the next applied up-to-date clears it; shorter outages leave health unchanged. A restarted Collection no longer resolves `awaitTxId` from a txid the previous sync run delivered. An `onError` handler that throws now gives up the stream without an unhandled rejection. +Report an extended Electric outage in `collection.health`. After the Collection is ready, five failed requests to the shape log in a row, retried by the client or by an `onError` that returns a retry, show as the source node's `failureReason` with the fifth attempt's error, and the next applied up-to-date clears it. Shorter outages, and retried subset snapshot requests, leave health unchanged. A restarted or reloaded Collection resolves `awaitTxId` for a txid the previous sync run delivered only once it is ready again, instead of at once. When the stream fails, pending `awaitTxId` and `awaitMatch` calls reject with the failure instead of waiting for their timeout. An `onError` handler that throws now gives up the stream with its own error, without an unhandled rejection. diff --git a/docs/collections/electric-collection.md b/docs/collections/electric-collection.md index ecf4296a78..e537cc2a31 100644 --- a/docs/collections/electric-collection.md +++ b/docs/collections/electric-collection.md @@ -341,13 +341,16 @@ await todosCollection.utils.awaitMatch( ### Cleanup and resume safety -Transaction evidence and pending `awaitTxId`/`awaitMatch` calls belong to one -collection lifecycle. Explicit cleanup and automatic garbage collection reject -pending waits with `StreamAbortedError`; callbacks from the retired stream cannot -settle waits in a restarted collection, and a restarted collection does not -treat a txid the retired stream delivered as synced: `awaitTxId` waits for the -new stream's evidence. Observe these promises even when the -component or collection may be disposed before they settle. +Pending `awaitTxId`/`awaitMatch` calls belong to one collection lifecycle. +Explicit cleanup and automatic garbage collection reject pending waits with +`StreamAbortedError`; callbacks from the retired stream cannot settle waits in +a restarted collection. A txid the retired stream delivered still counts after +a restart or reload, but only once the restarted collection is ready again, +because its rows are read after that txid; until then `awaitTxId` waits. When +the stream fails or gives up, pending waits reject with the failure, and a +later `awaitTxId` or `awaitMatch` rejects unless its evidence was already +seen. Observe these promises even when the component or collection may be +disposed before they settle. An on-demand Collection with managed SQLite persistence can also replace its Electric provider session within the same Collection sync run when its cached @@ -433,12 +436,19 @@ source node shows an `error` in `collection.health` and in each live query's While the Electric stream retries a failed request, the Collection stays `loading` before its first up-to-date. After it, a short outage leaves the -health tree unchanged, so a blip does not flap it. Once five attempts in a row -have failed, whether the client retried them or your `onError` returned a -retry, the source node shows the outage as its `failureReason`; the next -applied up-to-date clears it. When the stream gives up, including when -`onError` throws, it calls `fail()`: a failed first load before readiness, or -the source node's `error` after it. Complete updates from an explicit `replica: 'full'` stream remain valid. +health tree unchanged, so a blip does not flap it. Once five requests to the +shape log in a row have failed, whether the client retried them or your +`onError` returned a retry, the source node shows the outage as its +`failureReason`, with the error of the fifth attempt; the next applied +up-to-date clears it. Any response the client accepts, such as a +`must-refetch`, starts the count again. With the default backoff, five failed +attempts take from under a second to about 15 seconds, because each retry +waits a random part of a doubling delay. A source that keeps failing five +times and then succeeding once shows and clears the outage each time. A +subset snapshot request that the client retries does not count: its failure +belongs to the live query that requested it. When the stream gives up, it +calls `fail()` with the error, or with the error your `onError` threw: a +failed first load before readiness, or the source node's `error` after it. Complete updates from an explicit `replica: 'full'` stream remain valid. On-demand streams can observe updates outside their loaded subsets; unknown partial rows are ignored, while transaction acknowledgement evidence is retained. Outside scoped recovery, complete rows published by persistence reloads or diff --git a/docs/contributing/glossary.md b/docs/contributing/glossary.md index e64d8c7214..feacf1523a 100644 --- a/docs/contributing/glossary.md +++ b/docs/contributing/glossary.md @@ -82,7 +82,7 @@ until cleanup ends the run. | loading error | The `fail()` that ends a sync run's first load in status `error` (`isLoadingError`). It is final for its sync run: `markReady()` afterward does nothing, and only a new sync run can reach `ready`. | Sync error or a retry. | | crash | Moving a Collection to status `crashed` (`isCrashed`) from any active status, because it can no longer serve reads or admit writes. Core uses it for IndexedDB closure, terminal local persistence failures, a throwing sync function, and internal invariant failures. It cannot be recovered in place: nothing restarts it and `reload()` rejects. Only cleanup leaves it, and cleanup discards all of the Collection's state, so a later start is a fresh start. After a crash, the sync adapter's status calls (`markReady()`, `fail()`, `reportError()`, `reportHealthy()`) do nothing. Sync data that the run commits still applies, because closure keeps accepted sync obligations such as the confirmation of a write admitted before it. | Loading error, sync error, or cleanup. | | sync error | A problem after Collection readiness on a source's health node, from the sync adapter: a failure reason (`reportError()`, still trying; `reportHealthy()` clears it, and `reportHealthy(error)` removes one failure) or an error (`fail()`, stopped; it stays for the sync run, every later report throws, and a second `fail()` does nothing). A live query's `isSyncError` is `true` when it is `ready` and its tree has any problem. It never changes status or rows. Cleanup resets it. | Loading error or subscription status. | -| failure reason | A node's `failureReason`: the problem while the sync adapter keeps trying, after TanStack Query's `failureReason`. Its frame is the sync run: it lists each `reportError()` since the slot was last clear, once per error value (a value already listed keeps its failure's place and `at`), `reportHealthy(error)` removes that value's failure, `reportHealthy()` ends it, and a reload's new run that reports starts a new one. The slot clears when its list empties. | An error or a retry count. | +| failure reason | A node's `failureReason`: the problem while the sync adapter keeps trying, after TanStack Query's `failureReason`. When to report is the adapter's choice of threshold: a connection-aware source can report at once, while Electric, which syncs over HTTP, reports only after five failed attempts in a row. Its frame is the sync run: it lists each `reportError()` since the slot was last clear, once per error value (a value already listed keeps its failure's place and `at`), `reportHealthy(error)` removes that value's failure, `reportHealthy()` ends it, and a reload's new run that reports starts a new one. The slot clears when its list empties. | An error or a retry count. | | problem | A health node's `failureReason` or `error` (`HealthProblem`). Its `message`, `cause`, and `at` are its newest failure's, and `errors` lists every failure of its frame still open, oldest first, each `{ name, message, code?, at, cause }`. A failure that clears leaves the list; one error value is one failure, which keeps the place of its first failure. Adding or removing a failure changes the problem, so its node is a new object. | A failure, which is one entry of a problem. | | failure | One reported value in a problem's `errors`: one `reportError()`, the `fail()` of a sync run, a crash, or one failed request. | A problem. | | request error | A failed subset load (a rejection, or a synchronous throw, which is a rejection) for demand that a live query or Effect started after its initial-query readiness, shown as the `error` on that query's own node. The source Collection's node does not change. The node's problem lists every request failure still open. The public name stays general on purpose: an application sees a failed request for more data, not the subset-load mechanism. It clears when a later subset load for that demand fulfills, including one that the query's own reload issues, or when the demand retires. A load that core aborted decides nothing. A truncate's or a source reload's replay decides every request error of the demand it replayed: its success clears them all, and its failure replaces them with the replay's failures, which stay until a replay succeeds for the demand that remains (a replay failure belongs to the logical demand that failed, ARCHITECTURE law 9) or the demand retires. | Sync error or request data. | diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 5325d71aa1..57556d420e 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6003,8 +6003,9 @@ commit per batch or per change. PUB-1 checks every change publication during a step, not only the settled checkpoint. The reference push adapter (`tests/reference-adapter-oracle.property.test.ts`, `adapter-conformance.eager-live-source.reference`) runs fixed and random -campaigns of 600 histories with declared reach, a per-change variant, and 35 -law-named hostile mutants, with ORC-004 reconstruction, exclusion, and +campaigns of 600 histories with declared reach, a per-change variant, an +outage-threshold variant with client-created values, and 41 law-named +hostile mutants, with ORC-004 reconstruction, exclusion, and ablation controls. The Neon Realtime receiving driver (`adapter-conformance.eager-live-source.neon-realtime`, `test:neon`) runs `@neon/realtime-tanstack` 0.1.1 vendored at neon-pkgs `7ac55e3e` and is RED on @@ -6048,24 +6049,37 @@ evidence: a direct write and read through `@powersync/node` 0.20.0. Electric (`adapter-conformance.eager-live-source.electric`, eager mode over the installed `@electric-sql/client` and a controlled `fetchClient` that answers the HTTP protocol, per-batch commits, typed Postgres columns for -nine kinds) is GREEN after three fixes. After ready, an outage of five -consecutive failed attempts that the client retries (network error, 429, -5xx) or the shape's `onError` retries is now reported in `failureReason`, -once, and the next applied commit point clears it; shorter outages show -nothing (HEALTH-1, HEALTH-2, RETRY-1, driver parameter -`retryReportAfter: 5`; maintainer decision 2026-10-30: a retrying source -must not flap its health, so only an extended outage reports). A restarted run no -longer settles `awaitTxId` from the retired run's txids (EVID-2). An -`onError` that throws now gives up without a detached rejection (HEALTH-5, -RETRY-1); the driver gives up alternately by returning nothing and by -throwing. Electric's -failures are values its client creates (`failureValue: any`), so the check -owes some error, at most one per attempt, and the outage's first failure -kept; it does not check identity. Its empty batches carry no txid -(`txidEvidence: change`). Excluded by `excludesValue` (column domains, not +nine kinds) is GREEN after fixes from two rounds. Maintainer decisions +(2026-10-30) that the shared model and Electric's driver encode: only an +extended outage reports, five shape-log attempts in a row for Electric +(`retryReportAfter: 5`; a threshold is each adapter's choice: a +connection-aware source can report at once), with the attempt that crossed +it as the failure, any accepted response restarting the count, and an +applied up-to-date clearing it at once; a sync run retires when it fails, +so its pending waits reject then and later unmet waits reject; a retired +run's evidence fulfills a wait once the current run is ready (revising law +set v18 EVID-2, which said never); and a throwing `onError` gives up with +its own error. Attempt kinds are generated (`transientVariants`: HTTP 503, +HTTP 429 with `Retry-After`, a rejected fetch, an HTTP 400 that `onError` +retries), and outages land anywhere in a history, also a second time in a +run, after a live batch, and in a later run. Failure values are compared +through the driver's `matchesFailure`: each failed response carries the +injected failure's index in a header, and a thrown give-up is compared by +identity. The on-demand subset lane of `electric-sdk-delivery-oracle` +(`electric.sdk-snapshot-delivery`) adds retried subset attempts (0..7 per +request), up to six live pauses, and a live outage after the first load, +and checks that the source node shows a problem exactly when the live +stream had an outage. Excluded by `excludesValue` (column domains, not adapter behavior): int4 values outside 32 bits, and a `-0` nested in an -array or object for the float8 column. Limits: no on-demand or progressive -mode, no persisted resume (so a held commit receipt that clears an outage -late is not exercised), and no real Electric server. Two generated scenario -prefixes reach EVID-2 (a batch, a restart, a wait for its txid) and the -second retried failure of one outage. +array or object for the float8 column. Limits: the eager grammar has no +accepted response that commits nothing, so restarting the count on one is +enforced only by source inspection (a mutant that never restarts it +survives); snapshot-visibility evidence (`snapshot-end`) across a restart is +not generated, so the retired-evidence rule is witnessed for txids only; +`awaitMatch` after a failure is not generated; a held commit receipt and +persisted resume are not exercised; there is no real Electric server; the +default configuration without `onError` and a give-up after `maxRetries` +are not generated. The on-demand lane runs 12 histories without declared +reach. The shared check refuses `retryReportAfter` above 1 with +`transport-reconnect` or `capability-renewal`: no law fixes whether a +reconnect or a renewal ends an outage a threshold is counting. diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index 12b0de0317..da9bb067f0 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -352,3 +352,56 @@ range`). The Collection shows what the database stored, so this is not domains (`excludesValue`). Waiting out a deadline after each give-up made histories take 1 s each; the driver now counts the give-ups its own `onError` made. +- **2026-10-30, Electric, review round (three oracle reviewers).** The + reviews found gaps in the previous round's oracle and three more Electric + defects; the maintainer decided the law questions they raised. + - **Decisions:** a sync run retires when it fails, so its pending waits + reject then; a retired run's evidence fulfills a wait once the current + run is ready (law set v18 EVID-2 said never); "five in a row" restarts + on any accepted response; a throwing `onError` reports its own error; + the outage shows the attempt that crossed the threshold; a threshold is + each adapter's choice (Electric syncs over HTTP, a connection-aware + source can report at once); recovery clears at once. + - **Oracle changes:** outages are generated anywhere in a history, with + lengths below and past the threshold and a recurring outage around a + live batch; attempt kinds are a generator dimension + (`transientVariants`); `failureValue: any` is replaced by the driver's + `matchesFailure` relation, so a stale or replaced value is rejected; + txids number batches across runs; waits are generated after a failure; + a field-only publication mismatch names FID-1. New reach states for + each attempt kind and for an outage reported again, after a batch, and + in a later run. A reference variant with an outage threshold of 3 and + client-created values conforms, and its mutants (stale value, replaced + value, early threshold, count kept across a delivery) are rejected; + new reference mutants for EVID-2 (retired evidence before ready, never, + and waits kept after a failure). The on-demand subset lane of + `electric-sdk-delivery-oracle` gains retried subset attempts, extra + pausing loads, a live outage, and a source-health check. + - **RED on 815854815:** EVID-2 at `[fail, wait, snapshot]` (a wait after + the failure stayed pending); in the on-demand lane, source health + showed a subset request's five retried 503s (`failures [0,5]`). + Mutants restoring each earlier behavior were rejected: retired txids + forgotten (EVID-2), the thrown error not reported (HEALTH-1, health + error cause), a subset's end ending a live outage (on-demand lane). + - **Fixes:** the lifecycle keeps retired evidence and counts it after the + run's ready; a failure rejects pending txid and match waits and later + unmet ones; the `fetchClient` wrapper skips subset snapshot requests + (they carry no abort signal), counts every failed shape-log attempt, + and restarts the count on an accepted response; only an applied + up-to-date ends an outage; a throwing `onError` fails the run with its + error. + - **GREEN:** Electric suite 1242 passed (4 skipped); SDK delivery oracle + 34/34; eager campaigns pass with every declared state reached at least + 15 times across four seeds; reference suite 77/77; TrailBase, RxDB, + PowerSync pass. + - **Mutants of `src/electric.ts`, rejected:** retired txids forgotten, + waits kept after a failure, retired evidence before ready (EVID-2); + 429 or network attempts not counted, threshold 6, thrown error not + reported (HEALTH-1); subset attempts counted, a subset's end ending the + outage, aborted requests counted (on-demand lane). Survives: never + restarting the count on an accepted response, because every accepted + eager response is also an up-to-date (recorded limit). + - **Size:** Electric +415 gzip bytes over the merge base (+4.2%), +192 + over the previous round, for the retired-evidence set, failure-time + wait rejection, and the merged export. The `app` scenario does not + include Electric. diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md index 824925d14a..ef2468c290 100644 --- a/packages/db-adapter-conformance/README.md +++ b/packages/db-adapter-conformance/README.md @@ -62,7 +62,7 @@ reach counter, so the run fails when histories miss a state the laws need. - **Reference adapter** (`tests/reference-adapter.ts`): a small push adapter written to conform. `tests/reference-adapter-oracle.property.test.ts` runs a fixed-seed and a random campaign with declared reach, a per-change variant, - and 35 hostile mutants of the adapter, each of which the check must reject + an outage-threshold variant, and 41 hostile mutants of the adapter, each of which the check must reject with its law. It runs in the default `pnpm test`. - **Neon Realtime** (`tests/neon/`): `@neon/realtime-tanstack` 0.1.1, vendored diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 669d74c4b1..549a0e2886 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -121,6 +121,15 @@ type ElectricLifecycleEvidence = { hydratedResumeState?: ElectricResumeState } +function hasEvidence(evidence: ElectricLifecycleEvidence, txId: Txid): boolean { + return ( + evidence.seenTxids.has(txId) || + evidence.seenSnapshots.some((snapshot) => + isVisibleInSnapshot(txId, snapshot), + ) + ) +} + type ElectricPendingMatch> = { matchFn: (message: Message) => boolean resolve: (value: boolean) => void @@ -917,6 +926,15 @@ class ElectricLifecycle> { private nextWaiterId = 0 private epoch = 0 private active = false + // Evidence of retired runs. It counts once the current run is ready, + // because that run's rows are read after it (EVID-2). + private readonly retired: ElectricLifecycleEvidence = { + seenTxids: new Set(), + seenSnapshots: [], + } + private runReady = false + // The current run's failure: no evidence can arrive after it (EVID-2). + private failure: { error: unknown } | undefined constructor(private readonly collectionId?: string) {} @@ -928,6 +946,8 @@ class ElectricLifecycle> { start(): number { if (this.active) this.retire() this.active = true + this.runReady = false + this.failure = undefined this.epoch++ this.matchBuffer = { committedMessages: [], pendingMessages: [] } return this.epoch @@ -941,31 +961,57 @@ class ElectricLifecycle> { if (epoch !== undefined && !this.isActive(epoch)) return this.active = false this.epoch++ + this.rejectWaits(new StreamAbortedError(this.collectionId)) + this.matchBuffer = { committedMessages: [], pendingMessages: [] } + for (const txid of this.evidence.seenTxids) this.retired.seenTxids.add(txid) + this.retired.seenSnapshots.push(...this.evidence.seenSnapshots) + this.evidence.seenTxids.clear() + this.evidence.seenSnapshots.length = 0 + this.evidence.hydratedResumeState = undefined + } - for (const match of this.pendingMatches.values()) { - clearTimeout(match.timeoutId) - match.reject(new StreamAbortedError(this.collectionId)) - } - this.pendingMatches.clear() + /** The run's first ready: retired evidence now counts (EVID-2). */ + markReady(epoch: number): void { + if (!this.isActive(epoch)) return + this.runReady = true + this.resolveTxidWaiters() + } - for (const waiter of this.pendingTxidWaits.values()) { + /** The run failed: its pending waits reject, as later unmet ones do. */ + fail(epoch: number, error: unknown): void { + if (!this.isActive(epoch)) return + this.failure ??= { error } + this.rejectWaits(error) + } + + private rejectWaits(error: unknown): void { + for (const waiter of [ + ...this.pendingMatches.values(), + ...this.pendingTxidWaits.values(), + ]) { clearTimeout(waiter.timeoutId) - waiter.reject(new StreamAbortedError(this.collectionId)) + waiter.reject(error as Error) } + this.pendingMatches.clear() this.pendingTxidWaits.clear() - this.matchBuffer = { committedMessages: [], pendingMessages: [] } - // A retired run's evidence does not settle a later run's waits (EVID-2). - this.forgetEvidence() } forgetEvidence(): void { - this.evidence.seenTxids.clear() - this.evidence.seenSnapshots.length = 0 + for (const evidence of [this.evidence, this.retired]) { + evidence.seenTxids.clear() + evidence.seenSnapshots.length = 0 + } this.evidence.hydratedResumeState = undefined } exportMeta(): ElectricSyncMeta { - return exportElectricSyncMeta(this.evidence) + return exportElectricSyncMeta({ + ...this.evidence, + seenTxids: new Set([ + ...this.retired.seenTxids, + ...this.evidence.seenTxids, + ]), + }) } importMeta(meta: unknown): void { @@ -992,10 +1038,8 @@ class ElectricLifecycle> { private hasTxid(txId: Txid): boolean { return ( - this.evidence.seenTxids.has(txId) || - this.evidence.seenSnapshots.some((snapshot) => - isVisibleInSnapshot(txId, snapshot), - ) + hasEvidence(this.evidence, txId) || + (this.runReady && hasEvidence(this.retired, txId)) ) } @@ -1020,6 +1064,7 @@ class ElectricLifecycle> { throw new ExpectedNumberInAwaitTxIdError(typeof txId, this.collectionId) } if (this.hasTxid(txId)) return true + if (this.failure) throw this.failure.error return new Promise((resolve, reject) => { const waitId = this.nextWaiterId++ @@ -1048,6 +1093,7 @@ class ElectricLifecycle> { if (!matchFn(message)) continue return true } + if (this.failure) throw this.failure.error for (const message of this.matchBuffer.pendingMessages) { if (matchFn(message)) return this.registerMatch(matchFn, timeout, true) } @@ -1639,7 +1685,6 @@ function createElectricSync>( begin, write, commit: commitSyncTransaction, - fail, truncate, collection, metadata, @@ -1650,14 +1695,21 @@ function createElectricSync>( let ready = false const markReady = () => { ready = true + lifecycle.markReady(lifecycleEpoch) params.markReady() } - // After the first ready, a retried outage shows only once it lasted - // five failed attempts in a row, so health does not flap on a blip - // (maintainer decision). It is reported once: later attempts keep its - // failure. A commit point ends the outage, and its failure clears once - // that commit applies (HEALTH-1, HEALTH-2, RETRY-1). A give-up after - // ready ends health reports; the other failures abort the run. + // A failure ends the run's waits (EVID-2). + const fail = (error: unknown) => { + lifecycle.fail(lifecycleEpoch, error) + params.fail(error) + } + // After the first ready, a retried outage shows only once five + // shape-log attempts in a row failed, so health does not flap on a blip + // (maintainer decision). The attempt that crossed the threshold reports + // it once; later attempts keep its failure. A response the client + // accepts ends the run of failures, and an applied up-to-date ends the + // outage, clearing its failure (HEALTH-1, HEALTH-2, RETRY-1). A give-up + // after ready ends health reports; the other failures abort the run. let failedAttempts = 0 let retrying: unknown let gaveUp = false @@ -1924,15 +1976,17 @@ function createElectricSync>( shapeOptions.handle ?? (canUsePersistedResume ? persistedResumeState.handle : undefined), signal: abortController.signal, - // The client retries a network error, 429, and 5xx without calling - // onError; onError decides the other failures. + // The client retries a failed network attempt, 429, or 5xx itself, and + // onError decides another 4xx; each is a failed attempt. A subset + // snapshot request carries no abort signal: its attempts belong to its + // demand, not to the source (glossary, request error). fetchClient: async (input, init) => { + const request = (shapeOptions.fetchClient ?? fetch)(input, init) + if (!init?.signal) return request try { - const response = await (shapeOptions.fetchClient ?? fetch)( - input, - init, - ) - if (response.status === 429 || response.status >= 500) { + const response = await request + if (response.ok || response.status === 409) failedAttempts = 0 + else { reportRetry( new FetchError( response.status, @@ -1945,7 +1999,7 @@ function createElectricSync>( } return response } catch (error) { - if (!init?.signal?.aborted) reportRetry(error) + if (!init.signal.aborted) reportRetry(error) throw error } }, @@ -1972,14 +2026,17 @@ function createElectricSync>( try { retry = await shapeOptions.onError(errorParams) } catch (error) { - // A throwing handler gives up. Rethrown, its error would reject - // the client's own start unobserved; returning nothing stops the - // stream, and the client reports its error to the subscriber. + // A throwing handler gives up with its error. Rethrown, it would + // reject the client's own start unobserved, so the run fails + // here and returning nothing stops the stream. failWaits(error) + if (ready) { + gaveUp = true + fail(error) + } return } if (!retry || typeof retry !== `object`) failWaits(errorParams) - else reportRetry(errorParams) return retry } else { const initialSyncFailed = !ready @@ -2611,18 +2668,23 @@ function createElectricSync>( () => undefined, ) } - const recover = endOutage() - if (applied === true) recover() + // Only the shape log's up-to-date shows the source recovered; a + // subset's end belongs to its demand. + const recover = commitPoint === `up-to-date` ? endOutage() : undefined + if (applied === true) recover?.() else { - void applied.then(recover, (error: unknown) => { - if (!isActiveLifecycle() || abortController.signal.aborted) { - return - } - streamErrorVersion++ - stream.unsubscribeAll() - abortController.abort() - fail(error) - }) + void applied.then( + () => recover?.(), + (error: unknown) => { + if (!isActiveLifecycle() || abortController.signal.aborted) { + return + } + streamErrorVersion++ + stream.unsubscribeAll() + abortController.abort() + fail(error) + }, + ) } if (finishesReset) { diff --git a/packages/electric-db-collection/tests/conformance-driver.ts b/packages/electric-db-collection/tests/conformance-driver.ts index 8edad5c7ea..417099325d 100644 --- a/packages/electric-db-collection/tests/conformance-driver.ts +++ b/packages/electric-db-collection/tests/conformance-driver.ts @@ -19,16 +19,18 @@ * for one it does, `delete` otherwise. The adapter commits at * `up-to-date`, so the driver declares `per-batch` commits; * - a failure answers with HTTP 400, which the client gives to the shape's - * `onError`. The handler gives up, alternately by returning nothing and by - * throwing, and the adapter reports the client's or the handler's error, - * so the driver declares `failureValue: any`. + * `onError`. The handler gives up by returning nothing, or for an odd + * failure index by throwing. Every failed response carries the injected failure's index in + * a header, so `matchesFailure` names the value owed: the client's + * FetchError for that response, or the error the handler threw. * The law set gives Electric a give-up after retries (`retryBound`), so the * driver declares `source-failure-after-ready`; - * - a retried failure (`retry-policy`) is one failed attempt; health shows - * an outage from its fifth consecutive attempt (`retryReportAfter`). It - * answers with HTTP 503, which the - * client retries (its backoff is set short), or, every other time, with - * HTTP 400, which the shape's `onError` retries; + * - a retried failure (`retry-policy`) is one failed attempt of a generated + * kind (`transientVariants`): an HTTP 503 or a 429 with `Retry-After`, + * which the client retries (its backoff is set short), a rejected fetch + * (a network failure, also retried by the client), or an HTTP 400 that + * the shape's `onError` retries. Health shows an outage from its fifth + * attempt in a row (`retryReportAfter`); * - a provider subscription is one sync run's request chain; its release is * the abort of the chain's pending request (LIFE-2, LIFE-10); * - the propagation step (`settle`) waits until the client asks again after @@ -42,6 +44,7 @@ * `note` is a nullable text column (`toSqlRow`, `fromSqlRow`). The driver * never reads the Collection. */ +import { FetchError } from '@electric-sql/client' import { createCollection } from '@tanstack/db' import { electricCollectionOptions } from '../src/electric' import { @@ -117,8 +120,21 @@ type Request = { url: URL signal: AbortSignal | undefined respond: (response: Response) => void + /** Rejects the fetch, as a network failure does. */ + reject: (error: Error) => void } +/** The oracle's index of an injected failure, from its message. */ +const failureIndex = (error: Error) => error.message.split(` `).at(-1)! + +/** The kinds of failed attempt the provider makes (`retry-policy`). */ +const TRANSIENT_VARIANTS = [ + `http-503`, + `http-429`, + `network`, + `http-400-retried`, +] as const + type Session = { handle: string latest: Request | undefined @@ -131,6 +147,20 @@ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) /** How long the driver waits for the client before letting the check decide. */ const clientTimeoutMs = 2_000 +/** The response header that carries an injected failure's index. */ +const FAILURE_HEADER = `x-conformance-failure` + +/** A failed response that carries the failure's index; a 429 asks for 1 ms. */ +function failureResponse(error: Error, status: number): Response { + return new Response(error.message, { + status, + headers: { + [FAILURE_HEADER]: failureIndex(error), + ...(status === 429 ? { 'retry-after': `0.001` } : {}), + }, + }) +} + class ElectricProvider implements ControlledProvider { private readonly queued: Array = [] private readonly sessions: Array = [] @@ -140,21 +170,30 @@ class ElectricProvider implements ControlledProvider { // Give-ups so far, and the count the latest failure must reach. private gaveUp = 0 private giveUpTarget = 0 - private transientFailures = 0 private retryNextError = false + /** The error `onError` threw to give up, by failure index. */ + readonly thrown = new Map() /** * An app's `onError`. It retries the HTTP 400 of a retried failure by - * returning an empty retry. It gives up on any other error, alternately - * by returning nothing and by throwing; both stop the stream (RETRY-1), - * and the adapter fails. + * returning an empty retry. It gives up on any other error by returning + * nothing, or, for an odd failure index, by throwing; both stop the + * stream (RETRY-1), and the adapter fails. */ - readonly onError = () => { + readonly onError = (error: Error) => { if (this.retryNextError) { this.retryNextError = false return {} } - if (this.gaveUp++ % 2 === 1) throw new Error(`conformance give-up`) + this.gaveUp++ + const index = (error as { headers?: Record }).headers?.[ + FAILURE_HEADER + ] + if (Number(index) % 2 === 1) { + const thrown = new Error(`conformance give-up ${index}`) + this.thrown.set(index!, thrown) + throw thrown + } return undefined } @@ -169,6 +208,10 @@ class ElectricProvider implements ControlledProvider { signal?.removeEventListener(`abort`, abort) resolve(response) }, + reject: (error) => { + signal?.removeEventListener(`abort`, abort) + reject(error) + }, } const abort = () => { const index = this.queued.indexOf(request) @@ -316,23 +359,29 @@ class ElectricProvider implements ControlledProvider { this.respondRows(request, messages) } - fail = async () => { + fail = async (error: Error) => { const request = await this.take() this.session(request).stopped = true this.giveUpTarget = this.gaveUp + 1 - request.respond(new Response(`conformance failure`, { status: 400 })) + request.respond(failureResponse(error, 400)) } - // Retried failures alternate: the client retries an HTTP 503 itself, and - // the app's `onError` retries an HTTP 400. - transientFailure = async () => { + // One failed attempt: an HTTP 503 or 429 the client retries, a network + // failure the client retries, or an HTTP 400 the shape's `onError` + // retries. + transientFailure = async (error: Error, variant: number) => { const request = await this.take() - const byApp = this.transientFailures++ % 2 === 1 - this.retryNextError = byApp + const kind = TRANSIENT_VARIANTS[variant] + if (kind === `network`) { + request.reject(new TypeError(error.message)) + return + } + this.retryNextError = kind === `http-400-retried` request.respond( - new Response(`conformance transient failure`, { - status: byApp ? 400 : 503, - }), + failureResponse( + error, + kind === `http-503` ? 503 : kind === `http-429` ? 429 : 400, + ), ) } @@ -379,6 +428,8 @@ function conformanceRow(row: ElectricRow): ConformanceRow { let sequence = 0 export function electricConformanceDriver(): EagerLiveSourceDriver { + // The provider of the history being checked. + let current: ElectricProvider | undefined return { adapter: `@tanstack/electric-db-collection (eager)`, capabilities: new Set([ @@ -388,8 +439,22 @@ export function electricConformanceDriver(): EagerLiveSourceDriver { `source-failure-after-ready`, ]), changeCommits: `per-batch`, - // The Collection reports the client's error for the failed response. - failureValue: `any`, + // The Collection reports the client's error for a failed response, the + // error a network failure rejected with, or the error `onError` threw to + // give up. Each carries the injected failure's index. + matchesFailure: (reported, failure) => { + const index = String(failure) + const thrown = current?.thrown.get(index) + if (thrown) return reported === thrown + if (reported instanceof FetchError) { + return reported.headers[FAILURE_HEADER] === index + } + return ( + reported instanceof TypeError && + reported.message.endsWith(` failure ${index}`) + ) + }, + transientVariants: TRANSIENT_VARIANTS, // Electric carries txids on row messages; an empty batch sends none. txidEvidence: `change`, // Only an extended outage shows (maintainer decision 2026-10-30). @@ -405,6 +470,7 @@ export function electricConformanceDriver(): EagerLiveSourceDriver { (kind === `negative-zero` && typeof value !== `number`), create(): EagerLiveSourceFixture { const provider = new ElectricProvider() + current = provider const collection = createCollection( electricCollectionOptions({ id: `electric-conformance-${++sequence}`, diff --git a/packages/electric-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/electric-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts index 3283ba1119..fb8a92e656 100644 --- a/packages/electric-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -36,10 +36,10 @@ import { electricConformanceDriver } from './conformance-driver' const property = `adapter-conformance.eager-live-source.electric` const driver = electricConformanceDriver() const historyArb = eagerLiveSourceHistoryArb(driver) -// The rarest state is `field-removed`: at 400 histories, six seeds reached -// it 19 to 31 times. +// At 400 histories, four seeds reached every declared state at least 15 +// times; the rarest is a reported outage in a later sync run. const runs = 400 -// A campaign takes about 20 seconds. +// A campaign takes about 35 seconds. const campaignTimeoutMs = 120_000 async function campaign(options: fc.Parameters<[unknown]>) { diff --git a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts index 99dbab2599..38fa4702e2 100644 --- a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts @@ -221,12 +221,25 @@ function resetResumeMetadata(): SyncMetadataApi { } } +/** + * Retried subset attempts: how many HTTP 503 responses each subset request + * gets before its snapshot, and how many empty loads follow the three + * ordered ones, each pausing the live request. + */ +type SubsetRetries = { + failures: ReadonlyArray + extraLoads: number + /** HTTP 503 responses to the live request after the first load. */ + liveFailures?: number +} + async function checkSnapshots( width: number, name: string, dropRows = false, cleanupFailure?: Error, setupFault?: { primary: Error; cleanup?: Error }, + retries: SubsetRetries = { failures: [], extraLoads: 0 }, ) { const http = controlledHttp() const batches: Array>> = [] @@ -273,6 +286,7 @@ async function checkSnapshots( url: `http://test-url/snapshot-${sequence}`, params: { table: `rows` }, fetchClient: http.fetchClient, + backoffOptions: { initialDelay: 1, maxDelay: 2, multiplier: 1 }, }, syncMode: `on-demand`, startSync: true, @@ -306,7 +320,22 @@ async function checkSnapshots( // poll is still pending; requestSnapshot owns pausing that poll. await atCheckpoint(http.take(), `initial live request`) expect(rows()).toEqual([]) - for (const [index, lowerBound] of [1, 2, width + 1].entries()) { + // After ready, five live attempts in a row that the client retries are + // a source outage (Electric's threshold, maintainer decision + // 2026-10-30). Only the live stream's up-to-date ends it; a subset's + // end does not. + let outage = false + const sourceHealth = (label: string) => + expect(collection.health.failureReason !== null, label).toBe(outage) + const lowerBounds = [ + 1, + 2, + ...Array.from( + { length: retries.extraLoads + 1 }, + (_, i) => width + 1 + i, + ), + ] + for (const [index, lowerBound] of lowerBounds.entries()) { // Distinct predicates prevent request deduplication; the second response // overlaps prior rows and changes their payload, and the last is empty. if (index === 1) for (const item of source) item.name += `-changed` @@ -326,7 +355,22 @@ async function checkSnapshots( () => ({ ok: true as const }), (error: unknown) => ({ ok: false as const, error }), ) - const request = await atCheckpoint(http.take(true), `snapshot request`) + let request = await atCheckpoint(http.take(true), `snapshot request`) + // A subset request's attempts that the client retries belong to that + // request, not to the source: the source node shows no problem + // (glossary, request error), however many fail in a row. + for ( + let attempt = 0; + attempt < (retries.failures[index] ?? 0); + attempt++ + ) { + request.respond(new Response(`transient`, { status: 503 })) + request = await atCheckpoint( + http.take(true), + `retried snapshot request`, + ) + sourceHealth(`snapshot ${index} attempt ${attempt} source health`) + } expect( request.url.searchParams.get(`subset__where`), `snapshot ${index} request predicate`, @@ -368,11 +412,21 @@ async function checkSnapshots( expect(rows(), `snapshot ${index} applied rows`).toEqual( [...expected.values()].sort((a, b) => a.id - b.id), ) + sourceHealth(`snapshot ${index} source health`) const outcome = await atCheckpoint( completion, `snapshot request completion`, ) if (!outcome.ok) throw outcome.error + if (index === 0 && (retries.liveFailures ?? 0) > 0) { + let live = await atCheckpoint(http.take(), `resumed live request`) + for (let attempt = 0; attempt < retries.liveFailures!; attempt++) { + live.respond(new Response(`transient`, { status: 503 })) + live = await atCheckpoint(http.take(), `retried live request`) + } + outage = retries.liveFailures! >= 5 + sourceHealth(`source health after live attempts`) + } } await atCheckpoint(http.take(), `resumed live request`) }, [ @@ -410,13 +464,33 @@ async function checkSnapshots( // requested lower bound is excluded by construction. This bounded grammar // does not claim arbitrary predicate orders or provider snapshots. The row at // id=2 distinguishes id>=2 from a misplaced exclusive boundary at snapshot 1. +// Retries: each subset request gets 0..7 HTTP 503 responses that the client +// retries before its snapshot, so five or more in a row are common; 0..3 +// extra empty loads (id>=width+2 on) pause the live request up to six times +// between its responses. After the first load, the live request gets 0..6 +// HTTP 503 responses the client retries; five or more are a source outage. +// The source node shows a problem exactly when the live stream had that +// outage: a subset request's retried attempts and a live pause are not +// failures of the source, and a subset's end does not end its outage. const snapshotViolation = sameViolation() const snapshotProperty = fc.asyncProperty( fc.integer({ min: 2, max: 4 }), fc.string({ maxLength: 10 }), - (width, name) => - snapshotViolation.check({ width, name }, () => - checkSnapshots(width, name, replayFault === `snapshot-rows`), + fc.record({ + failures: fc.array(fc.nat(7), { maxLength: 6 }), + extraLoads: fc.nat(3), + liveFailures: fc.nat(6), + }), + (width, name, retries) => + snapshotViolation.check({ width, name, retries }, () => + checkSnapshots( + width, + name, + replayFault === `snapshot-rows`, + undefined, + undefined, + retries, + ), ), ) sdkCampaigns( From 9401176e8158d55db9b6546172df0381f1f1c367 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 07:28:24 -0700 Subject: [PATCH 20/64] test(conformance): empty responses restart the outage count; missing columns reach the comparison - New capability and step empty-response: the provider accepts a request and delivers nothing (Electric: HTTP 204). It ends the run of failed attempts, so the count starts again, and leaves a reported outage in place (RETRY-1, maintainer decision 2026-10-30: five failed attempts "in a row"). Two outages below the threshold around one are a generated history element; reach state outage-count-reset. - fromSqlRow shows a row with no note column as a field instead of throwing, so an adapter that drops columns fails the comparison rather than the driver's mapping. - Reference adapter: handles empty responses. Mutants empty-response-keeps-count (HEALTH-1) and empty-response-recovers (RETRY-1) on the threshold variant. snapshot-merges now shrinks to a malformed delivery it also applies, so its entry accepts PUB-7 or PUB-4. Evidence: - Law: RETRY-1, HEALTH-1/2 with the "in a row" decision. - Reach: reference campaigns assert every declared state at least 5. - RED: the Electric mutant that never restarts the count, which survived the previous round, fails HEALTH-1 at [snapshot, attempt, empty-response, attempt x4] (next commit). - GREEN: conformance 79/79; TrailBase 151, RxDB 23, PowerSync 195. - Mutants: both new reference mutants rejected with their laws. Co-authored-by: Isaac --- .../src/eager-live-source-oracle.ts | 61 ++++++++++++++++++- .../src/observed-view.ts | 6 +- .../reference-adapter-oracle.property.test.ts | 6 +- .../tests/reference-adapter.ts | 11 ++++ 4 files changed, 80 insertions(+), 4 deletions(-) diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index feabab990d..0f4392df41 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -74,7 +74,10 @@ * `matchesFailure` the node shows the attempt that crossed the threshold * (maintainer decision 2026-10-30) and may list later attempts after it, * in order; a later attempt does not replace it (RETRY-1). Attempts come - * in each of the driver's `transientVariants` kinds. + * in each of the driver's `transientVariants` kinds. With capability + * `empty-response`, an accepted response that delivers nothing ends the + * run of failed attempts, so the count starts again, and leaves a + * reported outage in place (maintainer decision 2026-10-30). * - **RECON-1** (capability `transport-reconnect`). A transport disconnect and * reconnect keep status and rows, and later deliveries still apply. * - **RENEW-1** (capability `capability-renewal`). While the source's @@ -190,6 +193,7 @@ export type EagerLiveSourceCapability = | `listen-before-fetch` | `duplicate-key-delivery` | `source-failure-after-ready` + | `empty-response` /** * The public Collection surface the oracle reads. A driver passes the @@ -245,6 +249,11 @@ export interface ControlledProvider { * (capability `capability-renewal`). Rejects when the adapter never asked. */ completeRenewal?: () => Promise + /** + * An accepted response that carries no change, such as an HTTP 204 to a + * live request (capability `empty-response`). + */ + emptyResponse?: () => void | Promise /** Renewals the adapter completed on its latest subscription. */ renewals?: () => number /** @@ -386,6 +395,9 @@ export interface EagerLiveSourceDriver { * - `disconnect`, `reconnect`: transport loss and return. * - `renew`: the adapter's renewal of an expiring capability completes. Legal * once per sync run. + * - `empty-response`: the provider accepts a request and delivers nothing. + * Two outages below the threshold around one are generated as one + * element, so the reset it causes is reached. * - `restart`: cleanup, then `preload()` starts a new sync run. * * Keys range over 1..5 and titles over three values, so batches revisit @@ -429,6 +441,7 @@ export type EagerLiveSourceStep = | { readonly kind: `disconnect` } | { readonly kind: `reconnect` } | { readonly kind: `renew` } + | { readonly kind: `empty-response` } | { readonly kind: `restart` } /** @@ -678,6 +691,29 @@ export function eagerLiveSourceHistoryArb( ...second, ]), ) + if (capabilities.has(`empty-response`)) { + // Two outages below the threshold around an empty response: together + // they may reach it, but not in a row. + const below = fc.array(attempt, { + minLength: 1, + maxLength: Math.max(1, threshold - 1), + }) + outages.push( + fc + .tuple(below, below) + .map(([first, second]): Array => [ + ...first, + { kind: `empty-response` }, + ...second, + ]), + ) + } + } + if (capabilities.has(`empty-response`)) { + steps.push({ + weight: 1, + arbitrary: fc.constant({ kind: `empty-response` as const }), + }) } if (capabilities.has(`transport-reconnect`)) { steps.push( @@ -1153,6 +1189,7 @@ type AcceptedStep = | { kind: `disconnect` } | { kind: `reconnect` } | { kind: `renew` } + | { kind: `empty-response`; resetOutage: boolean } | { kind: `restart` } /** @@ -1457,6 +1494,15 @@ function stepModel( state.renewed = true return { done: { kind: `renew` }, publishable: [] } } + case `empty-response`: { + if (!canDeliver(state) || state.phase !== `ready`) return undefined + // RETRY-1: an accepted response ends the run of failed attempts, so a + // later outage counts from one; a reported outage stays until an + // applied delivery (maintainer decision 2026-10-30: "in a row"). + const resetOutage = state.outage > 0 + state.outage = 0 + return { done: { kind: `empty-response`, resetOutage }, publishable: [] } + } case `restart`: { // LIFE-5: cleanup retires the sync run; the next one starts empty. The // backend may change while no sync run listens: it then holds @@ -1535,6 +1581,7 @@ export const eagerLiveSourceReachStates = [ `outage-reported-again`, `outage-reported-after-batch`, `outage-reported-in-later-run`, + `outage-count-reset`, `transient-failure-recovered`, `reconnect`, `renewal`, @@ -1574,6 +1621,7 @@ const capabilityOfState: Partial< 'outage-reported-again': `retry-policy`, 'outage-reported-after-batch': `retry-policy`, 'outage-reported-in-later-run': `retry-policy`, + 'outage-count-reset': `empty-response`, 'transient-failure-repeated': `retry-policy`, 'transient-failure-recovered': `retry-policy`, reconnect: `transport-reconnect`, @@ -1713,6 +1761,8 @@ function lawFor(done: AcceptedStep, wasReady: boolean): string { return `RECON-1` case `renew`: return `RENEW-1` + case `empty-response`: + return `RETRY-1` case `restart`: return `LIFE-2/LIFE-5` } @@ -2249,6 +2299,10 @@ export async function checkEagerLiveSourceHistory( await provider.completeRenewal!() reach(`renewal`) break + case `empty-response`: + await provider.emptyResponse!() + if (done.resetOutage) reach(`outage-count-reset`) + break case `restart`: { const pendingBefore = waits.some((wait) => wait.current === `pending`) events.stop() @@ -2404,6 +2458,11 @@ function supports( return driver.capabilities.has(`transport-reconnect`) case `renew`: return driver.capabilities.has(`capability-renewal`) + case `empty-response`: + return ( + driver.capabilities.has(`empty-response`) && + !!fixture.provider.emptyResponse + ) default: return true } diff --git a/packages/db-adapter-conformance/src/observed-view.ts b/packages/db-adapter-conformance/src/observed-view.ts index 6b6e046e6e..12c3066b05 100644 --- a/packages/db-adapter-conformance/src/observed-view.ts +++ b/packages/db-adapter-conformance/src/observed-view.ts @@ -89,13 +89,15 @@ export function toSqlRow(row: ConformanceRow) { /** * A SQL row back as a conformance row: `NULL` is an absent `note`. Every - * other field passes through, so an extra or missing column still shows. + * other field passes through, so an extra or missing column still shows; a + * row with no `note` column shows a field no delivered row has, so the + * comparison reports it (FID-1). */ export function fromSqlRow( row: TRow, ): ConformanceRow { if (!Object.hasOwn(row, `note`)) { - throw new Error(`SQL row ${JSON.stringify(row)} has no note column`) + return { ...row, missingNoteColumn: true } as unknown as ConformanceRow } const { note, ...rest } = row return (note === null ? rest : { ...rest, note }) as unknown as ConformanceRow diff --git a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts index fc666d8791..3cf36adb32 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts @@ -98,7 +98,9 @@ describe(`eager live-source conformance: reference adapter`, () => { [`transient-fails-loading`, `RETRY-1`], [`splits-batch`, `PUB-1`], [`drops-deletes`, `PUB-3`], - [`snapshot-merges`, `PUB-7`], + // It merges a later delivery, so it also applies one that must be + // rejected for a duplicate key. + [`snapshot-merges`, /PUB-7|PUB-4/], [`leaks-subscription`, `LIFE-2`], [`restart-replays-rows`, `LIFE-5`], [`keeps-failed-source`, `LIFE-10`], @@ -210,6 +212,8 @@ describe(`eager live-source conformance: outage threshold, client values`, () => [`replaces-outage-value`, `RETRY-1`], [`reports-outage-early`, `HEALTH-1`], [`outage-count-survives-delivery`, `HEALTH-1`], + [`empty-response-keeps-count`, `HEALTH-1`], + [`empty-response-recovers`, `RETRY-1`], [`wraps-loading-failure`, `READY-4`], ] oracleReplayTest(it).each(outageMutants)( diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index cbf72d0893..3aa388e086 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -69,6 +69,10 @@ export type ReferenceMutant = | `reports-outage-early` /** HEALTH-1, threshold: a delivery does not reset the attempt count. */ | `outage-count-survives-delivery` + /** RETRY-1, threshold: an empty response does not restart the count. */ + | `empty-response-keeps-count` + /** HEALTH-2: an empty response clears a reported outage. */ + | `empty-response-recovers` /** EVID-3: a wait ignores its timeout. */ | `wait-ignores-timeout` /** RECON-1: a transport drop is reported as a failure. */ @@ -115,6 +119,7 @@ type ProviderEvent = | { type: `failed`; error: Error } | { type: `transient`; error: Error } | { type: `transport`; connected: boolean } + | { type: `empty` } interface ProviderSubscription { releases: number @@ -186,6 +191,7 @@ class ControlledPushProvider implements ControlledProvider { } } renewals = () => this.subscriptions.at(-1)?.renewed ?? 0 + emptyResponse = () => this.latest().listener({ type: `empty` }) } type Waiter = { resolve: () => void; reject: (error: Error) => void } @@ -467,6 +473,10 @@ function referenceCollection( if (receipt === true) settleWaits(event.txid) else void receipt.then(() => settleWaits(event.txid)) delivered() + } else if (event.type === `empty`) { + // An accepted response ends the run of failed attempts. + if (mutant !== `empty-response-keeps-count`) attempts = 0 + if (mutant === `empty-response-recovers`) delivered() } else if (event.type === `transport`) { if (mutant === `disconnect-fails` && !event.connected) { stop(subscription, new Error(`transport lost`)) @@ -604,6 +614,7 @@ export function referenceDriver( `listen-before-fetch`, `duplicate-key-delivery`, `source-failure-after-ready`, + `empty-response`, ]), // The same ten kinds as typed fields, one per row field. fieldKinds: [ From 2dd5f0e756840757d6909c824aeb53cf298ee6ec Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 07:28:43 -0700 Subject: [PATCH 21/64] fix(electric): fail the run when a give-up preceded the first snapshot's acceptance Close the Electric coverage gaps left by the oracle review, and fix the READY-9 defect the extended retry-decision model found. - READY-9, HEALTH-1, EVID-2: a give-up while a persistence wrapper is still accepting the first snapshot left the Collection ready, with no source error and pending waits. The adapter now remembers that give-up and fails the run once acceptance makes it ready, or at once when a truncate commit already made it ready (progressive). - Retry-decision model (shared by the controlled lane and the installed- client lane) observes an awaitTxId and an awaitMatch that reject when the run fails or is cleaned up, and the source node's error while ready. Health after an external abort after ready has no law yet and is not compared. The controlled driver now reports a give-up decision to the subscriber, as the installed client does. - Conformance driver: updates carry only the changed columns (default replica mode); empty-response is an HTTP 204; give-ups by HTTP 400 or by the client's maxRetries (85), returning nothing or throwing; a campaign without an app onError, declaring the adapter's give-up log. - On-demand lane: a restart checks that snapshot-boundary evidence and imported sync metadata stay pending until the restarted run is ready, then fulfill; declared reach; 40 histories. Evidence: - Law: READY-9, HEALTH-1, EVID-2 (law set v18/v19), RETRY-1 "in a row", FID-1/PUB-3 for partial updates. - Reach: eager campaigns assert reach (outage-count-reset included); on-demand lane asserts subset attempts past the threshold, five live pauses, a live outage, and the restart. - RED on e67c4bae9: retry-decision lane at [demand, hold-acceptance, up-to-date, error/sync-give-up, accept]: health none (law error), waits pending (law rejected). - GREEN: Electric suite 1243 passed (4 skipped). - Mutants rejected: acceptance not failing and truncate-ready not failing (retry-decision lane); count never restarted (HEALTH-1, eager); a partial update replacing the row (PUB-3/FID-4; it passed with whole-row updates); no-onError path retrying (LIFE-10); retired snapshots dropped, retired txids dropped, retired evidence before ready (on-demand lane). A no-onError adapter that stops logging times out: the driver detects that give-up from the log (recorded seam). - Size: electric-db-collection +443 gzip bytes over the merge base, +28 for READY-9; the app scenario is unchanged. Co-authored-by: Isaac --- .../electric-retry-health-and-evidence.md | 2 +- docs/collections/electric-collection.md | 4 +- docs/contributing/oracle-coverage.md | 39 ++++-- packages/db-adapter-conformance/MIGRATION.md | 40 ++++++ .../electric-db-collection/src/electric.ts | 16 ++- .../tests/conformance-driver.ts | 129 ++++++++++++++++-- ...source-conformance-oracle.property.test.ts | 35 ++++- .../tests/electric-oracle.property.test.ts | 32 ++++- .../tests/electric-retry-decision-model.ts | 36 ++++- ...ctric-sdk-delivery-oracle.property.test.ts | 114 +++++++++++++++- 10 files changed, 403 insertions(+), 44 deletions(-) diff --git a/.changeset/electric-retry-health-and-evidence.md b/.changeset/electric-retry-health-and-evidence.md index 413d218d9d..c032b73e93 100644 --- a/.changeset/electric-retry-health-and-evidence.md +++ b/.changeset/electric-retry-health-and-evidence.md @@ -2,4 +2,4 @@ '@tanstack/electric-db-collection': patch --- -Report an extended Electric outage in `collection.health`. After the Collection is ready, five failed requests to the shape log in a row, retried by the client or by an `onError` that returns a retry, show as the source node's `failureReason` with the fifth attempt's error, and the next applied up-to-date clears it. Shorter outages, and retried subset snapshot requests, leave health unchanged. A restarted or reloaded Collection resolves `awaitTxId` for a txid the previous sync run delivered only once it is ready again, instead of at once. When the stream fails, pending `awaitTxId` and `awaitMatch` calls reject with the failure instead of waiting for their timeout. An `onError` handler that throws now gives up the stream with its own error, without an unhandled rejection. +Report an extended Electric outage in `collection.health`. After the Collection is ready, five failed requests to the shape log in a row, retried by the client or by an `onError` that returns a retry, show as the source node's `failureReason` with the fifth attempt's error, and the next applied up-to-date clears it. Shorter outages, and retried subset snapshot requests, leave health unchanged. A restarted or reloaded Collection resolves `awaitTxId` for a txid the previous sync run delivered only once it is ready again, instead of at once. When the stream fails, pending `awaitTxId` and `awaitMatch` calls reject with the failure instead of waiting for their timeout. An `onError` handler that throws now gives up the stream with its own error, without an unhandled rejection. A give-up while the first snapshot's acceptance is pending, under a persistence wrapper, now fails the Collection once that acceptance makes it ready, instead of leaving a stopped stream with healthy status. diff --git a/docs/collections/electric-collection.md b/docs/collections/electric-collection.md index e537cc2a31..2c9081e452 100644 --- a/docs/collections/electric-collection.md +++ b/docs/collections/electric-collection.md @@ -448,7 +448,9 @@ times and then succeeding once shows and clears the outage each time. A subset snapshot request that the client retries does not count: its failure belongs to the live query that requested it. When the stream gives up, it calls `fail()` with the error, or with the error your `onError` threw: a -failed first load before readiness, or the source node's `error` after it. Complete updates from an explicit `replica: 'full'` stream remain valid. +failed first load before readiness, or the source node's `error` after it. A give-up while a persistence wrapper is +still accepting the first snapshot fails the Collection once that acceptance +makes it ready. Complete updates from an explicit `replica: 'full'` stream remain valid. On-demand streams can observe updates outside their loaded subsets; unknown partial rows are ignored, while transaction acknowledgement evidence is retained. Outside scoped recovery, complete rows published by persistence reloads or diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 57556d420e..609b7f27c0 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6071,15 +6071,30 @@ request), up to six live pauses, and a live outage after the first load, and checks that the source node shows a problem exactly when the live stream had an outage. Excluded by `excludesValue` (column domains, not adapter behavior): int4 values outside 32 bits, and a `-0` nested in an -array or object for the float8 column. Limits: the eager grammar has no -accepted response that commits nothing, so restarting the count on one is -enforced only by source inspection (a mutant that never restarts it -survives); snapshot-visibility evidence (`snapshot-end`) across a restart is -not generated, so the retired-evidence rule is witnessed for txids only; -`awaitMatch` after a failure is not generated; a held commit receipt and -persisted resume are not exercised; there is no real Electric server; the -default configuration without `onError` and a give-up after `maxRetries` -are not generated. The on-demand lane runs 12 histories without declared -reach. The shared check refuses `retryReportAfter` above 1 with -`transport-reconnect` or `capability-renewal`: no law fixes whether a -reconnect or a renewal ends an outage a threshold is counting. +array or object for the float8 column. Updates carry only the changed +columns, as Electric's default replica mode sends them, so the adapter's +merge is exercised. An `empty-response` step (HTTP 204, accepted, with no +change) restarts the attempt count. Give-ups come as an HTTP 400 or as the +client's `maxRetries` bound (85), with `onError` returning nothing or +throwing, and a second campaign runs the default configuration without an +app `onError`, declaring the adapter's give-up log. The on-demand lane also +restarts the Collection and checks that a retired run's snapshot-boundary +evidence (txid 5 below `xmin` 10) and imported sync metadata (txid 25) stay +pending until the restarted run is ready, then fulfill; it declares reach +for a subset request's five retried attempts, five live pauses, a live +outage, and that restart. The retry-decision model +(`electric-retry-decision-model.ts`, shared by the controlled lane of +`electric-oracle.property.test.ts` and the installed-client lane) now +observes an `awaitTxId` and an `awaitMatch` that reject when the run fails +or is cleaned up, and the source node's `error` while ready (READY-9: a +give-up while the first snapshot's acceptance is pending is a post-ready +failure once it is accepted). Its controlled driver now reports a give-up +decision to the subscriber, as the installed client does; before, a +give-up after ready never reached the adapter's failure path in that lane. +Limits: no persisted resume or held commit receipt in the eager lane; no +real Electric server; health after an external abort after ready has no +law yet and is not compared; without an app `onError`, the driver detects a +give-up from the adapter's log, so an adapter that stops logging times out +rather than failing a law. The shared check refuses `retryReportAfter` +above 1 with `transport-reconnect` or `capability-renewal`: no law fixes +whether a reconnect or a renewal ends an outage a threshold is counting. diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index da9bb067f0..a8045e2b95 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -405,3 +405,43 @@ range`). The Collection shows what the database stored, so this is not over the previous round, for the retired-evidence set, failure-time wait rejection, and the merged export. The `app` scenario does not include Electric. +- **2026-11-02, Electric, open coverage gaps.** Closing the gaps the review + round left open found one more Electric defect (READY-9). + - **Partial updates:** the driver now sends an update as Electric's + default replica mode does, the primary key and the changed columns. A + mutant that replaces the row with the partial value passed with + whole-row updates and is now rejected (PUB-3/FID-4). The shared + `fromSqlRow` shows a row with no `note` column as a field instead of + throwing, so such a row reaches the comparison. + - **Accepted response that commits nothing:** new capability and step + `empty-response` (Electric: HTTP 204 to the live request), with a + generated element of two below-threshold outages around one, and reach + `outage-count-reset`. The mutant that never restarts the count on an + accepted response, which survived last round, is rejected (HEALTH-1 at + `[snapshot, attempt, empty-response, attempt ×4]`). Reference mutants + `empty-response-keeps-count` and `empty-response-recovers`. + - **Give-ups:** by HTTP 400 or by the client's `maxRetries` bound, with + `onError` returning nothing or throwing; a second campaign runs without + an app `onError`. A mutant whose no-`onError` path retries is rejected + (LIFE-10). A mutant that stops logging times out instead: the driver + detects that give-up from the log (recorded seam). + - **Evidence across a restart (on-demand lane):** snapshot-boundary + evidence and imported sync metadata stay pending until the restarted + run is ready, then fulfill. Mutants: retired snapshots dropped, retired + txids dropped, retired evidence before ready, each rejected. The lane + declares reach and runs 40 histories. + - **READY-9 (retry-decision model):** the model now observes two waits + and the source's error while ready. RED on e67c4bae9: a give-up while + the first snapshot's acceptance was pending left a ready Collection with + no error and pending waits (`[demand, hold-acceptance, up-to-date, +error/sync-give-up, accept]`). Fix: remember that give-up and fail the + run once acceptance makes it ready, or at once when a truncate commit + already made it ready (progressive). Mutants removing each half are + rejected. The controlled driver now reports a give-up decision to the + subscriber, as the installed client does. + - **Law set:** `field-trip-adapter-conformance/grammar/v19-laws.md` + applies the 2026-10-30 decisions (untracked). + - **GREEN:** Electric 1243 passed (4 skipped); conformance 79/79; + TrailBase 151, RxDB 23, PowerSync 195. + - **Size:** Electric +443 gzip bytes over the merge base, +28 for READY-9. + - **Open:** health after an external abort after ready has no law. diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 549a0e2886..22607eb3a4 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1693,10 +1693,17 @@ function createElectricSync>( // Collection still shows the previous run's status, so status cannot say // whether this run has reached its first ready. let ready = false + // READY-9: a give-up while the first snapshot's acceptance is pending + // fails the run once that acceptance makes it ready. + let gaveUpBeforeReady: { cause: unknown } | undefined const markReady = () => { ready = true lifecycle.markReady(lifecycleEpoch) params.markReady() + if (gaveUpBeforeReady) { + gaveUp = true + fail(gaveUpBeforeReady.cause) + } } // A failure ends the run's waits (EVID-2). const fail = (error: unknown) => { @@ -1922,7 +1929,14 @@ function createElectricSync>( // claim on readiness; a replacement session decides it. let sessionCleanedUp = false const failInitialSync = (cause: unknown) => { - if (firstSnapshotCommitted) return + if (firstSnapshotCommitted) { + // A truncate commit already made the Collection ready. + if (collection.status === `ready`) { + gaveUp = true + fail(cause) + } else gaveUpBeforeReady ??= { cause } + return + } initialSyncFailedFinally = true fail(cause) } diff --git a/packages/electric-db-collection/tests/conformance-driver.ts b/packages/electric-db-collection/tests/conformance-driver.ts index 417099325d..55038ee2ab 100644 --- a/packages/electric-db-collection/tests/conformance-driver.ts +++ b/packages/electric-db-collection/tests/conformance-driver.ts @@ -16,7 +16,9 @@ * - a change batch answers the pending live request with one message per * change, each carrying the batch's txid (`await-evidence`), and * `up-to-date`. `insert` for a key the provider does not hold, `update` - * for one it does, `delete` otherwise. The adapter commits at + * for one it does, `delete` otherwise. As in Electric's default replica + * mode, an update carries the primary key and only the changed columns, + * so the adapter merges it over the row it holds. The adapter commits at * `up-to-date`, so the driver declares `per-batch` commits; * - a failure answers with HTTP 400, which the client gives to the shape's * `onError`. The handler gives up by returning nothing, or for an odd @@ -31,8 +33,13 @@ * (a network failure, also retried by the client), or an HTTP 400 that * the shape's `onError` retries. Health shows an outage from its fifth * attempt in a row (`retryReportAfter`); + * - an empty response (`empty-response`) answers the live request with + * HTTP 204, which the client accepts and which carries no change; * - a provider subscription is one sync run's request chain; its release is - * the abort of the chain's pending request (LIFE-2, LIFE-10); + * the abort of the chain's pending request (LIFE-2, LIFE-10), or the + * driver's own terminal response, after which the client holds nothing + * open. The driver serves eager mode only: it takes a request after an + * aborted one as a new sync run, which an on-demand pause would break; * - the propagation step (`settle`) waits until the client asks again after * a response, or, after a failure, until the shape's `onError` gave up. * @@ -116,6 +123,21 @@ function rowValue(row: ConformanceRow): Record { } } +/** + * An update's value in Electric's default replica mode: the primary key and + * the columns whose value changed, a column set to NULL included. + */ +function changedColumns( + before: Record, + after: Record, +): Record { + return Object.fromEntries( + Object.entries(after).filter( + ([name, value]) => name === `id` || before[name] !== value, + ), + ) +} + type Request = { url: URL signal: AbortSignal | undefined @@ -134,6 +156,13 @@ const TRANSIENT_VARIANTS = [ `network`, `http-400-retried`, ] as const +type TransientVariant = (typeof TRANSIENT_VARIANTS)[number] + +/** + * The client's retry bound. A history holds at most 12 outages of at most + * seven attempts, so only a give-up by exhaustion reaches it. + */ +const MAX_RETRIES = 85 type Session = { handle: string @@ -174,6 +203,13 @@ class ElectricProvider implements ControlledProvider { /** The error `onError` threw to give up, by failure index. */ readonly thrown = new Map() + constructor(readonly variants: ReadonlyArray) {} + + /** The adapter gave up; without an app `onError`, it logs that. */ + readonly noteGiveUp = () => { + this.gaveUp++ + } + /** * An app's `onError`. It retries the HTTP 400 of a retried failure by * returning an empty retry. It gives up on any other error by returning @@ -185,7 +221,7 @@ class ElectricProvider implements ControlledProvider { this.retryNextError = false return {} } - this.gaveUp++ + this.noteGiveUp() const index = (error as { headers?: Record }).headers?.[ FAILURE_HEADER ] @@ -336,11 +372,13 @@ class ElectricProvider implements ControlledProvider { const request = await this.take() const messages = changes.map((change) => { if (change.type === `upsert`) { - const held = this.rows.has(change.row.id) + const held = this.rows.get(change.row.id) this.rows.set(change.row.id, change.row) return { key: String(change.row.id), - value: rowValue(change.row), + value: held + ? changedColumns(rowValue(held), rowValue(change.row)) + : rowValue(change.row), headers: { operation: held ? `update` : `insert`, txids: [Number(txid)], @@ -359,11 +397,33 @@ class ElectricProvider implements ControlledProvider { this.respondRows(request, messages) } + // A failure is an HTTP 400 the client does not retry, or, for failure + // indexes 2 and 3 modulo 4, HTTP 503s until the client's `maxRetries` + // bound gives up. fail = async (error: Error) => { - const request = await this.take() - this.session(request).stopped = true this.giveUpTarget = this.gaveUp + 1 - request.respond(failureResponse(error, 400)) + if (Math.floor(Number(failureIndex(error)) / 2) % 2 === 0) { + const request = await this.take() + this.session(request).stopped = true + request.respond(failureResponse(error, 400)) + return + } + for (;;) { + const request = await this.take() + request.respond(failureResponse(error, 503)) + const deadline = Date.now() + clientTimeoutMs + while ( + this.queued.length === 0 && + this.gaveUp < this.giveUpTarget && + Date.now() < deadline + ) { + await sleep(1) + } + if (this.queued.length === 0) { + this.session(request).stopped = true + return + } + } } // One failed attempt: an HTTP 503 or 429 the client retries, a network @@ -371,7 +431,7 @@ class ElectricProvider implements ControlledProvider { // retries. transientFailure = async (error: Error, variant: number) => { const request = await this.take() - const kind = TRANSIENT_VARIANTS[variant] + const kind = this.variants[variant] if (kind === `network`) { request.reject(new TypeError(error.message)) return @@ -385,6 +445,17 @@ class ElectricProvider implements ControlledProvider { ) } + /** An HTTP 204 to the live request: accepted, with no change. */ + emptyResponse = async () => { + const request = await this.take() + request.respond( + new Response(null, { + status: 204, + headers: this.headers(this.session(request), false), + }), + ) + } + /** * The propagation step: the client asks again after a response, or, after * a terminal one, gives up through `onError`. @@ -427,16 +498,28 @@ function conformanceRow(row: ElectricRow): ConformanceRow { let sequence = 0 -export function electricConformanceDriver(): EagerLiveSourceDriver { +/** + * With `appOnError: false`, the shape has no `onError`, as in Electric's + * default configuration: the adapter gives up on every failure the client + * does not retry and logs it, so no HTTP 400 is retried and the driver + * declares that log. + */ +export function electricConformanceDriver({ + appOnError = true, +}: { appOnError?: boolean } = {}): EagerLiveSourceDriver { // The provider of the history being checked. let current: ElectricProvider | undefined + const variants = TRANSIENT_VARIANTS.filter( + (kind) => appOnError || kind !== `http-400-retried`, + ) return { - adapter: `@tanstack/electric-db-collection (eager)`, + adapter: `@tanstack/electric-db-collection (eager${appOnError ? `` : `, no onError`})`, capabilities: new Set([ `complete-snapshot`, `await-evidence`, `retry-policy`, `source-failure-after-ready`, + `empty-response`, ]), changeCommits: `per-batch`, // The Collection reports the client's error for a failed response, the @@ -454,7 +537,18 @@ export function electricConformanceDriver(): EagerLiveSourceDriver { reported.message.endsWith(` failure ${index}`) ) }, - transientVariants: TRANSIENT_VARIANTS, + transientVariants: variants, + // Without an app `onError`, the adapter logs each give-up; the log is + // the driver's sign that the client stopped. + declaredDiagnostic: appOnError + ? undefined + : (level, args) => { + const giveUp = + level === `error` && + String(args[0]).startsWith(`An error occurred while syncing`) + if (giveUp) current?.noteGiveUp() + return giveUp + }, // Electric carries txids on row messages; an empty batch sends none. txidEvidence: `change`, // Only an extended outage shows (maintainer decision 2026-10-30). @@ -469,7 +563,7 @@ export function electricConformanceDriver(): EagerLiveSourceDriver { (value > 2_147_483_647 || value < -2_147_483_648)) || (kind === `negative-zero` && typeof value !== `number`), create(): EagerLiveSourceFixture { - const provider = new ElectricProvider() + const provider = new ElectricProvider(variants) current = provider const collection = createCollection( electricCollectionOptions({ @@ -479,8 +573,13 @@ export function electricConformanceDriver(): EagerLiveSourceDriver { params: { table: `rows` }, fetchClient: provider.fetchClient, parser: { timestamptz: (value: string) => new Date(value) }, - backoffOptions: { initialDelay: 1, maxDelay: 2, multiplier: 1 }, - onError: provider.onError, + backoffOptions: { + initialDelay: 1, + maxDelay: 2, + multiplier: 1, + maxRetries: MAX_RETRIES, + }, + ...(appOnError ? { onError: provider.onError } : {}), }, getKey: (row) => row.id, }), diff --git a/packages/electric-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/electric-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts index fb8a92e656..74e78aa5dd 100644 --- a/packages/electric-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -26,10 +26,12 @@ import { electricConformanceDriver } from './conformance-driver' * * Electric runs here in eager mode over the installed `@electric-sql/client` * through a controlled `fetchClient`. It declares `complete-snapshot` - * (must-refetch), `await-evidence` (txids), `retry-policy` (an HTTP 503 the - * client retries, or an HTTP 400 the shape's `onError` retries), and - * `source-failure-after-ready` (a give-up through `onError`), and typed - * Postgres columns. The claim is limited to the controlled HTTP provider; a + * (must-refetch), `await-evidence` (txids), `retry-policy` (an HTTP 503 or + * 429 or a network failure the client retries, or an HTTP 400 the shape's + * `onError` retries), and `source-failure-after-ready` (a give-up through + * `onError`, after an HTTP 400 or after the client's `maxRetries`), and + * typed Postgres columns. A second campaign runs Electric's default + * configuration, with no app `onError`. The claim is limited to the controlled HTTP provider; a * claim about an Electric server needs a receiving witness against it. */ @@ -42,11 +44,15 @@ const runs = 400 // A campaign takes about 35 seconds. const campaignTimeoutMs = 120_000 -async function campaign(options: fc.Parameters<[unknown]>) { - const reach = createOracleReach(property, reachStatesFor(driver), { +async function campaign( + options: fc.Parameters<[unknown]>, + checked = driver, + name = property, +) { + const reach = createOracleReach(name, reachStatesFor(checked), { minimum: 5, }) - await fc.assert(eagerLiveSourceProperty(driver, reach), options) + await fc.assert(eagerLiveSourceProperty(checked, reach), options) return reach } @@ -68,6 +74,21 @@ describe(`eager live-source conformance: Electric`, () => { }, campaignTimeoutMs, ) + + // Electric's default configuration: no app `onError`, so the adapter + // gives up on every failure the client does not retry and logs it. + oracleReplayTest(it)( + `conforms without an app onError on a fixed-seed campaign`, + async () => { + const reach = await campaign( + { seed: 2118, numRuns: oracleRuns(runs) }, + electricConformanceDriver({ appOnError: false }), + `${property}.no-on-error`, + ) + reach.assertReached() + }, + campaignTimeoutMs, + ) }) declareOracle({ diff --git a/packages/electric-db-collection/tests/electric-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-oracle.property.test.ts index c88b0aa197..0b46013a32 100644 --- a/packages/electric-db-collection/tests/electric-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-oracle.property.test.ts @@ -3995,12 +3995,35 @@ describe(`Electric adapter laws`, () => { preload = `rejected:${retrySettlementTag(reason, tags)}` }, ) + // An `awaitTxId` and an `awaitMatch` that no evidence satisfies. + let waitsRejected = 0 + for (const wait of [ + trace.collection.utils.awaitTxId(2_000_000_000, 2_000_000_000), + trace.collection.utils.awaitMatch(() => false, 2_000_000_000), + ]) { + void wait.then( + () => undefined, + () => waitsRejected++, + ) + } const turn = () => new Promise((resolve) => setTimeout(resolve, 0)) const observe = (): RetryObservation => ({ status: trace.collection.status, demands: [...settlements], rowVisible: trace.collection.has(1), preload: trace.collection.status === `cleaned-up` ? undefined : preload, + waits: + waitsRejected === 2 + ? `rejected` + : waitsRejected === 0 + ? `pending` + : (`one of two rejected` as never), + health: + trace.collection.status === `ready` && !controller.signal.aborted + ? trace.collection.health.error + ? `error` + : `none` + : undefined, }) const decisions: Array> = [] const persistence = createDeferred() @@ -4035,8 +4058,15 @@ describe(`Electric adapter laws`, () => { decision = step.decision thrown = new Error(`onError threw ${tag}`) tags.set(thrown, `thrown-${tag}`) + // As the installed client does, a decision that is not a retry + // ends the stream and reports the error to the subscriber. decisions.push( - Promise.resolve(streamOnError(error)).catch(() => undefined), + Promise.resolve(streamOnError(error)).then( + (retry) => { + if (!retry || typeof retry !== `object`) trace.sdkError(error) + }, + () => undefined, + ), ) break } diff --git a/packages/electric-db-collection/tests/electric-retry-decision-model.ts b/packages/electric-db-collection/tests/electric-retry-decision-model.ts index 46d3f3db28..ed94ba7970 100644 --- a/packages/electric-db-collection/tests/electric-retry-decision-model.ts +++ b/packages/electric-db-collection/tests/electric-retry-decision-model.ts @@ -45,6 +45,12 @@ export type RetryObservation = { // A readiness wait started with the history. It rejects with the same // cause as every waiting demand. Not compared after cleanup. preload?: RetrySettlement + // An `awaitTxId` and an `awaitMatch` that no evidence satisfies, started + // with the history: both `pending`, or both `rejected`. + waits?: `pending` | `rejected` + // While `ready`, whether the source node shows an `error`. Not compared + // after an external abort, which no law gives a health effect yet. + health?: `error` | `none` } export type RetryModel = { @@ -58,6 +64,9 @@ export type RetryModel = { committed: boolean // The source stopped after its first snapshot was committed. stopped: boolean + // The source gave up after the Collection became ready, or while its + // first snapshot's acceptance was pending and that acceptance succeeded. + gaveUpAfterReady: boolean ready: boolean visible: boolean held: boolean @@ -77,7 +86,12 @@ export function retryModelFail(model: RetryModel, cause: string): void { model.stopped = true return } - if (model.ready || model.failed !== undefined) return + // HEALTH-1: a give-up after ready is the source node's error. + if (model.ready) { + model.gaveUpAfterReady = true + return + } + if (model.failed !== undefined) return model.failed = cause model.demands = model.demands.map((settlement): RetrySettlement => settlement === `pending` ? `rejected:${cause}` : settlement, @@ -174,8 +188,10 @@ export function applyRetryModel( return case `accept`: // The committed snapshot is usable: the Collection is ready, whatever - // errors, retries, or give-ups arrived while it waited. + // errors, retries, or give-ups arrived while it waited. READY-9: a + // give-up while it waited is then a post-ready failure (HEALTH-1). model.ready = true + if (model.stopped && !model.aborted) model.gaveUpAfterReady = true return case `accept-fails`: { // The snapshot never became usable, so this is an initial failure with @@ -240,6 +256,21 @@ export function expectedRetryObservation( : settlement, ), rowVisible: !cleanedUp && model.visible, + // EVID-2: a sync run retires when it fails or is cleaned up, and its + // pending waits reject then (maintainer decision 2026-10-30). + waits: + cleanedUp || + model.failed !== undefined || + model.gaveUpAfterReady || + model.aborted + ? `rejected` + : `pending`, + health: + !cleanedUp && model.ready && model.failed === undefined && !model.aborted + ? model.gaveUpAfterReady + ? `error` + : `none` + : undefined, } } @@ -263,6 +294,7 @@ export function createRetryModel( acceptanceArmed: false, committed: false, stopped: false, + gaveUpAfterReady: false, ready: false, visible: false, held: false, diff --git a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts index 38fa4702e2..2456d6209b 100644 --- a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts @@ -221,6 +221,21 @@ function resetResumeMetadata(): SyncMetadataApi { } } +type SnapshotReachState = + | `subset-attempts-past-threshold` + | `live-pauses-past-threshold` + | `live-outage` + | `retired-evidence-after-restart` +const createSnapshotReach = () => + createOracleReach(`electric.sdk-snapshot-delivery`, [ + `subset-attempts-past-threshold`, + `live-pauses-past-threshold`, + `live-outage`, + `retired-evidence-after-restart`, + ]) +// Each campaign declares its own reach. +let snapshotReach = createSnapshotReach() + /** * Retried subset attempts: how many HTTP 503 responses each subset request * gets before its snapshot, and how many empty loads follow the three @@ -241,6 +256,7 @@ async function checkSnapshots( setupFault?: { primary: Error; cleanup?: Error }, retries: SubsetRetries = { failures: [], extraLoads: 0 }, ) { + snapshotReach.beginHistory() const http = controlledHttp() const batches: Array>> = [] let delivery = deferred() @@ -335,7 +351,13 @@ async function checkSnapshots( (_, i) => width + 1 + i, ), ] + // Each load pauses the live request; five pause it past the threshold. + if (lowerBounds.length >= 5) + snapshotReach.reach(`live-pauses-past-threshold`) for (const [index, lowerBound] of lowerBounds.entries()) { + if ((retries.failures[index] ?? 0) >= 5) { + snapshotReach.reach(`subset-attempts-past-threshold`) + } // Distinct predicates prevent request deduplication; the second response // overlaps prior rows and changes their payload, and the last is empty. if (index === 1) for (const item of source) item.name += `-changed` @@ -425,10 +447,66 @@ async function checkSnapshots( live = await atCheckpoint(http.take(), `retried live request`) } outage = retries.liveFailures! >= 5 + if (outage) snapshotReach.reach(`live-outage`) sourceHealth(`source health after live attempts`) } } await atCheckpoint(http.take(), `resumed live request`) + + // EVID-2 across a restart (maintainer decision 2026-10-30): the + // snapshots' boundary (xmin 10) shows txid 5 committed, and imported sync + // metadata (META-1) shows txid 25, which no snapshot boundary covers + // (xmax 20). A restarted run counts that retired + // evidence once it is ready, because its rows are read after it. + collection.config.sync.importSyncMeta?.({ version: 1, seenTxids: [25] }) + for (const txid of [5, 25]) { + expect( + await collection.utils.awaitTxId(txid, 50), + `txid ${txid} before the restart`, + ).toBe(true) + } + await collection.cleanup() + void collection.preload() + const restartedLive = await atCheckpoint( + http.take(), + `restarted live request`, + ) + const retired = [5, 25].map((txid) => { + const wait = { + settled: `pending` as `pending` | `fulfilled` | `rejected`, + } + collection.utils.awaitTxId(txid, 5_000).then( + () => (wait.settled = `fulfilled`), + () => (wait.settled = `rejected`), + ) + return wait + }) + for (let turn = 0; turn < 5; turn++) + await new Promise((r) => setTimeout(r, 0)) + expect( + retired.map((wait) => wait.settled), + `retired evidence before the restarted run is ready`, + ).toEqual([`pending`, `pending`]) + // The restarted run becomes ready on its live up-to-date, which carries + // no evidence for either txid: only the retired evidence can fulfill + // the waits. + restartedLive.respond( + new Response( + JSON.stringify([ + { headers: { control: `up-to-date`, global_last_seen_lsn: `1` } }, + ]), + { headers: headers(lowerBounds.length + 2) }, + ), + ) + await vi.waitFor(() => expect(collection.status).toBe(`ready`)) + for (let turn = 0; turn < 5; turn++) + await new Promise((r) => setTimeout(r, 0)) + expect(collection.status, `restarted status`).toBe(`ready`) + expect( + retired.map((wait) => wait.settled), + `retired evidence once the restarted run is ready`, + ).toEqual([`fulfilled`, `fulfilled`]) + snapshotReach.reach(`retired-evidence-after-restart`) }, [ { label: `snapshot collection`, release: () => collection.cleanup() }, { @@ -498,11 +576,14 @@ sdkCampaigns( `applies installed-SDK snapshot deliveries against an independent overlapping source`, 20260917, snapshotViolation, - (seed) => - fc.assert(snapshotProperty, { - ...oraclePropertyOptions(12, `electric.sdk-snapshot-delivery`), + async (seed) => { + snapshotReach = createSnapshotReach() + await fc.assert(snapshotProperty, { + ...oraclePropertyOptions(40, `electric.sdk-snapshot-delivery`), ...(seed === undefined ? {} : { seed }), - }), + }) + snapshotReach.assertReached() + }, ) fixedCase(`detects dropped snapshot rows`, async () => { @@ -831,6 +912,8 @@ function clientObservation(observation: RetryObservation): RetryObservation { return { status: observation.status, rowVisible: observation.rowVisible, + waits: observation.waits, + health: observation.health, demands: observation.demands.map((settlement) => settlement.startsWith(`rejected`) ? `rejected:any` : settlement, ), @@ -887,6 +970,17 @@ async function runClientRetryCell( : originalSync, }) const model: RetryModel = createRetryModel(false) + // An `awaitTxId` and an `awaitMatch` that no evidence satisfies. + let waitsRejected = 0 + for (const wait of [ + collection.utils.awaitTxId(2_000_000_000, 2_000_000_000), + collection.utils.awaitMatch(() => false, 2_000_000_000), + ]) { + void wait.then( + () => undefined, + () => waitsRejected++, + ) + } const settlements: Array<`pending` | `fulfilled` | `rejected:any`> = [] const track = (result: unknown) => { const slot = settlements.push(`pending`) - 1 @@ -915,6 +1009,18 @@ async function runClientRetryCell( status: collection.status, demands: [...settlements], rowVisible: collection.has(1), + waits: + waitsRejected === 2 + ? `rejected` + : waitsRejected === 0 + ? `pending` + : (`one of two rejected` as never), + health: + collection.status === `ready` + ? collection.health.error + ? `error` + : `none` + : undefined, }), `${outcome} ${mode} ${decision}: ${label}`, ).toEqual(clientObservation(expectedRetryObservation(model, false))) From f4a03a41d0d5d92f869171cee53e091bef13e5a9 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 07:42:06 -0700 Subject: [PATCH 22/64] fix(electric): an external abort after ready is the source's error Maintainer decision 2026-11-02: aborting a stream is an error-ish state; deliberate or not, the Collection no longer syncs. After the first ready, aborting shapeOptions.signal now fails the sync run, so the source node shows an error and pending waits reject. Before ready it still fails the first load; while the first snapshot's acceptance is pending, it fails the run once the acceptance makes it ready (READY-9). The retry-decision model (controlled and installed-client lanes) now expects the source's error after an external abort, where it previously left health unchecked. Evidence: - Law: HEALTH-1 with the decision above; READY-9 for a pending acceptance. - RED on 2dd5f0e75: retry-decision lane at [demand, hold, up-to-date, abort, release]: health none, law error. - GREEN: retry-decision lane; Electric suite 1243 passed (4 skipped). - Mutant: dropping the pre-ready abort failure is rejected at [demand, error/async-retry, abort, decide]; the original behavior is the RED. - Size: electric-db-collection +457 gzip bytes over the merge base, +14. Co-authored-by: Isaac --- .../electric-retry-health-and-evidence.md | 2 +- docs/collections/electric-collection.md | 5 ++++- docs/contributing/oracle-coverage.md | 7 ++++--- packages/db-adapter-conformance/MIGRATION.md | 8 +++++++- packages/electric-db-collection/src/electric.ts | 11 ++++++++--- .../tests/electric-oracle.property.test.ts | 2 +- .../tests/electric-retry-decision-model.ts | 17 ++++++++++------- 7 files changed, 35 insertions(+), 17 deletions(-) diff --git a/.changeset/electric-retry-health-and-evidence.md b/.changeset/electric-retry-health-and-evidence.md index c032b73e93..4beaa9de73 100644 --- a/.changeset/electric-retry-health-and-evidence.md +++ b/.changeset/electric-retry-health-and-evidence.md @@ -2,4 +2,4 @@ '@tanstack/electric-db-collection': patch --- -Report an extended Electric outage in `collection.health`. After the Collection is ready, five failed requests to the shape log in a row, retried by the client or by an `onError` that returns a retry, show as the source node's `failureReason` with the fifth attempt's error, and the next applied up-to-date clears it. Shorter outages, and retried subset snapshot requests, leave health unchanged. A restarted or reloaded Collection resolves `awaitTxId` for a txid the previous sync run delivered only once it is ready again, instead of at once. When the stream fails, pending `awaitTxId` and `awaitMatch` calls reject with the failure instead of waiting for their timeout. An `onError` handler that throws now gives up the stream with its own error, without an unhandled rejection. A give-up while the first snapshot's acceptance is pending, under a persistence wrapper, now fails the Collection once that acceptance makes it ready, instead of leaving a stopped stream with healthy status. +Report an extended Electric outage in `collection.health`. After the Collection is ready, five failed requests to the shape log in a row, retried by the client or by an `onError` that returns a retry, show as the source node's `failureReason` with the fifth attempt's error, and the next applied up-to-date clears it. Shorter outages, and retried subset snapshot requests, leave health unchanged. A restarted or reloaded Collection resolves `awaitTxId` for a txid the previous sync run delivered only once it is ready again, instead of at once. When the stream fails, pending `awaitTxId` and `awaitMatch` calls reject with the failure instead of waiting for their timeout. An `onError` handler that throws now gives up the stream with its own error, without an unhandled rejection. A give-up while the first snapshot's acceptance is pending, under a persistence wrapper, now fails the Collection once that acceptance makes it ready, instead of leaving a stopped stream with healthy status. Aborting `shapeOptions.signal` after the Collection is ready now shows as the source node's `error`, since the Collection no longer syncs. diff --git a/docs/collections/electric-collection.md b/docs/collections/electric-collection.md index 2c9081e452..ac4ae258cf 100644 --- a/docs/collections/electric-collection.md +++ b/docs/collections/electric-collection.md @@ -450,7 +450,10 @@ belongs to the live query that requested it. When the stream gives up, it calls `fail()` with the error, or with the error your `onError` threw: a failed first load before readiness, or the source node's `error` after it. A give-up while a persistence wrapper is still accepting the first snapshot fails the Collection once that acceptance -makes it ready. Complete updates from an explicit `replica: 'full'` stream remain valid. +makes it ready. Aborting `shapeOptions.signal` stops the stream for +good: before readiness the first load fails, and after it the source node +shows an `error`, because the Collection no longer syncs, whether or not the +abort was deliberate. Complete updates from an explicit `replica: 'full'` stream remain valid. On-demand streams can observe updates outside their loaded subsets; unknown partial rows are ignored, while transaction acknowledgement evidence is retained. Outside scoped recovery, complete rows published by persistence reloads or diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 609b7f27c0..24b5e3b9cd 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6088,12 +6088,13 @@ outage, and that restart. The retry-decision model observes an `awaitTxId` and an `awaitMatch` that reject when the run fails or is cleaned up, and the source node's `error` while ready (READY-9: a give-up while the first snapshot's acceptance is pending is a post-ready -failure once it is accepted). Its controlled driver now reports a give-up +failure once it is accepted). An external abort after ready is the +source node's `error` too (maintainer decision 2026-11-02: deliberate or +not, an aborted stream no longer syncs). Its controlled driver now reports a give-up decision to the subscriber, as the installed client does; before, a give-up after ready never reached the adapter's failure path in that lane. Limits: no persisted resume or held commit receipt in the eager lane; no -real Electric server; health after an external abort after ready has no -law yet and is not compared; without an app `onError`, the driver detects a +real Electric server; without an app `onError`, the driver detects a give-up from the adapter's log, so an adapter that stops logging times out rather than failing a law. The shared check refuses `retryReportAfter` above 1 with `transport-reconnect` or `capability-renewal`: no law fixes diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index a8045e2b95..fd9bdcea8a 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -444,4 +444,10 @@ error/sync-give-up, accept]`). Fix: remember that give-up and fail the - **GREEN:** Electric 1243 passed (4 skipped); conformance 79/79; TrailBase 151, RxDB 23, PowerSync 195. - **Size:** Electric +443 gzip bytes over the merge base, +28 for READY-9. - - **Open:** health after an external abort after ready has no law. + - **External abort after ready (decided 2026-11-02):** deliberate or + not, an aborted stream no longer syncs, so it is the source node's + `error`. The retry-decision model now expects it, also for an abort + while the first snapshot's acceptance is pending once it is accepted. + RED on 2dd5f0e75 at `[demand, hold, up-to-date, abort, release]` + (health none); the adapter now fails the run on an external abort + after ready. diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 22607eb3a4..62ccf63708 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1950,11 +1950,16 @@ function createElectricSync>( } else markReady() } // An external abort stops the source for good. Before the first ready, - // that is the final initial failure; cleanup aborts internally instead. + // that is the final initial failure; after it, deliberate or not, the + // source no longer syncs, so it is the source's error (maintainer + // decision). Cleanup aborts internally instead. const forwardExternalAbort = () => { - const fails = isActiveLifecycle() && !ready + const active = isActiveLifecycle() abortController.abort() - if (fails) failInitialSync(new StreamAbortedError(collectionId)) + if (active && ready) { + gaveUp = true + fail(new StreamAbortedError(collectionId)) + } else if (active) failInitialSync(new StreamAbortedError(collectionId)) // A source that stops must end its reserved transaction. The initial // failure decides first: abandoning commits. abandonOpenTransaction() diff --git a/packages/electric-db-collection/tests/electric-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-oracle.property.test.ts index 0b46013a32..5cd93b13f7 100644 --- a/packages/electric-db-collection/tests/electric-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-oracle.property.test.ts @@ -4019,7 +4019,7 @@ describe(`Electric adapter laws`, () => { ? `pending` : (`one of two rejected` as never), health: - trace.collection.status === `ready` && !controller.signal.aborted + trace.collection.status === `ready` ? trace.collection.health.error ? `error` : `none` diff --git a/packages/electric-db-collection/tests/electric-retry-decision-model.ts b/packages/electric-db-collection/tests/electric-retry-decision-model.ts index ed94ba7970..56b3e3d307 100644 --- a/packages/electric-db-collection/tests/electric-retry-decision-model.ts +++ b/packages/electric-db-collection/tests/electric-retry-decision-model.ts @@ -48,8 +48,7 @@ export type RetryObservation = { // An `awaitTxId` and an `awaitMatch` that no evidence satisfies, started // with the history: both `pending`, or both `rejected`. waits?: `pending` | `rejected` - // While `ready`, whether the source node shows an `error`. Not compared - // after an external abort, which no law gives a health effect yet. + // While `ready`, whether the source node shows an `error`. health?: `error` | `none` } @@ -189,9 +188,10 @@ export function applyRetryModel( case `accept`: // The committed snapshot is usable: the Collection is ready, whatever // errors, retries, or give-ups arrived while it waited. READY-9: a - // give-up while it waited is then a post-ready failure (HEALTH-1). + // give-up or an external abort while it waited is then a post-ready + // failure (HEALTH-1). model.ready = true - if (model.stopped && !model.aborted) model.gaveUpAfterReady = true + if (model.stopped) model.gaveUpAfterReady = true return case `accept-fails`: { // The snapshot never became usable, so this is an initial failure with @@ -217,8 +217,11 @@ export function applyRetryModel( case `abort`: // An external abort is a provider failure. Before the first ready it // fails the initial sync. After it, the snapshot has already arrived, - // so a waiting demand still settles when its rows apply. - if (!model.ready) retryModelFail(model, `StreamAbortedError`) + // so a waiting demand still settles when its rows apply, and the + // source no longer syncs: the node's error (HEALTH-1; maintainer + // decision 2026-11-02, deliberate or not, an aborted stream is not + // syncing). + retryModelFail(model, `StreamAbortedError`) model.aborted = true if (model.committed) model.stopped = true return @@ -266,7 +269,7 @@ export function expectedRetryObservation( ? `rejected` : `pending`, health: - !cleanedUp && model.ready && model.failed === undefined && !model.aborted + !cleanedUp && model.ready && model.failed === undefined ? model.gaveUpAfterReady ? `error` : `none` From b2ecbcfd6b5d2e70b61e8e6497968d6066f146a7 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 09:08:42 -0700 Subject: [PATCH 23/64] test(conformance): delivery errors and reported rejected deliveries - New capability and step delivery-error: after ready, the source reports an error for its current result and keeps its subscription; the error joins failureReason and the next valid delivery clears it (HEALTH-1's rejected-delivery clause, HEALTH-2). Convex's QueryFailed is the first driver. - New driver parameter rejectedDelivery: fails (default) or reported. Under reported, the law set's default errorMapping, a delivery the adapter rejects after ready joins failureReason with the adapter's own error and the run and rows stay. failureReason entries can be an adapter error. - Reach states delivery-error-reported, delivery-error-recovered, rejected-delivery-reported; a recovery is credited only to the kinds of failure that were open. - The FID-1 relabel of a publication mismatch is narrowed to value-only differences: a split batch whose intermediate state differs in note is PUB-1, as the splits-batch mutant shows. - Reference adapter: handles delivery errors; its threshold variant also reports rejected deliveries. Mutants delivery-error-is-fatal (HEALTH-1) and rejected-delivery-fails (PUB-4). Evidence: - Law: HEALTH-1, HEALTH-2, PUB-4 with the default errorMapping (law set v18/v19). - Reach: reference campaigns assert every declared state at least 5. - RED: Convex's adapter fails LIFE-10 at [fail, changes []] (next commit). - GREEN: conformance 81/81; Electric 1243, TrailBase 151, RxDB 23, PowerSync 195. - Mutants: both new reference mutants rejected with their laws. Co-authored-by: Isaac --- .../src/eager-live-source-oracle.ts | 149 +++++++++++++++--- .../reference-adapter-oracle.property.test.ts | 4 +- .../tests/reference-adapter.ts | 41 ++++- 3 files changed, 166 insertions(+), 28 deletions(-) diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 0f4392df41..831a8b9ef6 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -78,6 +78,12 @@ * `empty-response`, an accepted response that delivers nothing ends the * run of failed attempts, so the count starts again, and leaves a * reported outage in place (maintainer decision 2026-10-30). + * - **HEALTH-1, HEALTH-2** (capability `delivery-error`). After ready, an + * error the source reports for its current result joins `failureReason`, + * and the rows, status, and subscription stay; the next valid delivery + * clears it. Under `rejectedDelivery: reported` (the law set's default + * `errorMapping`), a delivery the adapter rejects after ready is reported + * the same way, with the adapter's own error. * - **RECON-1** (capability `transport-reconnect`). A transport disconnect and * reconnect keep status and rows, and later deliveries still apply. * - **RENEW-1** (capability `capability-renewal`). While the source's @@ -194,6 +200,7 @@ export type EagerLiveSourceCapability = | `duplicate-key-delivery` | `source-failure-after-ready` | `empty-response` + | `delivery-error` /** * The public Collection surface the oracle reads. A driver passes the @@ -254,6 +261,12 @@ export interface ControlledProvider { * live request (capability `empty-response`). */ emptyResponse?: () => void | Promise + /** + * After ready, the source reports `error` for its current result and keeps + * its subscription; its next valid delivery clears it (capability + * `delivery-error`). + */ + deliveryError?: (error: Error) => void | Promise /** Renewals the adapter completed on its latest subscription. */ renewals?: () => number /** @@ -319,6 +332,12 @@ export interface EagerLiveSourceDriver { * holds; required with `duplicate-key-delivery`. */ readonly duplicateInsert?: `error` | `replace` + /** + * The law set's `errorMapping` for a delivery the adapter rejects after + * ready (PUB-4): `fails` the sync run (the default), or is `reported` in + * `failureReason` until the next valid delivery clears it. + */ + readonly rejectedDelivery?: `fails` | `reported` /** * For an adapter whose client library creates the reported failure value, * such as an HTTP client's error for a failed response: whether `reported` @@ -395,6 +414,8 @@ export interface EagerLiveSourceDriver { * - `disconnect`, `reconnect`: transport loss and return. * - `renew`: the adapter's renewal of an expiring capability completes. Legal * once per sync run. + * - `delivery-error`: after ready, the source reports an error for its + * current result and keeps its subscription. * - `empty-response`: the provider accepts a request and delivers nothing. * Two outages below the threshold around one are generated as one * element, so the reset it causes is reached. @@ -442,6 +463,7 @@ export type EagerLiveSourceStep = | { readonly kind: `reconnect` } | { readonly kind: `renew` } | { readonly kind: `empty-response` } + | { readonly kind: `delivery-error` } | { readonly kind: `restart` } /** @@ -571,6 +593,7 @@ export type GrammarDomain = Pick< | `excludesValue` | `retryReportAfter` | `transientVariants` + | `rejectedDelivery` > /** Generates histories whose steps use only the driver's declarations. */ @@ -715,6 +738,12 @@ export function eagerLiveSourceHistoryArb( arbitrary: fc.constant({ kind: `empty-response` as const }), }) } + if (capabilities.has(`delivery-error`)) { + steps.push({ + weight: 3, + arbitrary: fc.constant({ kind: `delivery-error` as const }), + }) + } if (capabilities.has(`transport-reconnect`)) { steps.push( // A reconnect needs a ready run and a disconnect before it, so both @@ -829,6 +858,9 @@ export function eagerLiveSourceHistoryArb( */ type Phase = `loading` | `ready` | `loading-error` | `stopped` +/** A listed failure: an injected failure's index, or the adapter's error. */ +type FailureEntry = number | `adapter` + /** A wait's expected settlement. */ type WaitExpectation = `pending` | `fulfilled` | `rejected` @@ -854,7 +886,8 @@ interface ModelState { * A provider failure's index; `adapter` for the adapter's own error. */ failure: number | `adapter` | undefined - transient: Array + /** `adapter` for a rejected delivery reported under `reported`. */ + transient: Array /** Consecutive failed attempts after ready in the current outage. */ outage: number /** Each wait, and the law that rejects it when it must reject. */ @@ -873,7 +906,7 @@ export interface ExpectedObservation { /** Index of the failure the health node's `error` must carry, or none. */ healthError: number | `adapter` | null /** Indices of the failures `failureReason.errors` must list, in order. */ - failureReason: ReadonlyArray + failureReason: ReadonlyArray preload: `pending` | `fulfilled` | { rejectedWith: number | `adapter` } waits: ReadonlyArray /** Unreleased provider subscriptions: one while the provider runs. */ @@ -1190,6 +1223,7 @@ type AcceptedStep = | { kind: `reconnect` } | { kind: `renew` } | { kind: `empty-response`; resetOutage: boolean } + | { kind: `delivery-error`; failure: number } | { kind: `restart` } /** @@ -1206,6 +1240,7 @@ interface ModelOptions { duplicateInsert: `error` | `replace` txidEvidence: `batch` | `change` retryReportAfter: number + rejectedDelivery: `fails` | `reported` } /** EVID-1: a wait for `txid` fulfills once its batch applied. */ @@ -1446,6 +1481,15 @@ function stepModel( publishable: [rowState(state.rows)], } } + if (afterReady && options.rejectedDelivery === `reported`) { + // HEALTH-1: a rejected delivery the next valid delivery can clear is + // the node's failure reason; the run and its rows stay. + state.transient.push(`adapter`) + return { + done: { kind: `malformed`, rows, afterReady, replaced: false }, + publishable: [], + } + } state.failure = `adapter` state.phase = afterReady ? `stopped` : `loading-error` endOutage(state) @@ -1494,6 +1538,14 @@ function stepModel( state.renewed = true return { done: { kind: `renew` }, publishable: [] } } + case `delivery-error`: { + if (!canDeliver(state) || state.phase !== `ready`) return undefined + const failure = failures.next++ + // HEALTH-1, HEALTH-2: the error joins `failureReason` until the next + // valid delivery; status, rows, and the subscription stay. + state.transient.push(failure) + return { done: { kind: `delivery-error`, failure }, publishable: [] } + } case `empty-response`: { if (!canDeliver(state) || state.phase !== `ready`) return undefined // RETRY-1: an accepted response ends the run of failed attempts, so a @@ -1582,6 +1634,9 @@ export const eagerLiveSourceReachStates = [ `outage-reported-after-batch`, `outage-reported-in-later-run`, `outage-count-reset`, + `delivery-error-reported`, + `delivery-error-recovered`, + `rejected-delivery-reported`, `transient-failure-recovered`, `reconnect`, `renewal`, @@ -1622,6 +1677,9 @@ const capabilityOfState: Partial< 'outage-reported-after-batch': `retry-policy`, 'outage-reported-in-later-run': `retry-policy`, 'outage-count-reset': `empty-response`, + 'delivery-error-reported': `delivery-error`, + 'delivery-error-recovered': `delivery-error`, + 'rejected-delivery-reported': `duplicate-key-delivery`, 'transient-failure-repeated': `retry-policy`, 'transient-failure-recovered': `retry-policy`, reconnect: `transport-reconnect`, @@ -1650,7 +1708,9 @@ export function reachStatesFor( (state !== `malformed-after-ready` || domain.capabilities.has(`complete-snapshot`)) && (state !== `transient-failure-below-threshold` || - (domain.retryReportAfter ?? 1) > 1), + (domain.retryReportAfter ?? 1) > 1) && + (state !== `rejected-delivery-reported` || + domain.rejectedDelivery === `reported`), ) const values = (domain.valueKinds ?? []).flatMap( (kind): Array => [ @@ -1763,6 +1823,8 @@ function lawFor(done: AcceptedStep, wasReady: boolean): string { return `RENEW-1` case `empty-response`: return `RETRY-1` + case `delivery-error`: + return `HEALTH-1` case `restart`: return `LIFE-2/LIFE-5` } @@ -1822,6 +1884,7 @@ export async function checkEagerLiveSourceHistory( duplicateInsert: driver.duplicateInsert ?? `error`, txidEvidence: driver.txidEvidence ?? `batch`, retryReportAfter: driver.retryReportAfter ?? 1, + rejectedDelivery: driver.rejectedDelivery ?? `fails`, } if ( driver.capabilities.has(`duplicate-key-delivery`) && @@ -1907,18 +1970,18 @@ export async function checkEagerLiveSourceHistory( (rows, commit) => commit > last && sameRows(rows, publication), ) if (index === -1) { - // FID-1: a publication with a committed state's keys and titles - // that differs only in a field names the field law. - const near = publishable.find((rows) => sameTitles(rows, publication)) - const valuesOnly = near?.every( - ([, value], row) => rowHead(value) === rowHead(publication[row]![1]), + // FID-1: a publication that differs from a committed state only in + // a value or typed field names the value law. + const valuesOnly = publishable.some( + (rows) => + sameTitles(rows, publication) && + rows.every( + ([, value], row) => + rowHead(value) === rowHead(publication[row]![1]), + ), ) throw violation( - near === undefined - ? `${law} (publication)` - : valuesOnly - ? `FID-1 (values)` - : `FID-1 (fields)`, + valuesOnly ? `FID-1 (values)` : `${law} (publication)`, stepIndex, publishable, publication, @@ -2013,7 +2076,14 @@ export async function checkEagerLiveSourceHistory( } const reasons = health.failureReason?.errors.map((entry) => entry.cause) ?? [] - const wantReasons = expected.failureReason.map(failure) + // `adapter`: the adapter's own error, not an injected failure value. + const matchesEntry = (cause: unknown, entry: FailureEntry) => + entry === `adapter` + ? cause instanceof Error && + failures.every((_, index) => !matches(cause, index)) + : matches(cause, entry) + const describeEntry = (entry: FailureEntry) => + entry === `adapter` ? `an adapter error` : failure(entry).message const reasonLaw = delivery ? `HEALTH-2` : law const checkReason = (ok: boolean, want: unknown, got: unknown) => { if (!ok) @@ -2025,12 +2095,12 @@ export async function checkEagerLiveSourceHistory( ) } if (!driver.matchesFailure) { + const want = expected.failureReason checkReason( - reasons.length === wantReasons.length && - reasons.every((cause, index) => cause === wantReasons[index]) && - (wantReasons.length === 0 || - health.failureReason?.cause === wantReasons.at(-1)), - wantReasons.map((error) => error.message), + reasons.length === want.length && + reasons.every((cause, index) => matchesEntry(cause, want[index]!)) && + (want.length === 0 || health.failureReason?.cause === reasons.at(-1)), + want.map(describeEntry), reasons.map(String), ) } else { @@ -2039,14 +2109,14 @@ export async function checkEagerLiveSourceHistory( // attempts after it, in order. RETRY-1: a later attempt does not // replace that first failure. const want = expected.failureReason - if (want.length > 0 && !matches(reasons[0], want[0]!)) { + if (want.length > 0 && !matchesEntry(reasons[0], want[0]!)) { throw violation( reasons.length > 0 && - want.slice(1).some((i) => matches(reasons[0], i)) + want.slice(1).some((i) => matchesEntry(reasons[0], i)) ? `RETRY-1 (health failureReason)` : `${reasonLaw} (health failureReason)`, stepIndex, - failure(want[0]!).message, + describeEntry(want[0]!), reasons.map(String), ) } @@ -2054,10 +2124,11 @@ export async function checkEagerLiveSourceHistory( checkReason( reasons.length <= want.length && reasons.every((cause) => { - while (next < want.length && !matches(cause, want[next]!)) next++ + while (next < want.length && !matchesEntry(cause, want[next]!)) + next++ return next++ < want.length }), - want.map((index) => failure(index).message), + want.map(describeEntry), reasons.map(String), ) } @@ -2153,6 +2224,14 @@ export async function checkEagerLiveSourceHistory( try { await compare(-1, `READY-2`) + // The kinds of failure open in the failure reason, which a delivery + // clears. + const open = new Set<`retried` | `delivery-error`>() + const reachRecovered = () => { + if (open.has(`retried`)) reach(`transient-failure-recovered`) + if (open.has(`delivery-error`)) reach(`delivery-error-recovered`) + open.clear() + } // Reported outages and applied live batches in the current run. let outagesThisRun = 0 let batchesThisRun = 0 @@ -2191,7 +2270,7 @@ export async function checkEagerLiveSourceHistory( if (done.missed > 0) reach(`read-missed-change`) if (done.superseded) reach(`seen-change-superseded`) if (done.removesField) reach(`field-removed`) - if (transientBefore > 0) reach(`transient-failure-recovered`) + if (transientBefore > 0) reachRecovered() break case `changes`: await provider.deliverChanges( @@ -2216,7 +2295,7 @@ export async function checkEagerLiveSourceHistory( if (done.removesField) reach(`field-removed`) if (done.flipped) reach(`zero-sign-flipped`) if (done.changes.length > 1) reach(`multi-change-batch`) - if (transientBefore > 0) reach(`transient-failure-recovered`) + if (transientBefore > 0) reachRecovered() if ( state.waits.some( (wait, waitIndex) => @@ -2264,6 +2343,9 @@ export async function checkEagerLiveSourceHistory( ? `malformed-after-ready` : `malformed-before-ready`, ) + if (done.afterReady && state.transient.at(-1) === `adapter`) { + reach(`rejected-delivery-reported`) + } break case `transient-failure`: { const error = new Error( @@ -2274,6 +2356,7 @@ export async function checkEagerLiveSourceHistory( const kind = driver.transientVariants?.[done.variant] if (wasReady && kind !== undefined) reach(`transient:${kind}`) if (state.transient.at(-1) === done.failure) { + open.add(`retried`) reach(`transient-failure-reported`) } else if (wasReady) reach(`transient-failure-below-threshold`) // RETRY-1: a later attempt in the same outage. @@ -2299,6 +2382,14 @@ export async function checkEagerLiveSourceHistory( await provider.completeRenewal!() reach(`renewal`) break + case `delivery-error`: { + const error = new Error(`conformance delivery error ${done.failure}`) + failures[done.failure] = error + await provider.deliveryError!(error) + reach(`delivery-error-reported`) + open.add(`delivery-error`) + break + } case `empty-response`: await provider.emptyResponse!() if (done.resetOutage) reach(`outage-count-reset`) @@ -2316,6 +2407,7 @@ export async function checkEagerLiveSourceHistory( preload = recordSettlement(collection.preload()) events = foldChanges(collection) reach(`restart`) + open.clear() outagesThisRun = 0 batchesThisRun = 0 if (pendingBefore) reach(`wait-rejected-at-retirement`) @@ -2463,6 +2555,11 @@ function supports( driver.capabilities.has(`empty-response`) && !!fixture.provider.emptyResponse ) + case `delivery-error`: + return ( + driver.capabilities.has(`delivery-error`) && + !!fixture.provider.deliveryError + ) default: return true } diff --git a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts index 3cf36adb32..a35ec7f3ec 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts @@ -94,6 +94,7 @@ describe(`eager live-source conformance: reference adapter`, () => { [`wraps-loading-failure`, `READY-4`], [`failure-clears-rows`, `HEALTH-1`], [`transient-is-fatal`, `HEALTH-1`], + [`delivery-error-is-fatal`, `HEALTH-1`], [`never-recovers`, `HEALTH-2`], [`transient-fails-loading`, `RETRY-1`], [`splits-batch`, `PUB-1`], @@ -187,7 +188,7 @@ describe(`eager live-source conformance: per-change commits`, () => { // policy must conform, and each mutant of its outage reporting must be // rejected with its law. describe(`eager live-source conformance: outage threshold, client values`, () => { - const outage = { reportAfter: 3, clientValues: true } + const outage = { reportAfter: 3, clientValues: true, reportsRejected: true } const thresholdDriver = referenceDriver(undefined, { outage }) oracleReplayTest(it)( @@ -214,6 +215,7 @@ describe(`eager live-source conformance: outage threshold, client values`, () => [`outage-count-survives-delivery`, `HEALTH-1`], [`empty-response-keeps-count`, `HEALTH-1`], [`empty-response-recovers`, `RETRY-1`], + [`rejected-delivery-fails`, `PUB-4`], [`wraps-loading-failure`, `READY-4`], ] oracleReplayTest(it).each(outageMutants)( diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index 3aa388e086..ed158fcea8 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -69,6 +69,10 @@ export type ReferenceMutant = | `reports-outage-early` /** HEALTH-1, threshold: a delivery does not reset the attempt count. */ | `outage-count-survives-delivery` + /** HEALTH-1: a delivery error fails the sync run. */ + | `delivery-error-is-fatal` + /** HEALTH-1, reported rejections: a rejected delivery fails the run. */ + | `rejected-delivery-fails` /** RETRY-1, threshold: an empty response does not restart the count. */ | `empty-response-keeps-count` /** HEALTH-2: an empty response clears a reported outage. */ @@ -120,6 +124,7 @@ type ProviderEvent = | { type: `transient`; error: Error } | { type: `transport`; connected: boolean } | { type: `empty` } + | { type: `delivery-error`; error: Error } interface ProviderSubscription { releases: number @@ -192,6 +197,8 @@ class ControlledPushProvider implements ControlledProvider { } renewals = () => this.subscriptions.at(-1)?.renewed ?? 0 emptyResponse = () => this.latest().listener({ type: `empty` }) + deliveryError = (error: Error) => + this.latest().listener({ type: `delivery-error`, error }) } type Waiter = { resolve: () => void; reject: (error: Error) => void } @@ -252,6 +259,11 @@ interface OutagePolicy { * client library's error would carry an HTTP failure. */ clientValues: boolean + /** + * After ready, reports a delivery it rejects in `failureReason` and keeps + * the run, as the law set's default `errorMapping` does. + */ + reportsRejected?: boolean } /** Builds the reference adapter's Collection over `provider`. */ @@ -434,6 +446,24 @@ function referenceCollection( } } catch (error) { if (mutant === `commits-malformed-prefix`) void commit() + if ( + ready && + policy.reportsRejected && + mutant !== `rejected-delivery-fails` + ) { + // The rejected delivery writes nothing: an aborted + // commit discards its transaction. + const discard = new AbortController() + discard.abort() + void commit(discard.signal) + failing = true + reportError( + new Error(`reference rejected a delivery`, { + cause: error, + }), + ) + return + } stop(subscription, error as Error) return } @@ -473,6 +503,13 @@ function referenceCollection( if (receipt === true) settleWaits(event.txid) else void receipt.then(() => settleWaits(event.txid)) delivered() + } else if (event.type === `delivery-error`) { + if (mutant === `delivery-error-is-fatal`) { + stop(subscription, event.error) + return + } + failing = true + reportError(wrap(event.error)) } else if (event.type === `empty`) { // An accepted response ends the run of failed attempts. if (mutant !== `empty-response-keeps-count`) attempts = 0 @@ -595,12 +632,13 @@ export function referenceDriver( return { adapter: `reference${perChange ? ` per-change` : ``}${variant ? ` outage ${outage.reportAfter}` : ``}${mutant ? ` (${mutant})` : ``}`, retryReportAfter: outage.reportAfter, + rejectedDelivery: outage.reportsRejected ? `reported` : `fails`, // The adapter's error carries the provider's failure as `cause`. matchesFailure: outage.clientValues ? (reported, failure) => reported instanceof Error && reported.cause instanceof Error && - reported.cause.message.endsWith(` failure ${failure}`) + reported.cause.message.endsWith(` ${failure}`) : undefined, // A threshold has no law across a reconnect or a renewal, so a driver // with one declares neither. @@ -615,6 +653,7 @@ export function referenceDriver( `duplicate-key-delivery`, `source-failure-after-ready`, `empty-response`, + `delivery-error`, ]), // The same ten kinds as typed fields, one per row field. fieldKinds: [ From 6aa2671daaae51ce97a52e4d9e1c7bc0de6861f4 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 09:08:45 -0700 Subject: [PATCH 24/64] fix(convex): release the query subscription when the first load fails Run the Convex adapter (eager mode, real Convex JavaScript client, a controlled WebSocket that answers the sync protocol) on the shared eager live-source conformance oracle, and fix the defect it found. - LIFE-10: after a failed first load, or a later acceptance failure that stops the run, the query subscription stayed open until cleanup. A stopping failure now releases it once, also when the subscription fails while it is being set up. - Driver: every delivery is the complete query result in one Transition; a failure before ready and a delivery error after it are QueryFailed; a result with two rows for one key is rejected and reported after ready; releases are counted at the client boundary; the driver acknowledges a query removal at once, as a server does. Evidence: - Law: LIFE-10, READY-4, HEALTH-1/2 (delivery-error), PUB-3/4/7, FID-1 for Convex value kinds (law set v18/v19). - Reach: 400 histories per campaign; lowest declared state 13 across four seeds. - RED: LIFE-10 (open provider subscriptions) at [fail, changes []]. - GREEN: Convex suite 114/114. - Mutants of src/options.ts rejected: the original defect (LIFE-10), a fatal post-ready error (HEALTH-1), no recovery on an equal result (HEALTH-2), skipped updates (PUB-3/FID-4), accepted duplicates (PUB-4). Skipping the truncate on a run's first result survives: an eager restart starts empty (recorded limit). - Limits: no awaitSync evidence, transport reconnect, on-demand or paginated mode, or deployment. A restart before the server acknowledges the previous removal can receive the client's cached result (recorded for a decision). - Size: convex-db-collection +26 gzip bytes. Co-authored-by: Isaac --- .changeset/convex-failed-load-release.md | 5 + docs/contributing/oracle-coverage.md | 16 +- packages/convex-db-collection/src/options.ts | 12 +- .../tests/conformance-driver.ts | 315 ++++++++++++++++++ ...source-conformance-oracle.property.test.ts | 75 +++++ packages/db-adapter-conformance/MIGRATION.md | 67 +++- packages/db-adapter-conformance/README.md | 7 +- packages/db/tests/oracle-config.ts | 1 + packages/db/tests/oracle-replay-manifest.ts | 6 + 9 files changed, 487 insertions(+), 17 deletions(-) create mode 100644 .changeset/convex-failed-load-release.md create mode 100644 packages/convex-db-collection/tests/conformance-driver.ts create mode 100644 packages/convex-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts diff --git a/.changeset/convex-failed-load-release.md b/.changeset/convex-failed-load-release.md new file mode 100644 index 0000000000..dfe7a10d03 --- /dev/null +++ b/.changeset/convex-failed-load-release.md @@ -0,0 +1,5 @@ +--- +'@tanstack/convex-db-collection': patch +--- + +Release the Convex query subscription when a Collection's first load fails, or when a later acceptance failure stops the sync run, instead of keeping it until cleanup. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 24b5e3b9cd..996df912f2 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6004,7 +6004,7 @@ a step, not only the settled checkpoint. The reference push adapter (`tests/reference-adapter-oracle.property.test.ts`, `adapter-conformance.eager-live-source.reference`) runs fixed and random campaigns of 600 histories with declared reach, a per-change variant, an -outage-threshold variant with client-created values, and 41 law-named +outage-threshold variant with client-created values, and 45 law-named hostile mutants, with ORC-004 reconstruction, exclusion, and ablation controls. The Neon Realtime receiving driver (`adapter-conformance.eager-live-source.neon-realtime`, `test:neon`) runs @@ -6099,3 +6099,17 @@ give-up from the adapter's log, so an adapter that stops logging times out rather than failing a law. The shared check refuses `retryReportAfter` above 1 with `transport-reconnect` or `capability-renewal`: no law fixes whether a reconnect or a renewal ends an outage a threshold is counting. +Convex (`adapter-conformance.eager-live-source.convex`, eager mode over the +real Convex JavaScript client and a controlled WebSocket that answers the +sync protocol, complete results, per-batch commits, Convex value kinds as +values and typed fields) is GREEN after a fix: a failed first load now +releases the query subscription (LIFE-10). It declares `delivery-error` (a +`QueryFailed` after ready, cleared by the next result) and reports a +rejected delivery after ready (`rejectedDelivery: reported`, the law set's +default `errorMapping`). Limits: no `awaitSync` evidence (the shared +oracle's waits are txid waits), no transport reconnect, no on-demand or +paginated mode, no deployment; the truncate on a restarted run's first +result is not distinguished, because an eager restart starts empty; a +restart before the server acknowledges the previous removal can receive the +client's cached result, which the driver avoids by acknowledging at once +(observation for a maintainer decision). diff --git a/packages/convex-db-collection/src/options.ts b/packages/convex-db-collection/src/options.ts index cb001231a6..d998c36a63 100644 --- a/packages/convex-db-collection/src/options.ts +++ b/packages/convex-db-collection/src/options.ts @@ -83,6 +83,8 @@ export function createConvexOptions< if (!ready || terminal) { stopped = { error } failSync(error) + // LIFE-10: the stopped run releases its subscription at once. + release() } else reportError(error) onError?.(error) } @@ -184,6 +186,12 @@ export function createConvexOptions< } } let subscription: ConvexSubscription | undefined + let released = false + const release = (): void => { + if (released || !subscription) return + released = true + subscription.unsubscribe() + } const refresh = async (): Promise => { if (stopped) throw stopped.error if (!active || !subscription) { @@ -209,6 +217,8 @@ export function createConvexOptions< readCurrent = refresh try { subscription = subscribe(receive, fail) + // A failure during subscription comes before it can be released. + if (stopped) release() // Validate subscription construction before readiness. The empty lazy // run still precedes subset activation, avoiding first-page replay. if (syncMode === `on-demand`) receive([] as TResult) @@ -247,7 +257,7 @@ export function createConvexOptions< active = false rejectFirst(new Error(`Convex sync ended before source acceptance.`)) if (readCurrent === refresh) readCurrent = undefined - subscription?.unsubscribe() + release() }, } }, diff --git a/packages/convex-db-collection/tests/conformance-driver.ts b/packages/convex-db-collection/tests/conformance-driver.ts new file mode 100644 index 0000000000..1d882372c4 --- /dev/null +++ b/packages/convex-db-collection/tests/conformance-driver.ts @@ -0,0 +1,315 @@ +/** + * Production driver for the shared eager live-source conformance oracle + * (`packages/db-adapter-conformance`). It runs the real Convex adapter in + * eager mode over the real Convex JavaScript client, whose WebSocket the + * driver controls, as `sdk.test.ts` does: the client's query set, cache, + * decoding of Convex values, and subscriptions are real, and the driver + * answers the sync protocol. + * + * The oracle's steps map onto Convex like this: + * + * - every delivery is the query's complete result, a `QueryUpdated` in one + * `Transition`. A change batch folds onto the provider's rows and sends the + * new result, so the adapter commits it as one batch (`per-batch`). A + * result equal to the one the client holds sends no update, as a server + * would; after an error or a rejected result, the valid result is a + * change. The driver declares `complete-snapshot`; + * - a failure before ready is a `QueryFailed` for the query. After ready, a + * `QueryFailed` is a delivery error (`delivery-error`): Convex keeps the + * subscription and its next result clears it. Convex has no terminal source + * end after ready, so the driver does not declare + * `source-failure-after-ready`. The client creates each reported error from + * the failure's message, so `matchesFailure` names the value by the + * injected failure's index; + * - a complete result with two rows for one key (`duplicate-key-delivery`) + * is rejected by the adapter: before ready it fails the first load, after + * it the adapter reports it and keeps the run (`rejectedDelivery: + * reported`, the law set's default `errorMapping`). A rejected result does + * not replace the provider's rows; + * - a provider subscription is one `onUpdate` registration, and its release + * is the adapter's call to the returned unsubscribe (LIFE-2, LIFE-10), + * observed at the client boundary because the client may merge a removal + * with a later subscription on the wire. The driver acknowledges a removal + * at once, as a server does, which drops the client's cached result. + * + * Rows are Convex values: `note` absent is a missing field, and typed fields + * hold the Convex kinds (no Date, which Convex values do not have). The + * driver never reads the Collection. + */ +import { createCollection } from '@tanstack/db' +import { ConvexClient } from 'convex/browser' +import { makeFunctionReference } from 'convex/server' +import { convexToJson } from 'convex/values' +import { convexCollectionOptions } from '../src/index' +import { observedView } from '../../db-adapter-conformance/src' +import type { Value } from 'convex/values' +import type { + ConformanceRow, + ControlledProvider, + EagerLiveSourceDriver, + EagerLiveSourceFixture, + ProviderChange, + ValueKind, +} from '../../db-adapter-conformance/src' + +/** The kinds Convex values hold; Convex has no Date. */ +const VALUE_KINDS: ReadonlyArray = [ + `string`, + `integer`, + `float`, + `negative-zero`, + `boolean`, + `null`, + `bigint`, + `array`, + `object`, +] + +const rowsQuery = makeFunctionReference< + `query`, + Record, + Array +>(`rows:list`) + +/** The WebSocket the Convex client opens; the driver plays the server. */ +class Socket { + static latest: Socket | undefined + onopen: (() => void) | null = null + onmessage: ((event: { data: string }) => void) | null = null + onclose: ((event: { code: number; reason: string }) => void) | null = null + onerror: (() => void) | null = null + readonly sent: Array> = [] + onSend: ((message: Record) => void) | undefined + constructor(_url: string) { + Socket.latest = this + } + send(data: string) { + const message = JSON.parse(data) as Record + this.sent.push(message) + this.onSend?.(message) + } + close() { + queueMicrotask(() => this.onclose?.({ code: 1000, reason: `` })) + } + receive(message: unknown) { + this.onmessage?.({ data: JSON.stringify(message) }) + } +} + +function timestamp(value: number): string { + const bytes = Buffer.alloc(8) + bytes.writeBigUInt64LE(BigInt(value)) + return bytes.toString(`base64`) +} + +const turn = () => new Promise((resolve) => setTimeout(resolve, 0)) + +/** A result with two rows for one key: the adapter rejects it. */ +const hasDuplicateKey = (rows: ReadonlyArray) => + new Set(rows.map((row) => row.id)).size !== rows.length + +class ConvexProvider implements ControlledProvider { + readonly client: ConvexClient + private readonly socket: Socket + private readonly registrations: Array<{ releases: number }> = [] + private rows = new Map() + private time = 0 + private querySet = 0 + // The result the client holds, encoded; undefined after a failure. + private sent: string | undefined + + constructor() { + this.client = new ConvexClient(`https://conformance.convex.cloud`, { + webSocketConstructor: Socket as unknown as typeof WebSocket, + unsavedChangesWarning: false, + logger: false, + }) + this.socket = Socket.latest! + this.socket.onopen?.() + // A server acknowledges a removal promptly, which drops the client's + // cached result for that query. + this.socket.onSend = (message) => { + const modifications = (message.modifications ?? []) as Array<{ + type: string + queryId: number + }> + if ( + message.type !== `ModifyQuerySet` || + modifications.length === 0 || + modifications.some((modification) => modification.type !== `Remove`) + ) { + return + } + queueMicrotask(() => + this.transition( + modifications.map(({ queryId }) => ({ + type: `QueryRemoved`, + queryId, + })), + ), + ) + } + // Count each `onUpdate` registration and the adapter's unsubscribe. + const onUpdate = this.client.onUpdate.bind(this.client) + this.client.onUpdate = ((...args: Parameters) => { + const registration = { releases: 0 } + this.registrations.push(registration) + const subscription = onUpdate(...args) + return Object.assign( + () => { + registration.releases++ + subscription() + }, + { getCurrentValue: () => subscription.getCurrentValue() }, + ) + }) as typeof onUpdate + } + + /** The query the client subscribed most recently and has not removed. */ + private queryId(): number { + const changes = this.socket.sent.filter( + (message) => message.type === `ModifyQuerySet`, + ) + const live = new Set() + for (const change of changes) { + for (const modification of change.modifications as Array<{ + type: string + queryId: number + }>) { + if (modification.type === `Add`) live.add(modification.queryId) + else live.delete(modification.queryId) + } + } + const id = [...live].at(-1) + if (id === undefined) throw new Error(`the client holds no query`) + return id + } + + private transition(modifications: Array): void { + const changes = this.socket.sent.filter( + (message) => message.type === `ModifyQuerySet`, + ) + const version = (changes.at(-1)?.newVersion as number | undefined) ?? 0 + this.socket.receive({ + type: `Transition`, + startVersion: { + querySet: this.querySet, + ts: timestamp(this.time), + identity: 0, + }, + endVersion: { + querySet: version, + ts: timestamp(++this.time), + identity: 0, + }, + modifications, + }) + this.querySet = version + } + + private sendResult(rows: ReadonlyArray): void { + this.sent = JSON.stringify(convexToJson(rows as unknown as Value)) + this.transition([ + { + type: `QueryUpdated`, + queryId: this.queryId(), + value: convexToJson(rows as unknown as Value), + logLines: [], + journal: null, + }, + ]) + } + + private sendFailure(error: Error): void { + this.sent = undefined + this.transition([ + { + type: `QueryFailed`, + queryId: this.queryId(), + errorMessage: error.message, + errorData: null, + logLines: [], + journal: null, + }, + ]) + } + + releases = () => + this.registrations.map((registration) => registration.releases) + + deliverSnapshot = (rows: ReadonlyArray) => { + if (!hasDuplicateKey(rows)) this.rows = new Map(rows.map((r) => [r.id, r])) + this.sendResult(rows) + } + + deliverChanges = (changes: ReadonlyArray) => { + for (const change of changes) { + if (change.type === `upsert`) this.rows.set(change.row.id, change.row) + else this.rows.delete(change.id) + } + const result = [...this.rows.values()].sort((a, b) => a.id - b.id) + // A result equal to the one the client holds sends nothing. + if ( + JSON.stringify(convexToJson(result as unknown as Value)) === this.sent + ) { + this.transition([]) + return + } + this.sendResult(result) + } + + fail = (error: Error) => this.sendFailure(error) + + deliveryError = (error: Error) => this.sendFailure(error) + + /** The client applies a transition and calls back within a few turns. */ + settle = async () => { + for (let index = 0; index < 3; index++) await turn() + } + + dispose = () => this.client.close() +} + +let sequence = 0 + +export function convexConformanceDriver(): EagerLiveSourceDriver { + return { + adapter: `@tanstack/convex-db-collection (eager)`, + capabilities: new Set([ + `complete-snapshot`, + `delivery-error`, + `duplicate-key-delivery`, + ]), + changeCommits: `per-batch`, + duplicateInsert: `error`, + rejectedDelivery: `reported`, + // The client creates each error from the failure's message, which ends + // with the injected failure's index. + matchesFailure: (reported, failure) => + reported instanceof Error && + new RegExp(`(failure|error) ${failure}\\b`).test(reported.message), + valueKinds: VALUE_KINDS, + fieldKinds: VALUE_KINDS, + create(): EagerLiveSourceFixture { + const provider = new ConvexProvider() + const collection = createCollection( + convexCollectionOptions({ + id: `convex-conformance-${++sequence}`, + client: provider.client, + query: rowsQuery, + queryArgs: {}, + getKey: (row) => row.id, + }), + ) + return { + collection: observedView(collection, { + row: (row) => row, + key: (key) => key as number, + }), + provider, + settle: provider.settle, + dispose: provider.dispose, + } + }, + } +} diff --git a/packages/convex-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/convex-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts new file mode 100644 index 0000000000..6f2aa6a797 --- /dev/null +++ b/packages/convex-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -0,0 +1,75 @@ +import { fc } from '@fast-check/vitest' +import { describe, it } from 'vitest' +import { + oraclePropertyOptions, + oracleReplayTest, + oracleRuns, +} from '../../db/tests/oracle-config' +import { declareOracle } from '../../db/tests/oracle-declaration' +import { createOracleReach } from '../../db/tests/oracle-reach' +import { + checkEagerLiveSourceHistory, + eagerLiveSourceHistoryArb, + eagerLiveSourceModel, + eagerLiveSourceProperty, + reachStatesFor, +} from '../../db-adapter-conformance/src' +import { convexConformanceDriver } from './conformance-driver' + +/** + * # Convex on the shared eager live-source conformance oracle + * + * A receiving driver for `packages/db-adapter-conformance`. The shared + * module states the laws, the model, the history grammar, and the check; + * this file supplies only the Convex production driver in + * `conformance-driver.ts` and the campaigns. It states no expected results. + * + * Convex runs here in eager mode over the real Convex JavaScript client, + * whose WebSocket the driver answers with the sync protocol. Every delivery + * is a complete query result. It declares `complete-snapshot`, + * `delivery-error` (a `QueryFailed` after ready, cleared by the next + * result), and `duplicate-key-delivery` with rejected deliveries reported + * after ready, and Convex value kinds as values and typed fields. The claim + * is limited to the controlled socket; a claim about a Convex deployment + * needs a receiving witness against it. + */ + +const property = `adapter-conformance.eager-live-source.convex` +const driver = convexConformanceDriver() +const historyArb = eagerLiveSourceHistoryArb(driver) +const runs = 400 +const campaignTimeoutMs = 120_000 + +async function campaign(options: fc.Parameters<[unknown]>) { + const reach = createOracleReach(property, reachStatesFor(driver), { + minimum: 5, + }) + await fc.assert(eagerLiveSourceProperty(driver, reach), options) + return reach +} + +describe(`eager live-source conformance: Convex`, () => { + oracleReplayTest(it)( + `conforms on a fixed-seed campaign`, + async () => { + const reach = await campaign({ seed: 2118, numRuns: oracleRuns(runs) }) + reach.assertReached() + }, + campaignTimeoutMs, + ) + + oracleReplayTest(it, property)( + `conforms on a random campaign or a replayed seed`, + async () => { + const reach = await campaign(oraclePropertyOptions(runs, property)) + reach.assertReached() + }, + campaignTimeoutMs, + ) +}) + +declareOracle({ + check: [checkEagerLiveSourceHistory, eagerLiveSourceProperty], + model: [eagerLiveSourceModel.step, eagerLiveSourceModel.expected], + grammar: [historyArb], +}) diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index fd9bdcea8a..ec3eee6122 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -27,23 +27,23 @@ reviewed change. ## Adapters -| Adapter | Families that apply (section A) | Driver status | -| ----------------- | ------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Neon Realtime | eager live source, mutations | Eager: RED (READY-4, HEALTH-1, and LIFE-10 behind them); adapter update pending | -| Electric | eager live source, mutations, on-demand, source shapes, reload and retry, persistence | Eager: GREEN (2026-10-30) after three fixes, per-batch commits, typed Postgres columns, over the installed client. Driver `electric-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.electric`. Other families: not started | -| PowerSync | eager live source, mutations, on-demand | Eager: GREEN (2026-10-30) after a fix, per-batch commits, over real local SQLite. Driver `powersync-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.powersync`. Mutations, on-demand: not started | -| RxDB | eager live source, mutations | Eager: GREEN (2026-10-29) after a fix, per-change commits, `listen-before-fetch`. Driver `rxdb-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.rxdb`. Mutations: not started | -| TrailBase | eager live source, mutations, on-demand | Eager: GREEN (2026-10-29), per-change commits. Driver `trailbase-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.trailbase`; declares no optional capability. Mutations, on-demand: not started | -| Convex | eager live source, mutations, on-demand, source shapes | Not started | -| query-db | client cache and refetch, on-demand, mutations, reload and retry, persistence | Not started; not a push source, so the eager family does not apply | -| local-only | local and shared stores, mutations | Not started | -| localStorage | local and shared stores, mutations | Not started | -| Persisted wrapper | persistence wrapper | Not started | -| Storage hosts (8) | storage hosts | Not started | +| Adapter | Families that apply (section A) | Driver status | +| ----------------- | ------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Neon Realtime | eager live source, mutations | Eager: RED (READY-4, HEALTH-1, and LIFE-10 behind them); adapter update pending | +| Electric | eager live source, mutations, on-demand, source shapes, reload and retry, persistence | Eager: GREEN (2026-10-30) after three fixes, per-batch commits, typed Postgres columns, over the installed client. Driver `electric-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.electric`. Other families: not started | +| PowerSync | eager live source, mutations, on-demand | Eager: GREEN (2026-10-30) after a fix, per-batch commits, over real local SQLite. Driver `powersync-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.powersync`. Mutations, on-demand: not started | +| RxDB | eager live source, mutations | Eager: GREEN (2026-10-29) after a fix, per-change commits, `listen-before-fetch`. Driver `rxdb-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.rxdb`. Mutations: not started | +| TrailBase | eager live source, mutations, on-demand | Eager: GREEN (2026-10-29), per-change commits. Driver `trailbase-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.trailbase`; declares no optional capability. Mutations, on-demand: not started | +| Convex | eager live source, mutations, on-demand, source shapes | Eager: GREEN (2026-11-02) after a fix, per-batch commits, complete results, over the real Convex JavaScript client and a controlled WebSocket. Driver `convex-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.convex`. Mutations, on-demand, source shapes: not started | +| query-db | client cache and refetch, on-demand, mutations, reload and retry, persistence | Not started; not a push source, so the eager family does not apply | +| local-only | local and shared stores, mutations | Not started | +| localStorage | local and shared stores, mutations | Not started | +| Persisted wrapper | persistence wrapper | Not started | +| Storage hosts (8) | storage hosts | Not started | ## Suggested order -1. **Push adapters on the existing family:** RxDB (done), TrailBase (done), PowerSync (done), Electric (eager mode, done), and Convex. Each needs only a controlled-provider driver. A RED result is a finding against that adapter. +1. **Push adapters on the existing family:** RxDB (done), TrailBase (done), PowerSync (done), Electric (eager mode, done), and Convex (eager mode, done). Each needs only a controlled-provider driver. A RED result is a finding against that adapter. 2. **The Neon update, run against its driver.** This shows an adapter moving from RED to GREEN. 3. **The mutations family.** It applies to every adapter except the storage hosts. 4. **The other families,** one at a time. Each family's new driver work goes to the adapters that declare it. @@ -451,3 +451,42 @@ error/sync-give-up, accept]`). Fix: remember that give-up and fail the RED on 2dd5f0e75 at `[demand, hold, up-to-date, abort, release]` (health none); the adapter now fails the run on an external abort after ready. +- **2026-11-02, Convex, eager live source: one Convex defect.** The driver + runs the real Convex JavaScript client over a controlled WebSocket, as + `sdk.test.ts` does, and answers the sync protocol. Every delivery is the + query's complete result in one `Transition`; a failure is a + `QueryFailed`; a release is the adapter's unsubscribe, counted at the + client boundary. + - **Oracle changes this exposed:** capability and step `delivery-error`: + after ready, the source reports an error for its current result and + keeps its subscription, and the next valid delivery clears it (HEALTH-1, + HEALTH-2). Parameter `rejectedDelivery: reported` (the law set's default + `errorMapping`): a delivery the adapter rejects after ready joins + `failureReason` with the adapter's own error and the run continues. + Reach states `delivery-error-reported`, `delivery-error-recovered`, + `rejected-delivery-reported`. Reference mutants + `delivery-error-is-fatal` and `rejected-delivery-fails`. The FID-1 + relabel of a publication mismatch is narrowed to value-only + differences: a split batch differing in `note` is PUB-1. + - **RED:** LIFE-10 at `[fail, changes []]`: after a failed first load, + the query subscription stayed open until cleanup. + - **Fix:** a stopping failure releases the subscription once, also when + the subscription fails while it is being set up. + - **GREEN:** Convex suite 114/114; 400 histories per campaign, lowest + reach 13 across four seeds; about 15 seconds per campaign. + - **Mutants of `src/options.ts`:** the original defect (LIFE-10), a fatal + post-ready error (HEALTH-1), no recovery on an equal result (HEALTH-2), + skipped updates (PUB-3/FID-4), accepted duplicates (PUB-4), each + rejected. Skipping the truncate on a run's first result survives: an + eager restart already starts empty; it matters for retained rows, which + this lane does not generate. + - **Driver lessons:** a real server acknowledges a query removal at once, + which drops the client's cached result; without that, a restart reused + the previous run's cached error. A result equal to the one the client + holds sends no update, but after an error or a rejected result the + valid result is a change. + - **Observed, for a decision:** a restart that comes before the server + acknowledges the previous run's removal receives the client's cached + result or error for the query at once (the client serves results by + query token). + - **Size:** convex-db-collection +26 gzip bytes. diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md index ef2468c290..3e4a3da6bc 100644 --- a/packages/db-adapter-conformance/README.md +++ b/packages/db-adapter-conformance/README.md @@ -62,7 +62,7 @@ reach counter, so the run fails when histories miss a state the laws need. - **Reference adapter** (`tests/reference-adapter.ts`): a small push adapter written to conform. `tests/reference-adapter-oracle.property.test.ts` runs a fixed-seed and a random campaign with declared reach, a per-change variant, - an outage-threshold variant, and 41 hostile mutants of the adapter, each of which the check must reject + an outage-threshold variant, and 45 hostile mutants of the adapter, each of which the check must reject with its law. It runs in the default `pnpm test`. - **Neon Realtime** (`tests/neon/`): `@neon/realtime-tanstack` 0.1.1, vendored @@ -74,6 +74,11 @@ reach counter, so the run fails when histories miss a state the laws need. until the adapter is updated, so it has its own config: `pnpm --filter @tanstack/db-adapter-conformance test:neon`. +- **Convex** (`packages/convex-db-collection/tests/conformance-driver.ts`): + the published adapter in eager mode over the real Convex JavaScript + client, through a controlled WebSocket that answers the sync protocol, + with complete results and per-batch commits. It runs in that package's + default `pnpm test`. - **Electric** (`packages/electric-db-collection/tests/conformance-driver.ts`): the published adapter in eager mode over the installed `@electric-sql/client`, through a controlled `fetchClient` that answers diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 65e425b8dd..61c4433950 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -148,6 +148,7 @@ const staticOracleProperties = [ `adapter-conformance.eager-live-source.rxdb`, `adapter-conformance.eager-live-source.powersync`, `adapter-conformance.eager-live-source.electric`, + `adapter-conformance.eager-live-source.convex`, `trailbase.lifecycle`, `source-health.history`, `source-health.required-history`, diff --git a/packages/db/tests/oracle-replay-manifest.ts b/packages/db/tests/oracle-replay-manifest.ts index 5c0c6de6ae..e88fa1514d 100644 --- a/packages/db/tests/oracle-replay-manifest.ts +++ b/packages/db/tests/oracle-replay-manifest.ts @@ -668,6 +668,11 @@ const ownerGroups: ReadonlyArray< `adapter-conformance.eager-live-source`, `electric`, ], + [ + `convex-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts`, + `adapter-conformance.eager-live-source`, + `convex`, + ], ] const owners = new Map() @@ -761,6 +766,7 @@ const replayTestNames = new Map([ `rxdb`, `powersync`, `electric`, + `convex`, ].map( (driver) => [ From c25a5bb721c210bc06f0b85de9f09737ba47d5f8 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 09:32:38 -0700 Subject: [PATCH 25/64] test(conformance): declare reach for a restart after a failed run LIFE-5 holds for a restart after a failed run as for any restart: the new run starts loading. Declare the state restart-after-failure, reached when a restart follows a failure before or after ready, so every driver's campaign shows it was exercised. It reaches the Convex client's cached error (next commit). Evidence: every eager campaign (reference, TrailBase, RxDB, PowerSync, Electric, Convex) reaches the state at least five times. Co-authored-by: Isaac --- .../db-adapter-conformance/src/eager-live-source-oracle.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 831a8b9ef6..9212c5c6ca 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -1623,6 +1623,7 @@ export const eagerLiveSourceReachStates = [ `failure-before-ready`, `failure-after-ready`, `restart`, + `restart-after-failure`, `wait-before-evidence-fulfilled`, `wait-timed-out`, `wait-rejected-at-retirement`, @@ -2237,6 +2238,7 @@ export async function checkEagerLiveSourceHistory( let batchesThisRun = 0 for (const [index, step] of history.entries()) { const wasReady = state.phase === `ready` + const phaseBefore = state.phase const transientBefore = state.transient.length // Support is decided before the model moves, so a skipped step leaves // the model unchanged. @@ -2396,6 +2398,9 @@ export async function checkEagerLiveSourceHistory( break case `restart`: { const pendingBefore = waits.some((wait) => wait.current === `pending`) + // LIFE-5: a failed run's restart starts loading, as any does. + const failedBefore = + phaseBefore === `loading-error` || phaseBefore === `stopped` events.stop() await collection.cleanup() // The backend may change while no sync run listens. @@ -2407,6 +2412,7 @@ export async function checkEagerLiveSourceHistory( preload = recordSettlement(collection.preload()) events = foldChanges(collection) reach(`restart`) + if (failedBefore) reach(`restart-after-failure`) open.clear() outagesThisRun = 0 batchesThisRun = 0 From 759ddede0a091a874f88841baf8847fd392148fc Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 09:32:41 -0700 Subject: [PATCH 26/64] fix(convex): a cached query error does not fail a new subscription Maintainer decision 2026-11-02: the Convex client serves a query's cached result to a new subscription at once. A cached valid result is accepted client-cache behavior; a cached error belongs to an earlier subscription, so it must not fail the new one, which waits for the server's result. A restart right after a failed run no longer fails again from the stale error. subscribeQuery treats an error the client holds when it subscribes as cached: it reports no current value and skips the client's replay of that error, for the JavaScript client (onUpdate) and the React client (watchQuery). The first real result clears it. The driver now acknowledges a query removal only with its next transition while the client's result is a failure, so a restart finds the cached error, and a fixed-seed campaign runs the React client. Evidence: - Law: LIFE-5 with the decision above. - Reach: restart-after-failure in all three campaigns. - RED on 6aa2671da: LIFE-2/LIFE-5 (status) at [fail, restart, fail], expected loading, observed error, for both clients. - GREEN: Convex suite 115/115; Electric, TrailBase, RxDB, PowerSync, conformance suites pass. - Mutants rejected: the client's replay not skipped; the check skipped on the React path. - Limits: a cached valid result served to a restarted run is not generated (the model starts a restarted run empty). - Size: convex-db-collection +48 gzip bytes (+74 on the branch). Co-authored-by: Isaac --- .changeset/convex-failed-load-release.md | 2 +- docs/collections/convex-collection.md | 4 +- docs/contributing/oracle-coverage.md | 13 +- .../convex-db-collection/src/subscription.ts | 25 +++- .../tests/conformance-driver.ts | 131 ++++++++++++------ ...source-conformance-oracle.property.test.ts | 29 +++- packages/db-adapter-conformance/MIGRATION.md | 18 ++- 7 files changed, 163 insertions(+), 59 deletions(-) diff --git a/.changeset/convex-failed-load-release.md b/.changeset/convex-failed-load-release.md index dfe7a10d03..7b1c99bb81 100644 --- a/.changeset/convex-failed-load-release.md +++ b/.changeset/convex-failed-load-release.md @@ -2,4 +2,4 @@ '@tanstack/convex-db-collection': patch --- -Release the Convex query subscription when a Collection's first load fails, or when a later acceptance failure stops the sync run, instead of keeping it until cleanup. +Release the Convex query subscription when a Collection's first load fails, or when a later acceptance failure stops the sync run, instead of keeping it until cleanup. A new subscription no longer fails from an error the Convex client cached for its query before it subscribed, such as a restart right after a failed run; it waits for the server's result. diff --git a/docs/collections/convex-collection.md b/docs/collections/convex-collection.md index 36aca89604..d5c8bdfda6 100644 --- a/docs/collections/convex-collection.md +++ b/docs/collections/convex-collection.md @@ -321,9 +321,9 @@ convexCollectionOptions({ }) ``` -An initial subscription or projection error rejects `preload()` with the original error and ends that sync run. Call `collection.reload()` to try again. Later recoverable errors preserve existing rows, set `collection.health.failureReason`, and call `onError`. A valid query result clears that failure reason. `collection.utils.getSyncError()` returns the latest error, but this getter is not reactive. A rejected source acceptance stops the run and sets its health error; it also requires `reload()`. +An initial subscription or projection error rejects `preload()` with the original error, ends that sync run, and releases its query subscription. Call `collection.reload()` to try again. Later recoverable errors preserve existing rows, set `collection.health.failureReason`, and call `onError`. A valid query result clears that failure reason. `collection.utils.getSyncError()` returns the latest error, but this getter is not reactive. A rejected source acceptance stops the run and sets its health error; it also requires `reload()`. -Collection cleanup rejects while live queries or Effects read it. Unmount those consumers first, or use `collection.cleanup({ dependents: true })`. Cleanup unsubscribes and invalidates callbacks from that sync run. It does not close the shared Convex client. After a restart, the first result replaces retained data, including when the result is empty. Close the client separately when your application no longer needs it. +Collection cleanup rejects while live queries or Effects read it. Unmount those consumers first, or use `collection.cleanup({ dependents: true })`. Cleanup unsubscribes and invalidates callbacks from that sync run. It does not close the shared Convex client. After a restart, the first result replaces retained data, including when the result is empty. The Convex client serves a query's cached result to a new subscription at once, so a restart soon after cleanup can start from the previous run's rows until the server's next result. A cached error does not fail the restarted run: it waits for the server's result instead, so a query that has recovered is not reported failed again. The same holds for a Collection that subscribes to a query another subscription on the same client has in error. Close the client separately when your application no longer needs it. ## Scope diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 996df912f2..e08f925156 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6109,7 +6109,12 @@ rejected delivery after ready (`rejectedDelivery: reported`, the law set's default `errorMapping`). Limits: no `awaitSync` evidence (the shared oracle's waits are txid waits), no transport reconnect, no on-demand or paginated mode, no deployment; the truncate on a restarted run's first -result is not distinguished, because an eager restart starts empty; a -restart before the server acknowledges the previous removal can receive the -client's cached result, which the driver avoids by acknowledging at once -(observation for a maintainer decision). +result is not distinguished, because an eager restart starts empty. While +the client's result is a failure, the driver acknowledges a query removal +only with its next transition, so a restart finds the failed query's cached +error: maintainer decision 2026-11-02, a cached error does not fail a new +subscription, which waits for the server (reach `restart-after-failure`, +both clients); a cached valid result served to a restarted run is accepted +client-cache behavior and is not generated, because the model starts a +restarted run empty. A fixed-seed campaign runs the React client +(`watchQuery`). diff --git a/packages/convex-db-collection/src/subscription.ts b/packages/convex-db-collection/src/subscription.ts index ff50036ca4..d1fe656d43 100644 --- a/packages/convex-db-collection/src/subscription.ts +++ b/packages/convex-db-collection/src/subscription.ts @@ -27,10 +27,22 @@ export function subscribeQuery>( fail: (error: unknown) => void, ): ConvexSubscription> { let active = true + // The client serves a query's cached result to a new subscription at + // once. A cached error belongs to an earlier subscription, so this one + // waits for the server instead (maintainer decision). + let cachedError = false const stop = (unsubscribe: () => void) => () => { active = false unsubscribe() } + const hasCachedError = (read: () => unknown) => { + try { + read() + return false + } catch { + return true + } + } if (`disabled` in client && client.disabled === true) { throw new Error(`A disabled Convex client cannot sync a collection.`) } @@ -42,19 +54,26 @@ export function subscribeQuery>( query, args, (value) => { + cachedError = false if (active) receive(value) }, (error) => { - if (active) fail(error) + // The client's replay of the cached error. + if (cachedError) cachedError = false + else if (active) fail(error) }, ) + cachedError = hasCachedError(() => subscription.getCurrentValue()) return { - getCurrentValue: () => subscription.getCurrentValue(), + getCurrentValue: () => + cachedError ? undefined : subscription.getCurrentValue(), unsubscribe: stop(subscription), } } const watch = client.watchQuery(query, args) + cachedError = hasCachedError(() => watch.localQueryResult()) const unsubscribe = watch.onUpdate(() => { + cachedError = false if (!active) return try { const result = watch.localQueryResult() @@ -64,7 +83,7 @@ export function subscribeQuery>( } }) return { - getCurrentValue: () => watch.localQueryResult(), + getCurrentValue: () => (cachedError ? undefined : watch.localQueryResult()), unsubscribe: stop(unsubscribe), } } diff --git a/packages/convex-db-collection/tests/conformance-driver.ts b/packages/convex-db-collection/tests/conformance-driver.ts index 1d882372c4..6a65aa6b18 100644 --- a/packages/convex-db-collection/tests/conformance-driver.ts +++ b/packages/convex-db-collection/tests/conformance-driver.ts @@ -30,7 +30,9 @@ * is the adapter's call to the returned unsubscribe (LIFE-2, LIFE-10), * observed at the client boundary because the client may merge a removal * with a later subscription on the wire. The driver acknowledges a removal - * at once, as a server does, which drops the client's cached result. + * at once, as a server does, which drops the client's cached result; after + * a failure it acknowledges it only with its next transition, so a restart + * can find the failed query's cached error. * * Rows are Convex values: `note` absent is a missing field, and typed fields * hold the Convex kinds (no Date, which Convex values do not have). The @@ -38,6 +40,7 @@ */ import { createCollection } from '@tanstack/db' import { ConvexClient } from 'convex/browser' +import { ConvexReactClient } from 'convex/react' import { makeFunctionReference } from 'convex/server' import { convexToJson } from 'convex/values' import { convexCollectionOptions } from '../src/index' @@ -73,7 +76,11 @@ const rowsQuery = makeFunctionReference< /** The WebSocket the Convex client opens; the driver plays the server. */ class Socket { - static latest: Socket | undefined + /** + * The provider that the next socket serves. The React client opens its + * socket lazily, so a socket takes its provider when it is created. + */ + static provider: { attach: (socket: Socket) => void } | undefined onopen: (() => void) | null = null onmessage: ((event: { data: string }) => void) | null = null onclose: ((event: { code: number; reason: string }) => void) | null = null @@ -81,7 +88,9 @@ class Socket { readonly sent: Array> = [] onSend: ((message: Record) => void) | undefined constructor(_url: string) { - Socket.latest = this + Socket.provider?.attach(this) + // The client sets its handlers after construction; then it connects. + queueMicrotask(() => this.onopen?.()) } send(data: string) { const message = JSON.parse(data) as Record @@ -109,26 +118,30 @@ const hasDuplicateKey = (rows: ReadonlyArray) => new Set(rows.map((row) => row.id)).size !== rows.length class ConvexProvider implements ControlledProvider { - readonly client: ConvexClient - private readonly socket: Socket + readonly client: ConvexClient | ConvexReactClient + private socketOrUndefined: Socket | undefined private readonly registrations: Array<{ releases: number }> = [] private rows = new Map() private time = 0 private querySet = 0 + private ackLate = false + private pendingAcks: Array = [] // The result the client holds, encoded; undefined after a failure. private sent: string | undefined - constructor() { - this.client = new ConvexClient(`https://conformance.convex.cloud`, { + constructor(kind: ClientKind) { + Socket.provider = this + const options = { webSocketConstructor: Socket as unknown as typeof WebSocket, unsavedChangesWarning: false, - logger: false, - }) - this.socket = Socket.latest! - this.socket.onopen?.() - // A server acknowledges a removal promptly, which drops the client's - // cached result for that query. - this.socket.onSend = (message) => { + logger: false as const, + } + const url = `https://conformance.convex.cloud` + this.client = + kind === `javascript` + ? new ConvexClient(url, options) + : new ConvexReactClient(url, options) + this.onSend = (message) => { const modifications = (message.modifications ?? []) as Array<{ type: string queryId: number @@ -140,29 +153,60 @@ class ConvexProvider implements ControlledProvider { ) { return } - queueMicrotask(() => - this.transition( - modifications.map(({ queryId }) => ({ - type: `QueryRemoved`, - queryId, - })), - ), - ) + const removed = modifications.map(({ queryId }) => ({ + type: `QueryRemoved`, + queryId, + })) + // While the client's result is a failure, the acknowledgement comes + // with the next transition, so a restart can come first: the client + // then holds the failed query's cached error. A server that re-runs + // the query may now succeed; a cached result, valid or rejected, is + // what it would send again. + if (this.ackLate) this.pendingAcks.push(...removed) + else queueMicrotask(() => this.transition(removed)) } - // Count each `onUpdate` registration and the adapter's unsubscribe. - const onUpdate = this.client.onUpdate.bind(this.client) - this.client.onUpdate = ((...args: Parameters) => { + // Count each registration and the adapter's unsubscribe: `onUpdate` + // for the JavaScript client, a watch's `onUpdate` for the React one. + const counted = (unsubscribe: () => void) => { const registration = { releases: 0 } this.registrations.push(registration) - const subscription = onUpdate(...args) - return Object.assign( - () => { - registration.releases++ - subscription() - }, - { getCurrentValue: () => subscription.getCurrentValue() }, - ) - }) as typeof onUpdate + return () => { + registration.releases++ + unsubscribe() + } + } + const client = this.client + if (client instanceof ConvexClient) { + const onUpdate = client.onUpdate.bind(client) + client.onUpdate = ((...args: Parameters) => { + const subscription = onUpdate(...args) + return Object.assign(counted(subscription), { + getCurrentValue: () => subscription.getCurrentValue(), + }) + }) as typeof onUpdate + } else { + const watchQuery = client.watchQuery.bind(client) as ( + ...args: Array + ) => ReturnType + client.watchQuery = ((...args: Array) => { + const watch = watchQuery(...args) + const onUpdate = watch.onUpdate.bind(watch) + watch.onUpdate = (callback: () => void) => counted(onUpdate(callback)) + return watch + }) as typeof watchQuery + } + } + + private onSend: (message: Record) => void = () => {} + + attach(socket: Socket): void { + this.socketOrUndefined = socket + socket.onSend = (message) => this.onSend(message) + } + + private get socket(): Socket { + if (!this.socketOrUndefined) throw new Error(`the client has no socket`) + return this.socketOrUndefined } /** The query the client subscribed most recently and has not removed. */ @@ -202,12 +246,13 @@ class ConvexProvider implements ControlledProvider { ts: timestamp(++this.time), identity: 0, }, - modifications, + modifications: [...this.pendingAcks.splice(0), ...modifications], }) this.querySet = version } private sendResult(rows: ReadonlyArray): void { + this.ackLate = false this.sent = JSON.stringify(convexToJson(rows as unknown as Value)) this.transition([ { @@ -258,7 +303,10 @@ class ConvexProvider implements ControlledProvider { this.sendResult(result) } - fail = (error: Error) => this.sendFailure(error) + fail = (error: Error) => { + this.ackLate = true + this.sendFailure(error) + } deliveryError = (error: Error) => this.sendFailure(error) @@ -272,9 +320,14 @@ class ConvexProvider implements ControlledProvider { let sequence = 0 -export function convexConformanceDriver(): EagerLiveSourceDriver { +type ClientKind = `javascript` | `react` + +/** `client` selects the Convex JavaScript client or the React client. */ +export function convexConformanceDriver({ + client = `javascript`, +}: { client?: ClientKind } = {}): EagerLiveSourceDriver { return { - adapter: `@tanstack/convex-db-collection (eager)`, + adapter: `@tanstack/convex-db-collection (eager, ${client} client)`, capabilities: new Set([ `complete-snapshot`, `delivery-error`, @@ -291,7 +344,7 @@ export function convexConformanceDriver(): EagerLiveSourceDriver { valueKinds: VALUE_KINDS, fieldKinds: VALUE_KINDS, create(): EagerLiveSourceFixture { - const provider = new ConvexProvider() + const provider = new ConvexProvider(client) const collection = createCollection( convexCollectionOptions({ id: `convex-conformance-${++sequence}`, diff --git a/packages/convex-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/convex-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts index 6f2aa6a797..02241b2698 100644 --- a/packages/convex-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts +++ b/packages/convex-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -24,8 +24,9 @@ import { convexConformanceDriver } from './conformance-driver' * this file supplies only the Convex production driver in * `conformance-driver.ts` and the campaigns. It states no expected results. * - * Convex runs here in eager mode over the real Convex JavaScript client, - * whose WebSocket the driver answers with the sync protocol. Every delivery + * Convex runs here in eager mode over the real Convex JavaScript client, and + * in a fixed-seed campaign over the React client, whose WebSocket the driver + * answers with the sync protocol. Every delivery * is a complete query result. It declares `complete-snapshot`, * `delivery-error` (a `QueryFailed` after ready, cleared by the next * result), and `duplicate-key-delivery` with rejected deliveries reported @@ -40,11 +41,15 @@ const historyArb = eagerLiveSourceHistoryArb(driver) const runs = 400 const campaignTimeoutMs = 120_000 -async function campaign(options: fc.Parameters<[unknown]>) { - const reach = createOracleReach(property, reachStatesFor(driver), { +async function campaign( + options: fc.Parameters<[unknown]>, + checked = driver, + name = property, +) { + const reach = createOracleReach(name, reachStatesFor(checked), { minimum: 5, }) - await fc.assert(eagerLiveSourceProperty(driver, reach), options) + await fc.assert(eagerLiveSourceProperty(checked, reach), options) return reach } @@ -66,6 +71,20 @@ describe(`eager live-source conformance: Convex`, () => { }, campaignTimeoutMs, ) + + // The React client reaches the adapter through `watchQuery`. + oracleReplayTest(it)( + `conforms over the React client on a fixed-seed campaign`, + async () => { + const reach = await campaign( + { seed: 2118, numRuns: oracleRuns(runs) }, + convexConformanceDriver({ client: `react` }), + `${property}.react`, + ) + reach.assertReached() + }, + campaignTimeoutMs, + ) }) declareOracle({ diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index ec3eee6122..008a09ef07 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -485,8 +485,16 @@ error/sync-give-up, accept]`). Fix: remember that give-up and fail the the previous run's cached error. A result equal to the one the client holds sends no update, but after an error or a rejected result the valid result is a change. - - **Observed, for a decision:** a restart that comes before the server - acknowledges the previous run's removal receives the client's cached - result or error for the query at once (the client serves results by - query token). - - **Size:** convex-db-collection +26 gzip bytes. + - **Cached error (decided 2026-11-02):** a restart that comes before the + server acknowledges the previous run's removal receives the client's + cached result. A cached valid result is accepted client-cache behavior; + a cached error must not fail the new run, which waits for the server. + The driver now acknowledges a removal late while the client's result is + a failure; new shared reach state `restart-after-failure`. RED on + 6aa2671da at `[fail, restart, fail]` (status `error`, law `loading`), + for both the JavaScript and React clients. Fix: `subscribeQuery` treats + an error the client holds at subscription as cached, reports no current + value, and skips the client's replay of it. Mutants that keep the + replay, or skip the check on the React path, are rejected. A fixed-seed + campaign now runs the React client. + - **Size:** convex-db-collection +26 gzip bytes for LIFE-10. From ab648b90d1e293403b2914d481abdd757e0e34d9 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 09:36:46 -0700 Subject: [PATCH 27/64] chore: add the db-adapter-conformance package to the lockfile The new private workspace package needs its importer entry, with its dev dependency @neon/realtime (and its postgres-array dependency), so a frozen install accepts it. Co-authored-by: Isaac --- pnpm-lock.yaml | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 386af2630a..9822357ab0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1318,6 +1318,28 @@ importers: specifier: ^0.3.0 version: 0.3.0 + packages/db-adapter-conformance: + dependencies: + '@tanstack/db': + specifier: workspace:* + version: link:../db + fast-check: + specifier: ^3.23.0 + version: 3.23.2 + devDependencies: + '@fast-check/vitest': + specifier: ^0.4.1 + version: 0.4.1(vitest@4.1.12) + '@neon/realtime': + specifier: 0.1.2 + version: 0.1.2 + '@standard-schema/spec': + specifier: ^1.1.0 + version: 1.1.0 + '@vitest/coverage-istanbul': + specifier: ^4.1.12 + version: 4.1.12(vitest@4.1.12) + packages/db-collection-e2e: dependencies: '@tanstack/db': @@ -4935,6 +4957,15 @@ packages: '@napi-rs/wasm-runtime@1.1.0': resolution: {integrity: sha512-Fq6DJW+Bb5jaWE69/qOE0D1TUN9+6uWhCeZpdnSBk14pjLcCWR7Q8n49PTSPHazM37JqrsdpEthXy2xn6jWWiA==} + '@neon/realtime@0.1.2': + resolution: {integrity: sha512-82PkgG/iGZOHjO1znVTpBhUMTxpAd/82pFw141zeIS11SlTQLYsuMD5Emc/noatuMA99fZspokP8hnggip4rAA==} + engines: {node: '>=20.19.0'} + peerDependencies: + ws: ^8.21.2 + peerDependenciesMeta: + ws: + optional: true + '@next/env@16.4.1': resolution: {integrity: sha512-5sUUrrSvs/2/2LnLoesybMGx12EtPB46fdZ6RhAw0XDGSPxU2LFyQBB7yuEiOFFJPKmBY6+q2jTr1/AMbnp0/g==} @@ -11224,6 +11255,10 @@ packages: resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} engines: {node: '>=4'} + postgres-array@3.0.4: + resolution: {integrity: sha512-nAUSGfSDGOaOAEGwqsRY27GPOea7CNipJPOA7lPbdEpx5Kg3qzdP0AaWC5MlhTWV9s4hFX39nomVZ+C4tnGOJQ==} + engines: {node: '>=12'} + postgres-bytea@1.0.0: resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==} engines: {node: '>=0.10.0'} @@ -16558,6 +16593,11 @@ snapshots: '@tybys/wasm-util': 0.10.1 optional: true + '@neon/realtime@0.1.2': + dependencies: + base64-js: 1.5.1 + postgres-array: 3.0.4 + '@next/env@16.4.1': {} '@next/swc-darwin-arm64@16.4.1': @@ -23982,6 +24022,8 @@ snapshots: postgres-array@2.0.0: {} + postgres-array@3.0.4: {} + postgres-bytea@1.0.0: {} postgres-date@1.0.7: {} From 6d31ddfc5002dde040f3b30c7f4267f9134eee71 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 09:37:49 -0700 Subject: [PATCH 28/64] docs(conformance): document the driver fields added since Electric Writing a driver now covers retry-policy's transientVariants, retryReportAfter, and empty-response; delivery-error; rejectedDelivery; matchesFailure; txidEvidence; and how to run eagerLiveSourceProperty with a reach counter. Co-authored-by: Isaac --- packages/db-adapter-conformance/README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md index 3e4a3da6bc..1c52dc65c7 100644 --- a/packages/db-adapter-conformance/README.md +++ b/packages/db-adapter-conformance/README.md @@ -50,12 +50,20 @@ Implement `EagerLiveSourceDriver`: - **`valueKinds`:** the kinds of FID-1's `valueDomain` your backend represents; `jsonValueKinds` for a JSON backend. The driver delivers each value in the backend's own form and the Collection must show it decoded. - **Asynchronous steps and `settle`:** provider methods may return a promise, which the check awaits. A fixture whose deliveries reach the Collection later than a few event-loop turns supplies `settle()`, its law-set `propagationStep`, which the check awaits before every comparison. - **`source-failure-after-ready`:** declare it when your provider can fail terminally after the first ready (the law set's `terminal-source-end`, or an `errorMapping` that reports a terminal failure); otherwise `fail` is generated only before ready. +- **`retry-policy` and its parameters:** `transientFailure(error, variant)` makes one failed attempt the adapter retries, of the kind `transientVariants[variant]` (for example an HTTP 503, a 429, or a network error). `retryReportAfter` is your adapter's outage threshold: the failed attempts in a row after ready from which `failureReason` shows the outage (default 1, report at once; Electric 5). `empty-response` with `emptyResponse()`: an accepted response with no change, which restarts that count. +- **`delivery-error`:** `deliveryError(error)`, after ready: the source reports an error for its current result and keeps its subscription; the next valid delivery clears it (Convex's `QueryFailed`). +- **`rejectedDelivery`:** how your adapter reports a delivery it rejects after ready, such as two rows with one key: `fails` the sync run (default) or `reported` in `failureReason` until the next valid delivery. +- **`matchesFailure`:** when your client library creates the error value the Collection reports (an HTTP client's error for a response), the relation that names the value owed for each injected failure, for example by an index the driver carried in a response header. Without it, failures compare by identity with the provider's error. +- **`txidEvidence`:** `change` when txids travel on each row message, so a batch with no change carries none; default `batch`. - **`observedView`:** if your rows or keys differ from the conformance row, wrap the Collection with `observedView` and a one-to-one mapping. `conformanceKey` maps string keys; `toSqlRow` and `fromSqlRow` map an absent `note` to and from SQL `NULL`. Declare only the capabilities your adapter supports; the grammar generates -only the steps they allow. Then run `checkEagerLiveSourceHistory` over -`eagerLiveSourceHistoryArb(capabilities)` in a fast-check property. Pass a -reach counter, so the run fails when histories miss a state the laws need. +only the steps they allow. Then run `eagerLiveSourceProperty(driver, reach)` +with `fc.assert`. `reach` is a `ReachCounter` (`beginHistory`, `reach`); +count the states `reachStatesFor(driver)` lists and fail the run when one +was reached fewer than five times, so histories that miss a state the laws +need do not pass silently. The in-repo campaigns use `createOracleReach` +from `packages/db/tests/oracle-reach.ts`. ## Drivers From 1f2f44a9e38461cfbbfdb6febbe9734636c38521 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 09:40:40 -0700 Subject: [PATCH 29/64] docs(conformance): plan the duplicated-test audit and defer publishing Co-authored-by: Isaac --- packages/db-adapter-conformance/MIGRATION.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index 008a09ef07..3a7d0f9026 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -44,9 +44,10 @@ reviewed change. ## Suggested order 1. **Push adapters on the existing family:** RxDB (done), TrailBase (done), PowerSync (done), Electric (eager mode, done), and Convex (eager mode, done). Each needs only a controlled-provider driver. A RED result is a finding against that adapter. -2. **The Neon update, run against its driver.** This shows an adapter moving from RED to GREEN. -3. **The mutations family.** It applies to every adapter except the storage hosts. -4. **The other families,** one at a time. Each family's new driver work goes to the adapters that declare it. +2. **Retire duplicated adapter tests (separate PR, maintainer decision 2026-11-02).** For each adapter on the eager family, classify its existing eager-mode tests and oracles against what the shared oracle generates and observes: remove a test only when the shared family checks the same law, history, path, and observation; narrow one it covers in part; keep the rest. Known overlap: TrailBase's lifecycle oracle and the shared family both own eager startup failure and stream-end health. +3. **The Neon update, run against its driver.** This shows an adapter moving from RED to GREEN. Publishing the package for drivers outside this repository (a public package, `@tanstack/db` as a peer dependency, an exported reach counter) is deferred (maintainer decision 2026-11-02). +4. **The mutations family.** It applies to every adapter except the storage hosts. +5. **The other families,** one at a time. Each family's new driver work goes to the adapters that declare it. ## Migration log From 877f1a14e0f508974790700bc11709efe14652ba Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 11:10:04 -0700 Subject: [PATCH 30/64] fix(db): keep -0 distinct in transaction merges and ordered Effects Review findings CR4 and CR8 under the maintainer decision that -0 and 0 are distinct row values, for synced rows and user writes (2026-10-29). - A transaction that deletes a row and reinserts it with only a zero's sign changed merged to no request (deepEquals), and a field that changed only its sign was left out of the net update. The merge now compares with sameRowValue. - An ordered Effect suppressed an update whose only change was a zero's sign. Its comparison with the previous publication now uses sameRowValue. Oracles first: - optimistic-transaction-oracle: the model compares fields with Object.is; the generated values include -0 and the original note, so a reinsert can differ only by a zero's sign (reach sign-only-reinsert). Updates now expect exactly the differing fields: writing a field's own value back records no change, which the old grammar never exercised. - effect-callback-oracle (core and React drivers): an ordered dimension and a signed-zero payload. An ordered Effect delivers an update only when its projection changed. Evidence: - RED on f06ee8fdd: the same-key campaign dropped value -0 from [delete-insert, -0]; 10 ordered signed-zero callback histories failed. - GREEN: db suite 25,508; react-db 4,328. - Mutants: an unsigned whole-row cancellation check is rejected at a sign-only reinsert; the original effect.ts fails the ordered histories. - Open (coverage map): an ordered Effect drops an update whose change is private or non-enumerable state; those ordered pairs await a decision. Co-authored-by: Isaac --- .changeset/sync-negative-zero-change.md | 2 +- docs/contributing/oracle-coverage.md | 15 ++++ packages/db/src/query/effect.ts | 7 +- packages/db/src/transactions.ts | 7 +- ...mistic-transaction-oracle.property.test.ts | 52 +++++++---- .../db/tests/query/effect-callback-oracle.ts | 88 +++++++++++++++---- .../virtual-row-fields-effect-oracle.test.ts | 12 +-- .../tests/effect-callback-oracle.test.tsx | 9 +- 8 files changed, 143 insertions(+), 49 deletions(-) diff --git a/.changeset/sync-negative-zero-change.md b/.changeset/sync-negative-zero-change.md index 306e2ca42f..753c43b4d5 100644 --- a/.changeset/sync-negative-zero-change.md +++ b/.changeset/sync-negative-zero-change.md @@ -2,4 +2,4 @@ '@tanstack/db': patch --- -Treat `-0` and `0` as distinct row values. A synced update, a replay that replaces a subscription's rows, and an optimistic update that change a field only from `-0` to `0` (or back) now publish a change; before, the Collection could store the new value while subscribers kept the old one. A draft write of `0` over `-0` is now recorded as a change instead of being dropped. +Treat `-0` and `0` as distinct row values. A synced update, a replay that replaces a subscription's rows, and an optimistic update that change a field only from `-0` to `0` (or back) now publish a change; before, the Collection could store the new value while subscribers kept the old one. A draft write of `0` over `-0` is now recorded as a change instead of being dropped. A transaction that deletes a row and reinserts it with only a zero's sign changed now sends that update instead of dropping it, and an ordered Effect now delivers an update that changes only a zero's sign. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 9953aec8d9..bae2456f42 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6114,6 +6114,21 @@ sources and actual React hook commits; other providers/frameworks, arbitrary schedules and ordered repair semantics retain their existing limits. See the [key-only review evidence](oracle-reviews/effect-startup-delta.md#review-of-2b3dd8c2d-key-only-unordered-results). +Ordered callback histories (2026-11-02): the callback-profile grammar now +crosses an unordered and an ordered query (`orderBy` with `limit`) and adds a +`signed-zero` payload that alternates 0 and -0, distinct row values (maintainer +decision 2026-10-29); 120 histories on each path. An ordered Effect delivers an +update only when its projection changed. On unchanged `f06ee8fdd` the ordered +`signed-zero` histories fail (an update that flips only a zero's sign was +dropped); comparing with `sameRowValue` repairs them. Open, for a maintainer +decision: an ordered Effect compares an update by enumerable fields, so it +drops an update whose change is private or non-enumerable state, and the +consumer keeps a stale value. Trace: `private` or `prototype` payload, +ordered, any profile, update version 0 to 1 at the `unobserved` cut, expected +an update, received none; these 20 ordered pairs are not generated until the +decision (owner: this callback-profile oracle; witness: the same histories +with `ordered: true`). + ### RC bound schema mutation receivers (2026-10-28) Owner: `packages/db/tests/live-query-bound-options-oracle.ts`; shared driver: diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 4355c7a753..975901253b 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -3,7 +3,7 @@ import { createSourceRecord } from '../utils/source-record.js' import { createDeferred } from '../deferred.js' import { runAllCallbacks } from '../utils/callbacks.js' import { normalizeError } from '../utils/error.js' -import { deepEquals } from '../utils.js' +import { sameRowValue } from '../utils.js' import { codedMessage, devBuild } from '../error-message.js' import { buildHealthTree, healthNode } from '../collection/health-tree.js' import { isFailedStatus } from '../live-query-primitives.js' @@ -1346,10 +1346,11 @@ function classifyDelta( wasPresent && isPresent && insertValue !== undefined && - (publishedRows instanceof Set || !deepEquals(previousValue, insertValue)) + (publishedRows instanceof Set || !sameRowValue(previousValue, insertValue)) ) { // Unordered Effects preserve all graph updates without comparing values. - // Ordered repairs compare with the preceding coherent publication. + // Ordered repairs compare with the preceding coherent publication; -0 + // and 0 are distinct row values (FID-1). return { type: `update`, key, diff --git a/packages/db/src/transactions.ts b/packages/db/src/transactions.ts index 2c2dc23e11..0e882a9525 100644 --- a/packages/db/src/transactions.ts +++ b/packages/db/src/transactions.ts @@ -1,5 +1,5 @@ import { createDeferred } from './deferred' -import { deepEquals } from './utils' +import { sameRowValue } from './utils' import { safeRandomUUID } from './utils/uuid' import { normalizeError } from './utils/error.js' import './duplicate-instance-check' @@ -228,7 +228,8 @@ function mergePendingMutations( if (existing.type !== `delete`) break const original = existing.collection._state.syncedData.get(existing.key) if (original === undefined) return incoming - if (deepEquals(original, incoming.modified)) { + // -0 and 0 are distinct row values, as user writes (FID-1). + if (sameRowValue(original, incoming.modified)) { return null } @@ -238,7 +239,7 @@ function mergePendingMutations( for (const key of keys) { if ( Object.hasOwn(original, key) !== Object.hasOwn(modified, key) || - !deepEquals(original[key as keyof T], modified[key as keyof T]) + !sameRowValue(original[key as keyof T], modified[key as keyof T]) ) { changes[key as keyof T] = modified[key as keyof T] } diff --git a/packages/db/tests/optimistic-transaction-oracle.property.test.ts b/packages/db/tests/optimistic-transaction-oracle.property.test.ts index b3536770b1..6917cfbdd5 100644 --- a/packages/db/tests/optimistic-transaction-oracle.property.test.ts +++ b/packages/db/tests/optimistic-transaction-oracle.property.test.ts @@ -12,6 +12,7 @@ import { } from './optimistic-history-oracle.js' import { oraclePropertyOptions, oracleRuns } from './oracle-config.js' import { declareOracle } from './oracle-declaration.js' +import { createOracleReach } from './oracle-reach.js' import type { Collection } from '../src/collection/index.js' import type { ChangeMessage, SyncConfig } from '../src/types.js' import type { ReplacementRow, Row } from './optimistic-transaction-support.js' @@ -88,10 +89,14 @@ const sameKeySequences = [ // insert/update followed by update/delete, or a delete followed by a reinsert. // The applyMutations truth table gives the net request: a reinsert that // restores the original row cancels the pair; otherwise it is an update from -// the original row whose changes hold exactly the differing fields. Each of -// the five sequences and both +// the original row whose changes hold exactly the differing fields. A field +// differs by `Object.is`: -0 and 0 are distinct row values, for user writes +// too (maintainer decision 2026-10-29), so a reinsert of -0 over the original +// 0 is an update. Each of the five sequences and both // handler outcomes runs in the fixed matrix. The generated campaign varies -// the changed number (1..50) and nonempty note (1..8 chars). Sequence is needed +// the changed number (1..50, or -0) and the note (1..8 chars, or the +// original note, so a reinsert can differ from the original row only by a +// zero's sign; reach `sign-only-reinsert`). Sequence is needed // to distinguish cancellation from a delivered net request; outcome separates // payload correctness from later rollback; value and note expose dropped fields. // An update of an absent key or an insert on an occupied key is invalid here. @@ -113,10 +118,12 @@ async function runSameKey( const original = { id: 1, value: 0, note: `original` } const first = { id: 1, value, note: startsAbsent ? `created` : original.note } const final = { ...first, note } - // Fields of the reinserted row that differ from the original row. - const reinsertChanges = Object.fromEntries( + // Fields of the final row that differ from the original row: a reinsert's + // or an update's changes. An update that writes a field's own value back + // records no change for it (the proxy's write law). + const changedFields = Object.fromEntries( ([`value`, `note`] as const) - .filter((field) => final[field] !== original[field]) + .filter((field) => !Object.is(final[field], original[field])) .map((field) => [field, final[field]]), ) let sync!: Parameters[`sync`]>[0] @@ -186,7 +193,7 @@ async function runSameKey( await Promise.resolve() const canceled = sequence === `insert-delete` || - (reinserts && Object.keys(reinsertChanges).length === 0) + (reinserts && Object.keys(changedFields).length === 0) if (fault === `dropped-field`) request = request.map((mutation) => ({ ...mutation, @@ -207,9 +214,7 @@ async function runSameKey( ? first : startsAbsent ? final - : reinserts - ? reinsertChanges - : { value, note }, + : changedFields, }, ] expect(request, `net authored payload`).toStrictEqual(expected) @@ -280,22 +285,39 @@ describe(`Same-key transaction laws`, () => { it.each([505201, undefined])( `varies merged request fields, seed=%s`, async (seed) => { + const reach = createOracleReach( + `collection-state.same-key`, + [`sign-only-reinsert`], + { minimum: 3 }, + ) await fc.assert( fc.asyncProperty( fc.record({ sequence: fc.constantFrom(...sameKeySequences), success: fc.boolean(), - value: fc.integer({ min: 1, max: 50 }), - note: fc - .string({ minLength: 1, maxLength: 8 }) - .map((s) => `new:${s}`), + value: fc.oneof(fc.integer({ min: 1, max: 50 }), fc.constant(-0)), + note: fc.oneof( + fc.string({ minLength: 1, maxLength: 8 }).map((s) => `new:${s}`), + fc.constant(`original`), + ), }), - (scenario) => runSameKey(scenario), + (scenario) => { + reach.beginHistory() + if ( + scenario.sequence === `delete-insert` && + Object.is(scenario.value, -0) && + scenario.note === `original` + ) { + reach.reach(`sign-only-reinsert`) + } + return runSameKey(scenario) + }, ), seed === undefined ? oraclePropertyOptions(40, `collection-state.same-key`) : { numRuns: oracleRuns(40), seed }, ) + reach.assertReached() }, ) it.each([`dropped-field`, `canceled-persistence`] as const)( diff --git a/packages/db/tests/query/effect-callback-oracle.ts b/packages/db/tests/query/effect-callback-oracle.ts index 333a98959e..8544b65d0c 100644 --- a/packages/db/tests/query/effect-callback-oracle.ts +++ b/packages/db/tests/query/effect-callback-oracle.ts @@ -8,10 +8,15 @@ * An authored scalar source version independently predicts each readable * payload. The real query projection creates classes/Sets. The driver invokes * production. The finite grammar crosses payload representation, callback profile and initial - * suppression. Every history updates before and after callback replacement, - * then exits, reenters and updates again. An update of an absent source is not - * legal. These eager single-source histories do not cover ordered repairs or - * arbitrary provider schedules. Both receiving drivers use this same judgment. + * suppression, and an unordered or an ordered query (`orderBy` with `limit`). + * Every history updates before and after callback replacement, then exits, + * reenters and updates again. An update of an absent source is not legal. + * An ordered Effect compares an update with its previous publication, so an + * update whose projection did not change is not delivered; a `signed-zero` + * payload alternates 0 and -0, which are distinct row values (maintainer + * decision 2026-10-29), so each of its updates is delivered in both orders. + * These eager single-source histories do not cover ordered repairs across + * windows or arbitrary provider schedules. Both receiving drivers use this same judgment. * The maintainer's 2026-10-30 decision preserves unordered notifications even * for equal projections and permits key-only membership tracking. A synthesized * Set counts leaf reads outside observation capture: these unordered Effects @@ -26,17 +31,41 @@ import type { DeltaEvent, EffectConfig } from '../../src/query/effect.js' export type CallbackRow = { id: number; version: number } export type CallbackOutput = { id: number; payload: unknown } type Representation = - `scalar` | `opaque` | `private` | `prototype` | `set` | `equal` + | `scalar` + | `opaque` + | `private` + | `prototype` + | `set` + | `equal` + | `signed-zero` export const callbackHistories = ( - [`scalar`, `opaque`, `private`, `prototype`, `set`, `equal`] as const + [ + `scalar`, + `opaque`, + `private`, + `prototype`, + `set`, + `equal`, + `signed-zero`, + ] as const ).flatMap((representation) => ([`all`, `enter`, `exit`, `late-update`, `late-batch`] as const).flatMap( (profile) => - [false, true].map((skipInitial) => ({ - representation, - profile, - skipInitial, - })), + [false, true].flatMap((skipInitial) => + // Open: an ordered Effect compares an update by its enumerable fields, + // so it drops one whose change is private or non-enumerable state + // (coverage map, Effect callback delivery). Those pairs wait on the + // maintainer's decision about ordered equality. + (representation === `private` || representation === `prototype` + ? [false] + : [false, true] + ).map((ordered) => ({ + representation, + profile, + skipInitial, + ordered, + })), + ), ), ) export type CallbackHistory = (typeof callbackHistories)[number] @@ -47,6 +76,8 @@ export type CallbackHandlers = Pick< export interface CallbackDriver { mount: (options: { skipInitial: boolean + /** The query orders by `id` with a limit, so the Effect is ordered. */ + ordered: boolean project: (row: CallbackRow) => CallbackOutput handlers: CallbackHandlers }) => Promise<{ @@ -76,7 +107,7 @@ const cuts = [ `disposed`, ] export function callbackHistoryId(h: CallbackHistory) { - return `${h.representation}/${h.profile}/${h.skipInitial}` + return `${h.representation}/${h.profile}/${h.skipInitial}/${h.ordered ? `ordered` : `unordered`}` } export function callbackReach(path: string) { return createOracleReach( @@ -121,7 +152,11 @@ function expectedPayload( ? `0,0,${version}` : representation === `equal` ? `0` - : String(version) + : representation === `signed-zero` + ? version % 2 === 1 + ? `-0` + : `0` + : String(version) } function readPayload(payload: unknown): string { if (payload instanceof PrivatePayload || payload instanceof PrototypePayload) @@ -131,10 +166,11 @@ function readPayload(payload: unknown): string { .map((v: { value: number }) => v.value) .sort() .join(`,`) - return String(payload) + return Object.is(payload, -0) ? `-0` : String(payload) } function expectedDelivery( representation: Representation, + ordered: boolean, before: number | undefined, after: number | undefined, channels: ReadonlySet, @@ -143,6 +179,15 @@ function expectedDelivery( if (suppressed || before === after) return [] const type = before === undefined ? `enter` : after === undefined ? `exit` : `update` + // An ordered Effect delivers an update only when its projection changed. + if ( + ordered && + type === `update` && + expectedPayload(representation, before!) === + expectedPayload(representation, after!) + ) { + return [] + } const event = { type, value: expectedPayload(representation, after ?? before!), @@ -216,6 +261,8 @@ export async function runCallbackHistory( if (history.representation === `private`) payload = new KeyedPrivatePayload(version) if (history.representation === `equal`) payload = 0 + if (history.representation === `signed-zero`) + payload = version % 2 === 1 ? -0 : 0 if (history.representation === `prototype`) { payload = new PrototypePayload(version) } @@ -239,6 +286,7 @@ export async function runCallbackHistory( observed, expectedDelivery( history.representation, + history.ordered, before, after, channels, @@ -255,6 +303,7 @@ export async function runCallbackHistory( await checkWithCleanup(async () => { handle = await driver.mount({ skipInitial: history.skipInitial, + ordered: history.ordered, project, handlers: handlers(), }) @@ -288,10 +337,13 @@ export async function runCallbackHistory( callbackHistoryId(history), comparisonReads, ) - try { - expect(comparisonReads, `unordered output leaf traversal`).toBe(0) - } catch (error) { - failures.push(error) + // An ordered Effect compares values, so it may read them. + if (!history.ordered) { + try { + expect(comparisonReads, `unordered output leaf traversal`).toBe(0) + } catch (error) { + failures.push(error) + } } if (failures.length) throw new AggregateError(failures, `Effect callback delivery failed`, { diff --git a/packages/db/tests/query/virtual-row-fields-effect-oracle.test.ts b/packages/db/tests/query/virtual-row-fields-effect-oracle.test.ts index 6456720898..c93714938c 100644 --- a/packages/db/tests/query/virtual-row-fields-effect-oracle.test.ts +++ b/packages/db/tests/query/virtual-row-fields-effect-oracle.test.ts @@ -309,7 +309,7 @@ test.each(callbackHistories)( await runCallbackHistory( history, { - mount: async ({ skipInitial, project, handlers }) => { + mount: async ({ skipInitial, ordered, project, handlers }) => { const source = createCollection( mockSyncCollectionOptions({ id: `callback-core`, @@ -319,10 +319,12 @@ test.each(callbackHistories)( ) await source.preload() const config: EffectConfig = { - query: () => - new Query() - .from({ row: source }) - .fn.select(({ row }) => project(row)), + query: () => { + const rows = new Query().from({ row: source }) + return ( + ordered ? rows.orderBy(({ row }) => row.id).limit(10) : rows + ).fn.select(({ row }) => project(row)) + }, skipInitial, ...handlers, } diff --git a/packages/react-db/tests/effect-callback-oracle.test.tsx b/packages/react-db/tests/effect-callback-oracle.test.tsx index 8608fb34be..4c11249722 100644 --- a/packages/react-db/tests/effect-callback-oracle.test.tsx +++ b/packages/react-db/tests/effect-callback-oracle.test.tsx @@ -39,7 +39,7 @@ test.each(callbackHistories)( await runCallbackHistory( history, { - mount: async ({ skipInitial, project, handlers }) => { + mount: async ({ skipInitial, ordered, project, handlers }) => { const source = createCollection( mockSyncCollectionOptions({ id: `callback-react`, @@ -48,9 +48,10 @@ test.each(callbackHistories)( }), ) await source.preload() - const query = new Query() - .from({ row: source }) - .fn.select(({ row }) => project(row)) + const rows = new Query().from({ row: source }) + const query = ( + ordered ? rows.orderBy(({ row }) => row.id).limit(10) : rows + ).fn.select(({ row }) => project(row)) const mounted = renderHook( ({ config }: { config: EffectConfig }) => useLiveQueryEffect(config), From aaebae847da5649e8ff7760937c91a941e37b24d Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 11:10:09 -0700 Subject: [PATCH 31/64] fix(convex): awaitSync waits through a cached error and never re-reports a failure Review finding CR2 and a sibling found by the same witness. - CR2: after the client replayed a cached error, the flag cleared and getCurrentValue() read that stale error again, so an awaitSync (or a wrapped mutation handler) failed the restarted run. A subscription now has no current value until the server's first callback; the replay is skipped once. - Sibling (HEALTH-2): while the latest result was a failure, awaitSync re-read it and reported it again as a new value (the client builds a new error on each read, and a rejected result is rejected anew). refresh() now accepts only a result it has not received, and rejects with the reported failure (law set EVID-1: awaitSync rejects with that failure). Oracle first: new shared capability and step await-sync (EVID-1): a wait pending before the run's first accepted result, fulfilled while the latest result is accepted, rejected while it is a failure, after a failure, or at retirement. Reach sync-wait-pending and sync-wait-on-failure. The Convex driver declares it. Evidence: - RED on f06ee8fdd: [fail, restart, sync-wait] left the run in error; [snapshot, delivery-error, sync-wait] listed the error twice; the same for a rejected delivery. - GREEN: Convex 115/115, all three campaigns. - Mutants rejected: the replay clearing the stale flag (EVID-1), refresh reporting the failure again, refresh re-accepting a received result (EVID-1, health failureReason). - Size: convex-db-collection +42 gzip bytes (+116 on the branch). Co-authored-by: Isaac --- .changeset/convex-failed-load-release.md | 2 +- packages/convex-db-collection/src/options.ts | 23 +++++- .../convex-db-collection/src/subscription.ts | 18 ++-- .../tests/conformance-driver.ts | 5 ++ .../src/eager-live-source-oracle.ts | 82 +++++++++++++++++++ 5 files changed, 120 insertions(+), 10 deletions(-) diff --git a/.changeset/convex-failed-load-release.md b/.changeset/convex-failed-load-release.md index 7b1c99bb81..e164445469 100644 --- a/.changeset/convex-failed-load-release.md +++ b/.changeset/convex-failed-load-release.md @@ -2,4 +2,4 @@ '@tanstack/convex-db-collection': patch --- -Release the Convex query subscription when a Collection's first load fails, or when a later acceptance failure stops the sync run, instead of keeping it until cleanup. A new subscription no longer fails from an error the Convex client cached for its query before it subscribed, such as a restart right after a failed run; it waits for the server's result. +Release the Convex query subscription when a Collection's first load fails, or when a later acceptance failure stops the sync run, instead of keeping it until cleanup. A new subscription no longer fails from an error the Convex client cached for its query before it subscribed, such as a restart right after a failed run; it waits for the server's result. `awaitSync` during that wait no longer fails the run from the cached error. While the latest result is a failure, `awaitSync` rejects with the reported failure instead of reporting it again. diff --git a/packages/convex-db-collection/src/options.ts b/packages/convex-db-collection/src/options.ts index d998c36a63..2e0afbf307 100644 --- a/packages/convex-db-collection/src/options.ts +++ b/packages/convex-db-collection/src/options.ts @@ -178,7 +178,10 @@ export function createConvexOptions< } }, fail) } + // The latest result the source delivered, accepted or rejected. + let received: TResult | undefined const receive = (result: TResult): void => { + received = result try { accept(result) } catch (error) { @@ -204,10 +207,24 @@ export function createConvexOptions< // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (!active) throw new Error(`Convex sync ended before source acceptance.`) - const value = subscription.getCurrentValue() - if (value !== undefined) accept(value) + let value: TResult | undefined + try { + value = subscription.getCurrentValue() + } catch (error) { + // The latest result is a failure already reported: the client + // builds a new error on each read, so reject with the reported + // one rather than report it again (EVID-1, HEALTH-2). + if (syncError !== undefined) throw syncError + throw error + } + // A result already received is not delivered again: a rejected one + // keeps its reported failure. + if (value !== undefined && value !== received) { + received = value + accept(value) + } } catch (error) { - fail(error) + if (error !== syncError) fail(error) throw error } if (!initialized) await firstResult diff --git a/packages/convex-db-collection/src/subscription.ts b/packages/convex-db-collection/src/subscription.ts index d1fe656d43..f96fa660c4 100644 --- a/packages/convex-db-collection/src/subscription.ts +++ b/packages/convex-db-collection/src/subscription.ts @@ -29,8 +29,11 @@ export function subscribeQuery>( let active = true // The client serves a query's cached result to a new subscription at // once. A cached error belongs to an earlier subscription, so this one - // waits for the server instead (maintainer decision). + // waits for the server instead (maintainer decision): until the server's + // first callback it has no current value, and the client's replay of the + // cached error is skipped once. let cachedError = false + let replay = false const stop = (unsubscribe: () => void) => () => { active = false unsubscribe() @@ -54,16 +57,19 @@ export function subscribeQuery>( query, args, (value) => { - cachedError = false + cachedError = replay = false if (active) receive(value) }, (error) => { - // The client's replay of the cached error. - if (cachedError) cachedError = false - else if (active) fail(error) + if (replay) { + replay = false + return + } + cachedError = false + if (active) fail(error) }, ) - cachedError = hasCachedError(() => subscription.getCurrentValue()) + cachedError = replay = hasCachedError(() => subscription.getCurrentValue()) return { getCurrentValue: () => cachedError ? undefined : subscription.getCurrentValue(), diff --git a/packages/convex-db-collection/tests/conformance-driver.ts b/packages/convex-db-collection/tests/conformance-driver.ts index 6a65aa6b18..b56cccb14e 100644 --- a/packages/convex-db-collection/tests/conformance-driver.ts +++ b/packages/convex-db-collection/tests/conformance-driver.ts @@ -34,6 +34,9 @@ * a failure it acknowledges it only with its next transition, so a restart * can find the failed query's cached error. * + * `awaitSync` is the adapter's wait for the query's current result + * (`await-sync`). + * * Rows are Convex values: `note` absent is a missing field, and typed fields * hold the Convex kinds (no Date, which Convex values do not have). The * driver never reads the Collection. @@ -332,6 +335,7 @@ export function convexConformanceDriver({ `complete-snapshot`, `delivery-error`, `duplicate-key-delivery`, + `await-sync`, ]), changeCommits: `per-batch`, duplicateInsert: `error`, @@ -360,6 +364,7 @@ export function convexConformanceDriver({ key: (key) => key as number, }), provider, + awaitSync: () => collection.utils.awaitSync(), settle: provider.settle, dispose: provider.dispose, } diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 9212c5c6ca..2cbd5c7b54 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -84,6 +84,11 @@ * clears it. Under `rejectedDelivery: reported` (the law set's default * `errorMapping`), a delivery the adapter rejects after ready is reported * the same way, with the adapter's own error. + * - **EVID-1** (capability `await-sync`). An `awaitSync` call waits for the + * run's first accepted result, fulfills while the latest result is + * accepted, and rejects while it is a failure, after the run failed, or + * when the run retires. A result the client cached for an earlier + * subscription, such as a failed run's error, is not the new run's. * - **RECON-1** (capability `transport-reconnect`). A transport disconnect and * reconnect keep status and rows, and later deliveries still apply. * - **RENEW-1** (capability `capability-renewal`). While the source's @@ -201,6 +206,7 @@ export type EagerLiveSourceCapability = | `source-failure-after-ready` | `empty-response` | `delivery-error` + | `await-sync` /** * The public Collection surface the oracle reads. A driver passes the @@ -287,6 +293,11 @@ export interface EagerLiveSourceFixture { readonly provider: ControlledProvider /** The adapter's `awaitTxId` utility (capability `await-evidence`). */ awaitTxId?: (txid: string, timeout?: number) => Promise + /** + * The adapter's wait for its source's current result to be accepted + * (capability `await-sync`), such as Convex's `awaitSync`. + */ + awaitSync?: () => Promise /** Releases fixture resources after the history's final cleanup. */ dispose?: () => void | Promise /** @@ -464,6 +475,7 @@ export type EagerLiveSourceStep = | { readonly kind: `renew` } | { readonly kind: `empty-response` } | { readonly kind: `delivery-error` } + | { readonly kind: `sync-wait` } | { readonly kind: `restart` } /** @@ -738,6 +750,12 @@ export function eagerLiveSourceHistoryArb( arbitrary: fc.constant({ kind: `empty-response` as const }), }) } + if (capabilities.has(`await-sync`)) { + steps.push({ + weight: 2, + arbitrary: fc.constant({ kind: `sync-wait` as const }), + }) + } if (capabilities.has(`delivery-error`)) { steps.push({ weight: 3, @@ -890,6 +908,8 @@ interface ModelState { transient: Array /** Consecutive failed attempts after ready in the current outage. */ outage: number + /** Each `awaitSync` call and its expected settlement, across sync runs. */ + syncWaits: Array<{ run: number; expected: WaitExpectation }> /** Each wait, and the law that rejects it when it must reject. */ waits: Array<{ run: number @@ -909,6 +929,7 @@ export interface ExpectedObservation { failureReason: ReadonlyArray preload: `pending` | `fulfilled` | { rejectedWith: number | `adapter` } waits: ReadonlyArray + syncWaits: ReadonlyArray /** Unreleased provider subscriptions: one while the provider runs. */ /** Undefined when no law fixes it: after the adapter rejected a delivery. */ openSubscriptions: number | undefined @@ -1050,6 +1071,7 @@ function initialModel(seed: ReadonlyArray = []): ModelState { transient: [], outage: 0, waits: [], + syncWaits: [], } } @@ -1079,6 +1101,11 @@ function retireWaits(state: ModelState): void { wait.rejectLaw = `EVID-2` } } + for (const wait of state.syncWaits) { + if (wait.run === state.run && wait.expected === `pending`) { + wait.expected = `rejected` + } + } } /** @@ -1097,6 +1124,12 @@ function openGate(state: ModelState): void { wait.expected = `fulfilled` } } + // EVID-1: the first accepted result fulfills the run's `awaitSync` waits. + for (const wait of state.syncWaits) { + if (wait.run === state.run && wait.expected === `pending`) { + wait.expected = `fulfilled` + } + } } function rowState(rows: ReadonlyMap): RowState { @@ -1224,6 +1257,7 @@ type AcceptedStep = | { kind: `renew` } | { kind: `empty-response`; resetOutage: boolean } | { kind: `delivery-error`; failure: number } + | { kind: `sync-wait` } | { kind: `restart` } /** @@ -1538,6 +1572,23 @@ function stepModel( state.renewed = true return { done: { kind: `renew` }, publishable: [] } } + case `sync-wait`: { + // EVID-1 (`awaitSync`): while the run's latest source result is a + // failure the wait rejects with it; it fulfills after an accepted + // result, so before the first one it waits. A result the client + // cached for an earlier subscription is not this run's (maintainer + // decision 2026-11-02). + state.syncWaits.push({ + run: state.run, + expected: + state.phase === `loading` + ? `pending` + : state.phase === `ready` && state.transient.length === 0 + ? `fulfilled` + : `rejected`, + }) + return { done: { kind: `sync-wait` }, publishable: [] } + } case `delivery-error`: { if (!canDeliver(state) || state.phase !== `ready`) return undefined const failure = failures.next++ @@ -1599,6 +1650,7 @@ function expectedObservation(state: ModelState): ExpectedObservation { ? { rejectedWith: state.failure! } : `fulfilled`, waits: state.waits.map((wait) => wait.expected), + syncWaits: state.syncWaits.map((wait) => wait.expected), // LIFE-10: a failed provider's source is released at its end. openSubscriptions: running ? 1 @@ -1638,6 +1690,8 @@ export const eagerLiveSourceReachStates = [ `delivery-error-reported`, `delivery-error-recovered`, `rejected-delivery-reported`, + `sync-wait-pending`, + `sync-wait-on-failure`, `transient-failure-recovered`, `reconnect`, `renewal`, @@ -1681,6 +1735,8 @@ const capabilityOfState: Partial< 'delivery-error-reported': `delivery-error`, 'delivery-error-recovered': `delivery-error`, 'rejected-delivery-reported': `duplicate-key-delivery`, + 'sync-wait-pending': `await-sync`, + 'sync-wait-on-failure': `await-sync`, 'transient-failure-repeated': `retry-policy`, 'transient-failure-recovered': `retry-policy`, reconnect: `transport-reconnect`, @@ -1811,6 +1867,8 @@ function lawFor(done: AcceptedStep, wasReady: boolean): string { return done.beforeReady ? `READY-10/PUB-9` : `PUB-3/FID-4` case `wait`: return done.timeout ? `EVID-3` : `EVID-1` + case `sync-wait`: + return `EVID-1` case `fail`: return wasReady ? `HEALTH-1` : `READY-4` case `malformed`: @@ -1921,6 +1979,7 @@ export async function checkEagerLiveSourceHistory( const failures: Array = [] const failureCounter = { next: 0 } const waits: Array<{ current: Settlement }> = [] + const syncWaits: Array<{ current: Settlement }> = [] if (listenBeforeFetch) await provider.seed?.(structuredClone(seedFrom(0))) // READY-3, READY-10: the rows at the moment status becomes `ready`. let readyRows: RowState | undefined @@ -2145,6 +2204,18 @@ export async function checkEagerLiveSourceHistory( want, typeof got === `object` ? { rejected: String(got.rejected) } : got, ) + expected.syncWaits.forEach((wantWait, index) => { + const settled = syncWaits[index]!.current + const gotKind = typeof settled === `object` ? `rejected` : settled + if (gotKind !== wantWait) { + throw violation( + `EVID-1 (awaitSync ${index})`, + stepIndex, + wantWait, + gotKind, + ) + } + }) expected.waits.forEach((wantWait, index) => { const gotWait = waits[index]!.current const gotKind = typeof gotWait === `object` ? `rejected` : gotWait @@ -2309,6 +2380,15 @@ export async function checkEagerLiveSourceHistory( reach(`wait-before-evidence-fulfilled`) } break + case `sync-wait`: { + syncWaits.push(recordSettlement(fixture.awaitSync!())) + if (state.syncWaits.at(-1)!.expected === `pending`) { + reach(`sync-wait-pending`) + } else if (state.syncWaits.at(-1)!.expected === `rejected`) { + reach(`sync-wait-on-failure`) + } + break + } case `wait`: { const pending = fixture.awaitTxId!( String(done.txid), @@ -2546,6 +2626,8 @@ function supports( ) case `wait`: return driver.capabilities.has(`await-evidence`) && !!fixture.awaitTxId + case `sync-wait`: + return driver.capabilities.has(`await-sync`) && !!fixture.awaitSync case `transient-failure`: return ( driver.capabilities.has(`retry-policy`) && From 370e029db65eb4cf975d9017d87a7c593d059f94 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 11:10:12 -0700 Subject: [PATCH 32/64] test: declare zero-sign publication reach; fix the Neon config pointer Review findings CR9 (part) and CR10. The fixed-campaign zero-sign check in the lifecycle publication oracle now uses createOracleReach with a minimum of five instead of a counter above zero (ORC-015). The db-adapter-conformance vite config pointed to a missing tests/neon/README.md; it now names the README's Neon Realtime driver entry. Co-authored-by: Isaac --- packages/db-adapter-conformance/vite.config.ts | 3 ++- ...tion-lifecycle-publication-oracle.property.test.ts | 11 +++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/db-adapter-conformance/vite.config.ts b/packages/db-adapter-conformance/vite.config.ts index e94ee7e1f7..5377269f36 100644 --- a/packages/db-adapter-conformance/vite.config.ts +++ b/packages/db-adapter-conformance/vite.config.ts @@ -9,7 +9,8 @@ const config = defineConfig({ environment: `node`, // The Neon receiving driver runs a vendored adapter that targets an older // TanStack DB sync API. It has its own config and is expected to fail - // until that adapter is updated (see tests/neon/README.md). + // until that adapter is updated (see "Neon Realtime" under Drivers in + // README.md). exclude: [`**/node_modules/**`, `tests/neon/**`, `**/neon-*.test.ts`], coverage: { enabled: true, provider: `istanbul`, include: [`src/**/*`] }, }, diff --git a/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts b/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts index fdfe5287a6..88b2f2ea24 100644 --- a/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts @@ -2974,14 +2974,21 @@ describe(`CollectionSubscription lifecycle publication oracle`, () => { // carried row): the random grammar inserts source writes into green // lifecycle histories. The pinned zero-sign history below covers that path. it(`reaches a zero-sign source write in the fixed campaign`, async () => { - zeroSignWrites = 0 + const reach = createOracleReach( + `lifecycle-publication.zero-sign`, + [`zero-sign-write`], + { minimum: 5 }, + ) for (const history of fc.sample(publicationCommandHistoryArbitrary, { seed: 1_657_005, numRuns: runs, })) { + reach.beginHistory() + const before = zeroSignWrites await runPublicationHistory(history) + if (zeroSignWrites > before) reach.reach(`zero-sign-write`) } - expect(zeroSignWrites).toBeGreaterThan(0) + reach.assertReached() }, 120_000) fcTest.prop([publicationCommandHistoryArbitrary], { numRuns: runs, From 9c0f0598aa65cbda2d47e11b29d5464d81454cc4 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 11:51:54 -0700 Subject: [PATCH 33/64] fix(db): -0 inside typed arrays, and ordered Effects compare class instances by identity Maintainer decisions of 2026-11-02 on three review findings. - Typed arrays (CR5): -0 and 0 are distinct inside them as row values too, as for plain numbers. The row comparison checks typed-array elements with Object.is. - Ordered Effects (sibling of CR8): an ordered Effect compared an update by enumerable fields, so it dropped one whose change was private or non-enumerable state, and the consumer kept a stale value. Its comparison (samePublishedValue) now treats a class instance as equal only to itself. - Reload give-up (CR3): a give-up while a reload's first commit is being accepted fails at once, because the Collection is already ready; READY-9's deferral applies to the first load. No behavior change; the Electric comment says why it checks Collection status. deepEqualsInternal's signedZero flag becomes a mode: row (row values, -0 distinct) or publication (also class instances by identity), so recursive calls carry it unchanged. Oracles first: - Shared eager oracle: value kind float-array (a Float64Array), with zeros flipped on a retouch; the reference adapter declares it. - effect-callback-oracle: the ordered private and prototype pairs are generated again. - optimistic-transaction-oracle: a weighted branch for a sign-only reinsert; at random only one history in 20 reached it, below its reach minimum. Evidence: - RED on 370e029db: reference campaign PUB-2 after a retouch that flipped a zero inside a Float64Array; 20 ordered private/prototype callback histories. - GREEN: db 25,528; react-db 4,348; solid 342; conformance 81; Electric, Convex, TrailBase, RxDB, PowerSync, sqlite persistence core, query-db. - Size: db +47 gzip bytes this round (+69 on the branch); app +67. Co-authored-by: Isaac --- .changeset/sync-negative-zero-change.md | 2 +- docs/contributing/oracle-coverage.md | 28 ++++++++----- .../src/eager-live-source-oracle.ts | 16 ++++++++ .../tests/reference-adapter.ts | 2 + packages/db/src/proxy.ts | 2 +- packages/db/src/query/effect.ts | 10 +++-- packages/db/src/utils.ts | 41 +++++++++++++------ ...mistic-transaction-oracle.property.test.ts | 39 ++++++++++++++---- .../db/tests/query/effect-callback-oracle.ts | 14 +++---- .../electric-db-collection/src/electric.ts | 5 ++- 10 files changed, 110 insertions(+), 49 deletions(-) diff --git a/.changeset/sync-negative-zero-change.md b/.changeset/sync-negative-zero-change.md index 753c43b4d5..58295daeb9 100644 --- a/.changeset/sync-negative-zero-change.md +++ b/.changeset/sync-negative-zero-change.md @@ -2,4 +2,4 @@ '@tanstack/db': patch --- -Treat `-0` and `0` as distinct row values. A synced update, a replay that replaces a subscription's rows, and an optimistic update that change a field only from `-0` to `0` (or back) now publish a change; before, the Collection could store the new value while subscribers kept the old one. A draft write of `0` over `-0` is now recorded as a change instead of being dropped. A transaction that deletes a row and reinserts it with only a zero's sign changed now sends that update instead of dropping it, and an ordered Effect now delivers an update that changes only a zero's sign. +Treat `-0` and `0` as distinct row values. A synced update, a replay that replaces a subscription's rows, and an optimistic update that change a field only from `-0` to `0` (or back) now publish a change; before, the Collection could store the new value while subscribers kept the old one. A draft write of `0` over `-0` is now recorded as a change instead of being dropped. A transaction that deletes a row and reinserts it with only a zero's sign changed now sends that update instead of dropping it, and an ordered Effect now delivers an update that changes only a zero's sign. The same holds inside typed arrays. An ordered Effect also delivers an update whose value is a class instance, which can change in private state its enumerable fields do not show. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index bae2456f42..567b5f0906 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6120,14 +6120,13 @@ crosses an unordered and an ordered query (`orderBy` with `limit`) and adds a decision 2026-10-29); 120 histories on each path. An ordered Effect delivers an update only when its projection changed. On unchanged `f06ee8fdd` the ordered `signed-zero` histories fail (an update that flips only a zero's sign was -dropped); comparing with `sameRowValue` repairs them. Open, for a maintainer -decision: an ordered Effect compares an update by enumerable fields, so it -drops an update whose change is private or non-enumerable state, and the -consumer keeps a stale value. Trace: `private` or `prototype` payload, -ordered, any profile, update version 0 to 1 at the `unobserved` cut, expected -an update, received none; these 20 ordered pairs are not generated until the -decision (owner: this callback-profile oracle; witness: the same histories -with `ordered: true`). +dropped); comparing with `sameRowValue` repairs them. An ordered Effect also +dropped an update whose change is private or non-enumerable state, so the +consumer kept a stale value (`private` and `prototype` payloads, 20 ordered +histories, RED on `370e029db`). Maintainer decision 2026-11-02: the ordered +comparison (`samePublishedValue`) treats a class instance as equal only to +itself, so its update is delivered; an ordered repair may then deliver an +update for an unchanged class instance. ### RC bound schema mutation receivers (2026-10-28) @@ -6591,8 +6590,8 @@ READY-4 and HEALTH-1 until that adapter moves to `fail()`; probes show LIFE-10 (subscription kept open after an error state) behind them. Limits: no on-demand loading, mutation handlers, or reload; publications are observed through `subscribeChanges` only; EVID-3 does not distinguish a timed wait -that rejects before its timeout; FID-1 covers ten value kinds, not Temporal, -Map, Set, typed arrays, class instances, or field identity; PUB-4 covers a +that rejects before its timeout; FID-1 covers eleven value kinds (with `Float64Array`), not +Temporal, Map, Set, other typed arrays, class instances, or field identity; PUB-4 covers a complete delivery with two rows for one key, under each driver's `duplicateInsert` (TrailBase `replace`, reference `error`), not malformed keys, undecodable rows, or FID-3 out-of-domain values; the Neon and TrailBase claims cover @@ -6676,7 +6675,14 @@ real Electric server; without an app `onError`, the driver detects a give-up from the adapter's log, so an adapter that stops logging times out rather than failing a law. The shared check refuses `retryReportAfter` above 1 with `transport-reconnect` or `capability-renewal`: no law fixes -whether a reconnect or a renewal ends an outage a threshold is counting. +whether a reconnect or a renewal ends an outage a threshold is counting. Typed arrays (maintainer decision 2026-11-02): -0 and 0 +are distinct inside them as row values too; the `float-array` value kind +(a `Float64Array`) makes the reference driver the generated witness (RED on +`370e029db`: PUB-2 after a retouch that flipped a zero inside one). A +reload's give-up while its first commit's acceptance is pending fails at once +when the Collection is already ready (maintainer decision 2026-11-02: READY-9's +deferral applies to the first load); no generated history reloads with a held +acceptance. Convex (`adapter-conformance.eager-live-source.convex`, eager mode over the real Convex JavaScript client and a controlled WebSocket that answers the sync protocol, complete results, per-batch commits, Convex value kinds as diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 2cbd5c7b54..5d585c63e9 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -178,6 +178,8 @@ export type ValueKind = | `bigint` | `array` | `object` + /** A `Float64Array`: its elements follow the number rules, -0 included. */ + | `float-array` export const jsonValueKinds: ReadonlyArray = [ `string`, @@ -518,6 +520,13 @@ const valueExamples: Record unknown>> = { () => ({ nested: { deep: [1, { k: null }] } }), () => ({ '': false }), ], + // -0 and 0 are distinct inside typed arrays too (maintainer decision + // 2026-11-02). + 'float-array': [ + () => Float64Array.of(0), + () => Float64Array.of(-0, 1.5), + () => new Float64Array(0), + ], } /** A value the driver's backend cannot store as given; see `excludesValue`. */ @@ -969,6 +978,9 @@ export function describeValue(value: unknown): string { if (Array.isArray(value) && prototype === Array.prototype) { return `[${value.map(describeValue).join(`,`)}]` } + if (prototype === Float64Array.prototype) { + return `f[${[...(value as Float64Array)].map(describeValue).join(`,`)}]` + } if (prototype === Object.prototype || prototype === null) { const fields = Object.keys(value) .sort() @@ -988,6 +1000,7 @@ export function describeValue(value: unknown): string { /** The kind of a generated value, for reach. */ function kindOf(value: unknown): ValueKind { if (value === null) return `null` + if (value instanceof Float64Array) return `float-array` if (describeValue(value).includes(`n-0`)) return `negative-zero` if (typeof value === `number`) { return Number.isInteger(value) ? `integer` : `float` @@ -1142,6 +1155,9 @@ function flipZeros(value: unknown): unknown { return value === 0 ? (Object.is(value, -0) ? 0 : -0) : value } if (Array.isArray(value)) return value.map(flipZeros) + if (value instanceof Float64Array) { + return value.map((item) => flipZeros(item) as number) + } if ( typeof value === `object` && value !== null && diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index ed158fcea8..e717c12ab3 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -667,6 +667,7 @@ export function referenceDriver( `bigint`, `array`, `object`, + `float-array`, ], valueKinds: [ `string`, @@ -679,6 +680,7 @@ export function referenceDriver( `bigint`, `array`, `object`, + `float-array`, ], changeCommits: perChange ? `per-change` : `per-batch`, // A delivery holding two rows with one key fails the run (PUB-4). diff --git a/packages/db/src/proxy.ts b/packages/db/src/proxy.ts index ec5116f476..29cc386f80 100644 --- a/packages/db/src/proxy.ts +++ b/packages/db/src/proxy.ts @@ -364,7 +364,7 @@ function draftValuesEqual( right: unknown, paired?: Map, ): boolean { - return deepEqualsInternal(left, right, paired, true, false, true) + return deepEqualsInternal(left, right, paired, true, false, `row`) } /** diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 975901253b..3d4aa3b94e 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -3,7 +3,7 @@ import { createSourceRecord } from '../utils/source-record.js' import { createDeferred } from '../deferred.js' import { runAllCallbacks } from '../utils/callbacks.js' import { normalizeError } from '../utils/error.js' -import { sameRowValue } from '../utils.js' +import { samePublishedValue } from '../utils.js' import { codedMessage, devBuild } from '../error-message.js' import { buildHealthTree, healthNode } from '../collection/health-tree.js' import { isFailedStatus } from '../live-query-primitives.js' @@ -1346,11 +1346,13 @@ function classifyDelta( wasPresent && isPresent && insertValue !== undefined && - (publishedRows instanceof Set || !sameRowValue(previousValue, insertValue)) + (publishedRows instanceof Set || + !samePublishedValue(previousValue, insertValue)) ) { // Unordered Effects preserve all graph updates without comparing values. - // Ordered repairs compare with the preceding coherent publication; -0 - // and 0 are distinct row values (FID-1). + // Ordered repairs compare with the preceding coherent publication: -0 + // and 0 are distinct row values (FID-1), and a class instance equals + // only itself. return { type: `update`, key, diff --git a/packages/db/src/utils.ts b/packages/db/src/utils.ts index e71f028714..fc12936093 100644 --- a/packages/db/src/utils.ts +++ b/packages/db/src/utils.ts @@ -50,7 +50,17 @@ export function deepEquals(a: any, b: any): boolean { * tells -0 from 0, as row values do (FID-1; maintainer decision 2026-10-29). */ export function sameRowValue(a: unknown, b: unknown): boolean { - return deepEqualsInternal(a, b, undefined, false, false, true) + return deepEqualsInternal(a, b, undefined, false, false, `row`) +} + +/** + * @internal Whether an ordered Effect's update repeats its previous + * publication: row-value equality, except that a class instance equals only + * itself, since it can hold readable state outside its enumerable fields + * (maintainer decision 2026-11-02). + */ +export function samePublishedValue(a: unknown, b: unknown): boolean { + return deepEqualsInternal(a, b, undefined, false, false, `publication`) } /** @internal Compare persisted public and durable value shapes before skipping a reload. */ @@ -86,11 +96,13 @@ export function deepEqualsInternal( visited: Map | undefined, draft = false, persistedSnapshot = false, - // Row change detection tells -0 from 0; general equality does not. - signedZero = false, + // Row change detection (`row`) tells -0 from 0; general equality does not. + // An ordered Effect's publication (`publication`) also compares a class + // instance by identity. + rowValue: false | `row` | `publication` = false, ): boolean { // Handle strict equality (primitives, same reference) - if (signedZero ? Object.is(a, b) : a === b || Object.is(a, b)) return true + if (rowValue ? Object.is(a, b) : a === b || Object.is(a, b)) return true // Handle null/undefined if (a == null || b == null) return false @@ -151,7 +163,7 @@ export function deepEqualsInternal( visited, true, persistedSnapshot, - signedZero, + rowValue, ) : b.has(key) && deepEqualsInternal( @@ -160,7 +172,7 @@ export function deepEqualsInternal( visited, draft, persistedSnapshot, - signedZero, + rowValue, ), ) @@ -194,7 +206,7 @@ export function deepEqualsInternal( visited, draft, persistedSnapshot, - signedZero, + rowValue, ), ) visited.delete(a) @@ -225,7 +237,7 @@ export function deepEqualsInternal( candidateVisited, draft, persistedSnapshot, - signedZero, + rowValue, ) && matchValues( index + 1, @@ -258,8 +270,9 @@ export function deepEqualsInternal( ) { const typedA = a as unknown as TypedArray const typedB = b as unknown as TypedArray - // Elements compare like numbers: -0 equals 0, NaN equals NaN. Only a - // draft treats a change of typed-array class as a change. + // Elements compare like numbers: NaN equals NaN, and -0 equals 0 except + // as a row value. Only a draft treats a change of typed-array class as a + // change. if ( (draft && Object.getPrototypeOf(a) !== Object.getPrototypeOf(b)) || typedA.length !== typedB.length @@ -269,7 +282,8 @@ export function deepEqualsInternal( for (let i = 0; i < typedA.length; i++) { const x = typedA[i]! const y = typedB[i]! - if (x !== y && !(x !== x && y !== y)) return false + if (rowValue ? !Object.is(x, y) : x !== y && !(x !== x && y !== y)) + return false } return true @@ -331,7 +345,7 @@ export function deepEqualsInternal( visited, draft, persistedSnapshot, - signedZero, + rowValue, ), ) visited.delete(a) @@ -347,6 +361,7 @@ export function deepEqualsInternal( const plain = isPlainPrototype(prototype) const plainB = isPlainPrototype(prototypeB) if (prototype !== prototypeB && !plain && !plainB) return false + if (rowValue === `publication` && !(plain && plainB)) return false // Check for circular references if (visited?.has(a)) { @@ -388,7 +403,7 @@ export function deepEqualsInternal( visited, draft, persistedSnapshot, - signedZero, + rowValue, ) ) { result = false diff --git a/packages/db/tests/optimistic-transaction-oracle.property.test.ts b/packages/db/tests/optimistic-transaction-oracle.property.test.ts index 6917cfbdd5..3d3d3add79 100644 --- a/packages/db/tests/optimistic-transaction-oracle.property.test.ts +++ b/packages/db/tests/optimistic-transaction-oracle.property.test.ts @@ -292,15 +292,36 @@ describe(`Same-key transaction laws`, () => { ) await fc.assert( fc.asyncProperty( - fc.record({ - sequence: fc.constantFrom(...sameKeySequences), - success: fc.boolean(), - value: fc.oneof(fc.integer({ min: 1, max: 50 }), fc.constant(-0)), - note: fc.oneof( - fc.string({ minLength: 1, maxLength: 8 }).map((s) => `new:${s}`), - fc.constant(`original`), - ), - }), + fc.oneof( + { + weight: 4, + arbitrary: fc.record({ + sequence: fc.constantFrom(...sameKeySequences), + success: fc.boolean(), + value: fc.oneof( + fc.integer({ min: 1, max: 50 }), + fc.constant(-0), + ), + note: fc.oneof( + fc + .string({ minLength: 1, maxLength: 8 }) + .map((s) => `new:${s}`), + fc.constant(`original`), + ), + }), + }, + // A reinsert that differs from the original row only by a + // zero's sign; at random, one history in 20 is one. + { + weight: 1, + arbitrary: fc.record({ + sequence: fc.constant(`delete-insert` as const), + success: fc.boolean(), + value: fc.constant(-0), + note: fc.constant(`original`), + }), + }, + ), (scenario) => { reach.beginHistory() if ( diff --git a/packages/db/tests/query/effect-callback-oracle.ts b/packages/db/tests/query/effect-callback-oracle.ts index 8544b65d0c..cda300479b 100644 --- a/packages/db/tests/query/effect-callback-oracle.ts +++ b/packages/db/tests/query/effect-callback-oracle.ts @@ -12,7 +12,10 @@ * Every history updates before and after callback replacement, then exits, * reenters and updates again. An update of an absent source is not legal. * An ordered Effect compares an update with its previous publication, so an - * update whose projection did not change is not delivered; a `signed-zero` + * update whose projection did not change is not delivered. A class instance + * can hold readable state outside its enumerable fields, so the comparison + * treats it as equal only to itself (maintainer decision 2026-11-02): its + * update is delivered. A `signed-zero` * payload alternates 0 and -0, which are distinct row values (maintainer * decision 2026-10-29), so each of its updates is delivered in both orders. * These eager single-source histories do not cover ordered repairs across @@ -52,14 +55,7 @@ export const callbackHistories = ( ([`all`, `enter`, `exit`, `late-update`, `late-batch`] as const).flatMap( (profile) => [false, true].flatMap((skipInitial) => - // Open: an ordered Effect compares an update by its enumerable fields, - // so it drops one whose change is private or non-enumerable state - // (coverage map, Effect callback delivery). Those pairs wait on the - // maintainer's decision about ordered equality. - (representation === `private` || representation === `prototype` - ? [false] - : [false, true] - ).map((ordered) => ({ + [false, true].map((ordered) => ({ representation, profile, skipInitial, diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 5ff5fcd6ae..772c042a00 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1959,7 +1959,10 @@ function createElectricSync>( let sessionCleanedUp = false const failInitialSync = (cause: unknown) => { if (firstSnapshotCommitted) { - // A truncate commit already made the Collection ready. + // The Collection is already ready: a truncate commit made it so, or + // a reload's run started on a ready Collection. The give-up is then + // a post-ready failure; READY-9 defers only a first load's + // (maintainer decision). if (collection.status === `ready`) { gaveUp = true fail(cause) From 7fba045bbd4b4dbd60d16e25899f0a06523ab9fa Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 14:12:08 -0700 Subject: [PATCH 34/64] Apply code simplification review From the prep-pr simplification pass, each behavior-preserving: - Electric: one giveUp(cause) helper for the five give-up sites; forwardExternalAbort builds its error once; the outage's reported value is named outageFailure; a shared clearEvidence helper. - Core: deepEqualsInternal's identity fast path is (!rowValue && a === b) || Object.is(a, b). - Shared oracle: sameRows(a, b, project) replaces sameTitles and two rowHead loops; the reference mutant table, which names FID-1 (values) and FID-1 (fields), still matches every law. Checked: Electric 1243 (4 skipped); db 25,528; conformance 81; every adapter's eager campaign. Co-authored-by: Isaac --- .../src/eager-live-source-oracle.ts | 48 +++++++------- packages/db/src/utils.ts | 2 +- .../electric-db-collection/src/electric.ts | 62 +++++++++---------- 3 files changed, 53 insertions(+), 59 deletions(-) diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 5d585c63e9..357d30e810 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -2048,13 +2048,8 @@ export async function checkEagerLiveSourceHistory( if (index === -1) { // FID-1: a publication that differs from a committed state only in // a value or typed field names the value law. - const valuesOnly = publishable.some( - (rows) => - sameTitles(rows, publication) && - rows.every( - ([, value], row) => - rowHead(value) === rowHead(publication[row]![1]), - ), + const valuesOnly = publishable.some((rows) => + sameRows(rows, publication, rowHead), ) throw violation( valuesOnly ? `FID-1 (values)` : `${law} (publication)`, @@ -2109,13 +2104,14 @@ export async function checkEagerLiveSourceHistory( ) // FID-1: rows that differ only in a field, with the right keys and // titles, name the field law. - if (!sameRows(rows, expected.rows) && sameTitles(rows, expected.rows)) { - const valuesOnly = rows.every( - ([, value], index) => - rowHead(value) === rowHead(expected.rows[index]![1]), - ) + if ( + !sameRows(rows, expected.rows) && + sameRows(rows, expected.rows, rowTitle) + ) { throw violation( - valuesOnly ? `FID-1 (values)` : `FID-1 (fields)`, + sameRows(rows, expected.rows, rowHead) + ? `FID-1 (values)` + : `FID-1 (fields)`, stepIndex, expected.rows, rows, @@ -2669,23 +2665,25 @@ function supports( } } -/** Same keys and titles, whatever the other fields. */ -function sameTitles(a: RowState, b: RowState): boolean { - const title = (value: string) => rowHead(value).split(`/`)[0] - return ( - a.length === b.length && - a.every( - ([id, value], index) => - b[index]![0] === id && title(b[index]![1]) === title(value), - ) - ) +/** A row's title, without its note and fields. */ +function rowTitle(value: string): string { + return rowHead(value).split(`/`)[0]! } -function sameRows(a: RowState, b: RowState): boolean { +/** + * Same keys in order, and rows equal under `project`: the whole row by + * default, `rowHead` for title and note, or `rowTitle`. + */ +function sameRows( + a: RowState, + b: RowState, + project: (value: string) => string = (value) => value, +): boolean { return ( a.length === b.length && a.every( - ([id, title], index) => b[index]![0] === id && b[index]![1] === title, + ([id, value], index) => + b[index]![0] === id && project(b[index]![1]) === project(value), ) ) } diff --git a/packages/db/src/utils.ts b/packages/db/src/utils.ts index fc12936093..0680ab5c4f 100644 --- a/packages/db/src/utils.ts +++ b/packages/db/src/utils.ts @@ -102,7 +102,7 @@ export function deepEqualsInternal( rowValue: false | `row` | `publication` = false, ): boolean { // Handle strict equality (primitives, same reference) - if (rowValue ? Object.is(a, b) : a === b || Object.is(a, b)) return true + if ((!rowValue && a === b) || Object.is(a, b)) return true // Handle null/undefined if (a == null || b == null) return false diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 772c042a00..421bc27ffc 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -130,6 +130,12 @@ function hasEvidence(evidence: ElectricLifecycleEvidence, txId: Txid): boolean { ) } +function clearEvidence(evidence: ElectricLifecycleEvidence): void { + evidence.seenTxids.clear() + evidence.seenSnapshots.length = 0 + evidence.hydratedResumeState = undefined +} + type ElectricPendingMatch> = { matchFn: (message: Message) => boolean resolve: (value: boolean) => void @@ -965,9 +971,7 @@ class ElectricLifecycle> { this.matchBuffer = { committedMessages: [], pendingMessages: [] } for (const txid of this.evidence.seenTxids) this.retired.seenTxids.add(txid) this.retired.seenSnapshots.push(...this.evidence.seenSnapshots) - this.evidence.seenTxids.clear() - this.evidence.seenSnapshots.length = 0 - this.evidence.hydratedResumeState = undefined + clearEvidence(this.evidence) } /** The run's first ready: retired evidence now counts (EVID-2). */ @@ -997,11 +1001,8 @@ class ElectricLifecycle> { } forgetEvidence(): void { - for (const evidence of [this.evidence, this.retired]) { - evidence.seenTxids.clear() - evidence.seenSnapshots.length = 0 - } - this.evidence.hydratedResumeState = undefined + clearEvidence(this.evidence) + clearEvidence(this.retired) } exportMeta(): ElectricSyncMeta { @@ -1700,10 +1701,7 @@ function createElectricSync>( ready = true lifecycle.markReady(lifecycleEpoch) params.markReady() - if (gaveUpBeforeReady) { - gaveUp = true - fail(gaveUpBeforeReady.cause) - } + if (gaveUpBeforeReady) giveUp(gaveUpBeforeReady.cause) } // A failure ends the run's waits (EVID-2). const fail = (error: unknown) => { @@ -1718,19 +1716,24 @@ function createElectricSync>( // outage, clearing its failure (HEALTH-1, HEALTH-2, RETRY-1). A give-up // after ready ends health reports; the other failures abort the run. let failedAttempts = 0 - let retrying: unknown + let outageFailure: unknown let gaveUp = false + // A give-up after ready fails the run and ends its health reports. + const giveUp = (cause: unknown) => { + gaveUp = true + fail(cause) + } const reportsHealth = () => ready && !gaveUp && isActiveLifecycle() const reportRetry = (cause: unknown) => { if (!reportsHealth() || abortController.signal.aborted) return if (++failedAttempts < 5) return - retrying ??= cause - params.reportError(retrying) + outageFailure ??= cause + params.reportError(outageFailure) } const endOutage = () => { - const recovered = retrying + const recovered = outageFailure failedAttempts = 0 - retrying = undefined + outageFailure = undefined return () => { if (recovered !== undefined && reportsHealth()) { params.reportHealthy(recovered) @@ -1963,10 +1966,8 @@ function createElectricSync>( // a reload's run started on a ready Collection. The give-up is then // a post-ready failure; READY-9 defers only a first load's // (maintainer decision). - if (collection.status === `ready`) { - gaveUp = true - fail(cause) - } else gaveUpBeforeReady ??= { cause } + if (collection.status === `ready`) giveUp(cause) + else gaveUpBeforeReady ??= { cause } return } initialSyncFailedFinally = true @@ -1988,10 +1989,11 @@ function createElectricSync>( const forwardExternalAbort = () => { const active = isActiveLifecycle() abortController.abort() - if (active && ready) { - gaveUp = true - fail(new StreamAbortedError(collectionId)) - } else if (active) failInitialSync(new StreamAbortedError(collectionId)) + if (active) { + const error = new StreamAbortedError(collectionId) + if (ready) giveUp(error) + else failInitialSync(error) + } // A source that stops must end its reserved transaction. The initial // failure decides first: abandoning commits. abandonOpenTransaction() @@ -2081,10 +2083,7 @@ function createElectricSync>( // reject the client's own start unobserved, so the run fails // here and returning nothing stops the stream. failWaits(error) - if (ready) { - gaveUp = true - fail(error) - } + if (ready) giveUp(error) return } if (!retry || typeof retry !== `object`) failWaits(errorParams) @@ -2797,10 +2796,7 @@ function createElectricSync>( rejectFullSnapshot(error) if (isActiveLifecycle() && !abortController.signal.aborted) { if (!ready) failInitialSync(error) - else { - gaveUp = true - fail(error) - } + else giveUp(error) } // A source that stops must end its reserved transaction. The initial // failure decides first: abandoning commits. From f3b761f8eb219ccae44529aecd686ebbc1c13417 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:32:18 +0000 Subject: [PATCH 35/64] ci: apply automated fixes --- ...ger-live-source-conformance-oracle.property.test.ts | 10 +++------- ...ger-live-source-conformance-oracle.property.test.ts | 10 +++------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts index 54eede25fe..8001dfc4f8 100644 --- a/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts +++ b/packages/rxdb-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -44,13 +44,9 @@ const runs = 400 const campaignTimeoutMs = 120_000 async function campaign(options: fc.Parameters<[unknown]>) { - const reach = createOracleReach( - property, - reachStatesFor(driver), - { - minimum: 5, - }, - ) + const reach = createOracleReach(property, reachStatesFor(driver), { + minimum: 5, + }) await fc.assert(eagerLiveSourceProperty(driver, reach), options) return reach } diff --git a/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts index 50fdc47519..79ed5e71a7 100644 --- a/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts +++ b/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -42,13 +42,9 @@ const runs = 600 const campaignTimeoutMs = 120_000 async function campaign(options: fc.Parameters<[unknown]>) { - const reach = createOracleReach( - property, - reachStatesFor(driver), - { - minimum: 5, - }, - ) + const reach = createOracleReach(property, reachStatesFor(driver), { + minimum: 5, + }) await fc.assert(eagerLiveSourceProperty(driver, reach), options) return reach } From 3b530292c8112a32be531a20d1389d4647906c07 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 14:39:56 -0700 Subject: [PATCH 36/64] Fix CI failure: assign the conformance package a test group - scripts/ci-tests.mjs: put @tanstack/db-adapter-conformance in the collections group, beside the adapters its drivers run; every test group failed with 'Assign @tanstack/db-adapter-conformance to exactly one CI test group'. - Convex conformance driver: rename the provider's attach method to bindSocket; check:mangle rejected it because core mangles a private member named attach. Checked: node scripts/ci-tests.mjs check (27 package test scripts); pnpm run check:mangle; Convex conformance campaigns 3/3. Co-authored-by: Isaac --- packages/convex-db-collection/tests/conformance-driver.ts | 6 +++--- scripts/ci-tests.mjs | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/convex-db-collection/tests/conformance-driver.ts b/packages/convex-db-collection/tests/conformance-driver.ts index b56cccb14e..ecdaceabdf 100644 --- a/packages/convex-db-collection/tests/conformance-driver.ts +++ b/packages/convex-db-collection/tests/conformance-driver.ts @@ -83,7 +83,7 @@ class Socket { * The provider that the next socket serves. The React client opens its * socket lazily, so a socket takes its provider when it is created. */ - static provider: { attach: (socket: Socket) => void } | undefined + static provider: { bindSocket: (socket: Socket) => void } | undefined onopen: (() => void) | null = null onmessage: ((event: { data: string }) => void) | null = null onclose: ((event: { code: number; reason: string }) => void) | null = null @@ -91,7 +91,7 @@ class Socket { readonly sent: Array> = [] onSend: ((message: Record) => void) | undefined constructor(_url: string) { - Socket.provider?.attach(this) + Socket.provider?.bindSocket(this) // The client sets its handlers after construction; then it connects. queueMicrotask(() => this.onopen?.()) } @@ -202,7 +202,7 @@ class ConvexProvider implements ControlledProvider { private onSend: (message: Record) => void = () => {} - attach(socket: Socket): void { + bindSocket(socket: Socket): void { this.socketOrUndefined = socket socket.onSend = (message) => this.onSend(message) } diff --git a/scripts/ci-tests.mjs b/scripts/ci-tests.mjs index bef09c8fac..a5f96486ca 100644 --- a/scripts/ci-tests.mjs +++ b/scripts/ci-tests.mjs @@ -36,6 +36,7 @@ const groups = { `@tanstack/rxdb-db-collection`, `@tanstack/trailbase-db-collection`, `@tanstack/db-collection-e2e`, + `@tanstack/db-adapter-conformance`, ], } From 2c78d9ccd4309edf053039b01793879295305d4b Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 14:54:54 -0700 Subject: [PATCH 37/64] Fix CI failure: build an outage FetchError's headers without iterating Headers TrailBase type-checks the Electric source with a DOM lib that has no iterable Headers, so Object.fromEntries(response.headers) failed its vitest typecheck. Copy the headers with forEach instead. Co-authored-by: Isaac --- packages/electric-db-collection/src/electric.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 421bc27ffc..63b3667ab6 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -2040,12 +2040,16 @@ function createElectricSync>( const response = await request if (response.ok || response.status === 409) failedAttempts = 0 else { + // Not `Object.fromEntries`: a DOM lib without iterable Headers + // also type-checks this source. + const headers: Record = {} + response.headers.forEach((value, name) => (headers[name] = value)) reportRetry( new FetchError( response.status, undefined, undefined, - Object.fromEntries(response.headers), + headers, String(input), ), ) From 94a85bb31899ab29a80ded571fd6155b1fb30a6a Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 15:14:56 -0700 Subject: [PATCH 38/64] fix(convex): keep a fresh server error that arrives before the cached replay A restarted run that found a cached error skipped the first error callback, taking it for the client's setTimeout replay. When the server's answer came first, the client never replayed (the listener had run), so the fresh failure was lost and the Collection stayed loading. The adapter now names the cached result by the identity of its log lines, which belong to the client's result object. It reads them through the base client, because the onUpdate handle's getQueryLogs looks up the wrong query in Convex 1.47.0. Without the accessor nothing is skipped. The shared oracle gains a `prompt-after-restart` capability: the next step can run in the same task as a restart's preload(), with no checkpoint between them, plus a scenario (fail, prompt restart, fail or a complete delivery). The reference adapter declares it, with a hostile mutant that skips one early failure after a failed run. - Law: READY-4 and LIFE-5 for a restarted run; maintainer decision 2026-11-02 that a cached error does not fail a new subscription, which waits for the server. - Reach (Convex, seed 2118, 400 histories, both clients): prompt-restart 113, prompt-failure-after-failed-run 36, restart-after-failure 94; campaign minimum 5. - RED (unchanged adapter): READY-4 (status) at step 2 of [fail, prompt restart, fail]: expected "error", observed "loading", fixed and random JavaScript-client campaigns. - GREEN: convex-db-collection 115/115; db-adapter-conformance 82/82. - Mutants: the original skip-once rule fails READY-4 as above; a version that skips nothing (the handle's getQueryLogs) fails LIFE-2/LIFE-5 (status) at [fail, restart, changes]; the reference swallows-prompt-failure mutant fails READY-4. Label change: wait-off-by-one now also matches EVID-2, the law its new shrink ([snapshot, changes, restart, wait]) breaks. - Limits: relies on two untyped client accessors (feature-detected); no real Convex deployment ordering; the React client has no replay, so its campaign is a neighboring check. Co-authored-by: Isaac --- .changeset/convex-failed-load-release.md | 2 +- docs/contributing/oracle-coverage.md | 14 +++- .../convex-db-collection/src/subscription.ts | 47 ++++++++------ .../tests/conformance-driver.ts | 11 +++- packages/db-adapter-conformance/README.md | 1 + .../src/eager-live-source-oracle.ts | 64 +++++++++++++++++-- .../reference-adapter-oracle.property.test.ts | 5 +- .../tests/reference-adapter.ts | 13 ++++ 8 files changed, 124 insertions(+), 33 deletions(-) diff --git a/.changeset/convex-failed-load-release.md b/.changeset/convex-failed-load-release.md index e164445469..b526e67ce6 100644 --- a/.changeset/convex-failed-load-release.md +++ b/.changeset/convex-failed-load-release.md @@ -2,4 +2,4 @@ '@tanstack/convex-db-collection': patch --- -Release the Convex query subscription when a Collection's first load fails, or when a later acceptance failure stops the sync run, instead of keeping it until cleanup. A new subscription no longer fails from an error the Convex client cached for its query before it subscribed, such as a restart right after a failed run; it waits for the server's result. `awaitSync` during that wait no longer fails the run from the cached error. While the latest result is a failure, `awaitSync` rejects with the reported failure instead of reporting it again. +Release the Convex query subscription when a Collection's first load fails, or when a later acceptance failure stops the sync run, instead of keeping it until cleanup. A new subscription no longer fails from an error the Convex client cached for its query before it subscribed, such as a restart right after a failed run; it waits for the server's result, also when the server's answer arrives before the client replays the cached error. `awaitSync` during that wait no longer fails the run from the cached error. While the latest result is a failure, `awaitSync` rejects with the reported failure instead of reporting it again. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index a72795df92..426af1e3ca 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -7018,15 +7018,23 @@ values and typed fields) is GREEN after a fix: a failed first load now releases the query subscription (LIFE-10). It declares `delivery-error` (a `QueryFailed` after ready, cleared by the next result) and reports a rejected delivery after ready (`rejectedDelivery: reported`, the law set's -default `errorMapping`). Limits: no `awaitSync` evidence (the shared -oracle's waits are txid waits), no transport reconnect, no on-demand or +default `errorMapping`). Limits: no transport reconnect, no on-demand or paginated mode, no deployment; the truncate on a restarted run's first result is not distinguished, because an eager restart starts empty. While the client's result is a failure, the driver acknowledges a query removal only with its next transition, so a restart finds the failed query's cached error: maintainer decision 2026-11-02, a cached error does not fail a new subscription, which waits for the server (reach `restart-after-failure`, -both clients); a cached valid result served to a restarted run is accepted +both clients). With `prompt-after-restart`, the new run's first transition +can arrive before the client's replay of that cached error (reach +`prompt-failure-after-failed-run`, 36 of 400 histories per client at seed +2118). Skipping the first error after a cached one lost a fresh failure +there: RED `READY-4 (status)` at `[fail, prompt restart, fail]`, status +`loading` and not `error`. The adapter now names the cached result by the +identity of its log lines, read through the base client, because the +`onUpdate` handle's `getQueryLogs` looks up the wrong query in Convex +1.47.0; a version that skips nothing fails `LIFE-2/LIFE-5 (status)` at +`[fail, restart, changes]`. A cached valid result served to a restarted run is accepted client-cache behavior and is not generated, because the model starts a restarted run empty. A fixed-seed campaign runs the React client (`watchQuery`). diff --git a/packages/convex-db-collection/src/subscription.ts b/packages/convex-db-collection/src/subscription.ts index f96fa660c4..7206fcfac1 100644 --- a/packages/convex-db-collection/src/subscription.ts +++ b/packages/convex-db-collection/src/subscription.ts @@ -1,3 +1,4 @@ +import { getFunctionName } from 'convex/server' import type { LoadSubsetFn, UnloadSubsetFn } from '@tanstack/db' import type { ConvexClient } from 'convex/browser' import type { ConvexReactClient } from 'convex/react' @@ -19,6 +20,11 @@ export interface ConvexSubscription { waitForCurrent?: () => Promise } +/** The clients' untyped accessor for a query result's log lines. */ +type Logged> = { + localQueryLogs?: (...args: TArgs) => unknown +} + export function subscribeQuery>( client: ConvexCollectionClient, query: TQuery, @@ -28,12 +34,17 @@ export function subscribeQuery>( ): ConvexSubscription> { let active = true // The client serves a query's cached result to a new subscription at - // once. A cached error belongs to an earlier subscription, so this one - // waits for the server instead (maintainer decision): until the server's - // first callback it has no current value, and the client's replay of the - // cached error is skipped once. - let cachedError = false - let replay = false + // once, and replays it on a later turn. A cached error belongs to an + // earlier subscription, so this one waits for the server instead + // (maintainer decision): while the client still holds that result, the + // subscription has no current value and ignores its replay. The client + // stores a new result object for each server answer, and the result's log + // lines belong to that object, so they name the cached result, also when + // the server answers before the replay. The log accessors are untyped, and + // the `onUpdate` handle's own one looks up the wrong query, so the + // JavaScript client's base client is read. Without them nothing is skipped. + let cached: unknown + const isCached = (logs: unknown) => cached !== undefined && logs === cached const stop = (unsubscribe: () => void) => () => { active = false unsubscribe() @@ -57,30 +68,27 @@ export function subscribeQuery>( query, args, (value) => { - cachedError = replay = false if (active) receive(value) }, (error) => { - if (replay) { - replay = false - return - } - cachedError = false - if (active) fail(error) + if (active && !isCached(logs())) fail(error) }, ) - cachedError = replay = hasCachedError(() => subscription.getCurrentValue()) + const base = (client as { client?: Logged<[string, unknown]> }).client + const name = getFunctionName(query) + const logs = () => base?.localQueryLogs?.(name, args) + if (hasCachedError(() => subscription.getCurrentValue())) cached = logs() return { getCurrentValue: () => - cachedError ? undefined : subscription.getCurrentValue(), + isCached(logs()) ? undefined : subscription.getCurrentValue(), unsubscribe: stop(subscription), } } const watch = client.watchQuery(query, args) - cachedError = hasCachedError(() => watch.localQueryResult()) + const logs = () => (watch as Logged<[]>).localQueryLogs?.() + if (hasCachedError(() => watch.localQueryResult())) cached = logs() const unsubscribe = watch.onUpdate(() => { - cachedError = false - if (!active) return + if (!active || isCached(logs())) return try { const result = watch.localQueryResult() if (result !== undefined) receive(result) @@ -89,7 +97,8 @@ export function subscribeQuery>( } }) return { - getCurrentValue: () => (cachedError ? undefined : watch.localQueryResult()), + getCurrentValue: () => + isCached(logs()) ? undefined : watch.localQueryResult(), unsubscribe: stop(unsubscribe), } } diff --git a/packages/convex-db-collection/tests/conformance-driver.ts b/packages/convex-db-collection/tests/conformance-driver.ts index ecdaceabdf..1626c4cb97 100644 --- a/packages/convex-db-collection/tests/conformance-driver.ts +++ b/packages/convex-db-collection/tests/conformance-driver.ts @@ -37,6 +37,10 @@ * `awaitSync` is the adapter's wait for the query's current result * (`await-sync`). * + * A prompt restart (`prompt-after-restart`) sends the new run's first + * transition before the client's `setTimeout` replay of a cached result, as + * a server answer that is already queued on the socket arrives first. + * * Rows are Convex values: `note` absent is a missing field, and typed fields * hold the Convex kinds (no Date, which Convex values do not have). The * driver never reads the Collection. @@ -183,9 +187,9 @@ class ConvexProvider implements ControlledProvider { const onUpdate = client.onUpdate.bind(client) client.onUpdate = ((...args: Parameters) => { const subscription = onUpdate(...args) - return Object.assign(counted(subscription), { - getCurrentValue: () => subscription.getCurrentValue(), - }) + const release = counted(subscription) + // The handle keeps its own fields; only its release is counted. + return Object.assign(release, subscription, { unsubscribe: release }) }) as typeof onUpdate } else { const watchQuery = client.watchQuery.bind(client) as ( @@ -336,6 +340,7 @@ export function convexConformanceDriver({ `delivery-error`, `duplicate-key-delivery`, `await-sync`, + `prompt-after-restart`, ]), changeCommits: `per-batch`, duplicateInsert: `error`, diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md index 1c52dc65c7..ab31709b1b 100644 --- a/packages/db-adapter-conformance/README.md +++ b/packages/db-adapter-conformance/README.md @@ -52,6 +52,7 @@ Implement `EagerLiveSourceDriver`: - **`source-failure-after-ready`:** declare it when your provider can fail terminally after the first ready (the law set's `terminal-source-end`, or an `errorMapping` that reports a terminal failure); otherwise `fail` is generated only before ready. - **`retry-policy` and its parameters:** `transientFailure(error, variant)` makes one failed attempt the adapter retries, of the kind `transientVariants[variant]` (for example an HTTP 503, a 429, or a network error). `retryReportAfter` is your adapter's outage threshold: the failed attempts in a row after ready from which `failureReason` shows the outage (default 1, report at once; Electric 5). `empty-response` with `emptyResponse()`: an accepted response with no change, which restarts that count. - **`delivery-error`:** `deliveryError(error)`, after ready: the source reports an error for its current result and keeps its subscription; the next valid delivery clears it (Convex's `QueryFailed`). +- **`prompt-after-restart`:** your provider can take its next step in the same task as a restart's `preload()`, before any event-loop turn, as an answer already queued on a socket does. Declare it when a client may replay a cached result on a later turn (Convex), so a failure that arrives first is checked. - **`rejectedDelivery`:** how your adapter reports a delivery it rejects after ready, such as two rows with one key: `fails` the sync run (default) or `reported` in `failureReason` until the next valid delivery. - **`matchesFailure`:** when your client library creates the error value the Collection reports (an HTTP client's error for a response), the relation that names the value owed for each injected failure, for example by an index the driver carried in a response header. Without it, failures compare by identity with the provider's error. - **`txidEvidence`:** `change` when txids travel on each row message, so a batch with no change carries none; default `batch`. diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index 357d30e810..f7d494b1d1 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -44,7 +44,13 @@ * replaces the rows: keys it omits are gone. * - **LIFE-2, LIFE-5.** Cleanup releases the provider subscription. A restart * is a fresh sync run: one new provider subscription, status `loading`, and - * no rows until its own first delivery. + * no rows until its own first delivery. With capability + * `prompt-after-restart`, the provider's next step can come in the same + * task as the restart, before any event-loop turn, as an answer already + * queued when the new subscription opens does. The new run still owes that + * step's law: a failure fails it (READY-4) and a delivery applies. A result + * a client cached for the retired run, which the client may replay on a + * later turn, does not take the step's place. * - **LIFE-10.** A provider that fails ends its source: the adapter releases * that subscription at the failure, and releases each subscription at most * once across that end and the later cleanup. @@ -209,6 +215,7 @@ export type EagerLiveSourceCapability = | `empty-response` | `delivery-error` | `await-sync` + | `prompt-after-restart` /** * The public Collection surface the oracle reads. A driver passes the @@ -432,7 +439,12 @@ export interface EagerLiveSourceDriver { * - `empty-response`: the provider accepts a request and delivers nothing. * Two outages below the threshold around one are generated as one * element, so the reset it causes is reached. - * - `restart`: cleanup, then `preload()` starts a new sync run. + * - `restart`: cleanup, then `preload()` starts a new sync run. With + * `prompt-after-restart`, half the restarts are `prompt`: the next step + * runs in the same task as `preload()`, and no checkpoint comes between + * them. One history in six with this capability starts with a failure, a + * prompt restart, and a failure or a complete delivery, because random + * steps rarely put a failed run's restart next to a failure. * * Keys range over 1..5 and titles over three values, so batches revisit * keys, deletes hit live rows, and complete deliveries omit held keys. A row @@ -478,7 +490,7 @@ export type EagerLiveSourceStep = | { readonly kind: `empty-response` } | { readonly kind: `delivery-error` } | { readonly kind: `sync-wait` } - | { readonly kind: `restart` } + | { readonly kind: `restart`; readonly prompt?: boolean } /** * A change in a generated batch. `retouch` re-delivers a row the source @@ -656,7 +668,12 @@ export function eagerLiveSourceHistoryArb( .map((changes) => ({ kind: `changes` as const, changes })), }, { weight: 2, arbitrary: fc.constant({ kind: `fail` as const }) }, - { weight: 1, arbitrary: fc.constant({ kind: `restart` as const }) }, + { + weight: 1, + arbitrary: capabilities.has(`prompt-after-restart`) + ? fc.boolean().map((prompt) => ({ kind: `restart` as const, prompt })) + : fc.constant({ kind: `restart` as const }), + }, ] if (capabilities.has(`duplicate-key-delivery`)) { steps.push({ @@ -825,6 +842,18 @@ export function eagerLiveSourceHistoryArb( ]), ) } + if (capabilities.has(`prompt-after-restart`)) { + // A failed run, its prompt restart, and the new run's first step. + scenarios.push( + fc + .oneof(fc.constant({ kind: `fail` as const }), snapshot) + .map((next): Array => [ + { kind: `fail` }, + { kind: `restart`, prompt: true }, + next, + ]), + ) + } const history = scenarios.length === 0 ? generated @@ -1274,7 +1303,7 @@ type AcceptedStep = | { kind: `empty-response`; resetOutage: boolean } | { kind: `delivery-error`; failure: number } | { kind: `sync-wait` } - | { kind: `restart` } + | { kind: `restart`; prompt: boolean } /** * Applies one step to the model. Returns the accepted step and the row @@ -1641,7 +1670,10 @@ function stepModel( state.renewed = false state.failure = undefined endOutage(state) - return { done: { kind: `restart` }, publishable: [] } + return { + done: { kind: `restart`, prompt: step.prompt === true }, + publishable: [], + } } } } @@ -1692,6 +1724,8 @@ export const eagerLiveSourceReachStates = [ `failure-after-ready`, `restart`, `restart-after-failure`, + `prompt-restart`, + `prompt-failure-after-failed-run`, `wait-before-evidence-fulfilled`, `wait-timed-out`, `wait-rejected-at-retirement`, @@ -1738,6 +1772,8 @@ const capabilityOfState: Partial< Record<(typeof eagerLiveSourceReachStates)[number], EagerLiveSourceCapability> > = { 'replacing-snapshot': `complete-snapshot`, + 'prompt-restart': `prompt-after-restart`, + 'prompt-failure-after-failed-run': `prompt-after-restart`, 'wait-before-evidence-fulfilled': `await-evidence`, 'wait-timed-out': `await-evidence`, 'wait-rejected-at-retirement': `await-evidence`, @@ -2319,6 +2355,9 @@ export async function checkEagerLiveSourceHistory( // Reported outages and applied live batches in the current run. let outagesThisRun = 0 let batchesThisRun = 0 + // A prompt restart whose checkpoint waits for the next accepted step, + // which runs in the same task. + let prompt: { index: number; afterFailure: boolean } | undefined for (const [index, step] of history.entries()) { const wasReady = state.phase === `ready` const phaseBefore = state.phase @@ -2340,6 +2379,10 @@ export async function checkEagerLiveSourceHistory( } accepted++ const { done, publishable } = result + if (prompt?.afterFailure && done.kind === `fail`) { + reach(`prompt-failure-after-failed-run`) + } + prompt = undefined switch (done.kind) { case `snapshot`: await provider.deliverSnapshot(structuredClone(done.rows)) @@ -2505,10 +2548,17 @@ export async function checkEagerLiveSourceHistory( events = foldChanges(collection) reach(`restart`) if (failedBefore) reach(`restart-after-failure`) + if (done.prompt) reach(`prompt-restart`) open.clear() outagesThisRun = 0 batchesThisRun = 0 if (pendingBefore) reach(`wait-rejected-at-retirement`) + if (done.prompt) { + // No turn before the next step: its checkpoint covers this one. + events.take() + prompt = { index, afterFailure: failedBefore } + continue + } break } } @@ -2534,6 +2584,8 @@ export async function checkEagerLiveSourceHistory( events.take() } } + // A prompt restart with no later accepted step still has its checkpoint. + if (prompt) await compare(prompt.index, `LIFE-2/LIFE-5`) } catch (error) { primary = error } diff --git a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts index a35ec7f3ec..b714aee292 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts @@ -104,11 +104,14 @@ describe(`eager live-source conformance: reference adapter`, () => { [`snapshot-merges`, /PUB-7|PUB-4/], [`leaks-subscription`, `LIFE-2`], [`restart-replays-rows`, `LIFE-5`], + [`swallows-prompt-failure`, `READY-4`], [`keeps-failed-source`, `LIFE-10`], [`double-release`, `LIFE-10`], [`detached-rejection`, `HEALTH-5`], [`logs-failure`, `HEALTH-5`], - [`wait-off-by-one`, `EVID-1`], + // Its early fulfillment also answers a wait for a retired run's txid + // before the new run is ready. + [`wait-off-by-one`, /EVID-1|EVID-2/], [`retired-evidence-before-ready`, `EVID-2`], [`forgets-retired-evidence`, `EVID-2`], [`leaks-waits-after-fail`, `EVID-2`], diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index e717c12ab3..99eeb8ff6a 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -45,6 +45,11 @@ export type ReferenceMutant = | `leaks-subscription` /** LIFE-5: a new sync run starts from the previous run's rows. */ | `restart-replays-rows` + /** + * READY-4, after a failed run: the new run skips one failure that comes + * before its first turn, taking it for a cached failure's replay. + */ + | `swallows-prompt-failure` /** LIFE-10: a failed provider's subscription stays open until cleanup. */ | `keeps-failed-source` /** LIFE-10: a failed provider's subscription is released twice. */ @@ -332,6 +337,9 @@ function referenceCollection( waiters = new Map() highestTxid = 0 runReady = false + let replayExpected = + mutant === `swallows-prompt-failure` && runFailed + if (replayExpected) setTimeout(() => (replayExpected = false), 0) runFailed = false // Failed attempts after ready since the last delivery, and the value // that reports the outage. @@ -549,6 +557,10 @@ function referenceCollection( reportError(outage) } } else if (!ready) { + if (replayExpected) { + replayExpected = false + return + } stop( subscription, mutant === `wraps-loading-failure` @@ -654,6 +666,7 @@ export function referenceDriver( `source-failure-after-ready`, `empty-response`, `delivery-error`, + `prompt-after-restart`, ]), // The same ten kinds as typed fields, one per row field. fieldKinds: [ From 564a3b5c2125f5817a7d4aa3d8b1243ca195e207 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 15:32:51 -0700 Subject: [PATCH 39/64] fix(electric): reject a wait made after an abort stopped the sync run Several failure paths abort the stream before they fail it. The abort retires the lifecycle first, and the retired run's fail() did nothing, so the failure was not recorded and a later awaitTxId or awaitMatch waited for its timeout. The run's epoch now stays until the next run starts, so fail() still records the latest run's failure after an abort. An external abort also records the failure for the waits while READY-9 defers the Collection's failure. The outage is reported once, at the attempt that crosses the threshold, as the comment said; core already ignored the repeats. The retry-decision model and both of its lanes now start a second pair of unmet waits at every checkpoint, after its step, and declare reach for each way a run stops before cleanup. - Law: EVID-2, a wait started after the sync run failed rejects (maintainer decision 2026-10-30); an external abort is a source error (maintainer decision 2026-11-02). - Reach (electric.retry-decision, fixed + random): loading failure 16, give-up after ready 14, abort after ready 4, abort during READY-9 deferral 2 (named examples). - RED (unchanged adapter): after [demand, hold, up-to-date, abort] the late waits are "pending", owed "rejected"; fixed and random campaigns. - GREEN: electric-db-collection 1243 passed, 4 skipped. - Mutants: retire() advancing the epoch again fails at [demand, hold, up-to-date, abort, release]; an abort that does not record the failure fails at [demand, hold-acceptance, up-to-date, abort, accept]. - Limits: a rejected applied receipt and a rejected or invalidated resume abort before failing too; they share the repaired rule, but no generated history reaches them. Co-authored-by: Isaac --- .../electric-retry-health-and-evidence.md | 2 +- docs/contributing/oracle-coverage.md | 14 ++++ .../electric-db-collection/src/electric.ts | 16 +++-- .../tests/electric-oracle-lifecycle.ts | 27 +++++++ .../tests/electric-oracle.property.test.ts | 71 ++++++++++++++----- .../tests/electric-retry-decision-model.ts | 17 ++--- ...ctric-sdk-delivery-oracle.property.test.ts | 30 ++++---- 7 files changed, 125 insertions(+), 52 deletions(-) diff --git a/.changeset/electric-retry-health-and-evidence.md b/.changeset/electric-retry-health-and-evidence.md index 4beaa9de73..f7876868b6 100644 --- a/.changeset/electric-retry-health-and-evidence.md +++ b/.changeset/electric-retry-health-and-evidence.md @@ -2,4 +2,4 @@ '@tanstack/electric-db-collection': patch --- -Report an extended Electric outage in `collection.health`. After the Collection is ready, five failed requests to the shape log in a row, retried by the client or by an `onError` that returns a retry, show as the source node's `failureReason` with the fifth attempt's error, and the next applied up-to-date clears it. Shorter outages, and retried subset snapshot requests, leave health unchanged. A restarted or reloaded Collection resolves `awaitTxId` for a txid the previous sync run delivered only once it is ready again, instead of at once. When the stream fails, pending `awaitTxId` and `awaitMatch` calls reject with the failure instead of waiting for their timeout. An `onError` handler that throws now gives up the stream with its own error, without an unhandled rejection. A give-up while the first snapshot's acceptance is pending, under a persistence wrapper, now fails the Collection once that acceptance makes it ready, instead of leaving a stopped stream with healthy status. Aborting `shapeOptions.signal` after the Collection is ready now shows as the source node's `error`, since the Collection no longer syncs. +Report an extended Electric outage in `collection.health`. After the Collection is ready, five failed requests to the shape log in a row, retried by the client or by an `onError` that returns a retry, show as the source node's `failureReason` with the fifth attempt's error, and the next applied up-to-date clears it. Shorter outages, and retried subset snapshot requests, leave health unchanged. A restarted or reloaded Collection resolves `awaitTxId` for a txid the previous sync run delivered only once it is ready again, instead of at once. When the stream fails, pending `awaitTxId` and `awaitMatch` calls reject instead of waiting for their timeout, and so do calls made after the failure, also when an abort stopped the stream. An `onError` handler that throws now gives up the stream with its own error, without an unhandled rejection. A give-up while the first snapshot's acceptance is pending, under a persistence wrapper, now fails the Collection once that acceptance makes it ready, instead of leaving a stopped stream with healthy status. Aborting `shapeOptions.signal` after the Collection is ready now shows as the source node's `error`, since the Collection no longer syncs. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 426af1e3ca..09ec79e37e 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6998,6 +6998,20 @@ source node's `error` too (maintainer decision 2026-11-02: deliberate or not, an aborted stream no longer syncs). Its controlled driver now reports a give-up decision to the subscriber, as the installed client does; before, a give-up after ready never reached the adapter's failure path in that lane. +Both lanes also start a second pair of those waits at every checkpoint, +after its step: a run that stopped rejects them too (EVID-2), except after +cleanup. Reach (`electric.retry-decision`, fixed and random campaigns +together): a loading failure 16, a give-up after ready 14, an external +abort after ready 4, and an abort while READY-9 defers the failure 2, the +last two from named examples. An abort retired the run before its failure +was recorded, so a later wait waited for its timeout: RED at +`[demand, hold, up-to-date, abort]` (late waits `pending`). Two mutants +fail it: a retirement that advances the run's epoch again (abort after +ready), and an abort that does not record the failure for the waits +(`[demand, hold-acceptance, up-to-date, abort, accept]`). The other paths +that abort before failing, a rejected applied receipt and a rejected or +invalidated resume, share the repaired lifecycle rule; no generated history +reaches them. Limits: no persisted resume or held commit receipt in the eager lane; no real Electric server; without an app `onError`, the driver detects a give-up from the adapter's log, so an adapter that stops logging times out diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 63b3667ab6..6c5a51f9ac 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -966,7 +966,6 @@ class ElectricLifecycle> { retire(epoch?: number): void { if (epoch !== undefined && !this.isActive(epoch)) return this.active = false - this.epoch++ this.rejectWaits(new StreamAbortedError(this.collectionId)) this.matchBuffer = { committedMessages: [], pendingMessages: [] } for (const txid of this.evidence.seenTxids) this.retired.seenTxids.add(txid) @@ -981,9 +980,13 @@ class ElectricLifecycle> { this.resolveTxidWaiters() } - /** The run failed: its pending waits reject, as later unmet ones do. */ + /** + * The run failed: its pending waits reject, as later unmet ones do. An + * abort that stops the run can retire it first; the failure still counts + * until the next run starts. + */ fail(epoch: number, error: unknown): void { - if (!this.isActive(epoch)) return + if (epoch !== this.epoch) return this.failure ??= { error } this.rejectWaits(error) } @@ -1726,9 +1729,7 @@ function createElectricSync>( const reportsHealth = () => ready && !gaveUp && isActiveLifecycle() const reportRetry = (cause: unknown) => { if (!reportsHealth() || abortController.signal.aborted) return - if (++failedAttempts < 5) return - outageFailure ??= cause - params.reportError(outageFailure) + if (++failedAttempts === 5) params.reportError((outageFailure ??= cause)) } const endOutage = () => { const recovered = outageFailure @@ -1991,6 +1992,9 @@ function createElectricSync>( abortController.abort() if (active) { const error = new StreamAbortedError(collectionId) + // The waits end with the stream, also while READY-9 defers the + // Collection's failure. + lifecycle.fail(lifecycleEpoch, error) if (ready) giveUp(error) else failInitialSync(error) } diff --git a/packages/electric-db-collection/tests/electric-oracle-lifecycle.ts b/packages/electric-db-collection/tests/electric-oracle-lifecycle.ts index 07579c618c..58b9654f4b 100644 --- a/packages/electric-db-collection/tests/electric-oracle-lifecycle.ts +++ b/packages/electric-db-collection/tests/electric-oracle-lifecycle.ts @@ -7,6 +7,33 @@ * remains primary, while cleanup failures stay visible as warnings; without a * primary error, the first cleanup failure rejects after later releases run. */ +/** + * Starts an `awaitTxId` and an `awaitMatch` that no evidence satisfies, and + * returns how they have settled: both `pending` or both `rejected`. A split + * outcome is returned as itself, so a comparison rejects it. + */ +export function startUnmetWaits(utils: { + awaitTxId: (txid: number, timeout?: number) => Promise + awaitMatch: (match: () => boolean, timeout?: number) => Promise +}): () => `pending` | `rejected` { + let rejected = 0 + for (const wait of [ + utils.awaitTxId(2_000_000_000, 2_000_000_000), + utils.awaitMatch(() => false, 2_000_000_000), + ]) { + void wait.then( + () => undefined, + () => rejected++, + ) + } + return () => + rejected === 2 + ? `rejected` + : rejected === 0 + ? `pending` + : (`one of two rejected` as never) +} + export async function atCheckpoint( promise: Promise, label: string, diff --git a/packages/electric-db-collection/tests/electric-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-oracle.property.test.ts index 5cd93b13f7..e01b3247cf 100644 --- a/packages/electric-db-collection/tests/electric-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-oracle.property.test.ts @@ -27,7 +27,11 @@ import { mutationConfirmationHistories, mutationConfirmationInputs, } from '../../db/tests/mutation-confirmation-oracle' -import { atCheckpoint, withElectricCleanup } from './electric-oracle-lifecycle' +import { + atCheckpoint, + startUnmetWaits, + withElectricCleanup, +} from './electric-oracle-lifecycle' import { createMockStream } from './electric-stream-mock' import { applyRetryModel, @@ -72,6 +76,7 @@ import type { ElectricCollectionUtils, ElectricSyncMode } from '../src/electric' import type { RetryDecision, RetryHistory, + RetryModel, RetryObservation, RetrySettlement, RetryStep, @@ -3839,7 +3844,13 @@ describe(`Electric adapter laws`, () => { * The driver runs the real adapter and Collection behind the controlled * ShapeStream. After each step and a timer turn it compares the status, * every demand's settlement and cause, a readiness wait's settlement and - * cause, and whether the snapshot's row is visible. The model lives in `electric-retry-decision-model.ts`. An + * cause, whether the snapshot's row is visible, and two pairs of waits + * that no evidence satisfies: one started with the history and one + * started at that checkpoint. A sync run that stopped, by a failure, a + * give-up, or an external abort, rejects both pairs, also while READY-9 + * defers the Collection's failure (EVID-2; maintainer decisions + * 2026-10-30 and 2026-11-02). After cleanup a later run could supply the + * evidence, so the late pair is not compared then. The model lives in `electric-retry-decision-model.ts`. An * installed-client lane in `electric-sdk-delivery-oracle.property.test.ts` * drives the same model through the real Electric client and HTTP. */ @@ -3897,6 +3908,34 @@ describe(`Electric adapter laws`, () => { return { wait, hasOnError, abortedAtStart, cleanupOnError, steps: legal } }) + // EVID-2: a wait started after the run stopped rejects. The states name + // each way the model stops a run before cleanup. + const retryReach = createOracleReach(`electric.retry-decision`, [ + `late-wait-after-loading-failure`, + `late-wait-after-ready-give-up`, + `late-wait-after-abort-after-ready`, + `late-wait-during-deferred-abort`, + ]) + afterAll(() => { + retryReach.assertReached() + console.info( + `electric.retry-decision reach`, + Object.fromEntries(retryReach.counts()), + ) + }) + const reachLateWait = (model: RetryModel) => { + if (model.cleanedUp) return + if (model.failed !== undefined) { + retryReach.reach(`late-wait-after-loading-failure`) + } else if (model.aborted && !model.ready && model.committed) { + retryReach.reach(`late-wait-during-deferred-abort`) + } else if (model.aborted && model.ready) { + retryReach.reach(`late-wait-after-abort-after-ready`) + } else if (model.gaveUpAfterReady) { + retryReach.reach(`late-wait-after-ready-give-up`) + } + } + function retrySettlementTag(error: unknown, tags: Map) { const tag = tags.get(error) if (tag !== undefined) return tag @@ -3995,29 +4034,19 @@ describe(`Electric adapter laws`, () => { preload = `rejected:${retrySettlementTag(reason, tags)}` }, ) - // An `awaitTxId` and an `awaitMatch` that no evidence satisfies. - let waitsRejected = 0 - for (const wait of [ - trace.collection.utils.awaitTxId(2_000_000_000, 2_000_000_000), - trace.collection.utils.awaitMatch(() => false, 2_000_000_000), - ]) { - void wait.then( - () => undefined, - () => waitsRejected++, - ) - } + // Waits that no evidence satisfies: one pair started with the history, + // and one started at each checkpoint, after its step. + const waits = startUnmetWaits(trace.collection.utils) + let lateWaits = waits const turn = () => new Promise((resolve) => setTimeout(resolve, 0)) const observe = (): RetryObservation => ({ status: trace.collection.status, demands: [...settlements], rowVisible: trace.collection.has(1), preload: trace.collection.status === `cleaned-up` ? undefined : preload, - waits: - waitsRejected === 2 - ? `rejected` - : waitsRejected === 0 - ? `pending` - : (`one of two rejected` as never), + waits: waits(), + lateWaits: + trace.collection.status === `cleaned-up` ? undefined : lateWaits(), health: trace.collection.status === `ready` ? trace.collection.health.error @@ -4123,6 +4152,9 @@ describe(`Electric adapter laws`, () => { applyRetryModel(model, step, history.hasOnError, tag) await turn() await turn() + lateWaits = startUnmetWaits(trace.collection.utils) + reachLateWait(model) + await turn() expect( observe(), `after ${JSON.stringify(history.steps.slice(0, index + 1))}`, @@ -4156,6 +4188,7 @@ describe(`Electric adapter laws`, () => { const consoleWarn = vi.spyOn(console, `warn`).mockImplementation(() => {}) try { resetElectricOracleMocks() + retryReach.beginHistory() await runRetryHistory(history, `retry-decision-${++retryHistoryRun}`) } finally { consoleError.mockRestore() diff --git a/packages/electric-db-collection/tests/electric-retry-decision-model.ts b/packages/electric-db-collection/tests/electric-retry-decision-model.ts index 56b3e3d307..4da43d636e 100644 --- a/packages/electric-db-collection/tests/electric-retry-decision-model.ts +++ b/packages/electric-db-collection/tests/electric-retry-decision-model.ts @@ -48,6 +48,9 @@ export type RetryObservation = { // An `awaitTxId` and an `awaitMatch` that no evidence satisfies, started // with the history: both `pending`, or both `rejected`. waits?: `pending` | `rejected` + // The same pair started at this checkpoint, after the step. Not compared + // after cleanup, when a later run could still supply the evidence. + lateWaits?: `pending` | `rejected` // While `ready`, whether the source node shows an `error`. health?: `error` | `none` } @@ -233,6 +236,8 @@ export function expectedRetryObservation( cleanupRequested: boolean, ): RetryObservation { const cleanedUp = cleanupRequested || model.cleanedUp + const stopped = + model.failed !== undefined || model.gaveUpAfterReady || model.aborted return { preload: cleanedUp ? undefined @@ -260,14 +265,10 @@ export function expectedRetryObservation( ), rowVisible: !cleanedUp && model.visible, // EVID-2: a sync run retires when it fails or is cleaned up, and its - // pending waits reject then (maintainer decision 2026-10-30). - waits: - cleanedUp || - model.failed !== undefined || - model.gaveUpAfterReady || - model.aborted - ? `rejected` - : `pending`, + // pending waits reject then (maintainer decision 2026-10-30). A wait + // started after the failure rejects as well: no evidence can arrive. + waits: cleanedUp || stopped ? `rejected` : `pending`, + lateWaits: cleanedUp ? undefined : stopped ? `rejected` : `pending`, health: !cleanedUp && model.ready && model.failed === undefined ? model.gaveUpAfterReady diff --git a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts index 2456d6209b..0f065e9eb1 100644 --- a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts @@ -18,6 +18,7 @@ import { } from './electric-sdk-delivery-support' import { atCheckpoint, + startUnmetWaits, withElectricCleanup, withElectricSetup, } from './electric-oracle-lifecycle' @@ -882,7 +883,9 @@ fixedCase(`detects dropped move-in at its visibility checkpoint`, async () => { * first ready, so its demand is the readiness wait (`preload()`). * * At each checkpoint the lane compares the Collection status, each demand's - * settlement, and whether the snapshot's row is visible with the model: + * settlement, whether the snapshot's row is visible, and two pairs of waits + * that no evidence satisfies, one started with the history and one started + * at the checkpoint, with the model: * after the first error reaches `onError`, after an asynchronous decision, * after the outcome, and after the later demand. It also checks that a * full-log Collection requests the full log, and that a client that gave up @@ -913,6 +916,7 @@ function clientObservation(observation: RetryObservation): RetryObservation { status: observation.status, rowVisible: observation.rowVisible, waits: observation.waits, + lateWaits: observation.lateWaits, health: observation.health, demands: observation.demands.map((settlement) => settlement.startsWith(`rejected`) ? `rejected:any` : settlement, @@ -970,17 +974,9 @@ async function runClientRetryCell( : originalSync, }) const model: RetryModel = createRetryModel(false) - // An `awaitTxId` and an `awaitMatch` that no evidence satisfies. - let waitsRejected = 0 - for (const wait of [ - collection.utils.awaitTxId(2_000_000_000, 2_000_000_000), - collection.utils.awaitMatch(() => false, 2_000_000_000), - ]) { - void wait.then( - () => undefined, - () => waitsRejected++, - ) - } + // Waits that no evidence satisfies: one pair started with the history, + // and one started at each checkpoint, after its step. + const waits = startUnmetWaits(collection.utils) const settlements: Array<`pending` | `fulfilled` | `rejected:any`> = [] const track = (result: unknown) => { const slot = settlements.push(`pending`) - 1 @@ -1004,17 +1000,15 @@ async function runClientRetryCell( const check = async (label: string) => { await turn() await turn() + const lateWaits = startUnmetWaits(collection.utils) + await turn() expect( clientObservation({ status: collection.status, demands: [...settlements], rowVisible: collection.has(1), - waits: - waitsRejected === 2 - ? `rejected` - : waitsRejected === 0 - ? `pending` - : (`one of two rejected` as never), + waits: waits(), + lateWaits: lateWaits(), health: collection.status === `ready` ? collection.health.error From 4ea88308bab0f310f7c2171b0816b785cb20c33f Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 15:40:38 -0700 Subject: [PATCH 40/64] fix(db): a sync insert that differs only by a zero's sign is a duplicate classifyProjectedInsert compared an insert of a held key with deep equality, so an insert of -0 over a held 0 (or the reverse) counted as an equal re-insert instead of a duplicate key. It now compares row values, as the other synced change paths do. The state-retention oracle's model wrote the same rule with !==, and its values never held -0, so both shared the blind spot. Values now include zeros of both signs, one history element in ten writes a zero and then inserts the other, and the model compares with Object.is. - Law: an insert of a held key is an equal re-insert only for an equal row value; -0 and 0 are distinct row values (maintainer decision 2026-10-29). - Reach: sign-only-reinsert, 92 histories over the fixed and random campaigns (minimum 5). - RED (unchanged state.ts): [update {0: 0}, insert {0: -0}, ...] expected DuplicateKeySyncError, the insert was accepted; both campaigns. - GREEN: @tanstack/db 30830 passed before the grammar element; the retention file 72/72 after it. - Mutant: the original deepEquals comparison (the RED above). - Limits: the changeset no longer claims that an ordered Effect delivers a source update that changes only a zero's sign. Ordered live queries drop it in top-K, whose D2 value identity treats -0 as 0; that conflict is open for a maintainer decision and recorded in the coverage map. Co-authored-by: Isaac --- .changeset/sync-negative-zero-change.md | 2 +- docs/contributing/oracle-coverage.md | 19 +++++- packages/db/src/collection/state.ts | 5 +- ...on-state-retention-oracle.property.test.ts | 58 ++++++++++++++++--- 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/.changeset/sync-negative-zero-change.md b/.changeset/sync-negative-zero-change.md index 58295daeb9..d270bd4ce5 100644 --- a/.changeset/sync-negative-zero-change.md +++ b/.changeset/sync-negative-zero-change.md @@ -2,4 +2,4 @@ '@tanstack/db': patch --- -Treat `-0` and `0` as distinct row values. A synced update, a replay that replaces a subscription's rows, and an optimistic update that change a field only from `-0` to `0` (or back) now publish a change; before, the Collection could store the new value while subscribers kept the old one. A draft write of `0` over `-0` is now recorded as a change instead of being dropped. A transaction that deletes a row and reinserts it with only a zero's sign changed now sends that update instead of dropping it, and an ordered Effect now delivers an update that changes only a zero's sign. The same holds inside typed arrays. An ordered Effect also delivers an update whose value is a class instance, which can change in private state its enumerable fields do not show. +Treat `-0` and `0` as distinct row values. A synced update, a replay that replaces a subscription's rows, and an optimistic update that change a field only from `-0` to `0` (or back) now publish a change; before, the Collection could store the new value while subscribers kept the old one. A draft write of `0` over `-0` is now recorded as a change instead of being dropped. A transaction that deletes a row and reinserts it with only a zero's sign changed now sends that update instead of dropping it, and an ordered Effect now delivers an update whose projected value changes only by a zero's sign. A sync insert of a held key whose row differs only by a zero's sign is now a duplicate key, not an equal re-insert. The same holds inside typed arrays. An ordered Effect also delivers an update whose value is a class instance, which can change in private state its enumerable fields do not show. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 09ec79e37e..78ca710cce 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6227,7 +6227,10 @@ crosses an unordered and an ordered query (`orderBy` with `limit`) and adds a decision 2026-10-29); 120 histories on each path. An ordered Effect delivers an update only when its projection changed. On unchanged `f06ee8fdd` the ordered `signed-zero` histories fail (an update that flips only a zero's sign was -dropped); comparing with `sameRowValue` repairs them. An ordered Effect also +dropped); comparing with `sameRowValue` repairs them. That payload's source +row also changes its `version`; a source update that changes only a zero's +sign is dropped earlier, by the ordered query's top-K (open, see the adapter +conformance entry's signed-zero paragraph). An ordered Effect also dropped an update whose change is private or non-enumerable state, so the consumer kept a stale value (`private` and `prototype` payloads, 20 ordered histories, RED on `370e029db`). Maintainer decision 2026-11-02: the ordered @@ -6890,7 +6893,19 @@ detection (`optimistic-history-oracle`, with a `flipZeros` edit and `zeroSignEditsOverPending` reach), subscription truncate replay (`collection-subscription-replay-oracle`), and stale-row reconciliation (`collection-subscription-lifecycle-publication-oracle`) all compare row -values with signed zeros. Open: generated publication histories never write +values with signed zeros. A source insert of a held key is an equal +re-insert only for an equal row value: the state-retention oracle +(`collection-state.retention`) generates zeros of both signs and a write of +one zero followed by an insert of the other (reach `sign-only-reinsert`, 92 +histories over its fixed and random campaigns); the deep-equality check +accepted such an insert as an equal re-insert instead of throwing +`DuplicateKeySyncError` (RED at `[update {0: 0}, insert {0: -0}]`). Open, +for a maintainer decision: an ordered live query (`orderBy`, with or without +`limit`) drops a source update that changes only a zero's sign, because its +top-K consolidates rows by D2 value identity, where -0 equals 0 (the +hash-identity oracle's law); an unordered live query delivers it. The +Effect-callback `signed-zero` payload changes the source row's `version` +too, so no generated history reaches that case. Open: generated publication histories never write over a row a failed replay retained (the random grammar inserts source writes into green lifecycle histories), so the stale reconcile is reached only by pinned replays, including a `-0` one. A generated failed-replay diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 83fbf4c8ca..3ecec40b7f 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -1,4 +1,4 @@ -import { deepEquals, sameRowValue } from '../utils' +import { sameRowValue } from '../utils' import { SortedMap } from '../SortedMap' import { enrichRowWithVirtualProps } from '../virtual-props.js' import { @@ -924,7 +924,8 @@ export class CollectionStateManager< // like any insert, holds and seeds notwithstanding. if (this.hydrationSeedKeys.has(key) && !projection.sourceKeys.has(key)) return `insert` - if (current.value !== undefined && deepEquals(current.value, value)) { + // An equal re-insert has the same row value: -0 and 0 differ (FID-1). + if (current.value !== undefined && sameRowValue(current.value, value)) { return `update` } return `duplicate` diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 05b4f26d20..2654eb93e5 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -1,9 +1,10 @@ import { fc, test as fcTest } from '@fast-check/vitest' -import { expect, it } from 'vitest' +import { afterAll, expect, it } from 'vitest' import { createCollection } from '../src/collection/index.js' import { DuplicateKeySyncError } from '../src/errors.js' import { oraclePropertyOptions, oracleRuns } from './oracle-config.js' import { declareOracle } from './oracle-declaration.js' +import { createOracleReach } from './oracle-reach.js' import { historyRow, optimisticHistory, @@ -32,6 +33,12 @@ import type { Collection } from '../src/collection/index.js' * restart, and while a reentrant restart's new transaction is still open, the * old run writes and commits a row. Cleanup ended that run, so the row never * appears and the commit returns `true`. + * + * A source insert of a held key is an equal re-insert only when its row value + * equals the held one; otherwise it throws `DuplicateKeySyncError`. Row values + * tell -0 from 0 (maintainer decision 2026-10-29), so values include -0, and + * an insert that differs from the held row only by a zero's sign is a + * duplicate (reach `sign-only-reinsert`). */ type RetentionAction = @@ -53,9 +60,22 @@ type RetentionHarness = { const retainedRowArbitrary = fc.record({ id: fc.integer({ min: 0, max: 3 }), - value: fc.integer({ min: -2, max: 2 }), + // Zeros of both signs are common, so an insert often differs from the + // held row only by a zero's sign. + value: fc.oneof( + { weight: 3, arbitrary: fc.integer({ min: -2, max: 2 }) }, + { weight: 1, arbitrary: fc.constant(0) }, + { weight: 1, arbitrary: fc.constant(-0) }, + ), }) +const retentionReach = createOracleReach( + `collection-state.retention`, + [`sign-only-reinsert`], + { minimum: 5 }, +) +afterAll(() => retentionReach.assertReached()) + function snapshotRetainedRow(row: RetainedRow): RetainedRow { return { id: row.id, value: row.value } } @@ -137,7 +157,17 @@ function applyAction( switch (action.type) { case `insert`: { const previous = model.get(action.row.id) - if (previous !== undefined && previous.value !== action.row.value) { + if ( + previous !== undefined && + previous.value === action.row.value && + !Object.is(previous.value, action.row.value) + ) { + retentionReach.reach(`sign-only-reinsert`) + } + if ( + previous !== undefined && + !Object.is(previous.value, action.row.value) + ) { expect(() => sync.write({ type: `insert`, @@ -212,6 +242,7 @@ declareOracle({ check: [expectRetainedState] }) async function runRetentionHistory( actions: ReadonlyArray, ): Promise { + retentionReach.beginHistory() const harness = createRetentionHarness() const { collection } = harness const model = new Map() @@ -434,10 +465,23 @@ it(`releases retained keys after long unique-key churn`, async () => { await runRetentionHistory(actions) }) -const retentionHistory = fc.array(retentionActionArbitrary, { - minLength: 1, - maxLength: 20, -}) +// One element in ten writes a zero to a key, then inserts the other zero: +// random actions rarely put the two next to each other. +const signFlipArbitrary = fc + .record({ id: fc.integer({ min: 0, max: 3 }), zero: fc.constantFrom(0, -0) }) + .map(({ id, zero }): Array => [ + { type: `update`, row: { id, value: zero } }, + { type: `insert`, row: { id, value: Object.is(zero, -0) ? 0 : -0 } }, + ]) +const retentionHistory = fc + .array( + fc.oneof( + { weight: 9, arbitrary: retentionActionArbitrary.map((a) => [a]) }, + { weight: 1, arbitrary: signFlipArbitrary }, + ), + { minLength: 1, maxLength: 20 }, + ) + .map((elements) => elements.flat()) fcTest.prop([retentionHistory], { numRuns: oracleRuns(100), seed: 1_902 })( `matches retained authoritative state with a fixed seed`, From 755af34e7d363b45676544f932bfbe55fa87d8a1 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 15:50:29 -0700 Subject: [PATCH 41/64] fix(db-adapter-conformance): remove the check's patches when a driver's setup fails The check patches console.error, console.warn, and the process's unhandledRejection listeners before driver.create() and the provider's seed(), outside the try that removes them. A driver whose setup threw left them installed for the rest of the worker, swallowing later tests' diagnostics. Setup now removes them before the failure propagates (ORC-010). - RED (committed check): an ordinary harness test with a driver whose create() or seed() fails finds console.error replaced and an extra unhandledRejection listener, both stages. - GREEN: db-adapter-conformance 84/84. Co-authored-by: Isaac --- .../src/eager-live-source-oracle.ts | 13 +++++- .../tests/eager-live-source-harness.test.ts | 45 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 packages/db-adapter-conformance/tests/eager-live-source-harness.test.ts diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index f7d494b1d1..be65dddcea 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -2025,14 +2025,23 @@ export async function checkEagerLiveSourceHistory( return [] } const detached = watchDetachedFailures(driver) - const fixture = driver.create() + let fixture: EagerLiveSourceFixture + try { + fixture = driver.create() + if (listenBeforeFetch) { + await fixture.provider.seed?.(structuredClone(seedFrom(0))) + } + } catch (error) { + // A setup failure leaves no console or rejection patch behind (ORC-010). + detached.stop() + throw error + } const { collection, provider } = fixture const state = initialModel(seedFrom(0)) const failures: Array = [] const failureCounter = { next: 0 } const waits: Array<{ current: Settlement }> = [] const syncWaits: Array<{ current: Settlement }> = [] - if (listenBeforeFetch) await provider.seed?.(structuredClone(seedFrom(0))) // READY-3, READY-10: the rows at the moment status becomes `ready`. let readyRows: RowState | undefined const watchReady = () => diff --git a/packages/db-adapter-conformance/tests/eager-live-source-harness.test.ts b/packages/db-adapter-conformance/tests/eager-live-source-harness.test.ts new file mode 100644 index 0000000000..29462fb056 --- /dev/null +++ b/packages/db-adapter-conformance/tests/eager-live-source-harness.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' +import { checkEagerLiveSourceHistory } from '../src' +import { referenceDriver } from './reference-adapter' +import type { EagerLiveSourceDriver } from '../src' + +// The check patches console.error, console.warn, and the process's +// unhandledRejection listeners for one history (HEALTH-5). A driver whose +// setup fails must leave all three as it found them, or later tests in the +// worker lose their diagnostics (ORC-010). +describe(`eager live-source check harness`, () => { + const failingSetup = ( + stage: `create` | `seed`, + ): EagerLiveSourceDriver => { + const driver = referenceDriver() + return { + ...driver, + create() { + if (stage === `create`) throw new Error(`driver setup failed`) + const fixture = driver.create() + return { + ...fixture, + provider: Object.assign(Object.create(fixture.provider), { + seed: () => Promise.reject(new Error(`driver setup failed`)), + }), + } + }, + } + } + + it.each([`create`, `seed`] as const)( + `restores console and rejection listeners when the driver's %s fails`, + async (stage) => { + const { error, warn } = console + const listeners = process.listenerCount(`unhandledRejection`) + await expect( + checkEagerLiveSourceHistory(failingSetup(stage), [ + { kind: `snapshot`, rows: [] }, + ]), + ).rejects.toThrow(`driver setup failed`) + expect(console.error).toBe(error) + expect(console.warn).toBe(warn) + expect(process.listenerCount(`unhandledRejection`)).toBe(listeners) + }, + ) +}) From ec576f807826cfd65ded1eb8e11e98ab75525c5e Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 15:50:36 -0700 Subject: [PATCH 42/64] fix(electric): name the status and request in an outage's error message The outage failure is built from the response without reading its body, which the client still reads, so FetchError's default message ended in ": undefined". It now passes the message "HTTP Error at ". The subset lane of the SDK delivery oracle now also checks the reported failure's message while an outage shows (HEALTH-4: the value owed is the client's error for the 503 response). - RED (committed adapter): both subset-lane campaigns fail the message check. - GREEN: electric-db-collection 1243 passed, 4 skipped. Co-authored-by: Isaac --- packages/electric-db-collection/src/electric.ts | 5 ++++- .../electric-sdk-delivery-oracle.property.test.ts | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 6c5a51f9ac..baa0936a85 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -2048,13 +2048,16 @@ function createElectricSync>( // also type-checks this source. const headers: Record = {} response.headers.forEach((value, name) => (headers[name] = value)) + // The client still reads the body, so the message names no text. + const url = String(input) reportRetry( new FetchError( response.status, undefined, undefined, headers, - String(input), + url, + `HTTP Error ${response.status} at ${url}`, ), ) } diff --git a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts index 0f065e9eb1..9425ed98a2 100644 --- a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts @@ -342,8 +342,16 @@ async function checkSnapshots( // 2026-10-30). Only the live stream's up-to-date ends it; a subset's // end does not. let outage = false - const sourceHealth = (label: string) => + // HEALTH-4: the outage shows the client's error for a 503 response, + // which names the status and the request. + const sourceHealth = (label: string) => { expect(collection.health.failureReason !== null, label).toBe(outage) + if (outage) { + expect(collection.health.failureReason?.message, label).toMatch( + /^HTTP Error 503 at http\S+$/, + ) + } + } const lowerBounds = [ 1, 2, From dac45c25ee3e6aadfda60d5846f8d3bd00051325 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 15:56:27 -0700 Subject: [PATCH 43/64] fix(electric): export a retired run's txids only once the current run is ready exportMeta merged the retired runs' txids whatever the current run's readiness, so metadata dehydrated before a restarted run was ready let a hydrating Collection accept txids that awaitTxId here still refused (EVID-2). Exported txids are now those awaitTxId accepts at that moment. The subset lane of the SDK delivery oracle now imports the exported metadata into a fresh Collection at both restart cuts and checks that it accepts no txid the source does not accept then. - Law: META-1 with EVID-2; a retired run's evidence counts once the current run is ready (maintainer decision 2026-10-30). - Reach: every subset-lane history ends with that restart (retired-evidence-after-restart). - RED (committed adapter): before the restarted run is ready, the import accepts txid 25 ([false, true], owed [false, false]); both campaigns. - GREEN: electric-db-collection 1243 passed, 4 skipped. - Mutant: never exporting retired txids fails after ready (txid 25 not accepted). - Limits: the metadata format carries txids only; snapshot evidence was never exported, on main either. No real hydration across processes. Co-authored-by: Isaac --- .../electric-retry-health-and-evidence.md | 2 +- docs/contributing/oracle-coverage.md | 8 ++++- .../electric-db-collection/src/electric.ts | 19 ++++++---- ...ctric-sdk-delivery-oracle.property.test.ts | 36 +++++++++++++++++++ 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/.changeset/electric-retry-health-and-evidence.md b/.changeset/electric-retry-health-and-evidence.md index f7876868b6..96067fd7db 100644 --- a/.changeset/electric-retry-health-and-evidence.md +++ b/.changeset/electric-retry-health-and-evidence.md @@ -2,4 +2,4 @@ '@tanstack/electric-db-collection': patch --- -Report an extended Electric outage in `collection.health`. After the Collection is ready, five failed requests to the shape log in a row, retried by the client or by an `onError` that returns a retry, show as the source node's `failureReason` with the fifth attempt's error, and the next applied up-to-date clears it. Shorter outages, and retried subset snapshot requests, leave health unchanged. A restarted or reloaded Collection resolves `awaitTxId` for a txid the previous sync run delivered only once it is ready again, instead of at once. When the stream fails, pending `awaitTxId` and `awaitMatch` calls reject instead of waiting for their timeout, and so do calls made after the failure, also when an abort stopped the stream. An `onError` handler that throws now gives up the stream with its own error, without an unhandled rejection. A give-up while the first snapshot's acceptance is pending, under a persistence wrapper, now fails the Collection once that acceptance makes it ready, instead of leaving a stopped stream with healthy status. Aborting `shapeOptions.signal` after the Collection is ready now shows as the source node's `error`, since the Collection no longer syncs. +Report an extended Electric outage in `collection.health`. After the Collection is ready, five failed requests to the shape log in a row, retried by the client or by an `onError` that returns a retry, show as the source node's `failureReason` with the fifth attempt's error, and the next applied up-to-date clears it. Shorter outages, and retried subset snapshot requests, leave health unchanged. A restarted or reloaded Collection resolves `awaitTxId` for a txid the previous sync run delivered only once it is ready again, instead of at once, and the sync metadata it exports for hydration carries those txids only from then on. When the stream fails, pending `awaitTxId` and `awaitMatch` calls reject instead of waiting for their timeout, and so do calls made after the failure, also when an abort stopped the stream. An `onError` handler that throws now gives up the stream with its own error, without an unhandled rejection. A give-up while the first snapshot's acceptance is pending, under a persistence wrapper, now fails the Collection once that acceptance makes it ready, instead of leaving a stopped stream with healthy status. Aborting `shapeOptions.signal` after the Collection is ready now shows as the source node's `error`, since the Collection no longer syncs. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 78ca710cce..d98c38e63b 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -7000,7 +7000,13 @@ throwing, and a second campaign runs the default configuration without an app `onError`, declaring the adapter's give-up log. The on-demand lane also restarts the Collection and checks that a retired run's snapshot-boundary evidence (txid 5 below `xmin` 10) and imported sync metadata (txid 25) stay -pending until the restarted run is ready, then fulfill; it declares reach +pending until the restarted run is ready, then fulfill. At both cuts a +fresh Collection imports the exported sync metadata, as hydration does, and +must accept no txid the source does not accept then: the export carried the +retired run's txids before the restarted run was ready (RED: txid 25 +accepted after import), and a mutant that never exports them fails after +ready. The metadata format carries txids only, so snapshot evidence is not +exported, as on `main`; it declares reach for a subset request's five retried attempts, five live pauses, a live outage, and that restart. The retry-decision model (`electric-retry-decision-model.ts`, shared by the controlled lane of diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index baa0936a85..ace9edff79 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1008,14 +1008,19 @@ class ElectricLifecycle> { clearEvidence(this.retired) } + /** Exported txids are those `awaitTxId` accepts now (EVID-2). */ exportMeta(): ElectricSyncMeta { - return exportElectricSyncMeta({ - ...this.evidence, - seenTxids: new Set([ - ...this.retired.seenTxids, - ...this.evidence.seenTxids, - ]), - }) + return exportElectricSyncMeta( + this.runReady + ? { + ...this.evidence, + seenTxids: new Set([ + ...this.retired.seenTxids, + ...this.evidence.seenTxids, + ]), + } + : this.evidence, + ) } importMeta(meta: unknown): void { diff --git a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts index 9425ed98a2..9c6ec25708 100644 --- a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts @@ -496,6 +496,38 @@ async function checkSnapshots( retired.map((wait) => wait.settled), `retired evidence before the restarted run is ready`, ).toEqual([`pending`, `pending`]) + // META-1 with EVID-2: exported sync metadata claims a txid only when + // this Collection's `awaitTxId` accepts it now, and claims each accepted + // txid evidence, so a Collection that imports it, as hydration does, + // accepts no more. The format carries txids only, so snapshot evidence + // (txid 5) is not exported. + const acceptedAfterImport = async () => { + const fresh = createCollection( + electricCollectionOptions({ + id: `snapshot-meta-import-${++sequence}`, + shapeOptions: { + url: `http://test-url/unused`, + params: { table: `rows` }, + }, + getKey: (row) => row.id, + }), + ) + fresh.config.sync.importSyncMeta?.( + collection.config.sync.exportSyncMeta?.(), + ) + return Promise.all( + [5, 25].map((txid) => + fresh.utils.awaitTxId(txid, 20).then( + () => true, + () => false, + ), + ), + ) + } + expect( + await acceptedAfterImport(), + `exported evidence before the restarted run is ready`, + ).toEqual([false, false]) // The restarted run becomes ready on its live up-to-date, which carries // no evidence for either txid: only the retired evidence can fulfill // the waits. @@ -515,6 +547,10 @@ async function checkSnapshots( retired.map((wait) => wait.settled), `retired evidence once the restarted run is ready`, ).toEqual([`fulfilled`, `fulfilled`]) + expect( + (await acceptedAfterImport())[1], + `exported txid evidence once the restarted run is ready`, + ).toBe(true) snapshotReach.reach(`retired-evidence-after-restart`) }, [ { label: `snapshot collection`, release: () => collection.cleanup() }, From a23b0394951e56ebc2446b6a17088f18b325b57e Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 15:59:07 -0700 Subject: [PATCH 44/64] docs(db-adapter-conformance): refresh the evidence block and TrailBase's capabilities - The README evidence block now reports the current fixed-campaign reach for all 76 declared states of the reference adapter (seed 2118, 600 histories; lowest 18), the 47 mutant calibrations with their laws, the 84-test GREEN count, and eleven value kinds. - TrailBase declares duplicate-key-delivery and source-failure-after-ready; the README, MIGRATION.md, TrailBase's ORACLE.md, and its property file said it declared none. - MIGRATION.md's persistence row names the P-* and S-* laws in code spans, so Markdown formatting no longer rewrites them as P-_ and S-_. Co-authored-by: Isaac --- packages/db-adapter-conformance/MIGRATION.md | 4 +- packages/db-adapter-conformance/README.md | 217 ++++++++++-------- .../trailbase-db-collection/tests/ORACLE.md | 8 +- ...source-conformance-oracle.property.test.ts | 9 +- 4 files changed, 137 insertions(+), 101 deletions(-) diff --git a/packages/db-adapter-conformance/MIGRATION.md b/packages/db-adapter-conformance/MIGRATION.md index 3a7d0f9026..403ab601d7 100644 --- a/packages/db-adapter-conformance/MIGRATION.md +++ b/packages/db-adapter-conformance/MIGRATION.md @@ -23,7 +23,7 @@ reviewed change. | Client cache and refetch | Needed | CACHE-1 to CACHE-8, REF-1 to REF-7, DIR-1 to DIR-5 | | Source shapes | Needed | RESET-1, RESUME-1 to RESUME-3, PROG-1, MEMB-1/2, PAGE-1 to PAGE-3, OWN-1/2, META-1 | | Local and shared stores | Needed | SELF-1 to SELF-3, STORE-1 to STORE-4, PEER-1 | -| Persistence wrapper and storage hosts | Needed | P-_, S-_ | +| Persistence wrapper and storage hosts | Needed | `P-*`, `S-*` | ## Adapters @@ -33,7 +33,7 @@ reviewed change. | Electric | eager live source, mutations, on-demand, source shapes, reload and retry, persistence | Eager: GREEN (2026-10-30) after three fixes, per-batch commits, typed Postgres columns, over the installed client. Driver `electric-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.electric`. Other families: not started | | PowerSync | eager live source, mutations, on-demand | Eager: GREEN (2026-10-30) after a fix, per-batch commits, over real local SQLite. Driver `powersync-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.powersync`. Mutations, on-demand: not started | | RxDB | eager live source, mutations | Eager: GREEN (2026-10-29) after a fix, per-change commits, `listen-before-fetch`. Driver `rxdb-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.rxdb`. Mutations: not started | -| TrailBase | eager live source, mutations, on-demand | Eager: GREEN (2026-10-29), per-change commits. Driver `trailbase-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.trailbase`; declares no optional capability. Mutations, on-demand: not started | +| TrailBase | eager live source, mutations, on-demand | Eager: GREEN (2026-10-29), per-change commits. Driver `trailbase-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.trailbase`; declares `duplicate-key-delivery` and `source-failure-after-ready`. Mutations, on-demand: not started | | Convex | eager live source, mutations, on-demand, source shapes | Eager: GREEN (2026-11-02) after a fix, per-batch commits, complete results, over the real Convex JavaScript client and a controlled WebSocket. Driver `convex-db-collection/tests/conformance-driver.ts`, property `adapter-conformance.eager-live-source.convex`. Mutations, on-demand, source shapes: not started | | query-db | client cache and refetch, on-demand, mutations, reload and retry, persistence | Not started; not a push source, so the eager family does not apply | | local-only | local and shared stores, mutations | Not started | diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md index ab31709b1b..688895aef5 100644 --- a/packages/db-adapter-conformance/README.md +++ b/packages/db-adapter-conformance/README.md @@ -99,103 +99,133 @@ from `packages/db/tests/oracle-reach.ts`. in that package's default `pnpm test`. - **TrailBase** (`packages/trailbase-db-collection/tests/conformance-driver.ts`): the published adapter in eager mode over a controlled `RecordApi`, with - per-change commits and no optional capability. It runs in that package's - default `pnpm test`. + per-change commits, `duplicate-key-delivery` (a paged list can return one + record twice; the later row wins), and `source-failure-after-ready` (the + change stream can error after ready). It runs in that package's default + `pnpm test`. -## Evidence (2026-10-29) +## Evidence (2026-11-02) - **Law:** the table above. Authority: the sync callback contract in `packages/db/src/types.ts`, the status and health terms in - `docs/contributing/glossary.md`, and the adapter law set v18. + `docs/contributing/glossary.md`, the adapter law set v18, and the + maintainer decisions recorded in `MIGRATION.md` and the coverage map. - **Reach:** in the fixed campaign (seed 2118, 600 histories) on the reference adapter, the histories that reached each declared state: - | Declared state | Histories | - | ------------------------------ | --------- | - | first-ready | 248 | - | replacing-snapshot | 77 | - | change-batch | 97 | - | multi-change-batch | 327 | - | failure-before-ready | 124 | - | failure-after-ready | 64 | - | restart | 116 | - | wait-before-evidence-fulfilled | 48 | - | wait-timed-out | 75 | - | wait-rejected-at-retirement | 188 | - | transient-failure-reported | 67 | - | transient-failure-recovered | 33 | - | reconnect | 62 | - | renewal | 141 | - | change-before-ready | 514 | - | read-missed-change | 101 | - | seen-change-superseded | 26 | - | field-removed | 27 | - | value:string:load | 66 | - | value:string:change | 95 | - | value:integer:load | 85 | - | value:integer:change | 130 | - | value:float:load | 49 | - | value:float:change | 61 | - | value:negative-zero:load | 54 | - | value:negative-zero:change | 103 | - | value:boolean:load | 64 | - | value:boolean:change | 104 | - | value:null:load | 62 | - | value:null:change | 95 | - | value:date:load | 51 | - | value:date:change | 93 | - | value:bigint:load | 74 | - | value:bigint:change | 91 | - | value:array:load | 71 | - | value:array:change | 99 | - | value:object:load | 56 | - | value:object:change | 99 | - | zero-sign-flipped | 74 | - - Each campaign fails when a state is reached fewer than five times. Across - 20 other seeds, the lowest count for any state was 12 (`field-removed`), across 12 seeds. - -- **GREEN:** `pnpm exec vitest run tests/reference-adapter-oracle.property.test.ts`: - 68 tests pass (both campaigns, the per-change pair, 32 calibrations, and the - grammar controls). TrailBase: its full suite, 151 tests, passes. -- **Mutants:** the fixed campaign rejects each with the law it breaks: - - | Mutant | Law | - | ----------------------------------------------- | -------------------------- | - | `ready-before-delivery` | READY-2 | - | `wraps-loading-failure` | READY-4 | - | `failure-clears-rows` | HEALTH-1 | - | `transient-is-fatal` | HEALTH-1 | - | `never-recovers` | HEALTH-2 | - | `transient-fails-loading` | RETRY-1 | - | `splits-batch` | PUB-1 | - | `reverses-batch` (per-change) | PUB-1 | - | `drops-deletes` | PUB-3 | - | `snapshot-merges` | PUB-7 | - | `leaks-subscription` | LIFE-2 | - | `restart-replays-rows` | LIFE-5 | - | `keeps-failed-source` | LIFE-10 | - | `double-release` | LIFE-10 | - | `detached-rejection` | HEALTH-5 | - | `logs-failure` | HEALTH-5 | - | `wait-off-by-one` | EVID-1 | - | `wait-ignores-timeout` | EVID-3 | - | `disconnect-fails` | RECON-1 | - | `renew-resubscribes` | RENEW-1 | - | `drops-buffered` | READY-10 | - | `buffer-before-rows` | READY-10 | - | `replays-per-batch` | PUB-5 | - | `ready-before-buffer` | READY-10 (rows when ready) | - | `merges-updates` | FID-1 | - | `relabels-inserts` (checker calibration) | PUB-2 | - | `dates-as-strings` (live changes only) | FID-1 (values) | - | `bigints-as-numbers` (initial load only) | FID-1 (values) | - | `drops-nulls` | FID-1 (values) | - | `loses-negative-zero` | FID-1 (values) | - | `typed-booleans-as-numbers` (live changes only) | FID-1 (values) | - | `commits-malformed-prefix` | PUB-4 | - | `ignores-malformed` | PUB-4 | - | `shows-buffered` | READY-10/PUB-9 | + | Declared state | Histories | + | ------------------------------- | --------- | + | first-ready | 291 | + | replacing-snapshot | 54 | + | change-batch | 169 | + | multi-change-batch | 332 | + | failure-before-ready | 172 | + | failure-after-ready | 36 | + | restart | 238 | + | restart-after-failure | 101 | + | prompt-restart | 124 | + | prompt-failure-after-failed-run | 40 | + | wait-before-evidence-fulfilled | 32 | + | wait-timed-out | 130 | + | wait-rejected-at-retirement | 148 | + | wait-on-retired-evidence | 92 | + | transient-failure-reported | 71 | + | transient-failure-repeated | 56 | + | outage-reported-again | 28 | + | outage-reported-after-batch | 39 | + | outage-reported-in-later-run | 27 | + | outage-count-reset | 21 | + | delivery-error-reported | 50 | + | delivery-error-recovered | 18 | + | transient-failure-recovered | 39 | + | reconnect | 52 | + | renewal | 129 | + | change-before-ready | 494 | + | read-missed-change | 103 | + | seen-change-superseded | 24 | + | field-removed | 22 | + | malformed-before-ready | 43 | + | malformed-after-ready | 23 | + | zero-sign-flipped | 79 | + + Each value kind, as a row's `value` and as a typed field, on the initial + load and on a live change: + + | Kind | Value, load | Value, change | Field, load | Field, change | + | ------------- | ----------- | ------------- | ----------- | ------------- | + | string | 70 | 82 | 256 | 446 | + | integer | 73 | 118 | 256 | 446 | + | float | 43 | 63 | 256 | 446 | + | negative-zero | 48 | 94 | 256 | 446 | + | boolean | 82 | 86 | 256 | 446 | + | null | 66 | 105 | 256 | 446 | + | date | 49 | 75 | 256 | 446 | + | bigint | 52 | 100 | 256 | 446 | + | array | 66 | 102 | 256 | 446 | + | object | 64 | 85 | 256 | 446 | + | float-array | 65 | 94 | 256 | 446 | + + Each campaign fails when a state is reached fewer than five times. The + lowest count above is 18. + +- **GREEN:** `pnpm exec vitest run` in this package: 84 tests pass. The + reference oracle file holds 82: both campaigns, the per-change pair, the + outage-threshold campaign, 47 mutant calibrations, and the grammar + controls. `tests/eager-live-source-harness.test.ts` holds 2 ordinary tests + of the check's own setup cleanup. +- **Mutants:** the fixed campaign rejects each with the law it breaks. Where + two laws are listed, the mutant breaks both, and the shrunk history shows + either: + + | Mutant | Law | + | ------------------------------------------------------------- | -------------------------- | + | `ready-before-delivery` | READY-2 | + | `wraps-loading-failure` | READY-4 | + | `failure-clears-rows` | HEALTH-1 | + | `transient-is-fatal` | HEALTH-1 | + | `delivery-error-is-fatal` | HEALTH-1 | + | `never-recovers` | HEALTH-2 | + | `transient-fails-loading` | RETRY-1 | + | `splits-batch` | PUB-1 | + | `drops-deletes` | PUB-3 | + | `snapshot-merges` | PUB-7 or PUB-4 | + | `leaks-subscription` | LIFE-2 | + | `restart-replays-rows` | LIFE-5 | + | `swallows-prompt-failure` | READY-4 | + | `keeps-failed-source` | LIFE-10 | + | `double-release` | LIFE-10 | + | `detached-rejection` | HEALTH-5 | + | `logs-failure` | HEALTH-5 | + | `wait-off-by-one` | EVID-1 or EVID-2 | + | `retired-evidence-before-ready` | EVID-2 | + | `forgets-retired-evidence` | EVID-2 | + | `leaks-waits-after-fail` | EVID-2 | + | `wait-ignores-timeout` | EVID-3 | + | `disconnect-fails` | RECON-1 | + | `renew-resubscribes` | RENEW-1 | + | `drops-buffered` | READY-10 | + | `buffer-before-rows` | PUB-9 or PUB-4 | + | `replays-per-batch` | PUB-5 | + | `ready-before-buffer` | READY-10 (rows when ready) | + | `shows-buffered` | READY-10/PUB-9 | + | `merges-updates` | FID-1 | + | `dates-as-strings` | FID-1 (values) | + | `bigints-as-numbers` | FID-1 (values) | + | `drops-nulls` | FID-1 (values) | + | `loses-negative-zero` | FID-1 (values) | + | `typed-booleans-as-numbers` | FID-1 (values) | + | `commits-malformed-prefix` | PUB-4 | + | `ignores-malformed` | PUB-4 | + | `relabels-inserts` | PUB-2 | + | `reverses-batch` (per-change) | PUB-1 | + | `stale-outage-value` (threshold 3, client values) | HEALTH-1 or HEALTH-2 | + | `replaces-outage-value` (threshold 3, client values) | RETRY-1 | + | `reports-outage-early` (threshold 3, client values) | HEALTH-1 | + | `outage-count-survives-delivery` (threshold 3, client values) | HEALTH-1 | + | `empty-response-keeps-count` (threshold 3, client values) | HEALTH-1 | + | `empty-response-recovers` (threshold 3, client values) | RETRY-1 | + | `rejected-delivery-fails` (threshold 3, client values) | PUB-4 | + | `wraps-loading-failure` (threshold 3, client values) | READY-4 | `reverses-batch` reverses only batches of distinct keys, so its final rows are right and only the intermediate publications are wrong. An earlier @@ -237,9 +267,10 @@ from `packages/db/tests/oracle-reach.ts`. timed wait settles within its own step, so no evidence arrives during it. An adapter that rejects a timed wait at once, before its timeout, is not distinguished from one that waits. - - FID-1 generates ten value kinds (string, integer, float, `-0`, boolean, - `null`, Date, bigint, array, plain object). Temporal values, Map, Set, - typed arrays, and class instances are not generated, and a stored + - FID-1 generates eleven value kinds (string, integer, float, `-0`, + boolean, `null`, Date, bigint, array, plain object, and a + `Float64Array`). Temporal values, Map, Set, other typed arrays, and class + instances are not generated, and a stored field's reference identity is not checked. - PUB-4 covers one malformed shape, two rows with one key in a complete delivery. A malformed key, a row the adapter cannot decode, and an diff --git a/packages/trailbase-db-collection/tests/ORACLE.md b/packages/trailbase-db-collection/tests/ORACLE.md index 608c35d5c0..8f54a58222 100644 --- a/packages/trailbase-db-collection/tests/ORACLE.md +++ b/packages/trailbase-db-collection/tests/ORACLE.md @@ -257,7 +257,9 @@ No service-backed E2E run is claimed. live-source oracle from `packages/db-adapter-conformance` through `conformance-driver.ts`, with the same controlled RecordApi. It checks READY-2/3/4, HEALTH-1, PUB-1 (per-change commits), PUB-2/3, FID-1/4, -LIFE-2/5/10, and HEALTH-5 in eager mode. TrailBase declares no optional -capability, so it is not checked for PUB-7, waits, retried failures, -reconnect, or renewal. This file's +LIFE-2/5/10, and HEALTH-5 in eager mode. TrailBase declares +`duplicate-key-delivery` (PUB-3 under `duplicateInsert: replace`) and +`source-failure-after-ready` (HEALTH-1), and no other optional capability, +so it is not checked for PUB-7, waits, retried failures, reconnect, or +renewal. This file's lifecycle oracle keeps its own laws; see `packages/db-adapter-conformance/MIGRATION.md`. diff --git a/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts index 79ed5e71a7..588003bde9 100644 --- a/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts +++ b/packages/trailbase-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -25,9 +25,12 @@ import { trailBaseConformanceDriver } from './conformance-driver' * TrailBase production driver in `conformance-driver.ts` and the campaigns; * it states no expected results. * - * TrailBase declares none of the oracle's optional capabilities: it lists - * once per sync run, carries no txid, and does not resubscribe. Those laws - * stay with `lifecycle-oracle.property.test.ts`, which also owns on-demand + * TrailBase declares two of the oracle's optional capabilities: + * `duplicate-key-delivery`, because a paged list can return one record twice + * (PUB-3 under `duplicateInsert: replace`), and `source-failure-after-ready`, + * because the change stream can error after ready (HEALTH-1). It declares no + * other: it lists once per sync run, carries no txid, and does not + * resubscribe. Those laws stay with `lifecycle-oracle.property.test.ts`, which also owns on-demand * loading, page and event fencing, cancellation, and timers. The claim is * limited to the controlled RecordApi. */ From 8aa144bc59ad3baad25fccb86d29079967ea8729 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 16:03:14 -0700 Subject: [PATCH 45/64] test(powersync): release the diff trigger when the first load's acceptance fails The eager conformance driver's failures come before the diff trigger exists, so a first load that fails after it, as when a wrapping sync rejects acceptance, was not covered. A focused test wraps commit with a rejected acceptance and checks that the trigger and the change watcher are released at the failure, and the trigger only once across the failure and cleanup (LIFE-10). - RED (origin/main adapter): the trigger is still held at the failure ([0], owed [1]). - GREEN: powersync-db-collection full suite. Co-authored-by: Isaac --- .../tests/load-hooks.test.ts | 67 ++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/packages/powersync-db-collection/tests/load-hooks.test.ts b/packages/powersync-db-collection/tests/load-hooks.test.ts index cf0b3d6abd..9b2a0ac8b4 100644 --- a/packages/powersync-db-collection/tests/load-hooks.test.ts +++ b/packages/powersync-db-collection/tests/load-hooks.test.ts @@ -1,7 +1,12 @@ import { randomUUID } from 'node:crypto' import { tmpdir } from 'node:os' import { PowerSyncDatabase, Schema, Table, column } from '@powersync/node' -import { createCollection, createLiveQueryCollection, eq } from '@tanstack/db' +import { + createCollection, + createLiveQueryCollection, + eq, + withAcceptedReceipt, +} from '@tanstack/db' import pDefer from 'p-defer' import { describe, expect, it, onTestFinished, vi } from 'vitest' import { powerSyncCollectionOptions } from '../src' @@ -97,6 +102,66 @@ describe(`Sync Streams`, () => { `) } + // LIFE-10 after the diff trigger exists: a wrapping sync, such as + // persistence, rejects the first load's acceptance. The conformance + // driver's failures come before the trigger, so this case is pinned here. + it(`eager mode: releases the diff trigger and watcher when the first load's acceptance fails`, async () => { + const db = await createDatabase() + await createTestProducts(db) + const disposed: Array = [] + const createDiffTrigger = db.triggers.createDiffTrigger.bind(db.triggers) + db.triggers.createDiffTrigger = async (options) => { + const dispose = await createDiffTrigger(options) + const index = disposed.length + disposed.push(0) + return async (...args) => { + disposed[index]!++ + return dispose(...args) + } + } + const signals: Array = [] + const onChange = db.onChangeWithCallback.bind(db) + db.onChangeWithCallback = (handler, options) => { + signals.push(options?.signal) + return onChange(handler, options) + } + const acceptanceFailure = new Error(`acceptance failed`) + const options = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + }) + const collection = createCollection({ + ...options, + sync: { + ...options.sync, + sync: (params) => + options.sync.sync({ + ...params, + commit: (signal?: AbortSignal) => { + const receipt = params.commit(signal) + const rejected = Promise.reject(acceptanceFailure) + rejected.catch(() => undefined) + return withAcceptedReceipt( + receipt === true ? Promise.resolve() : receipt, + rejected, + ) + }, + }), + }, + }) + + try { + await expect(collection.preload()).rejects.toBe(acceptanceFailure) + await vi.waitFor(() => expect(disposed).toEqual([1])) + expect(signals.map((signal) => signal?.aborted)).toEqual([true]) + } finally { + await collection.cleanup() + } + expect(disposed, `released once across the failure and cleanup`).toEqual([ + 1, + ]) + }) + it(`eager mode: should call onLoad on sync start and onUnload on cleanup`, async () => { const db = await createDatabase() await createTestProducts(db) From 51e0a8eccc60886ca60ef017f558feef78872219 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:06:23 +0000 Subject: [PATCH 46/64] ci: apply automated fixes --- .../tests/eager-live-source-harness.test.ts | 4 +--- packages/db-adapter-conformance/tests/reference-adapter.ts | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/db-adapter-conformance/tests/eager-live-source-harness.test.ts b/packages/db-adapter-conformance/tests/eager-live-source-harness.test.ts index 29462fb056..51aeebcb8d 100644 --- a/packages/db-adapter-conformance/tests/eager-live-source-harness.test.ts +++ b/packages/db-adapter-conformance/tests/eager-live-source-harness.test.ts @@ -8,9 +8,7 @@ import type { EagerLiveSourceDriver } from '../src' // setup fails must leave all three as it found them, or later tests in the // worker lose their diagnostics (ORC-010). describe(`eager live-source check harness`, () => { - const failingSetup = ( - stage: `create` | `seed`, - ): EagerLiveSourceDriver => { + const failingSetup = (stage: `create` | `seed`): EagerLiveSourceDriver => { const driver = referenceDriver() return { ...driver, diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index 99eeb8ff6a..32a23c965a 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -337,8 +337,7 @@ function referenceCollection( waiters = new Map() highestTxid = 0 runReady = false - let replayExpected = - mutant === `swallows-prompt-failure` && runFailed + let replayExpected = mutant === `swallows-prompt-failure` && runFailed if (replayExpected) setTimeout(() => (replayExpected = false), 0) runFailed = false // Failed attempts after ready since the last delivery, and the value From 683d4f40750b7b00080f832a544495a14d29b986 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 16:10:25 -0700 Subject: [PATCH 47/64] docs(electric): count a failed catch-up prefetch toward the outage threshold The client prefetches the next responses of a multi-response catch-up through the same retrying fetch, so each failed prefetch attempt counts toward the five failed shape-log attempts in a row. Maintainer decision 2026-11-02: they count, as any failed shape-log request does. The docs, the wrapper's comment, and the coverage map now say so; no driver history prefetches. Co-authored-by: Isaac --- docs/collections/electric-collection.md | 5 ++++- docs/contributing/oracle-coverage.md | 5 ++++- packages/electric-db-collection/src/electric.ts | 4 +++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/collections/electric-collection.md b/docs/collections/electric-collection.md index e5a04e27af..210615386c 100644 --- a/docs/collections/electric-collection.md +++ b/docs/collections/electric-collection.md @@ -453,7 +453,10 @@ attempts take from under a second to about 15 seconds, because each retry waits a random part of a doubling delay. A source that keeps failing five times and then succeeding once shows and clears the outage each time. A subset snapshot request that the client retries does not count: its failure -belongs to the live query that requested it. When the stream gives up, it +belongs to the live query that requested it. While the stream catches up on +a log that spans several responses, such as after a `must-refetch`, the +client also prefetches the next responses, and each failed prefetch attempt +counts too, so the outage can show sooner than five rounds of retries. When the stream gives up, it calls `fail()` with the error, or with the error your `onError` threw: a failed first load before readiness, or the source node's `error` after it. A give-up while a persistence wrapper is still accepting the first snapshot fails the Collection once that acceptance diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index d98c38e63b..c1ecfc2289 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -7006,7 +7006,10 @@ must accept no txid the source does not accept then: the export carried the retired run's txids before the restarted run was ready (RED: txid 25 accepted after import), and a mutant that never exports them fails after ready. The metadata format carries txids only, so snapshot evidence is not -exported, as on `main`; it declares reach +exported, as on `main`. Maintainer decision 2026-11-02: a failed catch-up +prefetch attempt counts toward the outage threshold, as any failed +shape-log request does; no driver response spans several chunks, so no +history prefetches. It declares reach for a subset request's five retried attempts, five live pauses, a live outage, and that restart. The retry-decision model (`electric-retry-decision-model.ts`, shared by the controlled lane of diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index ace9edff79..752eccb11d 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -2041,7 +2041,9 @@ function createElectricSync>( // The client retries a failed network attempt, 429, or 5xx itself, and // onError decides another 4xx; each is a failed attempt. A subset // snapshot request carries no abort signal: its attempts belong to its - // demand, not to the source (glossary, request error). + // demand, not to the source (glossary, request error). The client's + // catch-up prefetches carry one, and count (maintainer decision + // 2026-11-02). fetchClient: async (input, init) => { const request = (shapeOptions.fetchClient ?? fetch)(input, init) if (!init?.signal) return request From 3f851bf569e45f614b7a6bff9094caf619932b5e Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 16:16:33 -0700 Subject: [PATCH 48/64] Fix CI failure: reach every descriptor error-handoff history in the random campaign The error-handoff grammar has eight histories, and its random campaign drew 30 with fast-check's biased constantFrom, so a campaign could miss one and fail its reach check (throw-before-release and throw-during-release each reached 0 times on this PR's CI). The random campaign now runs all eight as examples first, so its reach no longer depends on the seed; the random draws still follow. - RED: the CI seed 2032704855 (react.descriptor-ownership) fails the committed campaign with throw-before-release (0). - GREEN: the same seed passes; all eight error-handoff campaigns pass. Co-authored-by: Isaac --- .../descriptor-query-binding-oracle.test.tsx | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/react-db/tests/descriptor-query-binding-oracle.test.tsx b/packages/react-db/tests/descriptor-query-binding-oracle.test.tsx index eaa5a39aa7..aaf89d4108 100644 --- a/packages/react-db/tests/descriptor-query-binding-oracle.test.tsx +++ b/packages/react-db/tests/descriptor-query-binding-oracle.test.tsx @@ -496,6 +496,11 @@ describe(`descriptor error handoff at React commit`, () => { return this.state.failed ? null : this.props.children } } + type ErrorHandoffHistory = { + outcome: `fail` | `throw` + cause: `error` | `undefined` + timing: `before-release` | `during-release` + } const properties = { 'error-surface': `react.descriptor-error-surface`, ownership: `react.descriptor-ownership`, @@ -513,7 +518,7 @@ describe(`descriptor error handoff at React commit`, () => { ]) await fc.assert( fc.asyncProperty( - fc.record({ + fc.record({ outcome: fc.constantFrom(`fail` as const, `throw` as const), cause: fc.constantFrom(`error` as const, `undefined` as const), timing: fc.constantFrom( @@ -641,9 +646,22 @@ describe(`descriptor error handoff at React commit`, () => { } }, ), + // The random campaign runs every one of the eight histories first, + // so its reach does not depend on the seed: 30 biased draws over + // eight histories can miss one. campaign === `fixed` ? { numRuns: oracleRuns(30), seed: 2229 } - : oraclePropertyOptions(30, properties[observation]), + : oraclePropertyOptions(30, properties[observation], { + examples: ([`fail`, `throw`] as const).flatMap((outcome) => + ([`error`, `undefined`] as const).flatMap((cause) => + ([`before-release`, `during-release`] as const).map( + (timing): [ErrorHandoffHistory] => [ + { outcome, cause, timing }, + ], + ), + ), + ), + }), ) reach.assertReached() console.info( From ba7c6b62ad9990e81c45eb05059e3c63de82b2da Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 16:22:48 -0700 Subject: [PATCH 49/64] fix(db-ivm): an ordered query delivers an update that changes only a zero's sign topKBatch merged a key's old and new versions by D2 value identity, where -0 equals 0, so a source update that changed a row only by a zero's sign cancelled out: an ordered live query or Effect kept the old value and published nothing, while an unordered query delivered it. Top-K now compares a key's versions as row values (equalRowValues: D2's structural comparison with Object.is at the leaves). D2 value identity and hashing, which join, distinct, and grouping keys use, still treat -0 as 0. The Effect-callback oracle gains a `source-zero` representation that stores the signed zero in the source row, so each update changes the row only by its sign. The top-K batch contract oracle's signed-zero relation, and a new nested one, now expect a replacement. - Law: live-query rows tell -0 from 0 (maintainer decision 2026-11-02, extending the 2026-10-29 row-value decision). - Reach: every Effect-callback history is enumerated (finite grammar); 20 `source-zero` histories per driver, 10 ordered. - RED (unchanged db-ivm): all 10 ordered `source-zero` histories fail in the core and React drivers; unordered ones pass. - GREEN: db-ivm 669/669, @tanstack/db 30850, @tanstack/react-db 4469. - Mutant: the previous topKBatch (equalHashValues) fails the revised `signed zero` and `nested signed zero` batch relations. - Size: app scenario +118 B gzip against origin/main for the whole PR. Co-authored-by: Isaac --- .changeset/sync-negative-zero-change.md | 3 +- docs/contributing/oracle-coverage.md | 26 ++++++++++------- packages/db-ivm/src/hashing/hash.ts | 13 ++++++++- packages/db-ivm/src/operators/topKState.ts | 10 ++++--- .../topk-batch-contract-oracle.test.ts | 5 +++- .../db/tests/query/effect-callback-oracle.ts | 29 ++++++++++++++----- 6 files changed, 62 insertions(+), 24 deletions(-) diff --git a/.changeset/sync-negative-zero-change.md b/.changeset/sync-negative-zero-change.md index d270bd4ce5..0153c0bbb9 100644 --- a/.changeset/sync-negative-zero-change.md +++ b/.changeset/sync-negative-zero-change.md @@ -1,5 +1,6 @@ --- '@tanstack/db': patch +'@tanstack/db-ivm': patch --- -Treat `-0` and `0` as distinct row values. A synced update, a replay that replaces a subscription's rows, and an optimistic update that change a field only from `-0` to `0` (or back) now publish a change; before, the Collection could store the new value while subscribers kept the old one. A draft write of `0` over `-0` is now recorded as a change instead of being dropped. A transaction that deletes a row and reinserts it with only a zero's sign changed now sends that update instead of dropping it, and an ordered Effect now delivers an update whose projected value changes only by a zero's sign. A sync insert of a held key whose row differs only by a zero's sign is now a duplicate key, not an equal re-insert. The same holds inside typed arrays. An ordered Effect also delivers an update whose value is a class instance, which can change in private state its enumerable fields do not show. +Treat `-0` and `0` as distinct row values. A synced update, a replay that replaces a subscription's rows, and an optimistic update that change a field only from `-0` to `0` (or back) now publish a change; before, the Collection could store the new value while subscribers kept the old one. A draft write of `0` over `-0` is now recorded as a change instead of being dropped. A transaction that deletes a row and reinserts it with only a zero's sign changed now sends that update instead of dropping it, and an ordered live query or Effect (`orderBy`, with or without `limit`) now delivers an update that changes only a zero's sign; before, it kept the old value. Join, distinct, and grouping keys still treat `-0` and `0` as equal. A sync insert of a held key whose row differs only by a zero's sign is now a duplicate key, not an equal re-insert. The same holds inside typed arrays. An ordered Effect also delivers an update whose value is a class instance, which can change in private state its enumerable fields do not show. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index c1ecfc2289..97057e4024 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6228,9 +6228,10 @@ decision 2026-10-29); 120 histories on each path. An ordered Effect delivers an update only when its projection changed. On unchanged `f06ee8fdd` the ordered `signed-zero` histories fail (an update that flips only a zero's sign was dropped); comparing with `sameRowValue` repairs them. That payload's source -row also changes its `version`; a source update that changes only a zero's -sign is dropped earlier, by the ordered query's top-K (open, see the adapter -conformance entry's signed-zero paragraph). An ordered Effect also +row also changes its `version`; the `source-zero` representation changes the +source row only by a zero's sign, which the ordered query's top-K dropped +before the 2026-11-02 decision (see the adapter conformance entry's +signed-zero paragraph). An ordered Effect also dropped an update whose change is private or non-enumerable state, so the consumer kept a stale value (`private` and `prototype` payloads, 20 ordered histories, RED on `370e029db`). Maintainer decision 2026-11-02: the ordered @@ -6899,13 +6900,18 @@ re-insert only for an equal row value: the state-retention oracle one zero followed by an insert of the other (reach `sign-only-reinsert`, 92 histories over its fixed and random campaigns); the deep-equality check accepted such an insert as an equal re-insert instead of throwing -`DuplicateKeySyncError` (RED at `[update {0: 0}, insert {0: -0}]`). Open, -for a maintainer decision: an ordered live query (`orderBy`, with or without -`limit`) drops a source update that changes only a zero's sign, because its -top-K consolidates rows by D2 value identity, where -0 equals 0 (the -hash-identity oracle's law); an unordered live query delivers it. The -Effect-callback `signed-zero` payload changes the source row's `version` -too, so no generated history reaches that case. Open: generated publication histories never write +`DuplicateKeySyncError` (RED at `[update {0: 0}, insert {0: -0}]`). An ordered +live query (`orderBy`, with or without `limit`) dropped a source update that +changes only a zero's sign: its top-K batch merged a key's versions by D2 +value identity, where -0 equals 0, while an unordered query delivered it. +Maintainer decision 2026-11-02: live-query rows tell -0 from 0, so top-K +compares a key's versions as row values (`equalRowValues`); D2 value +identity, which join, distinct, and grouping keys use, keeps -0 equal to 0. +The Effect-callback oracle's `source-zero` representation stores the signed +zero in the source row, so each update changes the row only by its sign +(RED: all 10 ordered `source-zero` histories in the core and React drivers; +unordered ones passed), and the top-K batch contract oracle's `signed zero` +and nested relations now expect a replacement. Open: generated publication histories never write over a row a failed replay retained (the random grammar inserts source writes into green lifecycle histories), so the stale reconcile is reached only by pinned replays, including a `-0` one. A generated failed-replay diff --git a/packages/db-ivm/src/hashing/hash.ts b/packages/db-ivm/src/hashing/hash.ts index e02d224edd..57c4787c33 100644 --- a/packages/db-ivm/src/hashing/hash.ts +++ b/packages/db-ivm/src/hashing/hash.ts @@ -237,6 +237,11 @@ export function equalHashValues(left: unknown, right: unknown): boolean { return compareHashValues(left, right) } +/** @internal `equalHashValues` for row values, which tell -0 from 0. */ +export function equalRowValues(left: unknown, right: unknown): boolean { + return compareHashValues(left, right, undefined, true) +} + function enumerableKeys(value: object): Array { return Reflect.ownKeys(value).filter((key) => Object.prototype.propertyIsEnumerable.call(value, key), @@ -249,8 +254,13 @@ function compareHashValues( a: unknown, b: unknown, compared?: Map>, + signedZero = false, ): boolean { - if (a === b || (Number.isNaN(a) && Number.isNaN(b))) return true + if ( + (signedZero ? Object.is(a, b) : a === b) || + (Number.isNaN(a) && Number.isNaN(b)) + ) + return true if ( a === null || b === null || @@ -289,6 +299,7 @@ function compareHashValues( aBody[key as keyof object], bBody[key as keyof object], compared, + signedZero, ), ) ) diff --git a/packages/db-ivm/src/operators/topKState.ts b/packages/db-ivm/src/operators/topKState.ts index b339ac0128..ee4e7f1220 100644 --- a/packages/db-ivm/src/operators/topKState.ts +++ b/packages/db-ivm/src/operators/topKState.ts @@ -1,4 +1,4 @@ -import { equalHashValues } from '../hashing/hash.js' +import { equalRowValues } from '../hashing/hash.js' import { TopKArray } from './topKArray.js' import type { MultiSet } from '../multiset.js' import type { IndexedValue } from './topKArray.js' @@ -8,8 +8,10 @@ type TopKOutput = Array<[[K, IndexedValue], number]> /** Apply a keyed batch's retractions before additions. An outer join can emit * a replacement in the opposite order; key multiplicity alone would hide it. * Consolidate first so a transient value added and removed in this same turn - * cannot be mistaken for the final replacement. Use structural relation - * identity: map can allocate a fresh value for each side of a cancelling pair. + * cannot be mistaken for the final replacement. Use structural identity of + * row values: map can allocate a fresh value for each side of a cancelling + * pair, and -0 and 0 are distinct row values (maintainer decision + * 2026-11-02), so a change of only a zero's sign is a replacement. * No state survives the run. */ export function* topKBatch(messages: Array>) { @@ -24,7 +26,7 @@ export function* topKBatch(messages: Array>) { // Compare only versions of the same key. A per-key scan keeps normal // replacements cheap and never merges keys on a hash collision. const previous = entries.find(([candidate]) => - equalHashValues(candidate[1], value[1]), + equalRowValues(candidate[1], value[1]), ) if (previous) previous[1] += weight else entries.push([value, weight]) diff --git a/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts b/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts index f76b452f72..1158593b9f 100644 --- a/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts +++ b/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts @@ -25,7 +25,10 @@ const valueRelations: Array<{ name: `different plain values`, pair: (n) => [{ a: n }, { a: n + 1 }, false], }, - { name: `signed zero`, pair: () => [-0, 0, true] }, + // -0 and 0 are distinct row values, so a version that changes only a + // zero's sign replaces the old one (maintainer decision 2026-11-02). + { name: `signed zero`, pair: () => [-0, 0, false] }, + { name: `nested signed zero`, pair: () => [{ a: -0 }, { a: 0 }, false] }, { name: `NaN`, pair: () => [NaN, Number.NaN, true] }, { name: `typed scalars`, pair: (n) => [n, String(n), false] }, { name: `dates`, pair: (n) => [new Date(n), new Date(n), true] }, diff --git a/packages/db/tests/query/effect-callback-oracle.ts b/packages/db/tests/query/effect-callback-oracle.ts index cda300479b..c8a1166cd6 100644 --- a/packages/db/tests/query/effect-callback-oracle.ts +++ b/packages/db/tests/query/effect-callback-oracle.ts @@ -18,6 +18,10 @@ * update is delivered. A `signed-zero` * payload alternates 0 and -0, which are distinct row values (maintainer * decision 2026-10-29), so each of its updates is delivered in both orders. + * Its source row also changes `version`. A `source-zero` history stores the + * signed zero in the source row itself, so each update changes the source + * row only by a zero's sign; live-query rows tell -0 from 0 as well + * (maintainer decision 2026-11-02), so it is delivered in both orders too. * These eager single-source histories do not cover ordered repairs across * windows or arbitrary provider schedules. Both receiving drivers use this same judgment. * The maintainer's 2026-10-30 decision preserves unordered notifications even @@ -41,6 +45,7 @@ type Representation = | `set` | `equal` | `signed-zero` + | `source-zero` export const callbackHistories = ( [ `scalar`, @@ -50,6 +55,7 @@ export const callbackHistories = ( `set`, `equal`, `signed-zero`, + `source-zero`, ] as const ).flatMap((representation) => ([`all`, `enter`, `exit`, `late-update`, `late-batch`] as const).flatMap( @@ -148,7 +154,7 @@ function expectedPayload( ? `0,0,${version}` : representation === `equal` ? `0` - : representation === `signed-zero` + : representation === `signed-zero` || representation === `source-zero` ? version % 2 === 1 ? `-0` : `0` @@ -259,6 +265,8 @@ export async function runCallbackHistory( if (history.representation === `equal`) payload = 0 if (history.representation === `signed-zero`) payload = version % 2 === 1 ? -0 : 0 + // The source row holds the signed zero (`stored`). + if (history.representation === `source-zero`) payload = version if (history.representation === `prototype`) { payload = new PrototypePayload(version) } @@ -273,6 +281,13 @@ export async function runCallbackHistory( ) return { id, payload } } + // The version the source row stores for an authored version. + const stored = (version: number) => + history.representation === `source-zero` + ? version % 2 === 1 + ? -0 + : 0 + : version const failures: Array = [] let handle: Awaited> | undefined let before: number | undefined @@ -304,28 +319,28 @@ export async function runCallbackHistory( handlers: handlers(), }) cut(`initial`, 0, history.skipInitial) - await handle.write(`update`, 1) + await handle.write(`update`, stored(1)) cut(`unobserved`, 1) if (history.profile === `late-update`) channels.add(`update`) if (history.profile === `late-batch`) channels.add(`batch`) await handle.replaceHandlers(handlers()) cut(`replacement`, 1) - await handle.write(`update`, 2) + await handle.write(`update`, stored(2)) cut(`update`, 2) // Installing exit after unobserved updates distinguishes current values // from an initial value retained only when update handlers were present. channels = new Set([`exit`]) await handle.replaceHandlers(handlers()) - await handle.write(`delete`, 2) + await handle.write(`delete`, stored(2)) cut(`exit`, undefined) channels = new Set([`enter`, `update`, `exit`, `batch`]) await handle.replaceHandlers(handlers()) - await handle.write(`insert`, 3) + await handle.write(`insert`, stored(3)) cut(`reentry`, 3) - await handle.write(`update`, 4) + await handle.write(`update`, stored(4)) cut(`next-use`, 4) await handle.dispose() - await handle.write(`update`, 5) + await handle.write(`update`, stored(5)) cut(`disposed`, 4) if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) console.info( From e0eca76b2c1bedbe745359465f2816df735bf96c Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 16:48:17 -0700 Subject: [PATCH 50/64] fix(electric): count a fetchClient that throws before it returns as a failed attempt The outage wrapper called the app's fetchClient before its try, so a fetchClient that threw synchronously, which the client still retries, skipped the attempt count. The call now runs inside the try. The subset lane of the SDK delivery oracle gains a generated number of synchronous throws (0 to 3) after the first live 503, and declares reach for an outage that only those throws carry past the threshold. Its campaigns also get an explicit 30 s timeout: under a loaded run one outlasted the 5 s default. - Law: HEALTH-1/RETRY-1, five failed shape-log attempts in a row show the outage (maintainer decision 2026-10-30). - RED (committed adapter): "source health after live attempts: expected false to be true", both campaigns. - GREEN: electric-db-collection 1243 passed, 4 skipped (twice). - Reach: live-outage-with-synchronous-throws, at least once per campaign. Co-authored-by: Isaac --- .../electric-db-collection/src/electric.ts | 10 +++-- ...ctric-sdk-delivery-oracle.property.test.ts | 41 +++++++++++++++++-- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 752eccb11d..b7294a124d 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -2045,10 +2045,14 @@ function createElectricSync>( // catch-up prefetches carry one, and count (maintainer decision // 2026-11-02). fetchClient: async (input, init) => { - const request = (shapeOptions.fetchClient ?? fetch)(input, init) - if (!init?.signal) return request + if (!init?.signal) + return (shapeOptions.fetchClient ?? fetch)(input, init) try { - const response = await request + // A fetch that throws before it returns is a failed attempt too. + const response = await (shapeOptions.fetchClient ?? fetch)( + input, + init, + ) if (response.ok || response.status === 409) failedAttempts = 0 else { // Not `Object.fromEntries`: a DOM lib without iterable Headers diff --git a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts index 9c6ec25708..f5fdba9e8f 100644 --- a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts @@ -136,6 +136,8 @@ declareOracle({ model: [createRetryModel, applyRetryModel, expectedRetryObservation], }) +const campaignTimeoutMs = 30_000 + function sdkCampaigns( propertyId: string, name: string, @@ -153,8 +155,18 @@ function sdkCampaigns( } // A path replay of the property runs only the random campaign, from its // path. The fixed campaign stands aside during every path replay. - oracleReplayTest(it)(`${name} (fixed)`, () => runCampaign(fixedSeed)) - oracleReplayTest(it, propertyId)(`${name} (random)`, () => runCampaign()) + // A campaign answers dozens of histories over real client backoff, so it + // can outlast the default timeout on a loaded runner. + oracleReplayTest(it)( + `${name} (fixed)`, + () => runCampaign(fixedSeed), + campaignTimeoutMs, + ) + oracleReplayTest(it, propertyId)( + `${name} (random)`, + () => runCampaign(), + campaignTimeoutMs, + ) } function violationCheckpoint(error: unknown): string { @@ -226,12 +238,14 @@ type SnapshotReachState = | `subset-attempts-past-threshold` | `live-pauses-past-threshold` | `live-outage` + | `live-outage-with-synchronous-throws` | `retired-evidence-after-restart` const createSnapshotReach = () => createOracleReach(`electric.sdk-snapshot-delivery`, [ `subset-attempts-past-threshold`, `live-pauses-past-threshold`, `live-outage`, + `live-outage-with-synchronous-throws`, `retired-evidence-after-restart`, ]) // Each campaign declares its own reach. @@ -247,6 +261,11 @@ type SubsetRetries = { extraLoads: number /** HTTP 503 responses to the live request after the first load. */ liveFailures?: number + /** + * After the first of those 503s, the app's `fetchClient` throws + * synchronously on this many live attempts: a failed attempt too. + */ + liveThrows?: number } async function checkSnapshots( @@ -259,6 +278,7 @@ async function checkSnapshots( ) { snapshotReach.beginHistory() const http = controlledHttp() + let liveThrows = 0 const batches: Array>> = [] let delivery = deferred() const subscribe = ShapeStream.prototype.subscribe @@ -302,7 +322,14 @@ async function checkSnapshots( shapeOptions: { url: `http://test-url/snapshot-${sequence}`, params: { table: `rows` }, - fetchClient: http.fetchClient, + // A live attempt can throw before it returns a request. + fetchClient: (input, init) => { + if (liveThrows > 0 && init?.signal) { + liveThrows-- + throw new TypeError(`offline`) + } + return http.fetchClient(input, init) + }, backoffOptions: { initialDelay: 1, maxDelay: 2, multiplier: 1 }, }, syncMode: `on-demand`, @@ -453,10 +480,15 @@ async function checkSnapshots( let live = await atCheckpoint(http.take(), `resumed live request`) for (let attempt = 0; attempt < retries.liveFailures!; attempt++) { live.respond(new Response(`transient`, { status: 503 })) + if (attempt === 0) liveThrows = retries.liveThrows ?? 0 live = await atCheckpoint(http.take(), `retried live request`) } - outage = retries.liveFailures! >= 5 + const attempts = retries.liveFailures! + (retries.liveThrows ?? 0) + outage = attempts >= 5 if (outage) snapshotReach.reach(`live-outage`) + if (outage && retries.liveFailures! < 5) { + snapshotReach.reach(`live-outage-with-synchronous-throws`) + } sourceHealth(`source health after live attempts`) } } @@ -603,6 +635,7 @@ const snapshotProperty = fc.asyncProperty( failures: fc.array(fc.nat(7), { maxLength: 6 }), extraLoads: fc.nat(3), liveFailures: fc.nat(6), + liveThrows: fc.nat(3), }), (width, name, retries) => snapshotViolation.check({ width, name, retries }, () => From a00cdcdacfba476bf476fddee86c8ee7d7ec5a34 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 16:48:54 -0700 Subject: [PATCH 51/64] fix(db-ivm): apply the signed-zero rule to object headers in equalRowValues equalRowValues compared primitive leaves with Object.is but object headers, such as a RegExp's lastIndex, with ===, so two versions of a row whose lastIndex differed only as -0 and 0 still merged in top-K. Headers now follow the same rule. - RED (committed hash): the new top-K batch relation `regexp position signed zero` fails in all three cancellation variants. - GREEN: db-ivm 672/672. Co-authored-by: Isaac --- packages/db-ivm/src/hashing/hash.ts | 4 +++- .../operators/topk-batch-contract-oracle.test.ts | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/db-ivm/src/hashing/hash.ts b/packages/db-ivm/src/hashing/hash.ts index 57c4787c33..4c1df4dd4c 100644 --- a/packages/db-ivm/src/hashing/hash.ts +++ b/packages/db-ivm/src/hashing/hash.ts @@ -283,7 +283,9 @@ function compareHashValues( const [bMarker, bHeader, bBody] = objectParts(b) if ( aMarker !== bMarker || - !aHeader.every((value, index) => value === bHeader[index]) + !aHeader.every((value, index) => + signedZero ? Object.is(value, bHeader[index]) : value === bHeader[index], + ) ) return false if (!aBody || !bBody) return true diff --git a/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts b/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts index 1158593b9f..830b2939d2 100644 --- a/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts +++ b/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts @@ -108,6 +108,17 @@ const valueRelations: Array<{ return [left, right, false] }, }, + { + // A header such as `lastIndex` follows the same signed-zero rule. + name: `regexp position signed zero`, + pair: () => { + const left = /same/g + const right = /same/g + left.lastIndex = -0 + right.lastIndex = 0 + return [left, right, false] + }, + }, { name: `equal regexp state`, pair: (n) => { From 58c5bd119d57bb171dca9024fe1925471f8f35f8 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 16:54:58 -0700 Subject: [PATCH 52/64] fix(db-adapter-conformance): check LIFE-5 at a prompt restart's own cut A prompt restart skipped its checkpoint, so an adapter that reported ready at the restart and then received an empty delivery matched the model at the next step's checkpoint. The check now reads status and rows synchronously after preload(), with no turn before the prompt step, and requires `loading` with no rows. - RED (previous check): the reference mutant ready-at-restart passes [fail, prompt restart, snapshot []]. - GREEN: that history now fails LIFE-5 (prompt restart) at step 1; db-adapter-conformance 85/85 with ready-at-restart in the calibration table; Convex's prompt campaigns pass. Co-authored-by: Isaac --- packages/db-adapter-conformance/README.md | 7 ++++--- .../src/eager-live-source-oracle.ts | 20 +++++++++++++++++-- .../reference-adapter-oracle.property.test.ts | 1 + .../tests/reference-adapter.ts | 4 ++++ 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/packages/db-adapter-conformance/README.md b/packages/db-adapter-conformance/README.md index 688895aef5..62beaab299 100644 --- a/packages/db-adapter-conformance/README.md +++ b/packages/db-adapter-conformance/README.md @@ -168,9 +168,9 @@ from `packages/db/tests/oracle-reach.ts`. Each campaign fails when a state is reached fewer than five times. The lowest count above is 18. -- **GREEN:** `pnpm exec vitest run` in this package: 84 tests pass. The - reference oracle file holds 82: both campaigns, the per-change pair, the - outage-threshold campaign, 47 mutant calibrations, and the grammar +- **GREEN:** `pnpm exec vitest run` in this package: 85 tests pass. The + reference oracle file holds 83: both campaigns, the per-change pair, the + outage-threshold campaign, 48 mutant calibrations, and the grammar controls. `tests/eager-live-source-harness.test.ts` holds 2 ordinary tests of the check's own setup cleanup. - **Mutants:** the fixed campaign rejects each with the law it breaks. Where @@ -192,6 +192,7 @@ from `packages/db/tests/oracle-reach.ts`. | `leaks-subscription` | LIFE-2 | | `restart-replays-rows` | LIFE-5 | | `swallows-prompt-failure` | READY-4 | + | `ready-at-restart` | LIFE-5 | | `keeps-failed-source` | LIFE-10 | | `double-release` | LIFE-10 | | `detached-rejection` | HEALTH-5 | diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index be65dddcea..afd35433cd 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -441,8 +441,8 @@ export interface EagerLiveSourceDriver { * element, so the reset it causes is reached. * - `restart`: cleanup, then `preload()` starts a new sync run. With * `prompt-after-restart`, half the restarts are `prompt`: the next step - * runs in the same task as `preload()`, and no checkpoint comes between - * them. One history in six with this capability starts with a failure, a + * runs in the same task as `preload()`, and only a synchronous check of + * LIFE-5 (status `loading`, no rows) comes between them. One history in six with this capability starts with a failure, a * prompt restart, and a failure or a complete delivery, because random * steps rarely put a failed run's restart next to a failure. * @@ -2555,6 +2555,22 @@ export async function checkEagerLiveSourceHistory( watchReady() preload = recordSettlement(collection.preload()) events = foldChanges(collection) + // LIFE-5 at a prompt restart's own cut: no turn passes before the + // next step, which may otherwise hide an early ready. + if (done.prompt) { + const restarted = { + status: collection.status, + rows: observedRows(collection), + } + if (restarted.status !== `loading` || restarted.rows.length > 0) { + throw violation( + `LIFE-5 (prompt restart)`, + index, + { status: `loading`, rows: [] }, + restarted, + ) + } + } reach(`restart`) if (failedBefore) reach(`restart-after-failure`) if (done.prompt) reach(`prompt-restart`) diff --git a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts index b714aee292..2928e506a0 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts @@ -105,6 +105,7 @@ describe(`eager live-source conformance: reference adapter`, () => { [`leaks-subscription`, `LIFE-2`], [`restart-replays-rows`, `LIFE-5`], [`swallows-prompt-failure`, `READY-4`], + [`ready-at-restart`, `LIFE-5`], [`keeps-failed-source`, `LIFE-10`], [`double-release`, `LIFE-10`], [`detached-rejection`, `HEALTH-5`], diff --git a/packages/db-adapter-conformance/tests/reference-adapter.ts b/packages/db-adapter-conformance/tests/reference-adapter.ts index 32a23c965a..dfcfee7cbb 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter.ts @@ -45,6 +45,8 @@ export type ReferenceMutant = | `leaks-subscription` /** LIFE-5: a new sync run starts from the previous run's rows. */ | `restart-replays-rows` + /** LIFE-5: a restarted sync run reports ready at its start. */ + | `ready-at-restart` /** * READY-4, after a failed run: the new run skips one failure that comes * before its first turn, taking it for a cached failure's replay. @@ -299,6 +301,7 @@ function referenceCollection( mutant !== `forgets-retired-evidence`) // Only the `restart-replays-rows` mutant reads this across sync runs. let previousRows: ReadonlyArray = [] + let runs = 0 const settleWaits = (txid: string) => { applied.add(txid) @@ -367,6 +370,7 @@ function referenceCollection( keys = new Set(previousRows.map((row) => row.id)) } if (mutant === `ready-before-delivery`) becomeReady() + if (mutant === `ready-at-restart` && ++runs > 1) becomeReady() const writeChange = (change: ProviderChange) => { if (change.type === `upsert`) { From ee012e195418d79384fbcc74cfdbec57fdd88622 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 16:56:09 -0700 Subject: [PATCH 53/64] test: assert declared reach only when its generated campaign ran The state-retention and Electric retry-decision reach checks ran in afterAll, so a focused run of an unrelated test in the same file failed with every state at 0. Each check now runs only after its generated campaign ran histories; a full run still enforces it. - RED: a focused pinned test failed afterAll in both files. - GREEN: both focused runs pass; the full files pass with reach checked. Co-authored-by: Isaac --- .../collection-state-retention-oracle.property.test.ts | 9 ++++++++- .../tests/electric-oracle.property.test.ts | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 2654eb93e5..534026a9d1 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -74,7 +74,12 @@ const retentionReach = createOracleReach( [`sign-only-reinsert`], { minimum: 5 }, ) -afterAll(() => retentionReach.assertReached()) +// Only the generated campaigns owe the reach; a focused run of a pinned +// test checks nothing here. +let generatedHistories = 0 +afterAll(() => { + if (generatedHistories > 0) retentionReach.assertReached() +}) function snapshotRetainedRow(row: RetainedRow): RetainedRow { return { id: row.id, value: row.value } @@ -486,6 +491,7 @@ const retentionHistory = fc fcTest.prop([retentionHistory], { numRuns: oracleRuns(100), seed: 1_902 })( `matches retained authoritative state with a fixed seed`, async (actions) => { + generatedHistories++ await runRetentionHistory(actions) }, ) @@ -495,6 +501,7 @@ fcTest.prop( )( `matches retained authoritative state with a random or replayed seed`, async (actions) => { + generatedHistories++ await runRetentionHistory(actions) }, ) diff --git a/packages/electric-db-collection/tests/electric-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-oracle.property.test.ts index e01b3247cf..e66c983cdf 100644 --- a/packages/electric-db-collection/tests/electric-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-oracle.property.test.ts @@ -3916,7 +3916,9 @@ describe(`Electric adapter laws`, () => { `late-wait-after-abort-after-ready`, `late-wait-during-deferred-abort`, ]) + // Only the retry-decision campaigns owe this reach. afterAll(() => { + if (retryHistoryRun === 0) return retryReach.assertReached() console.info( `electric.retry-decision reach`, From b7cff8f028913340f8db1340750208109d122c35 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 17:24:21 -0700 Subject: [PATCH 54/64] test(electric): scale the SDK campaign timeout with the run multiplier The fixed 30 s timeout overrode the global one and was too short for a stress run (TANSTACK_DB_ORACLE_RUNS_MULTIPLIER=25 runs 1,000 snapshot histories). It now scales with oracleRuns(). The metadata-import waits use a 1 ms timeout: imported evidence answers at once, so the timeout only bounds a refusal, and the always-export mutant still fails. Co-authored-by: Isaac --- .../tests/electric-sdk-delivery-oracle.property.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts index f5fdba9e8f..17c0cdce4e 100644 --- a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts @@ -7,6 +7,7 @@ import { electricCollectionOptions } from '../src/electric' import { oraclePropertyOptions, oracleReplayTest, + oracleRuns, } from '../../db/tests/oracle-config' import { createOracleReach } from '../../db/tests/oracle-reach' import { declareOracle } from '../../db/tests/oracle-declaration' @@ -136,7 +137,8 @@ declareOracle({ model: [createRetryModel, applyRetryModel, expectedRetryObservation], }) -const campaignTimeoutMs = 30_000 +// Scaled with the run multiplier, so stress campaigns keep their budget. +const campaignTimeoutMs = oracleRuns(30_000) function sdkCampaigns( propertyId: string, @@ -549,7 +551,8 @@ async function checkSnapshots( ) return Promise.all( [5, 25].map((txid) => - fresh.utils.awaitTxId(txid, 20).then( + // Imported evidence answers at once; 1 ms bounds a refusal. + fresh.utils.awaitTxId(txid, 1).then( () => true, () => false, ), From d8b2940c296dd29a34183a5ae697c3de7bb97d05 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 17:56:45 -0700 Subject: [PATCH 55/64] test(db-adapter-conformance): isolate the prompt-restart cut and check its health The ready-at-restart calibration was rejected in a campaign by an ordinary restart's checkpoint too, so it did not show that the synchronous check at a prompt restart is needed. A fixed calibration now runs [fail, prompt restart, snapshot []], where only that check can reject the mutant, and requires the exact law LIFE-5 (prompt restart). The cut also requires a health node with no problem from the failed run. No adapter can leave one (core resets health at cleanup and refuses reportError before ready), so that part guards core. - GREEN: db-adapter-conformance 86/86; Convex prompt campaigns pass. Co-authored-by: Isaac --- .../src/eager-live-source-oracle.ts | 13 ++++++++++--- .../reference-adapter-oracle.property.test.ts | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index afd35433cd..af935227de 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -2556,17 +2556,24 @@ export async function checkEagerLiveSourceHistory( preload = recordSettlement(collection.preload()) events = foldChanges(collection) // LIFE-5 at a prompt restart's own cut: no turn passes before the - // next step, which may otherwise hide an early ready. + // next step, which may otherwise hide an early ready. A fresh run + // also shows no problem from the failed one. if (done.prompt) { const restarted = { status: collection.status, rows: observedRows(collection), + problem: + collection.health.error ?? collection.health.failureReason, } - if (restarted.status !== `loading` || restarted.rows.length > 0) { + if ( + restarted.status !== `loading` || + restarted.rows.length > 0 || + restarted.problem !== null + ) { throw violation( `LIFE-5 (prompt restart)`, index, - { status: `loading`, rows: [] }, + { status: `loading`, rows: [], problem: null }, restarted, ) } diff --git a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts index 2928e506a0..6acb2655ff 100644 --- a/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts +++ b/packages/db-adapter-conformance/tests/reference-adapter-oracle.property.test.ts @@ -154,6 +154,25 @@ describe(`eager live-source conformance: reference adapter`, () => { ) }) +// The prompt restart's own cut (LIFE-5). A run that reports ready at its +// start and then receives an empty delivery matches the model at the next +// cut, so in this history only the synchronous check at the restart rejects +// it; in a campaign a non-prompt restart rejects it too. +describe(`eager live-source conformance: prompt restart cut`, () => { + oracleReplayTest(it)( + `rejects a run that is ready at its prompt restart`, + async () => { + await expect( + checkEagerLiveSourceHistory(referenceDriver(`ready-at-restart`), [ + { kind: `fail` }, + { kind: `restart`, prompt: true }, + { kind: `snapshot`, rows: [] }, + ]), + ).rejects.toMatchObject({ law: `LIFE-5 (prompt restart)` }) + }, + ) +}) + // PUB-1 has two commit granularities. The per-change reference adapter // commits each change of a batch alone; it must conform, and a variant that // commits them out of order must be rejected by the per-change comparison. From 4e0ef4c75704fd2918b3765c555d6153c97c6314 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 17:56:49 -0700 Subject: [PATCH 56/64] test(db-ivm): state that row-value headers treat NaN as equal to NaN equalRowValues compares headers with Object.is, so two RegExp versions whose lastIndex is NaN are one row value, as NaN is at a leaf (sameRowValue); D2 value identity keeps a NaN header distinct. A top-K batch relation and the function's comment now state it. - The relation passes on the current hash and fails on the version before a00cdcdac, which compared headers with ===. Co-authored-by: Isaac --- packages/db-ivm/src/hashing/hash.ts | 5 ++++- .../operators/topk-batch-contract-oracle.test.ts | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/db-ivm/src/hashing/hash.ts b/packages/db-ivm/src/hashing/hash.ts index 4c1df4dd4c..71176b0bb3 100644 --- a/packages/db-ivm/src/hashing/hash.ts +++ b/packages/db-ivm/src/hashing/hash.ts @@ -237,7 +237,10 @@ export function equalHashValues(left: unknown, right: unknown): boolean { return compareHashValues(left, right) } -/** @internal `equalHashValues` for row values, which tell -0 from 0. */ +/** + * @internal `equalHashValues` for row values, which tell -0 from 0 and treat + * NaN as equal to NaN, at a leaf or in a header such as `lastIndex`. + */ export function equalRowValues(left: unknown, right: unknown): boolean { return compareHashValues(left, right, undefined, true) } diff --git a/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts b/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts index 830b2939d2..dcbfd04d6c 100644 --- a/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts +++ b/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts @@ -119,6 +119,17 @@ const valueRelations: Array<{ return [left, right, false] }, }, + { + // Row values treat NaN as equal to NaN, in a header as at a leaf + // (`sameRowValue`); D2 value identity keeps a NaN header distinct. + name: `regexp position NaN`, + pair: () => { + const left = /same/g + const right = /same/g + left.lastIndex = right.lastIndex = NaN + return [left, right, true] + }, + }, { name: `equal regexp state`, pair: (n) => { From 0f5f75969926bda5ab2e6714d970d5f8c212d03c Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 17:56:52 -0700 Subject: [PATCH 57/64] test(electric): generate live attempts as any order of 503s and synchronous throws The subset lane placed synchronous throws only right after the first live 503, so a throw never led an outage or crossed the threshold, and the expected attempt count came from the inputs. Live attempts are now a generated sequence of 0..7 503s and throws in any order; the wrapper records each failed attempt the provider makes, and the expected outage, the crossing attempt's message (HEALTH-4: the 503's FetchError or the thrown error), and reach come from that record. New reach states: an outage led by a throw and one crossed by a throw. The lane's prose describes the dimension. The adapter's wrapper also calls fetchClient once, inside its try, and counts only signal-bearing requests. - RED: the adapter before e0eca76b2 fails "source health after live attempts" in both campaigns. - GREEN: electric-db-collection 1243 passed, 4 skipped. Co-authored-by: Isaac --- .../electric-db-collection/src/electric.ts | 6 +- ...ctric-sdk-delivery-oracle.property.test.ts | 66 ++++++++++++++----- 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index b7294a124d..209707913a 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -2045,14 +2045,14 @@ function createElectricSync>( // catch-up prefetches carry one, and count (maintainer decision // 2026-11-02). fetchClient: async (input, init) => { - if (!init?.signal) - return (shapeOptions.fetchClient ?? fetch)(input, init) + const attempt = init?.signal try { // A fetch that throws before it returns is a failed attempt too. const response = await (shapeOptions.fetchClient ?? fetch)( input, init, ) + if (!attempt) return response if (response.ok || response.status === 409) failedAttempts = 0 else { // Not `Object.fromEntries`: a DOM lib without iterable Headers @@ -2074,7 +2074,7 @@ function createElectricSync>( } return response } catch (error) { - if (!init.signal.aborted) reportRetry(error) + if (attempt && !attempt.aborted) reportRetry(error) throw error } }, diff --git a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts index 17c0cdce4e..9dd9b856e5 100644 --- a/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-sdk-delivery-oracle.property.test.ts @@ -241,6 +241,8 @@ type SnapshotReachState = | `live-pauses-past-threshold` | `live-outage` | `live-outage-with-synchronous-throws` + | `live-outage-led-by-throw` + | `live-outage-crossed-by-throw` | `retired-evidence-after-restart` const createSnapshotReach = () => createOracleReach(`electric.sdk-snapshot-delivery`, [ @@ -248,6 +250,8 @@ const createSnapshotReach = () => `live-pauses-past-threshold`, `live-outage`, `live-outage-with-synchronous-throws`, + `live-outage-led-by-throw`, + `live-outage-crossed-by-throw`, `retired-evidence-after-restart`, ]) // Each campaign declares its own reach. @@ -261,13 +265,11 @@ let snapshotReach = createSnapshotReach() type SubsetRetries = { failures: ReadonlyArray extraLoads: number - /** HTTP 503 responses to the live request after the first load. */ - liveFailures?: number /** - * After the first of those 503s, the app's `fetchClient` throws - * synchronously on this many live attempts: a failed attempt too. + * Failed live attempts after the first load, in order: an HTTP 503, or the + * app's `fetchClient` throwing before it returns a request. */ - liveThrows?: number + live?: ReadonlyArray<`http` | `throw`> } async function checkSnapshots( @@ -280,7 +282,10 @@ async function checkSnapshots( ) { snapshotReach.beginHistory() const http = controlledHttp() + // Live attempts the app's fetchClient still throws on, and every failed + // live attempt the provider made, in order. let liveThrows = 0 + const made: Array<`http` | `throw`> = [] const batches: Array>> = [] let delivery = deferred() const subscribe = ShapeStream.prototype.subscribe @@ -325,9 +330,12 @@ async function checkSnapshots( url: `http://test-url/snapshot-${sequence}`, params: { table: `rows` }, // A live attempt can throw before it returns a request. + // Only shape-log requests carry a signal. A prefetch is one too, and + // counts (maintainer decision 2026-11-02); `made` records it. fetchClient: (input, init) => { if (liveThrows > 0 && init?.signal) { liveThrows-- + made.push(`throw`) throw new TypeError(`offline`) } return http.fetchClient(input, init) @@ -371,13 +379,14 @@ async function checkSnapshots( // 2026-10-30). Only the live stream's up-to-date ends it; a subset's // end does not. let outage = false - // HEALTH-4: the outage shows the client's error for a 503 response, - // which names the status and the request. + // HEALTH-4: the outage shows the attempt that crossed the threshold: the + // client's error for a 503 response, which names the status and the + // request, or the error the app's fetchClient threw. const sourceHealth = (label: string) => { expect(collection.health.failureReason !== null, label).toBe(outage) if (outage) { expect(collection.health.failureReason?.message, label).toMatch( - /^HTTP Error 503 at http\S+$/, + made[4] === `throw` ? /^offline$/ : /^HTTP Error 503 at http\S+$/, ) } } @@ -473,24 +482,39 @@ async function checkSnapshots( [...expected.values()].sort((a, b) => a.id - b.id), ) sourceHealth(`snapshot ${index} source health`) + const attempts = index === 0 ? (retries.live ?? []) : [] + // Leading throws hit the live request that resumes after this load. + let next = attempts.findIndex((kind) => kind === `http`) + if (next === -1) next = attempts.length + liveThrows = next const outcome = await atCheckpoint( completion, `snapshot request completion`, ) if (!outcome.ok) throw outcome.error - if (index === 0 && (retries.liveFailures ?? 0) > 0) { + if (attempts.length > 0) { let live = await atCheckpoint(http.take(), `resumed live request`) - for (let attempt = 0; attempt < retries.liveFailures!; attempt++) { + // Each 503 is followed by the throws that come before the next one. + while (next < attempts.length) { live.respond(new Response(`transient`, { status: 503 })) - if (attempt === 0) liveThrows = retries.liveThrows ?? 0 + made.push(`http`) + let throws = 0 + while (attempts[++next] === `throw`) throws++ + liveThrows = throws live = await atCheckpoint(http.take(), `retried live request`) } - const attempts = retries.liveFailures! + (retries.liveThrows ?? 0) - outage = attempts >= 5 + // The outage follows the attempts the provider made, not the plan. + outage = made.length >= 5 if (outage) snapshotReach.reach(`live-outage`) - if (outage && retries.liveFailures! < 5) { + if (outage && made.slice(0, 5).includes(`throw`)) { snapshotReach.reach(`live-outage-with-synchronous-throws`) } + if (outage && made[0] === `throw`) { + snapshotReach.reach(`live-outage-led-by-throw`) + } + if (made[4] === `throw`) { + snapshotReach.reach(`live-outage-crossed-by-throw`) + } sourceHealth(`source health after live attempts`) } } @@ -625,8 +649,13 @@ async function checkSnapshots( // Retries: each subset request gets 0..7 HTTP 503 responses that the client // retries before its snapshot, so five or more in a row are common; 0..3 // extra empty loads (id>=width+2 on) pause the live request up to six times -// between its responses. After the first load, the live request gets 0..6 -// HTTP 503 responses the client retries; five or more are a source outage. +// between its responses. After the first load, the live stream makes 0..7 +// failed attempts the client retries, each an HTTP 503 or a throw of the +// app's fetchClient before it returns a request, in any order: a throw can +// lead, cross the threshold, or follow a 503. Five or more in a row are a +// source outage, which shows the attempt that crossed the threshold +// (HEALTH-4). The expected outage follows the attempts the provider +// actually made. // The source node shows a problem exactly when the live stream had that // outage: a subset request's retried attempts and a live pause are not // failures of the source, and a subset's end does not end its outage. @@ -637,8 +666,9 @@ const snapshotProperty = fc.asyncProperty( fc.record({ failures: fc.array(fc.nat(7), { maxLength: 6 }), extraLoads: fc.nat(3), - liveFailures: fc.nat(6), - liveThrows: fc.nat(3), + live: fc.array(fc.constantFrom(`http` as const, `throw` as const), { + maxLength: 7, + }), }), (width, name, retries) => snapshotViolation.check({ width, name, retries }, () => From 126b7a7fa146df84ce11c28de0e9de1d0ceba837 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 18:01:23 -0700 Subject: [PATCH 58/64] fix(convex): keep a cached error from failing a restart behind a client wrapper The cached result is named by the identity of its log lines, read through the client's untyped accessor. Without that accessor, as behind an app's wrapper that exposes only onUpdate, nothing was skipped, so the client's replay of a cached error failed the restarted run. The adapter now falls back to the earlier rule there: no current value until the server's first callback, and the first error is taken for the replay. The accessor is also read before onUpdate (no temporal dead zone if a client calls back at once), and the logs are read only while a cached result is held. The Convex driver gains a wrapped-client variant, which does not declare prompt-after-restart: without the accessor a server answer that comes before the replay can still be taken for it. - Law: maintainer decision 2026-11-02, a cached error does not fail a new subscription, which waits for the server. - RED (committed adapter): LIFE-2/LIFE-5 (status) at [fail, restart], "error" where "loading" is owed, on the wrapped client. - GREEN: convex-db-collection 116/116. - Mutant: a fallback that only skips the replay fails EVID-1 at [sync-wait, fail, restart, sync-wait] (awaitSync rejected). - Limits: behind a wrapper, the prompt-restart race stays open. Co-authored-by: Isaac --- .../convex-db-collection/src/subscription.ts | 39 +++++++++++++------ .../tests/conformance-driver.ts | 23 +++++++---- ...source-conformance-oracle.property.test.ts | 15 +++++++ 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/packages/convex-db-collection/src/subscription.ts b/packages/convex-db-collection/src/subscription.ts index 7206fcfac1..02e4b02ecd 100644 --- a/packages/convex-db-collection/src/subscription.ts +++ b/packages/convex-db-collection/src/subscription.ts @@ -42,9 +42,20 @@ export function subscribeQuery>( // lines belong to that object, so they name the cached result, also when // the server answers before the replay. The log accessors are untyped, and // the `onUpdate` handle's own one looks up the wrong query, so the - // JavaScript client's base client is read. Without them nothing is skipped. + // JavaScript client's base client is read. Without them, as behind an + // app's wrapper, the subscription has no current value until the server's + // first callback (`stale`) and skips the first error (`replay`), which a + // server answer that comes first can still take. let cached: unknown - const isCached = (logs: unknown) => cached !== undefined && logs === cached + let stale = false + let replay = false + const isCached = (read: () => unknown) => + cached !== undefined && read() === cached + const holdsCachedError = (read: () => unknown, logs: () => unknown) => { + if (!hasCachedError(read)) return + cached = logs() + stale = replay = cached === undefined + } const stop = (unsubscribe: () => void) => () => { active = false unsubscribe() @@ -64,31 +75,37 @@ export function subscribeQuery>( if (`closed` in client && client.closed === true) { throw new Error(`A closed Convex client cannot sync a collection.`) } + const base = (client as { client?: Logged<[string, unknown]> }).client + const name = getFunctionName(query) + const logs = () => base?.localQueryLogs?.(name, args) const subscription = client.onUpdate( query, args, (value) => { + stale = replay = false if (active) receive(value) }, (error) => { - if (active && !isCached(logs())) fail(error) + if (replay) replay = false + else { + stale = false + if (active && !isCached(logs)) fail(error) + } }, ) - const base = (client as { client?: Logged<[string, unknown]> }).client - const name = getFunctionName(query) - const logs = () => base?.localQueryLogs?.(name, args) - if (hasCachedError(() => subscription.getCurrentValue())) cached = logs() + holdsCachedError(() => subscription.getCurrentValue(), logs) return { getCurrentValue: () => - isCached(logs()) ? undefined : subscription.getCurrentValue(), + stale || isCached(logs) ? undefined : subscription.getCurrentValue(), unsubscribe: stop(subscription), } } const watch = client.watchQuery(query, args) const logs = () => (watch as Logged<[]>).localQueryLogs?.() - if (hasCachedError(() => watch.localQueryResult())) cached = logs() + holdsCachedError(() => watch.localQueryResult(), logs) const unsubscribe = watch.onUpdate(() => { - if (!active || isCached(logs())) return + stale = false + if (!active || isCached(logs)) return try { const result = watch.localQueryResult() if (result !== undefined) receive(result) @@ -98,7 +115,7 @@ export function subscribeQuery>( }) return { getCurrentValue: () => - isCached(logs()) ? undefined : watch.localQueryResult(), + stale || isCached(logs) ? undefined : watch.localQueryResult(), unsubscribe: stop(unsubscribe), } } diff --git a/packages/convex-db-collection/tests/conformance-driver.ts b/packages/convex-db-collection/tests/conformance-driver.ts index 1626c4cb97..d80574dd92 100644 --- a/packages/convex-db-collection/tests/conformance-driver.ts +++ b/packages/convex-db-collection/tests/conformance-driver.ts @@ -145,9 +145,9 @@ class ConvexProvider implements ControlledProvider { } const url = `https://conformance.convex.cloud` this.client = - kind === `javascript` - ? new ConvexClient(url, options) - : new ConvexReactClient(url, options) + kind === `react` + ? new ConvexReactClient(url, options) + : new ConvexClient(url, options) this.onSend = (message) => { const modifications = (message.modifications ?? []) as Array<{ type: string @@ -327,9 +327,15 @@ class ConvexProvider implements ControlledProvider { let sequence = 0 -type ClientKind = `javascript` | `react` +type ClientKind = `javascript` | `react` | `wrapped` -/** `client` selects the Convex JavaScript client or the React client. */ +/** + * `client` selects the Convex JavaScript client, the React client, or a + * wrapper that exposes only the JavaScript client's `onUpdate`, as an app's + * proxy may. Without the client's log accessor the adapter skips one + * replayed error, so a prompt restart's failure can still be lost there: + * the wrapper does not declare `prompt-after-restart`. + */ export function convexConformanceDriver({ client = `javascript`, }: { client?: ClientKind } = {}): EagerLiveSourceDriver { @@ -340,7 +346,7 @@ export function convexConformanceDriver({ `delivery-error`, `duplicate-key-delivery`, `await-sync`, - `prompt-after-restart`, + ...(client === `wrapped` ? [] : [`prompt-after-restart` as const]), ]), changeCommits: `per-batch`, duplicateInsert: `error`, @@ -357,7 +363,10 @@ export function convexConformanceDriver({ const collection = createCollection( convexCollectionOptions({ id: `convex-conformance-${++sequence}`, - client: provider.client, + client: + client === `wrapped` && provider.client instanceof ConvexClient + ? { onUpdate: provider.client.onUpdate.bind(provider.client) } + : provider.client, query: rowsQuery, queryArgs: {}, getKey: (row) => row.id, diff --git a/packages/convex-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts b/packages/convex-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts index 02241b2698..907df5309e 100644 --- a/packages/convex-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts +++ b/packages/convex-db-collection/tests/eager-live-source-conformance-oracle.property.test.ts @@ -85,6 +85,21 @@ describe(`eager live-source conformance: Convex`, () => { }, campaignTimeoutMs, ) + + // An app's wrapper that exposes only `onUpdate` hides the client's log + // accessor, so the adapter falls back to skipping one replayed error. + oracleReplayTest(it)( + `conforms over a wrapped JavaScript client on a fixed-seed campaign`, + async () => { + const reach = await campaign( + { seed: 2118, numRuns: oracleRuns(runs) }, + convexConformanceDriver({ client: `wrapped` }), + `${property}.wrapped`, + ) + reach.assertReached() + }, + campaignTimeoutMs, + ) }) declareOracle({ From 4e665f531025d4a0dad481f3acbf4544f89fe177 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 18:09:20 -0700 Subject: [PATCH 59/64] fix(db-adapter-conformance): dispose a fixture whose seed fails, and correct the scenario rate When a driver's create() succeeded and its seed() rejected, the setup guard removed the console and rejection patches but left the fixture's provider alive in the worker (ORC-010). It now disposes it. The grammar prose said one history in six opens with a scenario; with two scenarios it is one in seven each, after any startup burst. - RED (previous guard): the harness test's seed case finds the fixture created but not disposed. - GREEN: db-adapter-conformance 86/86. Co-authored-by: Isaac --- .../src/eager-live-source-oracle.ts | 25 ++++++++++++------- .../tests/eager-live-source-harness.test.ts | 9 +++++++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts index af935227de..ff802aeed4 100644 --- a/packages/db-adapter-conformance/src/eager-live-source-oracle.ts +++ b/packages/db-adapter-conformance/src/eager-live-source-oracle.ts @@ -442,9 +442,10 @@ export interface EagerLiveSourceDriver { * - `restart`: cleanup, then `preload()` starts a new sync run. With * `prompt-after-restart`, half the restarts are `prompt`: the next step * runs in the same task as `preload()`, and only a synchronous check of - * LIFE-5 (status `loading`, no rows) comes between them. One history in six with this capability starts with a failure, a - * prompt restart, and a failure or a complete delivery, because random - * steps rarely put a failed run's restart next to a failure. + * LIFE-5 (status `loading`, no rows, no problem) comes between them. + * Random steps rarely put a failed run's restart next to a failure, so a + * history can open with a failure, a prompt restart, and a failure or a + * complete delivery (see the scenario prefix below). * * Keys range over 1..5 and titles over three values, so batches revisit * keys, deletes hit live rows, and complete deliveries omit held keys. A row @@ -817,8 +818,11 @@ export function eagerLiveSourceHistoryArb( { minLength: 1, maxLength: 12 }, ) .map((elements) => elements.flat()) - // EVID-2 needs a sequence that random steps rarely produce, so one history - // in six starts with it. + // EVID-2 and a prompt restart after a failure need sequences that random + // steps rarely produce, so a history can open with one: no prefix weighs 5 + // and each declared scenario 1, so with one scenario one history in six + // opens with it, with two one in seven each. A listen-before-fetch startup + // burst still comes before the prefix. const snapshot = steps[0]!.arbitrary const scenarios: Array>> = [] if (capabilities.has(`await-evidence`)) { @@ -2025,17 +2029,20 @@ export async function checkEagerLiveSourceHistory( return [] } const detached = watchDetachedFailures(driver) - let fixture: EagerLiveSourceFixture + let created: EagerLiveSourceFixture | undefined try { - fixture = driver.create() + created = driver.create() if (listenBeforeFetch) { - await fixture.provider.seed?.(structuredClone(seedFrom(0))) + await created.provider.seed?.(structuredClone(seedFrom(0))) } } catch (error) { - // A setup failure leaves no console or rejection patch behind (ORC-010). + // A setup failure leaves no console or rejection patch, and no fixture, + // behind (ORC-010). detached.stop() + await created?.dispose?.() throw error } + const fixture = created const { collection, provider } = fixture const state = initialModel(seedFrom(0)) const failures: Array = [] diff --git a/packages/db-adapter-conformance/tests/eager-live-source-harness.test.ts b/packages/db-adapter-conformance/tests/eager-live-source-harness.test.ts index 51aeebcb8d..3835f5712f 100644 --- a/packages/db-adapter-conformance/tests/eager-live-source-harness.test.ts +++ b/packages/db-adapter-conformance/tests/eager-live-source-harness.test.ts @@ -8,6 +8,9 @@ import type { EagerLiveSourceDriver } from '../src' // setup fails must leave all three as it found them, or later tests in the // worker lose their diagnostics (ORC-010). describe(`eager live-source check harness`, () => { + // Fixtures the failing driver created and disposed. + let created = 0 + let disposed = 0 const failingSetup = (stage: `create` | `seed`): EagerLiveSourceDriver => { const driver = referenceDriver() return { @@ -15,11 +18,15 @@ describe(`eager live-source check harness`, () => { create() { if (stage === `create`) throw new Error(`driver setup failed`) const fixture = driver.create() + created++ return { ...fixture, provider: Object.assign(Object.create(fixture.provider), { seed: () => Promise.reject(new Error(`driver setup failed`)), }), + dispose: () => { + disposed++ + }, } }, } @@ -38,6 +45,8 @@ describe(`eager live-source check harness`, () => { expect(console.error).toBe(error) expect(console.warn).toBe(warn) expect(process.listenerCount(`unhandledRejection`)).toBe(listeners) + // A fixture that was created is disposed. + expect(disposed).toBe(created) }, ) }) From 3ae9018ff12616f554e5410848d9b58c41574570 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 18:09:24 -0700 Subject: [PATCH 60/64] refactor(electric): record an external abort's failure only where READY-9 defers it After ready, giveUp already records the run's failure, so the extra lifecycle.fail ran twice for every abort; it now runs only on the before-ready branch, where READY-9 can defer the Collection's failure. The late-wait lane's deferral example still covers it. Co-authored-by: Isaac --- packages/electric-db-collection/src/electric.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 209707913a..e91a767448 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1997,11 +1997,13 @@ function createElectricSync>( abortController.abort() if (active) { const error = new StreamAbortedError(collectionId) - // The waits end with the stream, also while READY-9 defers the - // Collection's failure. - lifecycle.fail(lifecycleEpoch, error) if (ready) giveUp(error) - else failInitialSync(error) + else { + // The waits end with the stream, also while READY-9 defers the + // Collection's failure. + lifecycle.fail(lifecycleEpoch, error) + failInitialSync(error) + } } // A source that stops must end its reserved transaction. The initial // failure decides first: abandoning commits. From 0ac99be9f060e92dcb4151f99c22c3876eb413f4 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 18:09:28 -0700 Subject: [PATCH 61/64] test(db): check the retention reach inside each generated campaign The reach ran in a gated afterAll with a module counter. Each campaign now declares its own reach and checks it when its fc.assert ends, so a pinned replay records none and a focused run of one passes. - GREEN: the file 72/72; a focused pinned run passes. - The campaigns still fail on classifyProjectedInsert before 4ea88308b. Co-authored-by: Isaac --- ...on-state-retention-oracle.property.test.ts | 65 ++++++++++--------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 534026a9d1..2d94e82a4f 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -1,5 +1,5 @@ import { fc, test as fcTest } from '@fast-check/vitest' -import { afterAll, expect, it } from 'vitest' +import { expect, it } from 'vitest' import { createCollection } from '../src/collection/index.js' import { DuplicateKeySyncError } from '../src/errors.js' import { oraclePropertyOptions, oracleRuns } from './oracle-config.js' @@ -69,17 +69,10 @@ const retainedRowArbitrary = fc.record({ ), }) -const retentionReach = createOracleReach( - `collection-state.retention`, - [`sign-only-reinsert`], - { minimum: 5 }, -) -// Only the generated campaigns owe the reach; a focused run of a pinned -// test checks nothing here. -let generatedHistories = 0 -afterAll(() => { - if (generatedHistories > 0) retentionReach.assertReached() -}) +// Each generated campaign declares its own reach and checks it when it +// ends; pinned replays record none. +let retentionReach: + ReturnType> | undefined function snapshotRetainedRow(row: RetainedRow): RetainedRow { return { id: row.id, value: row.value } @@ -167,7 +160,7 @@ function applyAction( previous.value === action.row.value && !Object.is(previous.value, action.row.value) ) { - retentionReach.reach(`sign-only-reinsert`) + retentionReach?.reach(`sign-only-reinsert`) } if ( previous !== undefined && @@ -247,7 +240,7 @@ declareOracle({ check: [expectRetainedState] }) async function runRetentionHistory( actions: ReadonlyArray, ): Promise { - retentionReach.beginHistory() + retentionReach?.beginHistory() const harness = createRetentionHarness() const { collection } = harness const model = new Map() @@ -488,23 +481,33 @@ const retentionHistory = fc ) .map((elements) => elements.flat()) -fcTest.prop([retentionHistory], { numRuns: oracleRuns(100), seed: 1_902 })( - `matches retained authoritative state with a fixed seed`, - async (actions) => { - generatedHistories++ - await runRetentionHistory(actions) - }, -) -fcTest.prop( - [retentionHistory], - oraclePropertyOptions(100, `collection-state.retention`), -)( - `matches retained authoritative state with a random or replayed seed`, - async (actions) => { - generatedHistories++ - await runRetentionHistory(actions) - }, -) +async function retentionCampaign( + campaign: string, + options: Parameters[1], +): Promise { + retentionReach = createOracleReach( + `collection-state.retention (${campaign})`, + [`sign-only-reinsert`], + { minimum: 5 }, + ) + try { + await fc.assert( + fc.asyncProperty(retentionHistory, runRetentionHistory), + options, + ) + retentionReach.assertReached() + } finally { + retentionReach = undefined + } +} + +it(`matches retained authoritative state with a fixed seed`, () => + retentionCampaign(`fixed`, { numRuns: oracleRuns(100), seed: 1_902 })) +it(`matches retained authoritative state with a random or replayed seed`, () => + retentionCampaign( + `random`, + oraclePropertyOptions(100, `collection-state.retention`), + )) // The partial-update lane reuses those histories and marks source batches // partial from a separate stream, so the full-mode campaigns keep their From 88f5ea2927fefa05193dad284a06065f22521125 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 18:09:49 -0700 Subject: [PATCH 62/64] docs: record the Convex wrapper fallback and the Electric live-attempt sequence in the coverage map Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 97057e4024..88717b8a2d 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -7006,7 +7006,13 @@ throwing, and a second campaign runs the default configuration without an app `onError`, declaring the adapter's give-up log. The on-demand lane also restarts the Collection and checks that a retired run's snapshot-boundary evidence (txid 5 below `xmin` 10) and imported sync metadata (txid 25) stay -pending until the restarted run is ready, then fulfill. At both cuts a +pending until the restarted run is ready, then fulfill. Its live attempts after the first load are a +generated sequence of 0..7 HTTP 503s and synchronous throws of the app's +`fetchClient`, in any order; the expected outage and the message of the +attempt that crossed the threshold (HEALTH-4) follow the attempts the +provider recorded, with reach for an outage led by a throw and one crossed +by a throw. A success between failed attempts is not generated in this lane; +the eager lane's empty response covers the reset. At both cuts a fresh Collection imports the exported sync metadata, as hydration does, and must accept no txid the source does not accept then: the export carried the retired run's txids before the restarted run was ready (RED: txid 25 @@ -7078,7 +7084,13 @@ there: RED `READY-4 (status)` at `[fail, prompt restart, fail]`, status identity of its log lines, read through the base client, because the `onUpdate` handle's `getQueryLogs` looks up the wrong query in Convex 1.47.0; a version that skips nothing fails `LIFE-2/LIFE-5 (status)` at -`[fail, restart, changes]`. A cached valid result served to a restarted run is accepted +`[fail, restart, changes]`. Behind an app's wrapper that exposes only +`onUpdate` the accessor is missing, so the adapter falls back to the earlier +rule (no current value until the server's first callback, and the first +error taken for the replay); a fixed campaign over such a wrapper checks it +(RED before the fallback: `[fail, restart]` ends in `error`). The wrapper does +not declare `prompt-after-restart`: there a server answer that comes before +the replay can still be taken for it (open limit). A cached valid result served to a restarted run is accepted client-cache behavior and is not generated, because the model starts a restarted run empty. A fixed-seed campaign runs the React client (`watchQuery`). From 195f3ec41d1fa2d40ff7389e24e9abc6d4128875 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 19:21:15 -0700 Subject: [PATCH 63/64] refactor(db): one comparison mode for deepEqualsInternal; typed arrays equal only typed arrays The row-value rule grew the typical app by 125 B gzip (+92 brotli). These changes bring the PR's app growth to +3 B gzip and +12 B brotli against origin/main, without changing any equality outcome except the fix below. - deepEqualsInternal takes one bit-flag mode (DRAFT, PERSISTED, ROW, PUBLICATION) instead of three positional flags, so every recursive call forwards one argument, and visited moves after it. - One cycle check before the container branches replaces the four copies in the Map, Set, array, and object branches. - Typed-array elements compare through the recursive call, which already holds the number rules; a draft's Set compares its contents as an array, as the separate loop did. - The flat change-tracking path compares every field with the draft equality, which already starts with Object.is, instead of guarding it with its own Object.is and object check. Fix: deepEquals(typedArray, record) was true when the record held the same index keys, while the reverse was false. A typed array is its own value class now, in both directions. - Law: change-event equality is symmetric. - RED: the new utils.41 property (a typed array against a record or an array of its elements, both directions) fails forward in both campaigns on the previous code. - GREEN: @tanstack/db 30852 passed; the equality oracles 143/143; react-db 4469; db-adapter-conformance 86. Co-authored-by: Isaac --- packages/db/src/proxy.ts | 16 +- packages/db/src/utils.ts | 180 ++++++------------ packages/db/tests/oracle-config.ts | 2 +- .../db/tests/utils-oracle.property.test.ts | 16 +- 4 files changed, 77 insertions(+), 137 deletions(-) diff --git a/packages/db/src/proxy.ts b/packages/db/src/proxy.ts index 29cc386f80..96ea928d21 100644 --- a/packages/db/src/proxy.ts +++ b/packages/db/src/proxy.ts @@ -3,7 +3,7 @@ * and provides a way to retrieve those changes. */ -import { deepEqualsInternal, isTemporal } from './utils' +import { DRAFT, ROW, deepEqualsInternal, isTemporal } from './utils' import { codedMessage, devBuild } from './error-message' // Resolve draft handles before calling native Map/Set membership methods. @@ -364,7 +364,7 @@ function draftValuesEqual( right: unknown, paired?: Map, ): boolean { - return deepEqualsInternal(left, right, paired, true, false, `row`) + return deepEqualsInternal(left, right, DRAFT | ROW, paired) } /** @@ -924,16 +924,8 @@ export function withFlatChangeTracking( for (const key in draft) { const value = (draft as Record)[key] const before = original[key] - // Only a hidden field can hold an object; compare it as the proxy does. - if ( - !Object.hasOwn(original, key) || - !( - Object.is(value, before) || - (typeof before === `object` && - before !== null && - draftValuesEqual(value, before)) - ) - ) { + // Compare as the proxy does; only a hidden field can hold an object. + if (!Object.hasOwn(original, key) || !draftValuesEqual(value, before)) { defineDataProperty(changes, key, value) if (value !== null && typeof value === `object`) assignedObject = true } diff --git a/packages/db/src/utils.ts b/packages/db/src/utils.ts index 0680ab5c4f..5b75b3857c 100644 --- a/packages/db/src/utils.ts +++ b/packages/db/src/utils.ts @@ -8,6 +8,7 @@ import type { CompareOptions } from './query/builder/types' interface TypedArray { length: number [index: number]: number + every: (predicate: (value: number, index: number) => boolean) => boolean } export const durationFields = [ @@ -42,7 +43,7 @@ export type DurationFields = Record<(typeof durationFields)[number], number> * ``` */ export function deepEquals(a: any, b: any): boolean { - return deepEqualsInternal(a, b, undefined) + return deepEqualsInternal(a, b) } /** @@ -50,7 +51,7 @@ export function deepEquals(a: any, b: any): boolean { * tells -0 from 0, as row values do (FID-1; maintainer decision 2026-10-29). */ export function sameRowValue(a: unknown, b: unknown): boolean { - return deepEqualsInternal(a, b, undefined, false, false, `row`) + return deepEqualsInternal(a, b, ROW) } /** @@ -60,12 +61,12 @@ export function sameRowValue(a: unknown, b: unknown): boolean { * (maintainer decision 2026-11-02). */ export function samePublishedValue(a: unknown, b: unknown): boolean { - return deepEqualsInternal(a, b, undefined, false, false, `publication`) + return deepEqualsInternal(a, b, ROW | PUBLICATION) } /** @internal Compare persisted public and durable value shapes before skipping a reload. */ export function equalPersistedSnapshotValues(a: unknown, b: unknown): boolean { - return deepEqualsInternal(a, b, undefined, true, true) + return deepEqualsInternal(a, b, DRAFT | PERSISTED) } function isPlainPrototype(prototype: object | null): boolean { @@ -80,29 +81,37 @@ function enumerableOwnKeys(value: object): Array { return keys } +// The comparison modes of `deepEqualsInternal`, combined as bits. +/** + * The stricter equality used by change-tracking drafts: Map and Set contents + * must match in order, RegExp match position must match, and arrays compare + * as keyed objects (holes and extra enumerable keys count). + */ +export const DRAFT = 1 +/** Persisted snapshots also distinguish prototypes after serialization. */ +const PERSISTED = 2 +/** Row change detection tells -0 from 0; general equality does not. */ +export const ROW = 4 +/** An ordered Effect's publication also compares a class instance by identity. */ +const PUBLICATION = 8 + /** * Internal implementation with cycle detection to prevent infinite recursion. * Internal callers can seed already-paired roots when comparing their children. * - * `draft` selects the stricter equality used by change-tracking drafts: Map - * and Set contents must match in order, RegExp match position must match, and - * arrays compare as keyed objects (holes and extra enumerable keys count). - * `persistedSnapshot` also distinguishes prototypes after serialization. + * `mode` combines the bits above; one recursive call forwards all of them. */ export function deepEqualsInternal( a: any, b: any, + mode = 0, // Created on the first container that descends into a child. - visited: Map | undefined, - draft = false, - persistedSnapshot = false, - // Row change detection (`row`) tells -0 from 0; general equality does not. - // An ordered Effect's publication (`publication`) also compares a class - // instance by identity. - rowValue: false | `row` | `publication` = false, + visited?: Map, ): boolean { - // Handle strict equality (primitives, same reference) - if ((!rowValue && a === b) || Object.is(a, b)) return true + const draft = mode & DRAFT + // Handle strict equality (primitives, same reference); a row value tells + // -0 from 0. + if ((!(mode & ROW) && a === b) || Object.is(a, b)) return true // Handle null/undefined if (a == null || b == null) return false @@ -113,13 +122,17 @@ export function deepEqualsInternal( // SQLite can turn an authored nested instance into an equal-keyed plain // record. A skipped persisted reload must not retain the old public shape. if ( - persistedSnapshot && + mode & PERSISTED && typeof a === `object` && Object.getPrototypeOf(a) !== Object.getPrototypeOf(b) ) { return false } + // A container already paired in this comparison closes a cycle: it equals + // only its partner. + if (visited?.has(a)) return visited.get(a) === b + // Handle Date objects if (a instanceof Date) { if (!(b instanceof Date)) return false @@ -144,36 +157,15 @@ export function deepEqualsInternal( if (a instanceof Map) { if (!(b instanceof Map)) return false if (a.size !== b.size) return false - - // Check for circular references - if (visited?.has(a)) { - return visited.get(a) === b - } - visited ??= new Map() - visited.set(a, b) + ;(visited ??= new Map()).set(a, b) // A draft compares entries in order; general equality looks keys up. const bEntries = draft && Array.from(b.entries()) const result = Array.from(a.entries()).every(([key, val], index) => bEntries ? Object.is(key, bEntries[index]![0]) && - deepEqualsInternal( - val, - bEntries[index]![1], - visited, - true, - persistedSnapshot, - rowValue, - ) - : b.has(key) && - deepEqualsInternal( - val, - b.get(key), - visited, - draft, - persistedSnapshot, - rowValue, - ), + deepEqualsInternal(val, bEntries[index]![1], mode, visited) + : b.has(key) && deepEqualsInternal(val, b.get(key), mode, visited), ) visited.delete(a) @@ -186,29 +178,15 @@ export function deepEqualsInternal( if (a instanceof Set) { if (!(b instanceof Set)) return false if (a.size !== b.size) return false - - // Check for circular references - if (visited?.has(a)) { - return visited.get(a) === b - } - visited ??= new Map() - visited.set(a, b) + ;(visited ??= new Map()).set(a, b) // Convert to arrays for comparison const aValues = Array.from(a) const bValues = Array.from(b) if (draft) { - const result = aValues.every((val, index) => - deepEqualsInternal( - val, - bValues[index], - visited, - draft, - persistedSnapshot, - rowValue, - ), - ) + // A draft compares contents in order, as it compares arrays. + const result = deepEqualsInternal(aValues, bValues, mode, visited) visited.delete(a) return result } @@ -234,10 +212,8 @@ export function deepEqualsInternal( deepEqualsInternal( value, bValues[candidateIndex], + mode, candidateVisited, - draft, - persistedSnapshot, - rowValue, ) && matchValues( index + 1, @@ -261,40 +237,23 @@ export function deepEqualsInternal( // Symmetric check: if b is Set but a is not, they're not equal if (b instanceof Set) return false - // Handle TypedArrays - if ( - ArrayBuffer.isView(a) && - ArrayBuffer.isView(b) && - !(a instanceof DataView) && - !(b instanceof DataView) - ) { + // Handle TypedArrays. A typed array is its own value class: it equals only + // another typed array, in both directions. + const typed = ArrayBuffer.isView(a) && !(a instanceof DataView) + if (typed !== (ArrayBuffer.isView(b) && !(b instanceof DataView))) { + return false + } + if (typed) { const typedA = a as unknown as TypedArray const typedB = b as unknown as TypedArray - // Elements compare like numbers: NaN equals NaN, and -0 equals 0 except - // as a row value. Only a draft treats a change of typed-array class as a - // change. - if ( - (draft && Object.getPrototypeOf(a) !== Object.getPrototypeOf(b)) || - typedA.length !== typedB.length + // Elements compare as the numbers they are: NaN equals NaN, and -0 + // equals 0 except as a row value. Only a draft treats a change of + // typed-array class as a change. + return ( + (!draft || Object.getPrototypeOf(a) === Object.getPrototypeOf(b)) && + typedA.length === typedB.length && + typedA.every((x, i) => deepEqualsInternal(x, typedB[i], mode)) ) - return false - - for (let i = 0; i < typedA.length; i++) { - const x = typedA[i]! - const y = typedB[i]! - if (rowValue ? !Object.is(x, y) : x !== y && !(x !== x && y !== y)) - return false - } - - return true - } - // Symmetric check: if b is TypedArray but a is not, they're not equal - if ( - ArrayBuffer.isView(b) && - !(b instanceof DataView) && - !ArrayBuffer.isView(a) - ) { - return false } // Handle Temporal objects @@ -331,22 +290,10 @@ export function deepEqualsInternal( if (Array.isArray(a) !== Array.isArray(b)) return false if (Array.isArray(a) && a.length !== b.length) return false if (Array.isArray(a) && !draft) { - // Check for circular references - if (visited?.has(a)) { - return visited.get(a) === b - } - visited ??= new Map() - visited.set(a, b) + ;(visited ??= new Map()).set(a, b) const result = a.every((item, index) => - deepEqualsInternal( - item, - b[index], - visited, - draft, - persistedSnapshot, - rowValue, - ), + deepEqualsInternal(item, b[index], mode, visited), ) visited.delete(a) return result @@ -361,12 +308,7 @@ export function deepEqualsInternal( const plain = isPlainPrototype(prototype) const plainB = isPlainPrototype(prototypeB) if (prototype !== prototypeB && !plain && !plainB) return false - if (rowValue === `publication` && !(plain && plainB)) return false - - // Check for circular references - if (visited?.has(a)) { - return visited.get(a) === b - } + if (mode & PUBLICATION && !(plain && plainB)) return false // Compare enumerable symbol keys as well as string keys. Query results may // use user-owned symbols, and a symbol-only update is still a value change. @@ -391,20 +333,12 @@ export function deepEqualsInternal( for (const key of keysA) { const value = a[key] if (!registered && value !== null && typeof value === `object`) { - visited ??= new Map() - visited.set(a, b) + ;(visited ??= new Map()).set(a, b) registered = true } if ( !Object.prototype.propertyIsEnumerable.call(b, key) || - !deepEqualsInternal( - value, - b[key], - visited, - draft, - persistedSnapshot, - rowValue, - ) + !deepEqualsInternal(value, b[key], mode, visited) ) { result = false break diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index e1109ec55b..357cf5fa49 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -432,7 +432,7 @@ const publicationProperties = [ // utils-oracle and comparison-oracle number their properties in file order. const utilsProperties = Array.from( - { length: 40 }, + { length: 41 }, (_, index) => `utils.${index + 1}`, ) const comparisonProperties = Array.from( diff --git a/packages/db/tests/utils-oracle.property.test.ts b/packages/db/tests/utils-oracle.property.test.ts index 322e197ef1..dc8852ee78 100644 --- a/packages/db/tests/utils-oracle.property.test.ts +++ b/packages/db/tests/utils-oracle.property.test.ts @@ -218,7 +218,8 @@ const arbitraryDefinedValue = arbitrarySingleValue.filter( // The fixed witnesses exercise those two outcomes; deleting either outcome // would leave an always-true or always-false comparator undetected. The // generated distinct-type grammar names Date/Temporal pairs, Date versus its -// ISO text and epoch number, and RegExp versus its source text; it does not +// ISO text and epoch number, RegExp versus its source text, and a typed array +// versus a record or an array of its elements; it does not // claim every cross-class combination. String lengths, integer // magnitude, and object keys are fast-check defaults; explicit array/record // bounds are 0–5 for equivalence and 0–10 for structural copies, bytes 0–50, @@ -815,3 +816,16 @@ describe(`deepEquals property-based tests`, () => { ) }) }) + +// Appended last: properties are numbered in file order (utils.41). +describe(`typed-array value class`, () => { + // A typed array is its own value class: a record or an array of its + // elements, with the same index keys, is not equal to it either way. + utilsProperty([arbitraryFloat32Array])( + `a typed array differs from a record or an array of its elements in both directions`, + (typed) => { + assertDistinctTypes(typed, { ...Array.from(typed) }, deepEquals) + assertDistinctTypes(typed, Array.from(typed), deepEquals) + }, + ) +}) From 77ef9a5eb90d5d77df3d9026dc8843c2a58f21ca Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 19:21:25 -0700 Subject: [PATCH 64/64] refactor(db-ivm): compare a row's versions without a separate wrapper; headers keep === Top-K now calls the structural comparison with its signed-zero flag directly, and equalHashValues is unchanged from main. The leaf check `(!signedZero && a === b) || Object.is(a, b)` replaces main's NaN test with one Object.is. The signed-zero rule for headers (a RegExp's lastIndex, a00cdcdac) is reverted: core's row equality ignores lastIndex outside drafts, so a sync update that changes only lastIndex is not a change and never reaches top-K. The top-K batch relations now state D2's header rule (=== for -0/0 and NaN) with that reason. - GREEN: db-ivm 675/675; the Effect-callback source-zero histories pass in core and React. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 3 +- packages/db-ivm/src/hashing/hash.ts | 29 +++++++------------ packages/db-ivm/src/operators/topKState.ts | 4 +-- .../topk-batch-contract-oracle.test.ts | 10 +++---- 4 files changed, 19 insertions(+), 27 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 88717b8a2d..e532fca667 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6905,7 +6905,8 @@ live query (`orderBy`, with or without `limit`) dropped a source update that changes only a zero's sign: its top-K batch merged a key's versions by D2 value identity, where -0 equals 0, while an unordered query delivered it. Maintainer decision 2026-11-02: live-query rows tell -0 from 0, so top-K -compares a key's versions as row values (`equalRowValues`); D2 value +compares a key's versions as row values (D2's structural comparison with +`signedZero`, which tells -0 from 0 at a leaf); D2 value identity, which join, distinct, and grouping keys use, keeps -0 equal to 0. The Effect-callback oracle's `source-zero` representation stores the signed zero in the source row, so each update changes the row only by its sign diff --git a/packages/db-ivm/src/hashing/hash.ts b/packages/db-ivm/src/hashing/hash.ts index 71176b0bb3..db8316758e 100644 --- a/packages/db-ivm/src/hashing/hash.ts +++ b/packages/db-ivm/src/hashing/hash.ts @@ -237,14 +237,6 @@ export function equalHashValues(left: unknown, right: unknown): boolean { return compareHashValues(left, right) } -/** - * @internal `equalHashValues` for row values, which tell -0 from 0 and treat - * NaN as equal to NaN, at a leaf or in a header such as `lastIndex`. - */ -export function equalRowValues(left: unknown, right: unknown): boolean { - return compareHashValues(left, right, undefined, true) -} - function enumerableKeys(value: object): Array { return Reflect.ownKeys(value).filter((key) => Object.prototype.propertyIsEnumerable.call(value, key), @@ -253,17 +245,18 @@ function enumerableKeys(value: object): Array { // Allocate pair memoization only when structural bodies need comparison. A // module-level recursion also avoids rebuilding the worker for every call. -function compareHashValues( +// With `signedZero`, it compares row values: a leaf tells -0 from 0. Headers +// still compare with `===`: core's row equality ignores a RegExp's +// `lastIndex`, so a change of only a header's zero sign never reaches here. +/** @internal */ +export function compareHashValues( a: unknown, b: unknown, + signedZero?: boolean, compared?: Map>, - signedZero = false, ): boolean { - if ( - (signedZero ? Object.is(a, b) : a === b) || - (Number.isNaN(a) && Number.isNaN(b)) - ) - return true + // `Object.is` also makes NaN equal to NaN; row values tell -0 from 0. + if ((!signedZero && a === b) || Object.is(a, b)) return true if ( a === null || b === null || @@ -286,9 +279,7 @@ function compareHashValues( const [bMarker, bHeader, bBody] = objectParts(b) if ( aMarker !== bMarker || - !aHeader.every((value, index) => - signedZero ? Object.is(value, bHeader[index]) : value === bHeader[index], - ) + !aHeader.every((value, index) => value === bHeader[index]) ) return false if (!aBody || !bBody) return true @@ -303,8 +294,8 @@ function compareHashValues( compareHashValues( aBody[key as keyof object], bBody[key as keyof object], - compared, signedZero, + compared, ), ) ) diff --git a/packages/db-ivm/src/operators/topKState.ts b/packages/db-ivm/src/operators/topKState.ts index ee4e7f1220..eba0acd6ce 100644 --- a/packages/db-ivm/src/operators/topKState.ts +++ b/packages/db-ivm/src/operators/topKState.ts @@ -1,4 +1,4 @@ -import { equalRowValues } from '../hashing/hash.js' +import { compareHashValues } from '../hashing/hash.js' import { TopKArray } from './topKArray.js' import type { MultiSet } from '../multiset.js' import type { IndexedValue } from './topKArray.js' @@ -26,7 +26,7 @@ export function* topKBatch(messages: Array>) { // Compare only versions of the same key. A per-key scan keeps normal // replacements cheap and never merges keys on a hash collision. const previous = entries.find(([candidate]) => - equalRowValues(candidate[1], value[1]), + compareHashValues(candidate[1], value[1], true), ) if (previous) previous[1] += weight else entries.push([value, weight]) diff --git a/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts b/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts index dcbfd04d6c..dbaae8c8c8 100644 --- a/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts +++ b/packages/db-ivm/tests/operators/topk-batch-contract-oracle.test.ts @@ -109,25 +109,25 @@ const valueRelations: Array<{ }, }, { - // A header such as `lastIndex` follows the same signed-zero rule. + // Headers compare with ===, as in D2 value identity. Core's row equality + // ignores a RegExp's lastIndex, so a version that changes only it never + // reaches top-K. name: `regexp position signed zero`, pair: () => { const left = /same/g const right = /same/g left.lastIndex = -0 right.lastIndex = 0 - return [left, right, false] + return [left, right, true] }, }, { - // Row values treat NaN as equal to NaN, in a header as at a leaf - // (`sameRowValue`); D2 value identity keeps a NaN header distinct. name: `regexp position NaN`, pair: () => { const left = /same/g const right = /same/g left.lastIndex = right.lastIndex = NaN - return [left, right, true] + return [left, right, false] }, }, {