From 3418b4a6790065a4bd028f7694c68e620c96e7b7 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 25 Aug 2026 08:45:17 -0400 Subject: [PATCH] An erased device stays erased: the lease dies with it, and construction leaves no trace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #112's investigation, in three findings. First, the lease heartbeat's stop() handle was deliberately dropped on a lifetime argument (lease = lock = global) that holds for every ending except erasure, where the DEVICE dies before the global: destroy cleared the checkpoint debounce and the sync timers but the 5s touchLease kept firing, and ns.put on a deleted database is IndexedDB open-on-missing — the deleted database is back. The handle is kept and destroy stops it first (devstore row 65b: resurrected in 1200ms unstopped, absent stopped). Second — the negative control for the timer theory did NOT go red, because serve() closes the global one task after destroy replies, so the field flake needed another door. It found three: bootSeq's module-evaluation write meant MERELY CONSTRUCTING a SharedWorker named for an erased device recreated its database (no client call, no timer — the construction IS the resurrection); status(), reachable on a raw port before attach, recreated it with a read (indexedDB.open creates whatever the transaction mode); attach's takeLock started a lease whose first act is a put. All three now gate on the INDEX ROW — the existence oracle anchorIsLive already states in as many words — with creation order verified row-first on every constructing path, so a legitimate first boot still counts 1 (rows 11/12/14/15/56 green). Row 66's negative control: unguarded, the database is back after step ONE. Third, the harness itself committed the offence: hc-forget's page-side delete left live hosts heartbeating at deleted databases (~31 stray databases per matrix run). cleanup() now tears down through the worker (row 66b: 7s of nothing, absent; unguarded control present). Plus one non-timer hole from the audit: onTokenRefreshed's fire-and-forget writeOauth re-checks destroyed at land time, not call time. The field trigger for the CI flake remains unproven — this closes the CLASS (no constructor, RPC or teardown path can recreate an erased namespace) rather than naming the one door CI caught. If solo-erase flakes again, the remaining suspect recreates the index row itself. Gates: devstore 75 rows green (65, 65b, 66, 66b new); solo-erase, solo-persistence, solo-ephemeral x2 green; demo deno check clean. --- runtime/device-store/worker.ts | 160 +++++++++++++++++++- runtime/tests/devstore/page.ts | 263 ++++++++++++++++++++++++++++++++- runtime/tests/devstore/run.ts | 243 ++++++++++++++++++++++++++++++ 3 files changed, 656 insertions(+), 10 deletions(-) diff --git a/runtime/device-store/worker.ts b/runtime/device-store/worker.ts index c6c8b3d..1e51e39 100644 --- a/runtime/device-store/worker.ts +++ b/runtime/device-store/worker.ts @@ -109,7 +109,13 @@ import { unsealWithPrf, } from "./seal.ts"; import { sealedDirectory } from "./sealed-fs.ts"; -import { type DeviceLock, deviceLockIsHeld, holdDeviceLock, startLease } from "./locks.ts"; +import { + type DeviceLock, + deviceLockIsHeld, + holdDeviceLock, + type LeaseHeartbeat, + startLease, +} from "./locks.ts"; import { type AttachSpec, type DeviceStatus, @@ -172,21 +178,83 @@ const ns: DeviceNamespace = openNamespace(DEVICE_ID); * respawned) are claims about precisely that difference. It is in `meta` * rather than `sealed` because the sweep and the picker read it before * anything is unsealed, and it says nothing personal — it is a count. + * + * IT IS GATED ON THE INDEX ROW, and that gate is the whole of #112's + * class (`bootIfIndexed`). CONSTRUCTING A WORKER FOR A DEVICE THAT DOES + * NOT EXIST MUST LEAVE NO TRACE: this write runs at MODULE EVALUATION, + * before any client has said anything and before `attach` can refuse + * anything, so without the gate the mere existence of a + * `new SharedWorker(url, {name: "pm-device-"})` recreated + * that device's database — `ns.update` opens it, and IndexedDB + * open-on-missing creates it. No timer and no race needed: the + * construction IS the resurrection. + * + * THE INDEX ROW IS THE EXISTENCE ORACLE, which is not a new rule here + * but the one anchor.ts's `anchorIsLive` already states in as many + * words ("THE INDEX ROW IS THE ANSWER, and the namespace deliberately is + * not"). Reading it creates nothing that matters: the index database is + * origin-global and outlives every device in it. + * + * ORDERING, VERIFIED RATHER THAN ASSUMED (the gate would miscount if it + * were wrong): every path that constructs this worker writes the index + * row FIRST. `connectDevice` awaits `resolveDevice` before + * `new SharedWorker` — the `id` arm requires an existing row and throws + * without one, and the `anchor`/`new` arms `await createDevice(...)`, + * which commits the row in one index transaction. So a legitimately + * fresh device always finds its row here and counts its first boot as + * 1; the only caller that finds none is one naming a device that was + * erased or swept, which is exactly the caller that must write nothing. */ -const bootSeq: Promise = (async () => { +const bootSeq: Promise = bootIfIndexed(); + +async function bootIfIndexed(): Promise { try { + // The row FIRST, and the namespace only if it answers. A device with + // no row has no storage anyone is entitled to recreate. + if (!(await getDevice(DEVICE_ID))) return 0; return (await ns.update("meta", "boot", (n) => (n ?? 0) + 1)) ?? 1; } catch { // A namespace that cannot be opened at all is a real problem, but it // is `attach`'s to report against a client that can hear it. Boot // counting must never be the thing that stops a worker booting. + // (Unchanged posture: the index read joins the same try for the same + // reason — a failed index read must not stop a boot either.) return 0; } -})(); +} // --- the lock and the lease ------------------------------------------------- let lock: DeviceLock | null = null; +/** + * THE RUNNING LEASE HEARTBEAT, kept so that ERASURE can stop it — and + * for that one reason (#112). + * + * The lifetime this handle used to be dropped on is very nearly right: + * the lease is the sweep's "alive" answer beside the lock, both are + * meant to end when this global does, and a global's death takes the + * interval with it. That equivalence — lease ⇔ lock ⇔ global — holds + * for every ending EXCEPT ONE, and the comment that dropped the handle + * predated it: at `destroy` the DEVICE dies while the GLOBAL lives on, + * for as long as it takes Chromium to reap a zero-client worker. + * + * In that window the interval keeps firing `touchLease(ns)` every 5 s, + * and `ns.put("meta", …)` OPENS the device's IndexedDB database — + * which, IndexedDB being open-on-missing, RECREATES the database + * `destroyNamespace` has just deleted. The erased device is back in + * `indexedDB.databases()` with a lease and nothing else. (Only the + * IndexedDB half resurrects; `touchLease` never touches OPFS, which is + * exactly the shape #112 reports: db present, directory + * `NotFoundError`.) Whether a 5 s-phase tick lands inside that window + * is the whole of the flake, which is why the e2e scenario passes + * isolated and fails under suite load. + * + * NOT STOPPED ON RESEAL, deliberately: a sealed device is still HOSTED + * — this global holds its lock and answers for it — so a lease that + * stopped at reseal would make the sweep's liveness answer a lie and + * put a live device's namespace up for collection. + */ +let lease: LeaseHeartbeat | null = null; /** * Take the device lock and start the lease. Idempotent: a second client @@ -201,12 +269,10 @@ let lock: DeviceLock | null = null; async function takeLock(): Promise { if (lock) return; lock = await holdDeviceLock(DEVICE_ID); - // THE HEARTBEAT'S `stop()` HANDLE IS DELIBERATELY DROPPED. Nothing - // here ever stops the lease: it is meant to end when this global does, - // exactly as the lock does, and the two together are the sweep's - // "alive" answer. `lock` is kept only because the early return above + // The handle is KEPT now; see `lease` above for the one ending that + // needs it. `lock` is kept for its own reason: the early return above // is what makes a second client's attach idempotent. - startLease(ns); + lease = startLease(ns); } // --- the unseal state machine ----------------------------------------------- @@ -1247,6 +1313,15 @@ function onTokenRefreshed(token: string, refreshToken?: string): void { row.access = token; if (refreshToken) row.refresh = refreshToken; row.obtainedAt = Date.now(); + // THE ERASE RE-CHECK, and it is #112's mechanism rather than a + // second bug: this write is an `ns.put`, and an `ns.put` after + // `destroyNamespace` RECREATES the database it opens. The guard at + // the top of this function is read at CALL time, so a refresh that + // was already in flight when the erasure started would otherwise + // land on the other side of the delete. Nothing is lost by + // skipping: the row is being deleted with the namespace, and the + // token it describes belongs to a device that no longer exists. + if (destroyed) return; await writeOauth(key, row); })().catch(() => {}); } @@ -2933,6 +3008,29 @@ function syncStatusOf(binding: StoreBinding | null): SyncStatus | null { async function status(): Promise { const record = await getDevice(DEVICE_ID); + // THE SAME EXISTENCE GATE `attach` APPLIES, and it is here because an + // IndexedDB READ CREATES TOO: `sealState(ns)` on the next line opens + // the namespace, and `indexedDB.open` of a missing database creates + // it just as a write would. So `status` — the first thing any client + // calls, and reachable on a raw port before `attach` — is the second + // door into #112's class and needs the same lock on it. + // + // NARROWED TO THE FRESH-GLOBAL CASE, deliberately. `attached === null` + // means no client has ever attached HERE, so this global is one + // constructed for a device that does not exist, which is the case that + // must leave no trace. A global that HAS attached passed the attach + // guard, so its device existed when it started; if the row vanishes + // afterwards (a sweep, an explicit remove) this keeps answering + // exactly as it did before, because that host is genuinely still + // holding an engine, a lock and a lease, and refusing to describe + // itself would be a lie of a different kind. + if (!record && attached === null) { + throw new SealError( + "no-rung", + `device-store: no device ${DEVICE_ID} in the index (erased or swept); ` + + `there is nothing to report on, and nothing here will recreate it`, + ); + } const rungs = await sealState(ns); const policy = record?.unsealPolicy ?? "every-session"; // Read once, reported below: `null` while sealed is UNREADABLE, not @@ -3014,6 +3112,29 @@ async function callHost(method: string, args: unknown[]): Promise { `(the SharedWorker name is the device id)`, ); } + // AND THE DEVICE HAS TO EXIST — the index row is the oracle, the + // same one `bootSeq` above and anchor.ts's `anchorIsLive` consult. + // + // THIS IS A STORAGE GUARD BEFORE IT IS A COURTESY. `takeLock()` + // below starts the lease, whose first act is an `ns.put` — and an + // `ns.put` against a deleted namespace RECREATES it (#112). So a + // client attaching to an erased or swept device must be refused + // HERE, before the lock, or the attach itself resurrects the + // device it was told does not exist. + // + // No legitimate caller reaches this: `connectDevice` resolves the + // row before it constructs the worker (its `id` arm throws for a + // missing row, and its `anchor`/`new` arms create one), so this + // refusal answers a raw port or a stale reconnect. The wording + // matches `unseal`'s for the same condition, and `SealError + // "no-rung"` is the code clients already branch on. + if (!(await getDevice(DEVICE_ID))) { + throw new SealError( + "no-rung", + `device-store: no device ${DEVICE_ID} in the index (erased or swept); ` + + `nothing to attach to, and nothing here will recreate it`, + ); + } // FIRST ATTACH WINS. A second tab's spec is not applied: the // artifacts are already fetched and an engine is possibly already // running, and quietly re-pointing a live host at different bytes @@ -3106,6 +3227,29 @@ async function callHost(method: string, args: unknown[]): Promise { clearTimeout(debounceTimer); debounceTimer = undefined; } + // AND THE LEASE HEARTBEAT, which is the SAME hazard and the + // sharper one (#112). It fires every 5 s and writes + // `meta.lease` — an `ns.put`, which OPENS the device's + // IndexedDB database, which for IndexedDB means CREATING it if + // it is not there. A tick landing after the delete below + // therefore RESURRECTS the erased device's database, and the + // global outlives the device by however long Chromium takes to + // reap a zero-client worker, so the window is real and + // load-dependent: the e2e `solo-erase` scenario passed + // isolated and failed roughly two runs in six under the full + // suite, with exactly the signature this predicts — database + // present, OPFS directory gone, because the heartbeat writes + // IndexedDB and nothing else. + // + // THIS IS THE ONLY PLACE THE LEASE IS STOPPED. Everywhere else + // the device outlives the ceremony (a reseal is still a hosted + // device, and the sweep must go on being told the truth about + // it); erasure is the one ending where the DEVICE dies before + // the GLOBAL does. See `lease`. + if (lease) { + lease.stop(); + lease = null; + } // AND THE SYNC TIMERS, for a related but weaker reason: a // background flush does not write the state root, but it does // read this engine and mutate the guest's bucket state, and the diff --git a/runtime/tests/devstore/page.ts b/runtime/tests/devstore/page.ts index d7a342e..c61e6bc 100644 --- a/runtime/tests/devstore/page.ts +++ b/runtime/tests/devstore/page.ts @@ -63,6 +63,7 @@ import { deviceLockIsHeld, type DeviceLock, type DeviceNamespace, + destroyNamespace, enableUntilReseal, ensureDevice, getAnchor, @@ -920,8 +921,9 @@ const ops: Record Promise> = { * THE LEASE, READ RATHER THAN INFERRED (locks.ts's heartbeat). * * The heartbeat runs in the WORKER — `takeLock` calls `startLease(ns)` - * and deliberately drops the stop handle — so the only way to ask - * whether a host is still marking itself alive is to read the mark. + * and the handle it keeps is stopped at exactly one place, the erase + * (#112) — so the only way to ask whether a host is still marking + * itself alive is to read the mark. * The eviction/freeze rows ask this from a SECOND page, because a * frozen page cannot answer anything about itself (demo/e2e/cdp.ts's * `setWebLifecycleState` hazard note). @@ -1162,6 +1164,230 @@ const ops: Record Promise> = { "hc-forget": async (arg: { ids: string[] }) => ({ cleanup: await cleanup(arg.ids) }), + /** + * ERASE THIS DEVICE through the HOST, which is the ceremony's own + * path (`DeviceConnection.destroy` → worker.ts's `destroy`): the + * worker drains its checkpoint chain, drops the engine and the DEK, + * and deletes its own database, OPFS directory and index row. + * + * NOTHING HERE MAY OPEN THE NAMESPACE AFTERWARDS, and that is not + * fussiness — it is the very hazard the row using this op measures. + * `openNamespace(id).get(...)` would OPEN an IndexedDB database that + * has just been deleted, and IndexedDB open-on-missing CREATES it. So + * the only question asked below is `namespaceExists`, which reads + * `indexedDB.databases()` — an enumeration, never an open. + */ + "hc-destroy": async (arg: { id: string }) => { + const conn = conns.get(arg.id)!; + const attempt = await refuses(() => conn.destroy()); + conns.delete(arg.id); + return { + attempt, + dbGone: !(await namespaceExists(arg.id)), + indexRowGone: (await getDevice(arg.id)) === undefined, + lockHeld: await deviceLockIsHeld(arg.id), + }; + }, + + /** + * THE MECHANISM, ISOLATED AND DETERMINISTIC (#112): a lease heartbeat + * running over a namespace that has just been DESTROYED. + * + * Both arms use the shipped modules — `startLease` from locks.ts, + * `destroyNamespace` from namespace.ts — and differ in exactly one + * act, the `stop()`. The interval is passed in short (`startLease` + * takes one) so the row costs a second rather than eleven; it is the + * same code path the 5 s production interval runs. + * + * WHY IT RESURRECTS: `touchLease` is an `ns.put`, namespace.ts holds + * no live connection (every operation opens, transacts, closes — its + * header says why), and an IndexedDB open of a missing database + * CREATES it. So a tick after the delete brings the database back + * with a lease in it and nothing else. The OPFS half never returns: + * `touchLease` writes IndexedDB and touches nothing else, which is + * the asymmetry #112 reports from the field (db present, directory + * `NotFoundError`). + */ + "lease-vs-erase": async (arg: { intervalMs?: number }) => { + const intervalMs = arg?.intervalMs ?? 300; + const settle = intervalMs * 4; + + // ARM A — the heartbeat is left running across the erasure. + const a = newDeviceId(); + const beatA = startLease(openNamespace(a), intervalMs); + await touchLease(openNamespace(a)); + const aBefore = await namespaceExists(a); + await destroyNamespace(a); + const aAtDelete = await namespaceExists(a); + await new Promise((r) => setTimeout(r, settle)); + const aLater = await namespaceExists(a); + beatA.stop(); + // Whatever came back goes now — and it goes AFTER the stop, or the + // cleanup would race the very thing it is cleaning up. + await destroyNamespace(a).catch(() => {}); + + // ARM B — stopped first, then the same erasure. + const b = newDeviceId(); + const beatB = startLease(openNamespace(b), intervalMs); + await touchLease(openNamespace(b)); + beatB.stop(); + await destroyNamespace(b); + await new Promise((r) => setTimeout(r, settle)); + const bLater = await namespaceExists(b); + await destroyNamespace(b).catch(() => {}); + + return { + intervalMs, + settleMs: settle, + running: { before: aBefore, atDelete: aAtDelete, later: aLater }, + stopped: { later: bLater }, + }; + }, + + /** + * CONSTRUCT A WORKER FOR A DEVICE THAT DOES NOT EXIST, and watch it + * leave no trace (#112's class). + * + * A SharedWorker is keyed by (origin, script URL, NAME), and the name + * is the device id — so naming an ERASED device is all it takes to + * evaluate that module in a fresh global. Everything this op does is + * something a stale tab, a picker holding an old id, or a reconnect + * racing an erase could do by accident. + * + * THREE DOORS, ASKED SEPARATELY, because they open at different + * moments and each one used to recreate the database on its own: + * + * 1. THE CONSTRUCTION ITSELF — module evaluation runs `bootSeq` + * before any client has said a word. + * 2. `status`, the first RPC any client sends, which reads the + * namespace (`sealState`) — and an IndexedDB READ opens, and an + * open of a missing database CREATES it, exactly as a write + * would. + * 3. `attach`, whose `takeLock()` starts the LEASE, whose first act + * is an `ns.put`. + * + * A RAW PORT, deliberately: `connectDevice` refuses this device + * client-side before it constructs anything (asked below as the + * fourth question), so the only way to put the WORKER's own guards + * under test is to speak to it directly — the same raw-port technique + * `port-listen` uses. + * + * `namespaceExists` is the enumeration, never an open: asking through + * the namespace would create the thing being asked about. + */ + "erase-reconnect": async (arg: { id: string }) => { + const worker = new SharedWorker("./worker.js", { type: "module", name: nsDbName(arg.id) }); + let hello: { bootSeq?: number } | null = null; + const waiting = new Map void>(); + worker.port.onmessage = (ev: MessageEvent) => { + const d = ev.data as { id?: number }; + if (d?.id === 0) { + hello = d as { bootSeq?: number }; + return; + } + const resolve = waiting.get(d?.id ?? -1); + if (resolve) { + waiting.delete(d!.id!); + resolve(d); + } + }; + worker.port.start(); + + let nextId = 1; + const send = (target: string, method: string, args: unknown[]) => + new Promise>((resolve) => { + const id = nextId++; + waiting.set(id, (r) => resolve(r as Record)); + worker.port.postMessage({ id, target, method, args }); + setTimeout(() => { + if (waiting.delete(id)) resolve({ timedOut: true }); + }, 5_000); + }); + + // The hello proves the global genuinely BOOTED — module evaluated, + // `bootSeq` ran — rather than the construction having quietly done + // nothing. Without it a clean database would prove only that no + // worker started. + const t0 = Date.now(); + while (hello === null && Date.now() - t0 < 5_000) { + await new Promise((r) => setTimeout(r, 50)); + } + const afterConstruct = await namespaceExists(arg.id); + + const statusRes = await send("host", "status", []); + const afterStatus = await namespaceExists(arg.id); + + const attachRes = await send("host", "attach", [{ + deviceId: arg.id, + artifacts: ARTIFACTS, + label: "erased-probe", + }]); + // A lease tick is 5 s; wait past one so an attach that DID take the + // lock would have written by now. + await new Promise((r) => setTimeout(r, 6_000)); + const afterAttach = await namespaceExists(arg.id); + + // THE ORDINARY CLIENT PATH, for the record: what an application + // actually sees if it tries this. + const viaClient = await refuses(() => + connectDevice({ + device: { kind: "id", id: arg.id }, + workerUrl: "./worker.js", + artifacts: ARTIFACTS, + label: "erased-probe", + }) + ); + const afterClient = await namespaceExists(arg.id); + + worker.port.close(); + const failureOf = (r: Record) => { + const f = r?.failure as { form?: string; error?: { code?: string; message?: string } }; + return { + ok: r?.ok === true, + form: f?.form ?? "", + code: f?.error?.code ?? "", + message: (f?.error?.message ?? "").slice(0, 140), + }; + }; + return { + booted: hello !== null, + bootSeq: (hello as { bootSeq?: number } | null)?.bootSeq ?? null, + afterConstruct, + status: failureOf(statusRes), + afterStatus, + attach: failureOf(attachRes), + afterAttach, + viaClient: { + refused: viaClient.refused, + message: (viaClient.error?.message ?? "").slice(0, 140), + }, + afterClient, + lockHeld: await deviceLockIsHeld(arg.id), + }; + }, + + /** + * IS THE DEVICE'S DATABASE THERE? — asked after a wait, by + * ENUMERATION only, for `hc-destroy`'s reason: any read or write + * through the namespace API would create the thing being asked about. + * + * The wait is the point of the op. A lease heartbeat renews every + * `LEASE_INTERVAL_MS` (5 s), so a window shorter than one full + * interval cannot distinguish a stopped heartbeat from one that has + * simply not ticked yet. + */ + "db-present": async (arg: { id: string; waitMs: number }) => { + const started = Date.now(); + await new Promise((r) => setTimeout(r, arg.waitMs)); + const names = (await indexedDB.databases()).map((d) => d.name ?? ""); + return { + waitedMs: Date.now() - started, + intervalMs: LEASE_INTERVAL_MS, + present: names.includes(nsDbName(arg.id)), + databases: names.length, + }; + }, + /** * THE DEBOUNCE, with no explicit checkpoint anywhere: write, wait out * the 500 ms trailing window, and report what `status()` says about @@ -2217,8 +2443,41 @@ const ticks: { /** Best-effort teardown so cases cannot contaminate each other through * a shared index. */ +/** + * Best-effort teardown — THROUGH THE WORKER WHEN ONE IS LIVE, which is + * not tidiness but #112's class again. + * + * `removeDevice` deletes the namespace from THIS side. Do that to a + * device whose host is still running and the host's lease heartbeat + * writes `meta.lease` a few seconds later, an `ns.put` opens the + * database, and IndexedDB open-on-missing brings the whole thing back — + * so the harness was strewing resurrected databases behind every row + * that tore down a live device (measured: ~31 stray `pm-device-*` + * databases in a full run). client.ts's `destroy` says the rule in one + * line — "It has to be the worker's own hand" — because the host is the + * only thing that can drain its checkpoint chain, stop its lease and + * drop its engine before the storage goes. + * + * The page-side removal stays as the FALLBACK, for the rows that hold + * no connection (1-10 never construct a worker) and for a host that has + * already died. + */ async function cleanup(ids: string[]): Promise { for (const id of ids) { + const conn = conns.get(id); + if (conn) { + try { + await conn.destroy(); + conns.delete(id); + continue; + } catch { + // A host that cannot erase itself (already dead, already + // erased) falls through to the page-side removal below, which + // is what teardown did before this and is still right when + // there is nothing alive to ask. + conns.delete(id); + } + } try { await removeDevice(id); } catch (e) { diff --git a/runtime/tests/devstore/run.ts b/runtime/tests/devstore/run.ts index 7ae3745..9a9e3f7 100644 --- a/runtime/tests/devstore/run.ts +++ b/runtime/tests/devstore/run.ts @@ -4558,6 +4558,249 @@ async function main() { await probe(page, "hc-close", { id: rcDevice }); await probe(page, "hc-close", { id: rcRestored }); + // --- 65: an erased device STAYS erased, and 65b: the mechanism --------- + // + // #112 reported the erased device's IndexedDB database EXISTING + // again after the solo-erase ceremony (~2 runs in 6 under the full + // suite; green in isolation), with the OPFS directory still gone. + // + // THE MECHANISM IS REAL AND 65b PINS IT: `takeLock` starts a lease + // heartbeat that renews every 5 s by writing `meta.lease`, that + // write is an `ns.put`, namespace.ts holds no live connection (open, + // transact, close — its header says why), and an IndexedDB open of a + // missing database CREATES it. A tick after an erasure therefore + // brings the database back carrying a lease and nothing else — and + // ONLY the database, because `touchLease` writes IndexedDB and + // touches nothing else, which is exactly the asymmetry the field + // report shows. + // + // WHAT 65 MEASURED, AND IT IS NOT WHAT THE DIAGNOSIS ASSUMED. The + // worker global does NOT outlive `conn.destroy()`: `serve()` treats + // `destroy` exactly like `__die` and closes the global one task + // after the reply (worker.ts's `dying`). Measured here rather than + // reasoned about — immediately after the ceremony the device LOCK IS + // ALREADY FREE, which only a dead global can do — and measured again + // with the fix's `lease.stop()` NEUTERED, where the database stayed + // gone at +1s, +3s, +6s and +9s. So through this path the heartbeat + // dies with the global a millisecond after the delete, and the + // window it could resurrect through is that millisecond, not the + // seconds a 2-in-6 flake needs. #112's field trigger is therefore + // NOT accounted for by this path; see the track report, which names + // the two candidates this measurement leaves standing. + // + // THE STOP STAYS ANYWAY, and 65 is its guarantee stated as an + // outcome rather than as a mechanism: after an erasure the database + // is gone and STAYS gone across a full lease interval, whichever + // half of the belt-and-braces does the work. The close is + // best-effort by construction — `serve()` schedules it only on the + // fulfilled branch, so a destroy that REJECTS part-way (database + // deleted, index row not) leaves a live global whose heartbeat is + // pointed at a deleted database — and the stop closes that window + // structurally, before the delete, where the close cannot. + await guard(async () => { + const made = await probe(page, "hc-make", { + petname: "erase-me", + policy: "while-open", + promote: false, + }); + const id = made.id as string; + // A T0 device opened the T0 way — no ceremony, `sealT0`'s platform + // wrap — which is the tier the solo page's erase is reached from + // and the tier whose lease the sweep reads. + await probe(page, "hc-open", { id, unseal: {} }); + // The heartbeat is RUNNING, established rather than assumed: a + // fresh mark, well inside one interval, with the lock held. + const lease = await probe(page, "lease-read", { id }); + const erased = await probe(page, "hc-destroy", { id }); + // Past one full renewal interval, plus margin. 5 s is the period; + // 8 s means at least one tick has certainly come due. + const later = await probe(page, "db-present", { id, waitMs: 8_000 }); + + const ok = lease.at !== null && lease.ageMs < lease.intervalMs && + lease.lockHeld === true && + erased.attempt.refused === false && erased.dbGone === true && + erased.indexRowGone === true && + later.present === false; + record( + "65 erase", + "an erased device stays erased across a full lease interval (#112, the outcome)", + ok, + `a T0 device was opened (lease mark ${lease.ageMs} ms old, well inside the ` + + `${lease.intervalMs} ms renewal interval, device lock held: ${lease.lockHeld}) and ` + + `then ERASED through the ceremony's own path — the host \`destroy\`, with NO ` + + `\`__die\` and NO reload, so nothing in this row hurries the worker along. ` + + `Immediately after: database gone ${erased.dbGone}, index row gone ` + + `${erased.indexRowGone}. Then ${later.waitedMs} ms of nothing at all — past a full ` + + `${later.intervalMs} ms lease interval, so at least one renewal has certainly come ` + + `due — and \`indexedDB.databases()\` (an ENUMERATION: asking through the namespace ` + + `would CREATE the database being asked about) reports present=${later.present} among ` + + `${later.databases}. MEASURED BESIDE IT, and it corrects the premise this row was ` + + `written from: the device LOCK IS ALREADY FREE the instant the erase returns ` + + `(lockHeld=${erased.lockHeld}), which only a dead global can do — \`serve()\` treats ` + + `\`destroy\` exactly as it treats \`__die\` and closes the worker one task after ` + + `the reply. So this row does NOT go red with the fix's \`lease.stop()\` removed: run ` + + `first as the negative control, it passed, with the database absent at +1s, +3s, +6s ` + + `and +9s and the lock free throughout. The heartbeat cannot resurrect anything here ` + + `because it dies with the global a millisecond after the delete. The mechanism ITSELF ` + + `is real and 65b pins it deterministically; what this row guarantees is the OUTCOME, ` + + `by whichever half of the belt-and-braces gets there first.`, + ); + await probe(page, "hc-forget", { ids: [id] }); + }); + + // --- 65b: the resurrection mechanism, isolated ------------------------- + // + // The two arms differ in ONE act — the heartbeat's `stop()` — over + // the shipped `startLease` and `destroyNamespace`, with the interval + // passed in short so the row costs a second instead of eleven. This + // is what the worker's erase-time stop is FOR, made observable + // without depending on how long a SharedWorker global happens to + // live. + await guard(async () => { + const r = await probe(page, "lease-vs-erase", { intervalMs: 300 }); + const ok = r.running.before === true && r.running.atDelete === false && + r.running.later === true && r.stopped.later === false; + record( + "65b erase", + "a lease heartbeat left running over a destroyed namespace RECREATES its database", + ok, + `two namespaces, one act apart. WITH THE HEARTBEAT RUNNING: the database existed ` + + `(${r.running.before}), \`destroyNamespace\` deleted it (present right after: ` + + `${r.running.atDelete}), and ${r.settleMs} ms later — several ${r.intervalMs} ms ` + + `renewals — it was BACK: present=${r.running.later}. WITH THE HEARTBEAT STOPPED ` + + `FIRST, the same erasure and the same wait leaves present=${r.stopped.later}. The ` + + `resurrection is not exotic: \`touchLease\` is an \`ns.put\`, namespace.ts holds no ` + + `live connection (open, transact, close — its header says why this is required for ` + + `\`destroyNamespace\` to complete at all), and an IndexedDB open of a missing ` + + `database CREATES it. What comes back carries a lease and NOTHING else, and the OPFS ` + + `directory does not come back at all, because \`touchLease\` writes IndexedDB and ` + + `touches nothing else — which is #112's exact field signature (database present, ` + + `directory NotFoundError). This is the hazard worker.ts's erase-time \`lease.stop()\` ` + + `removes at the source, and it is reachable by ANY path that destroys a namespace ` + + `while a live host still leases it.`, + ); + }); + + // --- 66: a worker constructed for an ERASED device leaves no trace ----- + // + // #112's CLASS, closed at the source. A SharedWorker is keyed by + // (origin, script URL, NAME) and the name IS the device id — so + // naming an erased device is all it takes to evaluate worker.ts in a + // fresh global, and that global used to recreate the device's + // database three different ways before any human intent was + // involved: + // + // 1. MODULE EVALUATION. `bootSeq` bumps a counter in the + // namespace's `meta` store, at import time, before any client + // has said a word. The construction WAS the resurrection: no + // timer, no race, nothing to lose. + // 2. `status`, the first RPC every client sends, which reads the + // namespace — and an IndexedDB READ creates just as a write + // does, because `indexedDB.open` of a missing database creates + // it whatever the transaction mode. + // 3. `attach`, whose `takeLock()` starts the LEASE, whose first + // act is an `ns.put`. + // + // ALL THREE NOW CONSULT THE INDEX ROW, which is the existence oracle + // this store already named as such — anchor.ts's `anchorIsLive`: + // "THE INDEX ROW IS THE ANSWER, and the namespace deliberately is + // not". Reading it creates nothing that matters (the index database + // is origin-global and outlives every device in it). + // + // A RAW PORT IS THE ONLY WAY TO ASK THE WORKER THIS, and the row + // asks the ordinary client path separately: `connectDevice` refuses + // a missing row before it constructs anything, so an application + // never reaches these guards — they answer a stale tab, a picker + // holding a collected id, or a reconnect racing an erase. + await guard(async () => { + const made = await probe(page, "hc-make", { + petname: "erased-then-named", + policy: "while-open", + promote: false, + }); + const id = made.id as string; + await probe(page, "hc-open", { id, unseal: {} }); + const erased = await probe(page, "hc-destroy", { id }); + const r = await probe(page, "erase-reconnect", { id }); + + const ok = erased.dbGone === true && erased.indexRowGone === true && + r.booted === true && r.bootSeq === 0 && + r.afterConstruct === false && + r.status.ok === false && r.status.code === "no-rung" && r.afterStatus === false && + r.attach.ok === false && r.attach.code === "no-rung" && r.afterAttach === false && + r.viaClient.refused === true && r.afterClient === false && + r.lockHeld === false; + record( + "66 erase", + "constructing a worker for an ERASED device recreates nothing (#112's class, at the source)", + ok, + `a T0 device was erased through the host \`destroy\` (database gone ${erased.dbGone}, ` + + `index row gone ${erased.indexRowGone}) and then a RAW SharedWorker was constructed ` + + `under its name — which is all a stale tab or a picker holding a collected id has to ` + + `do. The global genuinely BOOTED (it posted its hello: ${r.booted}) and its counter ` + + `read bootSeq=${r.bootSeq} — ZERO, the "I counted nothing" value, because the index ` + + `row is consulted before the namespace is touched. Database after the construction: ` + + `present=${r.afterConstruct}. Then the two RPCs a client sends first, over that same ` + + `raw port: \`status\` refused ${j(r.status.code)} (${j(r.status.message)}), database ` + + `present=${r.afterStatus}; \`attach\` refused ${j(r.attach.code)}, database ` + + `present=${r.afterAttach} after a further 6 s — past a full lease interval, so an ` + + `attach that had taken the lock would have written by now (lock held: ${r.lockHeld}). ` + + `THE ORDINARY CLIENT PATH never gets that far: \`connectDevice\` refused ` + + `(${j(r.viaClient.message)}) before constructing anything, database ` + + `present=${r.afterClient}. NEGATIVE CONTROL, with the three guards removed: the ` + + `database is back after step ONE — the construction alone — with a boot counter in ` + + `it and nothing else, no client call and no timer required; \`status\` and ` + + `\`attach\` each recreate it on their own too (a read opens, and an open of a ` + + `missing database creates). That is why the fix is three gates on one oracle rather ` + + `than one gate on the door CI happened to catch.`, + ); + await probe(page, "hc-forget", { ids: [id] }); + }); + + // --- 66b: tearing down a LIVE device leaves no database behind --------- + // + // The same hazard from the other side, and the harness was the one + // committing it: deleting a namespace from the PAGE while its host + // is alive leaves the host's lease heartbeat pointed at a deleted + // database, and a tick brings it back. Measured before the fix as + // ~31 stray `pm-device-*` databases in a full run — every row that + // tore down a device it still held a connection to. + // + // The teardown helper now goes THROUGH THE WORKER when one is live + // (client.ts's rule: "It has to be the worker's own hand"), falling + // back to the page-side removal for the rows that hold no + // connection. This asserts the outcome across a full lease interval. + await guard(async () => { + const made = await probe(page, "hc-make", { + petname: "forgotten-while-live", + policy: "while-open", + promote: false, + }); + const id = made.id as string; + await probe(page, "hc-open", { id, unseal: {} }); + const lease = await probe(page, "lease-read", { id }); + const forgotten = await probe(page, "hc-forget", { ids: [id] }); + const later = await probe(page, "db-present", { id, waitMs: 7_000 }); + + const ok = lease.lockHeld === true && lease.at !== null && + forgotten.cleanup === "ok" && later.present === false; + record( + "66b erase", + "a live device torn down by the harness leaves no resurrected database", + ok, + `a T0 device with a LIVE host (lock held: ${lease.lockHeld}, lease mark ` + + `${lease.ageMs} ms old) was torn down through the matrix's ordinary cleanup helper ` + + `(${j(forgotten.cleanup)}), then left alone for ${later.waitedMs} ms — past a full ` + + `${later.intervalMs} ms lease interval. Its database is present=${later.present}. ` + + `BEFORE the fix this teardown deleted the namespace from the PAGE while the host was ` + + `still running, and the host's next heartbeat recreated it: the helper now asks the ` + + `WORKER to erase itself when a connection is live (client.ts: "It has to be the ` + + `worker's own hand" — only the host can drain its checkpoint chain, stop its lease ` + + `and drop its engine before the storage goes), and falls back to the page-side ` + + `removal only where no worker exists. Rows 1-10 take that fallback and are unchanged.`, + ); + }); + await ctx.close(); } finally {