From 05d14ec00aa9daa97af3c2614cd7c91dafb5d9d8 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Mon, 2 Nov 2026 11:47:32 +0700 Subject: [PATCH 01/13] fix: serialize stale worker-lock recovery through atomic admission --- lib/runtime/store/file-lock.js | 124 +++++++++++-- test/runtime/file-lock.test.js | 329 +++++++++++++++++++++++++++++++++ 2 files changed, 438 insertions(+), 15 deletions(-) create mode 100644 test/runtime/file-lock.test.js diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index d122790..fbc2d06 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -183,27 +183,121 @@ function ownsLock(lockDir, token) { return contentOf(path.join(lockDir, 'owner')) === token; } -function tryLock(lockDir, now) { - fs.mkdirSync(path.dirname(lockDir), { recursive: true, mode: 0o700 }); - if (isStale(lockDir, now)) fs.rmSync(lockDir, { recursive: true, force: true }); +function reclaimAdmission(admission) { try { - fs.mkdirSync(lockDir); + if (!fs.lstatSync(admission).isDirectory()) return false; + const owners = fs.readdirSync(admission); + if (owners.length > 1) return false; + if (owners.length === 1) { + const owner = /^owner-(\d+)-[a-f0-9]+$/.exec(owners[0]); + if (!owner || !ownerIsGone(Number(owner[1]))) return false; + fs.unlinkSync(path.join(admission, owners[0])); + } + // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot + // remove it even if another reclaimer replaced the directory after our read. + fs.rmdirSync(admission); + return true; } catch { + return false; + } +} + +function admissionOf(lockDir) { + const admission = `${lockDir}.admission`; + const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}`; + const staging = `${admission}.${owner}.claim`; + fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); + fs.mkdirSync(staging, { mode: 0o700 }); + try { + fs.writeFileSync(path.join(staging, owner), '', { mode: 0o600, flag: 'wx' }); + // Publishing a prepopulated directory never exposes an empty fresh guard to a stale reclaimer. + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + fs.renameSync(staging, admission); + return { admission, owner }; + } catch { + if (attempt > 0 || !reclaimAdmission(admission)) return null; + } + } return null; + } finally { + fs.rmSync(staging, { recursive: true, force: true }); } - const token = crypto.randomBytes(12).toString('hex'); - fs.writeFileSync(path.join(lockDir, 'owner'), token); - return { - refresh: () => { - if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); - }, - release: () => { +} + +function retryAdmissionCleanup(remove) { + for (let attempt = 0; ; attempt += 1) { + try { + return remove(); + } catch (error) { + if (attempt === 2 || !['EPERM', 'EBUSY', 'EACCES'].includes(error?.code)) throw error; + pause(LOCK_RETRY_MS); + } + } +} + +function releaseAdmission({ admission, owner }) { + try { + retryAdmissionCleanup(() => fs.unlinkSync(path.join(admission, owner))); + try { + retryAdmissionCleanup(() => fs.rmdirSync(admission)); + } catch (error) { + if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error?.code)) throw error; + } + } catch (cause) { + throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + } +} + +// Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. +function admitted(lockDir, mutate) { + const admission = admissionOf(lockDir); + if (!admission) return null; + try { + return mutate(); + } finally { + releaseAdmission(admission); + } +} + +async function releaseWorkLock(lockDir, token) { + const deadline = Date.now() + LOCK_WAIT_MS; + for (;;) { + if (admitted(lockDir, () => { if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); - }, - }; + return true; + })) return; + if (Date.now() >= deadline) throw Object.assign(new Error(`Could not release lock ${lockDir}`), { code: 'ELOCKED' }); + await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS)); + } +} + +function tryLock(lockDir, now) { + return admitted(lockDir, () => { + if (isStale(lockDir, now)) { + const held = contentOf(path.join(lockDir, 'owner')); + // Old versions wrote random tokens without a pid. Keep their lease recovery, but a known live owner + // cannot lose its lock just because a slow or paused worker missed a refresh. A reused pid stays busy. + if (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))) return null; + fs.rmSync(lockDir, { recursive: true, force: true }); + } + try { + fs.mkdirSync(lockDir); + } catch { + return null; + } + const token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; + fs.writeFileSync(path.join(lockDir, 'owner'), token); + return { + refresh: () => admitted(lockDir, () => { + if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); + }), + release: () => releaseWorkLock(lockDir, token), + }; + }); } -/** Runs slow `work` while holding a lock directory no other process gets meanwhile; false when it is held. */ +/** Runs slow `work` under one owner's lock; dead or legacy directories recover after their lease expires. */ export async function withLock(lockDir, work, now = Date.now()) { const lock = tryLock(lockDir, now); if (!lock) return false; @@ -211,6 +305,6 @@ export async function withLock(lockDir, work, now = Date.now()) { await work(lock); return true; } finally { - lock.release(); + await lock.release(); } } diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js new file mode 100644 index 0000000..d2b8bdb --- /dev/null +++ b/test/runtime/file-lock.test.js @@ -0,0 +1,329 @@ +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, rmdirSync, statSync, unlinkSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; + +import { withLock } from '../../lib/runtime/store/file-lock.js'; + +const RUNTIME = new URL('../../lib/runtime/store/file-lock.js', import.meta.url).href; +const STALE = new Date(0); + +function scratch(t, token = null) { + const dir = mkdtempSync(join(tmpdir(), 'evolver-work-lock-')); + const lock = join(dir, 'work.lock'); + if (token !== null) { + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), token); + } + t.after(() => rmSync(dir, { recursive: true, force: true })); + return { dir, lock }; +} + +async function until(check) { + const deadline = Date.now() + 10_000; + while (!check()) { + if (Date.now() > deadline) throw new Error('timed out waiting for the lock contender'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +function holdAdmission(lock, pid = process.pid) { + const admission = `${lock}.admission`; + const owner = join(admission, `owner-${pid}-deadbeef`); + mkdirSync(admission); + writeFileSync(owner, 'fixture owner'); + return { admission, owner, release: () => { unlinkSync(owner); rmdirSync(admission); } }; +} + +test('stale legacy and dead-owner directories recover, but a fresh legacy lock stays held', async (t) => { + const { lock } = scratch(t, 'legacy token'); + let entered = 0; + assert.equal(await withLock(lock, async () => { entered += 1; }), false); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), `${exited.pid}:deadbeef`); + holdAdmission(lock, exited.pid); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + assert.equal(entered, 2); + assert.equal(existsSync(lock), false); + assert.equal(existsSync(`${lock}.admission`), false); +}); + +test('an old mtime never evicts a live owner, which can refresh and release its lock', async (t) => { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + utimesSync(lock, STALE, STALE); + let entered = false; + assert.equal(await withLock(lock, async () => { entered = true; }), false); + assert.equal(entered, false); + held.refresh(); + assert.ok(statSync(lock).mtimeMs > STALE.getTime()); + }), true); + assert.equal(existsSync(lock), false); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release waits asynchronously for transient admission contention', async (t) => { + const { lock } = scratch(t); + let guard; + let finished = false; + let refreshedAt; + const running = withLock(lock, async (held) => { + guard = holdAdmission(lock); + utimesSync(lock, STALE, STALE); + held.refresh(); + refreshedAt = statSync(lock).mtimeMs; + }).then(() => { finished = true; }); + try { + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(refreshedAt, STALE.getTime(), 'refresh also respects admission'); + assert.equal(finished, false); + assert.equal(existsSync(lock), true); + } finally { + guard.release(); + await running; + } + assert.equal(existsSync(lock), false); +}); + +test('release refuses bounded admission contention without deleting the held directory', async (t) => { + const { lock } = scratch(t); + let guard; + const started = Date.now(); + await assert.rejects(withLock(lock, async () => { + guard = holdAdmission(lock); + }), { code: 'ELOCKED' }); + assert.ok(Date.now() - started < 5_000, 'cleanup cannot wait forever'); + assert.equal(existsSync(lock), true); + assert.equal(readFileSync(guard.owner, 'utf8'), 'fixture owner'); +}); + +test('an admission cleanup failure is surfaced instead of silently orphaning a live guard', async (t) => { + const { lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + return unlink(target); + }; + let entered = false; + try { + await assert.rejects(withLock(lock, async () => { entered = true; }), { code: 'ELOCKED' }); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(entered, false); + assert.match(readdirSync(admission)[0], new RegExp(`^owner-${process.pid}-`)); +}); + +test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { + const { dir, lock } = scratch(t); + const unlink = fs.unlinkSync; + let refused = false; + fs.unlinkSync = (target) => { + if (dirname(target) === `${lock}.admission` && !refused) { + refused = true; + throw Object.assign(new Error('temporarily refused by fixture'), { code: 'EPERM' }); + } + return unlink(target); + }; + let entered = 0; + try { + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(refused, true); + assert.equal(entered, 1); + assert.deepEqual(readdirSync(dir), []); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + assert.equal(entered, 2); + assert.deepEqual(readdirSync(dir), []); +}); + +test('busy and unrecognised admission guards stay intact and leave no staging directories', async (t) => { + const { dir, lock } = scratch(t); + const guard = holdAdmission(lock); + assert.equal(await withLock(lock, async () => assert.fail('a live guard is held')), false); + assert.deepEqual(readdirSync(dir), ['work.lock.admission']); + writeFileSync(join(guard.admission, 'unexpected'), 'keep'); + assert.equal(await withLock(lock, async () => assert.fail('multiple owners are unknown')), false); + assert.equal(readdirSync(guard.admission).length, 2); + unlinkSync(guard.owner); + assert.equal(await withLock(lock, async () => assert.fail('an unknown owner is held')), false); + assert.equal(readFileSync(join(guard.admission, 'unexpected'), 'utf8'), 'keep'); + assert.deepEqual(readdirSync(dir), ['work.lock.admission']); +}); + +test('two contenders cannot replace the fresh lock while another reclaims a stale directory', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-work-lock-')); + const lock = join(dir, 'work.lock'); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), 'legacy token'); + utimesSync(lock, STALE, STALE); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { withLock } from ${JSON.stringify(RUNTIME)}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const wait = async (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + }; + if (role === 'A') { + const stat = fs.statSync; + let paused = false; + fs.statSync = (file, ...args) => { + const result = stat(file, ...args); + if (file === lock && !paused) { + paused = true; + fs.writeFileSync(marker('stale-read'), ''); + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker('resume'))) { + if (Date.now() > deadline) throw new Error('timed out waiting to resume'); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + } + return result; + }; + } + const result = await withLock(lock, async () => { + fs.writeFileSync(marker(role + '-entered'), ''); + await wait('release'); + }); + fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); + `; + const children = []; + const run = (role) => { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, lock, role]); + let stderr = ''; + child.stderr.on('data', (data) => { stderr += data; }); + const done = new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr }))); + children.push({ child, done }); + }; + t.after(async () => { + writeFileSync(marker('resume'), ''); + writeFileSync(marker('release'), ''); + await Promise.all(children.map(({ done }) => done)); + rmSync(dir, { recursive: true, force: true }); + }); + run('A'); + await until(() => existsSync(marker('stale-read'))); + run('B'); + run('C'); + await until(() => ['B', 'C'].every((role) => existsSync(marker(role + '-entered')) || existsSync(marker(role + '-result')))); + writeFileSync(marker('resume'), ''); + await until(() => existsSync(marker('A-entered'))); + const entered = ['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))); + writeFileSync(marker('release'), ''); + const results = await Promise.all(children.map(({ done }) => done)); + assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); + assert.deepEqual(entered, ['A'], 'only the process reclaiming the inspected stale directory may enter'); + assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [true, false, false]); +}); + +test('a delayed dead-admission reclaimer cannot unlink the fresh owner another contender published', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-admission-race-')); + const lock = join(dir, 'work.lock'); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), 'legacy token'); + utimesSync(lock, STALE, STALE); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + holdAdmission(lock, exited.pid); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { withLock } from ${JSON.stringify(RUNTIME)}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const waitSync = (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + }; + if (role === 'A') { + const list = fs.readdirSync; + let paused = false; + fs.readdirSync = (file, ...args) => { + const result = list(file, ...args); + if (file === lock + '.admission' && !paused) { + paused = true; + fs.writeFileSync(marker('dead-inspected'), ''); + waitSync('A-resume'); + } + return result; + }; + } + if (role === 'B') { + const rename = fs.renameSync; + let paused = false; + fs.renameSync = (source, target) => { + const result = rename(source, target); + if (target === lock + '.admission' && !paused) { + paused = true; + fs.writeFileSync(marker('fresh-held'), ''); + waitSync('B-resume'); + } + return result; + }; + } + const result = await withLock(lock, async () => { + fs.writeFileSync(marker(role + '-entered'), ''); + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker('release'))) { + if (Date.now() > deadline) throw new Error('timed out waiting to release'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + }); + fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); + `; + const children = []; + const run = (role) => { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, lock, role]); + let stderr = ''; + child.stderr.on('data', (data) => { stderr += data; }); + const done = new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr }))); + children.push({ child, done }); + return child; + }; + t.after(async () => { + for (const name of ['A-resume', 'B-resume', 'release']) writeFileSync(marker(name), ''); + await Promise.all(children.map(({ done }) => done)); + rmSync(dir, { recursive: true, force: true }); + }); + run('A'); + await until(() => existsSync(marker('dead-inspected'))); + const fresh = run('B'); + await until(() => existsSync(marker('fresh-held'))); + run('C'); + await until(() => existsSync(marker('C-result')) || existsSync(marker('C-entered'))); + assert.equal(existsSync(marker('C-entered')), false, 'publishing a fresh guard never exposes an empty directory'); + writeFileSync(marker('A-resume'), ''); + await until(() => existsSync(marker('A-result')) || existsSync(marker('A-entered'))); + assert.equal(existsSync(marker('A-entered')), false, 'the delayed reclaimer cannot displace the fresh guard'); + assert.equal(readdirSync(`${lock}.admission`).length, 1); + assert.match(readdirSync(`${lock}.admission`)[0], new RegExp(`^owner-${fresh.pid}-`)); + writeFileSync(marker('B-resume'), ''); + await until(() => existsSync(marker('B-entered'))); + assert.deepEqual(['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))), ['B']); + writeFileSync(marker('release'), ''); + const results = await Promise.all(children.map(({ done }) => done)); + assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); + assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [false, true, false]); +}); From 71f98f860e4f938f777eb0d0caffd444024376b7 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Mon, 2 Nov 2026 12:05:22 +0700 Subject: [PATCH 02/13] fix: preserve worker ownership when lock metadata is unreadable --- lib/runtime/store/file-lock.js | 35 ++++++-- test/runtime/file-lock.test.js | 146 +++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 9 deletions(-) diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index fbc2d06..e8732b1 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -179,8 +179,17 @@ function isStale(lockDir, now) { } } +function workOwnerOf(lockDir) { + try { + return retryLockAccess(() => fs.readFileSync(path.join(lockDir, 'owner'), 'utf8')); + } catch (cause) { + if (cause?.code === 'ENOENT') return null; + throw Object.assign(new Error(`Could not read lock owner ${lockDir}`, { cause }), { code: 'ELOCKED' }); + } +} + function ownsLock(lockDir, token) { - return contentOf(path.join(lockDir, 'owner')) === token; + return workOwnerOf(lockDir) === token; } function reclaimAdmission(admission) { @@ -225,10 +234,10 @@ function admissionOf(lockDir) { } } -function retryAdmissionCleanup(remove) { +function retryLockAccess(access) { for (let attempt = 0; ; attempt += 1) { try { - return remove(); + return access(); } catch (error) { if (attempt === 2 || !['EPERM', 'EBUSY', 'EACCES'].includes(error?.code)) throw error; pause(LOCK_RETRY_MS); @@ -238,9 +247,9 @@ function retryAdmissionCleanup(remove) { function releaseAdmission({ admission, owner }) { try { - retryAdmissionCleanup(() => fs.unlinkSync(path.join(admission, owner))); + retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); try { - retryAdmissionCleanup(() => fs.rmdirSync(admission)); + retryLockAccess(() => fs.rmdirSync(admission)); } catch (error) { if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error?.code)) throw error; } @@ -275,7 +284,8 @@ async function releaseWorkLock(lockDir, token) { function tryLock(lockDir, now) { return admitted(lockDir, () => { if (isStale(lockDir, now)) { - const held = contentOf(path.join(lockDir, 'owner')); + let held; + try { held = workOwnerOf(lockDir); } catch { return null; } // Old versions wrote random tokens without a pid. Keep their lease recovery, but a known live owner // cannot lose its lock just because a slow or paused worker missed a refresh. A reused pid stays busy. if (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))) return null; @@ -289,9 +299,16 @@ function tryLock(lockDir, now) { const token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; fs.writeFileSync(path.join(lockDir, 'owner'), token); return { - refresh: () => admitted(lockDir, () => { - if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); - }), + refresh: () => { + try { + return admitted(lockDir, () => { + if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); + }); + } catch { + // Timer-driven refresh is best effort; a live pid stays protected and final release still surfaces errors. + return null; + } + }, release: () => releaseWorkLock(lockDir, token), }; }); diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index d2b8bdb..1156254 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -70,6 +70,152 @@ test('an old mtime never evicts a live owner, which can refresh and release its assert.deepEqual(readdirSync(dir), []); }); +test('a transient owner read error never turns a stale live lock into a legacy lock', async (t) => { + for (const code of ['EBUSY', 'EPERM', 'EACCES', 'EIO']) { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async () => { + const owner = join(lock, 'owner'); + const token = readFileSync(owner, 'utf8'); + utimesSync(lock, STALE, STALE); + const read = fs.readFileSync; + let refused = false; + let entered = false; + fs.readFileSync = (target, ...args) => { + if (target === owner && !refused) { + refused = true; + throw Object.assign(new Error('transient owner read refusal'), { code }); + } + return read(target, ...args); + }; + try { + assert.equal(await withLock(lock, async () => { entered = true; }), false, code); + } finally { + fs.readFileSync = read; + } + assert.equal(refused, true); + assert.equal(entered, false, code); + assert.equal(readFileSync(owner, 'utf8'), token, 'the live owner stays intact'); + }), true); + assert.deepEqual(readdirSync(dir), [], 'the original owner still releases normally'); + } +}); + +test('a stale directory left before its owner was written is recoverable', async (t) => { + const { dir, lock } = scratch(t); + mkdirSync(lock); + utimesSync(lock, STALE, STALE); + let entered = false; + assert.equal(await withLock(lock, async () => { entered = true; }), true); + assert.equal(entered, true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release retries a transient owner read failure and really removes its lock', async (t) => { + const { dir, lock } = scratch(t); + const owner = join(lock, 'owner'); + const read = fs.readFileSync; + let refused = false; + try { + assert.equal(await withLock(lock, async () => { + fs.readFileSync = (target, ...args) => { + if (target === owner && !refused) { + refused = true; + throw Object.assign(new Error('transient read refusal'), { code: 'EBUSY' }); + } + return read(target, ...args); + }; + }), true); + } finally { + fs.readFileSync = read; + } + assert.equal(refused, true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release refuses an unreadable owner explicitly without claiming cleanup', async (t) => { + for (const code of ['EBUSY', 'EPERM', 'EIO']) { + const { lock } = scratch(t); + const owner = join(lock, 'owner'); + const read = fs.readFileSync; + let token; + try { + await assert.rejects(withLock(lock, async () => { + token = read(owner, 'utf8'); + fs.readFileSync = (target, ...args) => { + if (target === owner) throw Object.assign(new Error('persistent read refusal'), { code }); + return read(target, ...args); + }; + }), { code: 'ELOCKED' }); + } finally { + fs.readFileSync = read; + } + assert.equal(readFileSync(owner, 'utf8'), token, code); + } +}); + +test('refresh leaves an unreadable live lock untouched without crashing its timer caller', async (t) => { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + const owner = join(lock, 'owner'); + utimesSync(lock, STALE, STALE); + const read = fs.readFileSync; + fs.readFileSync = (target, ...args) => { + if (target === owner) throw Object.assign(new Error('unreadable owner'), { code: 'EPERM' }); + return read(target, ...args); + }; + try { + assert.doesNotThrow(() => held.refresh()); + assert.equal(statSync(lock).mtimeMs, STALE.getTime()); + } finally { + fs.readFileSync = read; + } + }), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('refresh tolerates file access refusals and the next refresh and release still work', async (t) => { + for (const method of ['utimesSync', 'mkdirSync', 'readFileSync']) { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + utimesSync(lock, STALE, STALE); + const access = fs[method]; + let refused = false; + fs[method] = (target, ...args) => { + const matches = method === 'mkdirSync' ? String(target).startsWith(`${lock}.admission.`) + : target === (method === 'readFileSync' ? join(lock, 'owner') : lock); + if (matches && !refused) { + refused = true; + throw Object.assign(new Error('transient access refusal'), { code: 'EPERM' }); + } + return access(target, ...args); + }; + try { + assert.doesNotThrow(() => held.refresh(), method); + if (method !== 'readFileSync') assert.equal(statSync(lock).mtimeMs, STALE.getTime()); + } finally { + fs[method] = access; + } + assert.equal(refused, true, method); + held.refresh(); + assert.ok(statSync(lock).mtimeMs > STALE.getTime()); + }), true); + assert.deepEqual(readdirSync(dir), []); + } +}); + +test('release does not delete a missing or different owner', async (t) => { + for (const replacement of [null, `${process.pid}:another-owner`]) { + const { lock } = scratch(t); + assert.equal(await withLock(lock, async () => { + if (replacement === null) unlinkSync(join(lock, 'owner')); + else writeFileSync(join(lock, 'owner'), replacement); + }), true); + assert.equal(existsSync(lock), true); + assert.equal(existsSync(join(lock, 'owner')), replacement !== null); + if (replacement !== null) assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), replacement); + } +}); + test('release waits asynchronously for transient admission contention', async (t) => { const { lock } = scratch(t); let guard; From 35ba6cc32a282b03d91a09e498504c3fad84f979 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Mon, 2 Nov 2026 12:05:28 +0700 Subject: [PATCH 03/13] fix: serialize state-file recovery attempts and refuse unlocked mutation --- lib/runtime/store/file-lock.js | 33 +++-- test/runtime/state-lock.test.js | 235 ++++++++++++++++++++++++++++++++ 2 files changed, 258 insertions(+), 10 deletions(-) create mode 100644 test/runtime/state-lock.test.js diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index e8732b1..d8c5e4f 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -77,17 +77,29 @@ function reclaim(lockPath, deadToken) { function acquire(lockPath, deadline) { const token = `${process.pid}:${crypto.randomBytes(8).toString('hex')}`; for (;;) { + if (Date.now() >= deadline) return null; + // All contenders admit the whole attempt, including creating a new owner, so reclaim's temporary + // missing-path window never lets a third process take a fresh file lock that another reclaimer moves. + let attempt; try { - fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 }); - fs.writeFileSync(lockPath, token, { mode: 0o600, flag: 'wx' }); - return token; + attempt = admitted(lockPath, () => { + try { + fs.writeFileSync(lockPath, token, { mode: 0o600, flag: 'wx' }); + return { status: 'acquired', token }; + } catch (error) { + if (error?.code !== 'EEXIST') return { status: 'failed', error }; + const held = contentOf(lockPath); + if (held !== null && ownerIsGone(ownerPidOf(held))) reclaim(lockPath, held); + return { status: 'busy' }; + } + }); } catch (error) { - if (error?.code !== 'EEXIST') return null; - const held = contentOf(lockPath); - if (held !== null && ownerIsGone(ownerPidOf(held))) reclaim(lockPath, held); - else if (Date.now() >= deadline) return null; - else pause(LOCK_RETRY_MS); + attempt = { status: 'failed', error }; } + if (attempt?.status === 'acquired') return attempt.token; + if (attempt?.status === 'failed' && !['EACCES', 'EPERM', 'EBUSY'].includes(attempt.error?.code)) return null; + if (Date.now() >= deadline) return null; + pause(LOCK_RETRY_MS); } } @@ -97,12 +109,13 @@ function release(lockPath, token) { /** * Applies `mutate` to a state file under an exclusive lock, so hook processes sharing it cannot overwrite each - * other's entries. A dead owner's lock is taken over; one that cannot be had within LOCK_WAIT_MS is given up on - * and the update runs unguarded, which never blocks a session. `mutate` returns undefined to leave the file. + * other's entries. A dead owner's lock is taken over; failed admission never permits an unlocked write. + * `mutate` returns undefined to leave the file. */ export function updateState(target, fallback, mutate) { const lockPath = `${target}.lock`; const token = acquire(lockPath, Date.now() + LOCK_WAIT_MS); + if (!token) throw Object.assign(new Error('State is busy or unavailable; no update was made'), { code: 'ELOCKED' }); try { const next = mutate(readState(target, fallback)); if (next !== undefined) writeState(target, next); diff --git a/test/runtime/state-lock.test.js b/test/runtime/state-lock.test.js new file mode 100644 index 0000000..03dd477 --- /dev/null +++ b/test/runtime/state-lock.test.js @@ -0,0 +1,235 @@ +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; + +import { updateState } from '../../lib/runtime/store/file-lock.js'; + +const RUNTIME = new URL('../../lib/runtime/store/file-lock.js', import.meta.url).href; +const BUSY = { code: 'ELOCKED', message: 'State is busy or unavailable; no update was made' }; + +function scratch(t) { + const dir = mkdtempSync(join(tmpdir(), 'evolver-state-lock-')); + const target = join(dir, 'state.json'); + writeFileSync(target, '{"count":0}'); + t.after(() => rmSync(dir, { recursive: true, force: true })); + return { dir, target, lock: `${target}.lock` }; +} + +test('a held state lock never runs an unguarded mutator after its wait expires', (t) => { + const { target, lock } = scratch(t); + const held = `${process.pid}:alive`; + writeFileSync(lock, held); + let called = false; + const started = Date.now(); + assert.throws(() => updateState(target, {}, () => { called = true; return { count: 1 }; }), BUSY); + assert.ok(Date.now() - started < 5_000); + assert.equal(called, false); + assert.equal(readFileSync(target, 'utf8'), '{"count":0}'); + assert.equal(readFileSync(lock, 'utf8'), held); +}); + +test('a legacy dead state owner is recovered, and file permission errors stop before mutation', (t) => { + const { target, lock } = scratch(t); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + writeFileSync(lock, `${exited.pid}:deadbeef`); + assert.deepEqual(updateState(target, {}, (stored) => ({ count: stored.count + 1 })), { count: 1 }); + assert.equal(existsSync(lock), false); + const write = fs.writeFileSync; + fs.writeFileSync = (file, ...args) => { + if (file === lock) throw Object.assign(new Error('refused by fixture'), { code: 'EROFS' }); + return write(file, ...args); + }; + let called = false; + try { + assert.throws(() => updateState(target, {}, () => { called = true; return {}; }), BUSY); + } finally { + fs.writeFileSync = write; + } + assert.equal(called, false); + assert.deepEqual(JSON.parse(readFileSync(target, 'utf8')), { count: 1 }); + assert.deepEqual(updateState(target, {}, (stored) => ({ count: stored.count + 1 })), { count: 2 }); +}); + +test('a transient sharing denial retries outside admission and runs the mutator only once', (t) => { + const { target, lock } = scratch(t); + const write = fs.writeFileSync; + let attempts = 0; + fs.writeFileSync = (file, ...args) => { + if (file === lock && ++attempts === 1) throw Object.assign(new Error('temporarily refused by fixture'), { code: 'EPERM' }); + return write(file, ...args); + }; + let called = 0; + try { + assert.deepEqual(updateState(target, {}, (stored) => { called += 1; return { count: stored.count + 1 }; }), { count: 1 }); + } finally { + fs.writeFileSync = write; + } + assert.equal(attempts, 2); + assert.equal(called, 1); + assert.equal(existsSync(lock), false); +}); + +test('an admission setup refusal leaves state unchanged and never calls the mutator', (t) => { + const { target, lock } = scratch(t); + const mkdir = fs.mkdirSync; + fs.mkdirSync = (file, ...args) => { + if (String(file).startsWith(`${lock}.admission.`)) throw Object.assign(new Error('read-only fixture'), { code: 'EROFS' }); + return mkdir(file, ...args); + }; + let called = false; + try { + assert.throws(() => updateState(target, {}, () => { called = true; return {}; }), BUSY); + } finally { + fs.mkdirSync = mkdir; + } + assert.equal(called, false); + assert.equal(readFileSync(target, 'utf8'), '{"count":0}'); + assert.deepEqual(updateState(target, {}, (stored) => ({ count: stored.count + 1 })), { count: 1 }); +}); + +test('a refused dead-owner reclaim obeys the same bounded wait without calling the mutator', async (t) => { + const { target, lock } = scratch(t); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + const held = `${exited.pid}:deadbeef`; + writeFileSync(lock, held); + const script = ` + import fs from 'node:fs'; + import { updateState } from ${JSON.stringify(RUNTIME)}; + const [target, lock] = process.argv.slice(1); + const rename = fs.renameSync; + let reclaims = 0; + fs.renameSync = (source, destination) => { + if (source === lock && destination.endsWith('.reclaimed')) { + reclaims += 1; + throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + } + return rename(source, destination); + }; + let called = false; + try { updateState(target, {}, () => { called = true; return {}; }); } + catch (error) { process.stdout.write(JSON.stringify({ code: error.code, called, reclaims })); } + `; + const child = spawn(process.execPath, ['--input-type=module', '-e', script, target, lock]); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (data) => { stdout += data; }); + child.stderr.on('data', (data) => { stderr += data; }); + const watchdog = setTimeout(() => child.kill('SIGKILL'), 5_000); + const code = await new Promise((resolve) => child.on('close', resolve)); + clearTimeout(watchdog); + assert.equal(code, 0, stderr || 'a failed reclaim must not spin forever'); + const result = JSON.parse(stdout); + assert.equal(result.code, 'ELOCKED'); + assert.equal(result.called, false); + assert.ok(result.reclaims > 0); + assert.equal(readFileSync(target, 'utf8'), '{"count":0}'); + assert.equal(readFileSync(lock, 'utf8'), held); + assert.deepEqual(updateState(target, {}, (stored) => ({ count: stored.count + 1 })), { count: 1 }); +}); + +test('three state updaters preserve every increment while recovering one dead file lock', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-state-lock-race-')); + const target = join(dir, 'state.json'); + const lock = `${target}.lock`; + writeFileSync(target, '{"count":0}'); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + const dead = `${exited.pid}:deadbeef`; + writeFileSync(lock, dead); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { updateState } from ${JSON.stringify(RUNTIME)}; + const [dir, target, lock, role, dead] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const wait = (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + }; + const read = fs.readFileSync; + let paused = false; + fs.readFileSync = (file, ...args) => { + const result = read(file, ...args); + if (role === 'A' && file === lock && !paused) { + paused = true; + fs.writeFileSync(marker('A-read-dead'), ''); + wait('A-resume'); + } else if (role === 'A' && String(file).startsWith(lock + '.') && String(file).endsWith('.reclaimed') && result !== dead) { + fs.writeFileSync(marker('A-moved-fresh'), ''); + wait('C-entered'); + } + return result; + }; + const rename = fs.renameSync; + fs.renameSync = (source, destination) => { + try { return rename(source, destination); } + catch (error) { + if (destination === lock + '.admission') fs.writeFileSync(marker(role + '-guard-busy'), ''); + throw error; + } + }; + const write = fs.writeFileSync; + fs.writeFileSync = (file, ...args) => { + if (file === lock) write(marker(role + '-attempted'), ''); + return write(file, ...args); + }; + updateState(target, {}, (stored) => { + try { fs.writeFileSync(marker('active'), role, { flag: 'wx' }); } + catch { fs.writeFileSync(marker('overlap'), ''); } + fs.writeFileSync(marker(role + '-entered'), ''); + wait('release'); + if (fs.readFileSync(marker('active'), 'utf8') === role) fs.unlinkSync(marker('active')); + return { count: stored.count + 1 }; + }); + `; + const children = []; + const run = (role) => { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, target, lock, role, dead]); + let stderr = ''; + child.stderr.on('data', (data) => { stderr += data; }); + children.push(new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr })))); + }; + const until = async (condition) => { + const deadline = Date.now() + 10000; + while (!condition()) { + if (Date.now() > deadline) throw new Error('timed out waiting for the state updater'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + }; + t.after(async () => { + writeFileSync(marker('A-resume'), ''); + writeFileSync(marker('release'), ''); + await Promise.all(children); + rmSync(dir, { recursive: true, force: true }); + }); + run('A'); + await until(() => existsSync(marker('A-read-dead'))); + run('B'); + await until(() => existsSync(marker('B-entered')) || existsSync(marker('B-guard-busy'))); + const unguarded = existsSync(marker('B-entered')); + writeFileSync(marker('A-resume'), ''); + if (unguarded) { + await until(() => existsSync(marker('A-moved-fresh'))); + run('C'); + await until(() => existsSync(marker('C-entered'))); + } else { + await until(() => ['A', 'B'].some((role) => existsSync(marker(role + '-entered')))); + run('C'); + await until(() => existsSync(marker('C-attempted')) || existsSync(marker('C-guard-busy'))); + } + writeFileSync(marker('release'), ''); + const results = await Promise.all(children); + assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); + assert.equal(existsSync(marker('overlap')), false, 'state mutators never overlap during dead-owner recovery'); + assert.deepEqual(JSON.parse(readFileSync(target, 'utf8')), { count: 3 }); +}); From a638041595b46c05f827793a2fb6ca61a9bb3fc3 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 10:29:28 +0700 Subject: [PATCH 04/13] fix: recover stranded worker lock admission safely --- lib/runtime/store/file-lock.js | 61 +++++++++++++++++++++++++++------- test/runtime/file-lock.test.js | 31 +++++++++++++++++ 2 files changed, 80 insertions(+), 12 deletions(-) diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index e8732b1..8cd0c19 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -4,6 +4,7 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; +import { spawnSync } from 'node:child_process'; import { ownerIsGone } from '../process/owner.js'; @@ -11,6 +12,23 @@ const LOCK_WAIT_MS = 2_000; const LOCK_RETRY_MS = 20; const WORK_LOCK_STALE_MS = 2 * 60 * 1000; const CLAIM_STALE_MS = 60 * 1000; +const ADMISSION_IDENTITY_CHECK_MS = 10_000; +const abandonedAdmissions = new Map(); +let ownBirth; + +function processBirth(pid) { + const windows = process.platform === 'win32'; + const systemRoot = process.env.SystemRoot ?? 'C:\\Windows'; + if (windows && !path.isAbsolute(systemRoot)) return null; + const executable = windows ? path.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') : 'ps'; + const args = windows + ? ['-NoProfile', '-NonInteractive', '-Command', `(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToUniversalTime().Ticks`] + : ['-p', String(pid), '-o', 'lstart=']; + const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; + const result = spawnSync(executable, args, { env, encoding: 'utf8', windowsHide: true, timeout: windows ? 2500 : 1000 }); + const started = result.status === 0 ? result.stdout.trim() : ''; + return started ? crypto.createHash('sha256').update(started).digest('hex').slice(0, 24) : null; +} export function readState(target, fallback) { try { @@ -194,13 +212,23 @@ function ownsLock(lockDir, token) { function reclaimAdmission(admission) { try { - if (!fs.lstatSync(admission).isDirectory()) return false; + const info = fs.lstatSync(admission); + if (!info.isDirectory()) return false; const owners = fs.readdirSync(admission); if (owners.length > 1) return false; if (owners.length === 1) { - const owner = /^owner-(\d+)-[a-f0-9]+$/.exec(owners[0]); - if (!owner || !ownerIsGone(Number(owner[1]))) return false; + const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}))?$/.exec(owners[0]); + if (!owner) return false; + const pid = Number(owner[1]); + if (!Number.isSafeInteger(pid) || pid <= 0) return false; + const abandonedHere = abandonedAdmissions.get(admission) === owners[0]; + if (!abandonedHere && !ownerIsGone(pid)) { + if (!owner[2] || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS) return false; + const actualBirth = pid === process.pid ? ownBirth : processBirth(pid); + if (!actualBirth || actualBirth === owner[2]) return false; + } fs.unlinkSync(path.join(admission, owners[0])); + if (abandonedHere) abandonedAdmissions.delete(admission); } // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot // remove it even if another reclaimer replaced the directory after our read. @@ -213,7 +241,9 @@ function reclaimAdmission(admission) { function admissionOf(lockDir) { const admission = `${lockDir}.admission`; - const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}`; + if (ownBirth === undefined) ownBirth = processBirth(process.pid); + const birth = ownBirth ? `-${ownBirth}` : ''; + const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}${birth}`; const staging = `${admission}.${owner}.claim`; fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); fs.mkdirSync(staging, { mode: 0o700 }); @@ -223,6 +253,7 @@ function admissionOf(lockDir) { for (let attempt = 0; attempt < 2; attempt += 1) { try { fs.renameSync(staging, admission); + abandonedAdmissions.delete(admission); return { admission, owner }; } catch { if (attempt > 0 || !reclaimAdmission(admission)) return null; @@ -248,24 +279,27 @@ function retryLockAccess(access) { function releaseAdmission({ admission, owner }) { try { retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); - try { - retryLockAccess(() => fs.rmdirSync(admission)); - } catch (error) { - if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error?.code)) throw error; - } } catch (cause) { + abandonedAdmissions.set(admission, owner); throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); } + // The guard is released when its unique owner is gone. An empty directory can be reclaimed. + try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} } // Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. -function admitted(lockDir, mutate) { +function admitted(lockDir, mutate, rollback = () => {}) { const admission = admissionOf(lockDir); if (!admission) return null; try { return mutate(); } finally { - releaseAdmission(admission); + try { + releaseAdmission(admission); + } catch (error) { + rollback(); + throw error; + } } } @@ -282,6 +316,7 @@ async function releaseWorkLock(lockDir, token) { } function tryLock(lockDir, now) { + let token; return admitted(lockDir, () => { if (isStale(lockDir, now)) { let held; @@ -296,7 +331,7 @@ function tryLock(lockDir, now) { } catch { return null; } - const token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; + token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; fs.writeFileSync(path.join(lockDir, 'owner'), token); return { refresh: () => { @@ -311,6 +346,8 @@ function tryLock(lockDir, now) { }, release: () => releaseWorkLock(lockDir, token), }; + }, () => { + if (token && ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); }); } diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index 1156254..855aa91 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -266,7 +266,10 @@ test('an admission cleanup failure is surfaced instead of silently orphaning a l fs.unlinkSync = unlink; } assert.equal(entered, false); + assert.equal(existsSync(lock), false, 'the failed acquire must not leave a work lock'); assert.match(readdirSync(admission)[0], new RegExp(`^owner-${process.pid}-`)); + assert.equal(await withLock(lock, async () => { entered = true; }), true); + assert.equal(entered, true); }); test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { @@ -294,6 +297,34 @@ test('a transient admission cleanup refusal retries and leaves later work unbloc assert.deepEqual(readdirSync(dir), []); }); +test('an old admission stays held by the same process, but a reused PID can be recovered', async (t) => { + const { lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + return unlink(target); + }; + try { + await assert.rejects(withLock(lock, async () => assert.fail()), { code: 'ELOCKED' }); + } finally { + fs.unlinkSync = unlink; + } + const [original] = readdirSync(admission); + const birth = original.split('-')[3]; + assert.match(birth ?? '', /^[a-f0-9]{24}$/); + const replaceOwner = (name) => { + unlinkSync(join(admission, readdirSync(admission)[0])); + writeFileSync(join(admission, name), ''); + utimesSync(admission, STALE, STALE); + }; + replaceOwner(`owner-${process.pid}-a1-${birth}`); + assert.equal(await withLock(lock, async () => assert.fail('same process still owns the guard')), false); + const differentBirth = (birth[0] === '0' ? '1' : '0') + birth.slice(1); + replaceOwner(`owner-${process.pid}-a2-${differentBirth}`); + assert.equal(await withLock(lock, async () => {}), true); +}); + test('busy and unrecognised admission guards stay intact and leave no staging directories', async (t) => { const { dir, lock } = scratch(t); const guard = holdAdmission(lock); From 68fab84ae688b9bfa7fa2ff4aff280c4b58cbaeb Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 10:37:30 +0700 Subject: [PATCH 05/13] test: accept buffer paths during state lock cleanup --- test/runtime/state-lock.test.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/runtime/state-lock.test.js b/test/runtime/state-lock.test.js index 9b29bf8..1abc7f3 100644 --- a/test/runtime/state-lock.test.js +++ b/test/runtime/state-lock.test.js @@ -23,7 +23,9 @@ test('a failed admission release after acquiring a state token lets the next upd const { target, lock } = scratch(t); const unlink = fs.unlinkSync; fs.unlinkSync = (file) => { - if (dirname(file) === `${lock}.admission`) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + if (typeof file === 'string' && dirname(file) === `${lock}.admission`) { + throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + } return unlink(file); }; try { From 48a7f8312b0c4028cedc5aa9ea395e9dfc7a89ae Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 15:54:18 +0700 Subject: [PATCH 06/13] fix: preserve work errors when lock cleanup also fails --- lib/runtime/store/file-lock.js | 14 +++++++++++--- test/runtime/file-lock.test.js | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index 8cd0c19..2d2e91c 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -239,7 +239,7 @@ function reclaimAdmission(admission) { } } -function admissionOf(lockDir) { +function enterAdmission(lockDir) { const admission = `${lockDir}.admission`; if (ownBirth === undefined) ownBirth = processBirth(process.pid); const birth = ownBirth ? `-${ownBirth}` : ''; @@ -289,7 +289,7 @@ function releaseAdmission({ admission, owner }) { // Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. function admitted(lockDir, mutate, rollback = () => {}) { - const admission = admissionOf(lockDir); + const admission = enterAdmission(lockDir); if (!admission) return null; try { return mutate(); @@ -355,10 +355,18 @@ function tryLock(lockDir, now) { export async function withLock(lockDir, work, now = Date.now()) { const lock = tryLock(lockDir, now); if (!lock) return false; + let workFailed = false; try { await work(lock); return true; + } catch (error) { + workFailed = true; + throw error; } finally { - await lock.release(); + try { + await lock.release(); + } catch (error) { + if (!workFailed) throw error; + } } } diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index 855aa91..5e84cbd 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -153,6 +153,23 @@ test('release refuses an unreadable owner explicitly without claiming cleanup', } }); +test('a release failure preserves the original work error', async (t) => { + const { lock } = scratch(t); + const read = fs.readFileSync; + const workError = new Error('work failed'); + try { + await assert.rejects(withLock(lock, async () => { + fs.readFileSync = (file, ...args) => { + if (String(file) === join(lock, 'owner')) throw Object.assign(new Error('unreadable owner'), { code: 'EIO' }); + return read(file, ...args); + }; + throw workError; + }), (error) => error === workError); + } finally { + fs.readFileSync = read; + } +}); + test('refresh leaves an unreadable live lock untouched without crashing its timer caller', async (t) => { const { dir, lock } = scratch(t); assert.equal(await withLock(lock, async (held) => { From ba5237f22aa299abc36349e5cc3bda836beac827 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 15:54:29 +0700 Subject: [PATCH 07/13] fix: unlink dead state owners under admission --- lib/runtime/store/file-lock.js | 47 +++++++++------------------------ test/runtime/state-lock.test.js | 29 +++++++------------- 2 files changed, 23 insertions(+), 53 deletions(-) diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index 9737111..a70aadc 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -10,6 +10,7 @@ import { ownerIsGone } from '../process/owner.js'; const LOCK_WAIT_MS = 2_000; const LOCK_RETRY_MS = 20; +const LOCK_ACCESS_ERRORS = new Set(['EPERM', 'EBUSY', 'EACCES']); const WORK_LOCK_STALE_MS = 2 * 60 * 1000; const CLAIM_STALE_MS = 60 * 1000; const ADMISSION_IDENTITY_CHECK_MS = 10_000; @@ -74,57 +75,35 @@ function contentOf(file) { const ownerPidOf = (token) => Number(token.split(':')[0]); -// Another process may reclaim the same dead lock and a third take a fresh one before this removal happens, so the -// lock is first moved aside, and only what was inspected as dead is deleted; anything else is put back. -function reclaim(lockPath, deadToken) { - const aside = `${lockPath}.${crypto.randomBytes(6).toString('hex')}.reclaimed`; - try { - fs.renameSync(lockPath, aside); - } catch { - return; - } - if (contentOf(aside) !== deadToken) { - try { - fs.linkSync(aside, lockPath); - } catch { - } - } - removeState(aside); -} - function acquire(lockPath, deadline) { const token = `${process.pid}:${crypto.randomBytes(8).toString('hex')}`; - for (;;) { - if (Date.now() >= deadline) return null; - // All contenders admit the whole attempt, including creating a new owner, so reclaim's temporary - // missing-path window never lets a third process take a fresh file lock that another reclaimer moves. - let attempt; + while (Date.now() < deadline) { let acquired = false; try { - attempt = admitted(lockPath, () => { + const result = admitted(lockPath, () => { try { fs.writeFileSync(lockPath, token, { mode: 0o600, flag: 'wx' }); acquired = true; - return { status: 'acquired', token }; + return token; } catch (error) { - if (error?.code !== 'EEXIST') return { status: 'failed', error }; + if (error?.code !== 'EEXIST') throw error; + // Admission serializes inspection and removal with every other acquisition. const held = contentOf(lockPath); - if (held !== null && ownerIsGone(ownerPidOf(held))) reclaim(lockPath, held); - return { status: 'busy' }; + if (held !== null && ownerIsGone(ownerPidOf(held))) fs.unlinkSync(lockPath); + return null; } }, () => { if (acquired && retryLockAccess(() => fs.readFileSync(lockPath, 'utf8')) === token) { retryLockAccess(() => fs.unlinkSync(lockPath)); } }); + if (result) return result; } catch (error) { - attempt = { status: 'failed', error }; + if (!LOCK_ACCESS_ERRORS.has(error?.code)) return null; } - if (attempt?.status === 'acquired') return attempt.token; - if (attempt?.status === 'failed' && !['EACCES', 'EPERM', 'EBUSY'].includes(attempt.error?.code)) return null; - if (Date.now() >= deadline) return null; - pause(LOCK_RETRY_MS); + if (Date.now() < deadline) pause(LOCK_RETRY_MS); } + return null; } function release(lockPath, token) { @@ -289,7 +268,7 @@ function retryLockAccess(access) { try { return access(); } catch (error) { - if (attempt === 2 || !['EPERM', 'EBUSY', 'EACCES'].includes(error?.code)) throw error; + if (attempt === 2 || !LOCK_ACCESS_ERRORS.has(error?.code)) throw error; pause(LOCK_RETRY_MS); } } diff --git a/test/runtime/state-lock.test.js b/test/runtime/state-lock.test.js index 1abc7f3..b519d17 100644 --- a/test/runtime/state-lock.test.js +++ b/test/runtime/state-lock.test.js @@ -120,14 +120,14 @@ test('a refused dead-owner reclaim obeys the same bounded wait without calling t import fs from 'node:fs'; import { updateState } from ${JSON.stringify(RUNTIME)}; const [target, lock] = process.argv.slice(1); - const rename = fs.renameSync; + const unlink = fs.unlinkSync; let reclaims = 0; - fs.renameSync = (source, destination) => { - if (source === lock && destination.endsWith('.reclaimed')) { + fs.unlinkSync = (file) => { + if (String(file) === lock) { reclaims += 1; throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); } - return rename(source, destination); + return unlink(file); }; let called = false; try { updateState(target, {}, () => { called = true; return {}; }); } @@ -165,7 +165,7 @@ test('three state updaters preserve every increment while recovering one dead fi import fs from 'node:fs'; import path from 'node:path'; import { updateState } from ${JSON.stringify(RUNTIME)}; - const [dir, target, lock, role, dead] = process.argv.slice(1); + const [dir, target, lock, role] = process.argv.slice(1); const marker = (name) => path.join(dir, name); const wait = (name) => { const deadline = Date.now() + 10000; @@ -186,9 +186,6 @@ test('three state updaters preserve every increment while recovering one dead fi paused = true; fs.writeFileSync(marker('A-read-dead'), ''); wait('A-resume'); - } else if (role === 'A' && String(file).startsWith(lock + '.') && String(file).endsWith('.reclaimed') && result !== dead) { - fs.writeFileSync(marker('A-moved-fresh'), ''); - wait('C-entered'); } return result; }; @@ -216,7 +213,7 @@ test('three state updaters preserve every increment while recovering one dead fi `; const children = []; const run = (role) => { - const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, target, lock, role, dead]); + const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, target, lock, role]); let stderr = ''; child.stderr.on('data', (data) => { stderr += data; }); children.push(new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr })))); @@ -241,17 +238,11 @@ test('three state updaters preserve every increment while recovering one dead fi await until(() => existsSync(marker('A-read-dead'))); writeFileSync(marker('B-start'), ''); await until(() => existsSync(marker('B-entered')) || existsSync(marker('B-guard-busy'))); - const unguarded = existsSync(marker('B-entered')); + assert.equal(existsSync(marker('B-entered')), false, 'the competing process waits for admission'); writeFileSync(marker('A-resume'), ''); - if (unguarded) { - await until(() => existsSync(marker('A-moved-fresh'))); - writeFileSync(marker('C-start'), ''); - await until(() => existsSync(marker('C-entered'))); - } else { - await until(() => ['A', 'B'].some((role) => existsSync(marker(role + '-entered')))); - writeFileSync(marker('C-start'), ''); - await until(() => existsSync(marker('C-attempted')) || existsSync(marker('C-guard-busy'))); - } + await until(() => ['A', 'B'].some((role) => existsSync(marker(role + '-entered')))); + writeFileSync(marker('C-start'), ''); + await until(() => existsSync(marker('C-attempted')) || existsSync(marker('C-guard-busy'))); writeFileSync(marker('release'), ''); const results = await Promise.all(children); assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); From 9733bd5943d0c853dcb2b6591d7d71149d05b50c Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 16:20:22 +0700 Subject: [PATCH 08/13] fix: preserve unreadable shared state and surface hook failures --- lib/agent/driver.js | 3 +- lib/channels/hook/runner.js | 11 +++- lib/runtime/store/file-lock.js | 38 ++++++++--- test/channels/hook.test.js | 22 ++++++- test/runtime/file-stores.test.js | 107 +++++++++++++++++++++++++++++-- 5 files changed, 163 insertions(+), 18 deletions(-) diff --git a/lib/agent/driver.js b/lib/agent/driver.js index 8ebe3e8..54e6e16 100644 --- a/lib/agent/driver.js +++ b/lib/agent/driver.js @@ -171,7 +171,8 @@ export function createDriver({ profile, ports, transcripts, env = process.env, l const at = { sessionId, projectDir: profile.payload.projectDir(payload, context) }; const waitLeft = (budgetOf = 'prompt') => waitBudgetOf(profile.budgets, budgetOf, { lifetime, startedAt }); return await HANDLERS[event]({ profile, ports, hostEvent, payload, context, at, waitLeft, startedAt, options }); - } catch { + } catch (error) { + options.onError?.(error); return profile.reply.empty(hostEvent, payload); } }, diff --git a/lib/channels/hook/runner.js b/lib/channels/hook/runner.js index a591080..565fe86 100644 --- a/lib/channels/hook/runner.js +++ b/lib/channels/hook/runner.js @@ -13,7 +13,7 @@ import { readPayload } from './stdin.js'; * One command-hook invocation: the host's payload on stdin, the agent's answer as JSON on stdout, exit 0. Inside * this plugin's own headless distillation the hook answers the agent's empty reply and does nothing else. */ -export async function runHookProcess(agent, hostEvent, { stdin = process.stdin, stdout = process.stdout, env = process.env, exit = process.exit, profileOf = profileNamed } = {}) { +export async function runHookProcess(agent, hostEvent, { stdin = process.stdin, stdout = process.stdout, stderr = process.stderr, env = process.env, exit = process.exit, profileOf = profileNamed } = {}) { const startedAt = Date.now(); const write = (answer) => stdout.write(JSON.stringify(answer ?? {}), () => exit(0)); const profile = await profileOf(agent).catch(() => null); @@ -22,5 +22,12 @@ export async function runHookProcess(agent, hostEvent, { stdin = process.stdin, if (isHeadlessRun(env)) return write(profile.reply.empty(hostEvent)); const { ports } = environmentFor({ profile, env, lifetime: EPHEMERAL, runBackground: runBackgroundTasks }); const driver = createDriver({ profile, ports, transcripts, env, lifetime: EPHEMERAL, startedAt }); - return write(await driver.handle(hostEvent, payload)); + const answer = await driver.handle(hostEvent, payload, { + onError(error) { + if (['EINVALIDSTATE', 'ESTATEIO', 'ELOCKED'].includes(error?.code)) { + stderr.write(`Evolver hook skipped a state operation (${error.code}). Check the plugin state directory and retry.\n`); + } + }, + }); + return write(answer); } diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index f730098..d591757 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -15,9 +15,13 @@ const CLAIM_STALE_MS = 60 * 1000; export function readState(target, fallback) { try { const parsed = JSON.parse(fs.readFileSync(target, 'utf8')); - return parsed && typeof parsed === 'object' && Array.isArray(parsed) === Array.isArray(fallback) ? parsed : fallback; - } catch { - return fallback; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed) !== Array.isArray(fallback)) throw Object.assign(new Error(`Invalid state shape in ${target}; repair it before retrying.`), { code: 'EINVALIDSTATE' }); + return parsed; + } catch (error) { + if (error?.code === 'ENOENT') return fallback; + if (error instanceof SyntaxError) throw Object.assign(new Error(`Invalid JSON in ${target}; repair it before retrying.`), { code: 'EINVALIDSTATE', cause: error }); + if (error?.code === 'EINVALIDSTATE') throw error; + throw Object.assign(new Error(`Could not read state in ${target}; check access and retry.`), { code: 'ESTATEIO', cause: error }); } } @@ -25,12 +29,19 @@ export function writeState(target, value) { fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 }); const temporary = `${target}.${process.pid}.${crypto.randomBytes(6).toString('hex')}.tmp`; fs.writeFileSync(temporary, JSON.stringify(value), { mode: 0o600, flag: 'wx' }); + const deadline = Date.now() + LOCK_WAIT_MS; try { - fs.renameSync(temporary, target); - } catch (error) { - if (!['EEXIST', 'EPERM'].includes(error?.code)) throw error; - fs.rmSync(target, { force: true }); - fs.renameSync(temporary, target); + for (;;) { + try { + fs.renameSync(temporary, target); + return; + } catch (error) { + if (!['EEXIST', 'EACCES', 'EPERM', 'EBUSY'].includes(error?.code) || Date.now() >= deadline) throw error; + pause(LOCK_RETRY_MS); + } + } + } finally { + removeState(temporary); } } @@ -92,7 +103,16 @@ function acquire(lockPath, deadline) { } function release(lockPath, token) { - if (contentOf(lockPath) === token) removeState(lockPath); + const deadline = Date.now() + LOCK_WAIT_MS; + for (;;) { + try { + if (fs.readFileSync(lockPath, 'utf8') === token) fs.rmSync(lockPath, { force: true }); + return; + } catch (error) { + if (!['EACCES', 'EPERM', 'EBUSY'].includes(error?.code) || Date.now() >= deadline) return; + pause(LOCK_RETRY_MS); + } + } } /** diff --git a/test/channels/hook.test.js b/test/channels/hook.test.js index ac20184..4046370 100644 --- a/test/channels/hook.test.js +++ b/test/channels/hook.test.js @@ -1,4 +1,7 @@ import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; import { Readable } from 'node:stream'; import { after, before, test } from 'node:test'; @@ -19,12 +22,12 @@ let proxy; before(async () => { proxy = await fakeProxy(); }); after(() => proxy.close()); -function invoke(hostEvent, payload, { env = proxy.env, raw } = {}) { +function invoke(hostEvent, payload, { env = proxy.env, raw, stderr } = {}) { return new Promise((resolve) => { let written = ''; const stdout = { write: (text, done) => { written += text; done(); } }; const stdin = Readable.from([raw ?? JSON.stringify(payload)]); - runHookProcess('testhook', hostEvent, { stdin, stdout, env, exit: () => resolve(JSON.parse(written)), profileOf: async () => profile }); + runHookProcess('testhook', hostEvent, { stdin, stdout, stderr, env, exit: () => resolve(JSON.parse(written)), profileOf: async () => profile }); }); } @@ -44,3 +47,18 @@ test('inside the plugin\'s own headless run, a hook answers the empty reply and test('a payload that is not JSON still gets an answer', async () => { assert.deepEqual(await invoke('UserPromptSubmit', null, { raw: '{not json' }), { empty: 'UserPromptSubmit' }); }); + +test('corrupt hook state stays intact and emits a diagnostic alongside the neutral reply', async () => { + const dir = profile.stateDir(proxy.env); + const key = createHash('sha256').update('s-corrupt').digest('hex').slice(0, 16); + const file = join(dir, `testhook-session-${key}.json`); + mkdirSync(dir, { recursive: true }); + writeFileSync(file, '{"private text":'); + let diagnostic = ''; + const stderr = { write: (text) => { diagnostic += text; } }; + + assert.deepEqual(await invoke('UserPromptSubmit', { session_id: 's-corrupt', cwd: '/proj', prompt: PROMPT }, { stderr }), { empty: 'UserPromptSubmit' }); + assert.match(diagnostic, /EINVALIDSTATE/); + assert.doesNotMatch(diagnostic, /private text/); + assert.equal(readFileSync(file, 'utf8'), '{"private text":'); +}); diff --git a/test/runtime/file-stores.test.js b/test/runtime/file-stores.test.js index 7f52193..b3e9219 100644 --- a/test/runtime/file-stores.test.js +++ b/test/runtime/file-stores.test.js @@ -1,11 +1,12 @@ import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; -import { mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import fs from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { after, test } from 'node:test'; -import { readState } from '../../lib/runtime/store/file-lock.js'; +import { readState, updateState } from '../../lib/runtime/store/file-lock.js'; import { fileStoresOf } from '../../lib/runtime/store/file-stores.js'; const dirs = []; @@ -42,6 +43,79 @@ test('the outbox takes queued reports and keeps what each delivery attempt settl assert.equal(seen[0].sentAt, 5); }); +test('an unreadable or invalid draft table is preserved and never passed to an update as empty', async t => { + for (const failure of ['EBUSY', 'EACCES', 'invalid-json', 'wrong-shape']) { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + const original = failure === 'invalid-json' ? '[unfinished' : failure === 'wrong-shape' ? '{}' : '[{"title":"keep this draft"}]'; + writeFileSync(target, original); + const originalRead = fs.readFileSync; + t.mock.method(fs, 'readFileSync', (file, ...args) => { + if (file === target && ['EBUSY', 'EACCES'].includes(failure)) throw Object.assign(new Error('fixture read denial'), { code: failure }); + return originalRead(file, ...args); + }); + let mutated = false; + await assert.rejects(cursor.drafts.update(drafts => { mutated = true; return [...drafts, { title: 'new draft' }]; }), failure); + assert.equal(mutated, false, failure); + t.mock.restoreAll(); + assert.equal(fs.readFileSync(target, 'utf8'), original, failure); + assert.equal(existsSync(`${target}.lock`), false, failure); + } + const { stores: cursor } = stores(); + assert.deepEqual(await cursor.drafts.read(), [], 'an absent table is still a fresh store'); + await cursor.drafts.update(() => [{ title: 'first draft' }]); + assert.deepEqual(await cursor.drafts.read(), [{ title: 'first draft' }]); +}); + +test('a corrupt draft file is preserved for every packaged host', async () => { + for (const agent of readdirSync(new URL('../../hosts/', import.meta.url))) { + if (!existsSync(new URL(`../../hosts/${agent}/profile.js`, import.meta.url))) continue; + const { dir, stores: host } = stores(agent); + const target = join(dir, `${agent}-distill-drafts.json`); + writeFileSync(target, '[unfinished'); + await assert.rejects(host.drafts.update(() => assert.fail('mutator must not run')), { code: 'EINVALIDSTATE' }); + assert.equal(fs.readFileSync(target, 'utf8'), '[unfinished', agent); + } +}); + +test('a failed draft replacement preserves the previous file, releases its lock and removes its temporary file', async t => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + await cursor.drafts.update(() => [{ title: 'keep this draft' }]); + const before = fs.readFileSync(target, 'utf8'); + const rename = fs.renameSync; + t.mock.method(fs, 'renameSync', (from, to) => { + if (to === target) throw Object.assign(new Error('fixture replacement denial'), { code: 'EPERM' }); + return rename(from, to); + }); + await assert.rejects(cursor.drafts.update(drafts => [...drafts, { title: 'new draft' }]), { code: 'EPERM' }); + t.mock.restoreAll(); + assert.equal(fs.readFileSync(target, 'utf8'), before); + assert.deepEqual(readdirSync(dir), ['cursor-distill-drafts.json']); + await cursor.drafts.update(drafts => [...drafts, { title: 'retry succeeded' }]); + assert.deepEqual(await cursor.drafts.read(), [{ title: 'keep this draft' }, { title: 'retry succeeded' }]); +}); + +test('a transient replacement denial retries while readers can still see the previous drafts', async t => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + const previous = [{ title: 'keep this draft' }]; + await cursor.drafts.update(() => previous); + const rename = fs.renameSync; + let attempts = 0; + t.mock.method(fs, 'renameSync', (from, to) => { + if (to === target) { + assert.deepEqual(JSON.parse(fs.readFileSync(target, 'utf8')), previous); + if (++attempts === 1) throw Object.assign(new Error('fixture sharing denial'), { code: 'EBUSY' }); + } + return rename(from, to); + }); + await cursor.drafts.update(drafts => [...drafts, { title: 'new draft' }]); + assert.equal(attempts, 2); + assert.deepEqual(await cursor.drafts.read(), [...previous, { title: 'new draft' }]); + assert.deepEqual(readdirSync(dir), ['cursor-distill-drafts.json']); +}); + test('a report appended after an interrupted queue write survives draining and commit', async () => { const { dir, stores: cursor } = stores(); const earlier = { report: { task_id: 'earlier', asset_id: 'a' }, sessionId: 's1' }; @@ -74,11 +148,36 @@ test('parallel processes updating one state file through the lock lose no update const lock = new URL('../../lib/runtime/store/file-lock.js', import.meta.url).href; const script = `import { updateState } from ${JSON.stringify(lock)}; for (let i = 0; i < 25; i += 1) updateState(${JSON.stringify(target)}, {}, (state) => ({ count: (state.count ?? 0) + 1 }));`; - const run = () => new Promise((resolve) => spawn(process.execPath, ['--input-type=module', '-e', script]).on('close', resolve)); - await Promise.all([run(), run(), run(), run()]); + const run = () => new Promise((resolve) => { + const child = spawn(process.execPath, ['--input-type=module', '-e', script]); + let stderr = ''; + child.stderr.on('data', chunk => { stderr += chunk; }); + child.on('close', code => resolve({ code, stderr })); + }); + const results = await Promise.all([run(), run(), run(), run()]); + assert.deepEqual(results.map(result => result.code), [0, 0, 0, 0], results.map(result => result.stderr).join('\n')); assert.equal(readState(target, {}).count, 100); }); +test('transient sharing denial while releasing a state lock leaves no orphaned lock', t => { + for (const operation of ['readFileSync', 'rmSync']) { + const { dir } = stores(); + const target = join(dir, 'counter.json'), lock = `${target}.lock`; + const original = fs[operation]; + let denied = false; + t.mock.method(fs, operation, (file, ...args) => { + if (file === lock && !denied) { denied = true; throw Object.assign(new Error('sharing violation'), { code: 'EPERM' }); } + return original(file, ...args); + }); + updateState(target, {}, state => ({ count: (state.count ?? 0) + 1 })); + assert.equal(denied, true); + assert.equal(existsSync(lock), false, operation); + updateState(target, {}, state => ({ count: state.count + 1 })); + assert.equal(readState(target, {}).count, 2); + t.mock.restoreAll(); + } +}); + test('a session file outlives the outbox: a session resumed within a month still counts its turns on', async () => { const dir = mkdtempSync(join(tmpdir(), 'evolver-prune-')); dirs.push(dir); From 25a7227eb1af3fc31cb024753b8c5ae12c6c1478 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 16:20:42 +0700 Subject: [PATCH 09/13] fix: flush the hook diagnostic before exiting --- lib/channels/hook/runner.js | 4 +++- test/channels/hook.test.js | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/channels/hook/runner.js b/lib/channels/hook/runner.js index 565fe86..fca96d4 100644 --- a/lib/channels/hook/runner.js +++ b/lib/channels/hook/runner.js @@ -22,12 +22,14 @@ export async function runHookProcess(agent, hostEvent, { stdin = process.stdin, if (isHeadlessRun(env)) return write(profile.reply.empty(hostEvent)); const { ports } = environmentFor({ profile, env, lifetime: EPHEMERAL, runBackground: runBackgroundTasks }); const driver = createDriver({ profile, ports, transcripts, env, lifetime: EPHEMERAL, startedAt }); + let diagnostic; const answer = await driver.handle(hostEvent, payload, { onError(error) { if (['EINVALIDSTATE', 'ESTATEIO', 'ELOCKED'].includes(error?.code)) { - stderr.write(`Evolver hook skipped a state operation (${error.code}). Check the plugin state directory and retry.\n`); + diagnostic = `Evolver hook skipped a state operation (${error.code}). Check the plugin state directory and retry.\n`; } }, }); + if (diagnostic) return stderr.write(diagnostic, () => write(answer)); return write(answer); } diff --git a/test/channels/hook.test.js b/test/channels/hook.test.js index 4046370..29cf3b2 100644 --- a/test/channels/hook.test.js +++ b/test/channels/hook.test.js @@ -55,7 +55,7 @@ test('corrupt hook state stays intact and emits a diagnostic alongside the neutr mkdirSync(dir, { recursive: true }); writeFileSync(file, '{"private text":'); let diagnostic = ''; - const stderr = { write: (text) => { diagnostic += text; } }; + const stderr = { write: (text, done) => { diagnostic += text; done(); } }; assert.deepEqual(await invoke('UserPromptSubmit', { session_id: 's-corrupt', cwd: '/proj', prompt: PROMPT }, { stderr }), { empty: 'UserPromptSubmit' }); assert.match(diagnostic, /EINVALIDSTATE/); From d1fc44ffe2c5a5ca0ab3d67aa5e064a91cb8a6c5 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 18:03:57 +0700 Subject: [PATCH 10/13] Recover admission without spawning on uncontended locks --- lib/runtime/process/owner.js | 22 +++++ lib/runtime/store/admission.js | 123 ++++++++++++++++++++++++++++ lib/runtime/store/file-lock.js | 140 ++++---------------------------- test/helpers/lock-contenders.js | 39 +++++++++ test/runtime/file-lock.test.js | 112 +++++++++++++++---------- 5 files changed, 270 insertions(+), 166 deletions(-) create mode 100644 lib/runtime/store/admission.js create mode 100644 test/helpers/lock-contenders.js diff --git a/lib/runtime/process/owner.js b/lib/runtime/process/owner.js index 8d39208..c9ea00e 100644 --- a/lib/runtime/process/owner.js +++ b/lib/runtime/process/owner.js @@ -8,6 +8,28 @@ import path from 'node:path'; const SHELLS = new Set(['sh', 'bash', 'zsh', 'dash', 'ksh', 'fish', 'cmd.exe', 'powershell', 'pwsh']); const PS_TIMEOUT_MS = 1000; const MAX_ANCESTORS = 16; +const ownStartTime = Math.round(Date.now() - process.uptime() * 1000); + +/** A bounded process start-time probe for admission recovery. The hot path needs no subprocess. */ +export function processStartTime(pid, timeout = PS_TIMEOUT_MS) { + if (pid === process.pid) return ownStartTime; + if (!Number.isSafeInteger(pid) || pid <= 0 || timeout <= 0) return null; + const windows = process.platform === 'win32'; + const systemRoot = process.env.SystemRoot ?? 'C:\\Windows'; + if (windows && !path.isAbsolute(systemRoot)) return null; + const executable = windows ? path.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') : '/bin/ps'; + const args = windows + ? ['-NoProfile', '-NonInteractive', '-Command', `([DateTimeOffset] (Get-Process -Id ${pid} -ErrorAction Stop).StartTime).ToUnixTimeMilliseconds()`] + : ['-p', String(pid), '-o', 'lstart=']; + const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; + const result = spawnSync(executable, args, { + env, encoding: 'utf8', windowsHide: true, timeout: Math.min(timeout, PS_TIMEOUT_MS), killSignal: 'SIGKILL', + }); + const started = result.status === 0 ? result.stdout.trim() : ''; + if (!started) return null; + const time = windows ? Number(started) : Date.parse(started + ' GMT'); + return Number.isSafeInteger(time) && time > 0 ? time : null; +} /** * Whether the process that wrote a state entry has exited. Signal 0 only probes: ESRCH means no such process; diff --git a/lib/runtime/store/admission.js b/lib/runtime/store/admission.js new file mode 100644 index 0000000..61f0e17 --- /dev/null +++ b/lib/runtime/store/admission.js @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 EvoMap + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { ownerIsGone, processStartTime } from '../process/owner.js'; + +export const LOCK_WAIT_MS = 2_000; +export const LOCK_RETRY_MS = 20; +export const LOCK_ACCESS_ERRORS = new Set(['EPERM', 'EBUSY', 'EACCES']); +const ADMISSION_IDENTITY_CHECK_MS = 10_000; +const START_TIME_TOLERANCE_MS = 2_000; +const abandonedAdmissions = new Map(); + +export function pause(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function reclaimAdmission(admission, deadline, starts) { + try { + const info = fs.lstatSync(admission); + if (!info.isDirectory()) return false; + const owners = fs.readdirSync(admission); + if (owners.length > 1) return false; + if (owners.length === 1) { + const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}|[0-9]{13}))?$/.exec(owners[0]); + if (!owner) return false; + const pid = Number(owner[1]); + if (!Number.isSafeInteger(pid) || pid <= 0) return false; + const abandonedHere = abandonedAdmissions.get(admission) === owners[0]; + if (!abandonedHere && !ownerIsGone(pid)) { + const birth = Number(owner[2]); + const remaining = deadline - Date.now(); + if (owner[2]?.length !== 13 || !Number.isSafeInteger(birth) || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS || remaining <= 0) return false; + if (!starts.has(pid)) starts.set(pid, processStartTime(pid, remaining)); + const actualBirth = starts.get(pid); + // Only a clearly newer process proves reuse. ps timestamps round to whole seconds. + if (!actualBirth || actualBirth <= birth + START_TIME_TOLERANCE_MS) return false; + } + fs.unlinkSync(path.join(admission, owners[0])); + if (abandonedHere) abandonedAdmissions.delete(admission); + } + // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot + // remove it even if another reclaimer replaced the directory after our read. + fs.rmdirSync(admission); + return true; + } catch { + return false; + } +} + +function enterAdmission(lockDir, deadline, starts) { + const admission = `${lockDir}.admission`; + const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}-${processStartTime(process.pid)}`; + const staging = `${admission}.${owner}.claim`; + fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); + fs.mkdirSync(staging, { mode: 0o700 }); + let published = false; + try { + fs.writeFileSync(path.join(staging, owner), '', { mode: 0o600, flag: 'wx' }); + // Publishing a prepopulated directory never exposes an empty fresh guard to a stale reclaimer. + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + fs.renameSync(staging, admission); + published = true; + abandonedAdmissions.delete(admission); + return { admission, owner }; + } catch { + if (attempt > 0 || !reclaimAdmission(admission, deadline, starts)) return null; + } + } + return null; + } finally { + if (!published) fs.rmSync(staging, { recursive: true, force: true }); + } +} + +export function retryLockAccess(access) { + for (let attempt = 0; ; attempt += 1) { + try { + return access(); + } catch (error) { + if (attempt === 2 || !LOCK_ACCESS_ERRORS.has(error?.code)) throw error; + pause(LOCK_RETRY_MS); + } + } +} + +function releaseAdmission({ admission, owner }) { + try { + retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); + } catch (cause) { + abandonedAdmissions.set(admission, owner); + throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + } + // The guard is released when its unique owner is gone. An empty directory can be reclaimed. + try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} +} + +// Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. +export function admitted(lockDir, mutate, rollback, deadline = Date.now() + LOCK_WAIT_MS, starts = new Map()) { + const admission = enterAdmission(lockDir, deadline, starts); + if (!admission) return null; + let result; + try { + result = mutate(); + } catch (error) { + try { releaseAdmission(admission); } catch {} + throw error; + } + try { + releaseAdmission(admission); + } catch (error) { + // After an irreversible release, report its result. The stranded guard is tracked for recovery. + if (rollback) { + try { rollback(); } catch (rollbackError) { error.rollbackError = rollbackError; } + throw error; + } + } + return result; +} diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index 2d2e91c..f28610a 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -4,31 +4,14 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import { spawnSync } from 'node:child_process'; import { ownerIsGone } from '../process/owner.js'; -const LOCK_WAIT_MS = 2_000; -const LOCK_RETRY_MS = 20; +import { admitted, LOCK_WAIT_MS, LOCK_RETRY_MS, pause, retryLockAccess } from './admission.js'; + const WORK_LOCK_STALE_MS = 2 * 60 * 1000; const CLAIM_STALE_MS = 60 * 1000; -const ADMISSION_IDENTITY_CHECK_MS = 10_000; -const abandonedAdmissions = new Map(); -let ownBirth; - -function processBirth(pid) { - const windows = process.platform === 'win32'; - const systemRoot = process.env.SystemRoot ?? 'C:\\Windows'; - if (windows && !path.isAbsolute(systemRoot)) return null; - const executable = windows ? path.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') : 'ps'; - const args = windows - ? ['-NoProfile', '-NonInteractive', '-Command', `(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToUniversalTime().Ticks`] - : ['-p', String(pid), '-o', 'lstart=']; - const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; - const result = spawnSync(executable, args, { env, encoding: 'utf8', windowsHide: true, timeout: windows ? 2500 : 1000 }); - const started = result.status === 0 ? result.stdout.trim() : ''; - return started ? crypto.createHash('sha256').update(started).digest('hex').slice(0, 24) : null; -} +const abandonedWork = new Map(); export function readState(target, fallback) { try { @@ -59,11 +42,6 @@ export function removeState(target) { } } -// Atomics.wait is the only sleep that holds a synchronous read-modify-write together. -function pause(ms) { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); -} - function contentOf(file) { try { return fs.readFileSync(file, 'utf8'); @@ -210,106 +188,14 @@ function ownsLock(lockDir, token) { return workOwnerOf(lockDir) === token; } -function reclaimAdmission(admission) { - try { - const info = fs.lstatSync(admission); - if (!info.isDirectory()) return false; - const owners = fs.readdirSync(admission); - if (owners.length > 1) return false; - if (owners.length === 1) { - const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}))?$/.exec(owners[0]); - if (!owner) return false; - const pid = Number(owner[1]); - if (!Number.isSafeInteger(pid) || pid <= 0) return false; - const abandonedHere = abandonedAdmissions.get(admission) === owners[0]; - if (!abandonedHere && !ownerIsGone(pid)) { - if (!owner[2] || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS) return false; - const actualBirth = pid === process.pid ? ownBirth : processBirth(pid); - if (!actualBirth || actualBirth === owner[2]) return false; - } - fs.unlinkSync(path.join(admission, owners[0])); - if (abandonedHere) abandonedAdmissions.delete(admission); - } - // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot - // remove it even if another reclaimer replaced the directory after our read. - fs.rmdirSync(admission); - return true; - } catch { - return false; - } -} - -function enterAdmission(lockDir) { - const admission = `${lockDir}.admission`; - if (ownBirth === undefined) ownBirth = processBirth(process.pid); - const birth = ownBirth ? `-${ownBirth}` : ''; - const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}${birth}`; - const staging = `${admission}.${owner}.claim`; - fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); - fs.mkdirSync(staging, { mode: 0o700 }); - try { - fs.writeFileSync(path.join(staging, owner), '', { mode: 0o600, flag: 'wx' }); - // Publishing a prepopulated directory never exposes an empty fresh guard to a stale reclaimer. - for (let attempt = 0; attempt < 2; attempt += 1) { - try { - fs.renameSync(staging, admission); - abandonedAdmissions.delete(admission); - return { admission, owner }; - } catch { - if (attempt > 0 || !reclaimAdmission(admission)) return null; - } - } - return null; - } finally { - fs.rmSync(staging, { recursive: true, force: true }); - } -} - -function retryLockAccess(access) { - for (let attempt = 0; ; attempt += 1) { - try { - return access(); - } catch (error) { - if (attempt === 2 || !['EPERM', 'EBUSY', 'EACCES'].includes(error?.code)) throw error; - pause(LOCK_RETRY_MS); - } - } -} - -function releaseAdmission({ admission, owner }) { - try { - retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); - } catch (cause) { - abandonedAdmissions.set(admission, owner); - throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); - } - // The guard is released when its unique owner is gone. An empty directory can be reclaimed. - try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} -} - -// Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. -function admitted(lockDir, mutate, rollback = () => {}) { - const admission = enterAdmission(lockDir); - if (!admission) return null; - try { - return mutate(); - } finally { - try { - releaseAdmission(admission); - } catch (error) { - rollback(); - throw error; - } - } -} - async function releaseWorkLock(lockDir, token) { const deadline = Date.now() + LOCK_WAIT_MS; + const starts = new Map(); for (;;) { if (admitted(lockDir, () => { if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); return true; - })) return; + }, null, deadline, starts)) return; if (Date.now() >= deadline) throw Object.assign(new Error(`Could not release lock ${lockDir}`), { code: 'ELOCKED' }); await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS)); } @@ -318,13 +204,15 @@ async function releaseWorkLock(lockDir, token) { function tryLock(lockDir, now) { let token; return admitted(lockDir, () => { - if (isStale(lockDir, now)) { + const stale = isStale(lockDir, now); + if (abandonedWork.has(lockDir) || stale) { let held; try { held = workOwnerOf(lockDir); } catch { return null; } // Old versions wrote random tokens without a pid. Keep their lease recovery, but a known live owner // cannot lose its lock just because a slow or paused worker missed a refresh. A reused pid stays busy. - if (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))) return null; + if (held !== abandonedWork.get(lockDir) && (!stale || (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))))) return null; fs.rmSync(lockDir, { recursive: true, force: true }); + abandonedWork.delete(lockDir); } try { fs.mkdirSync(lockDir); @@ -344,10 +232,16 @@ function tryLock(lockDir, now) { return null; } }, - release: () => releaseWorkLock(lockDir, token), + release: async () => { + try { await releaseWorkLock(lockDir, token); } + catch (error) { abandonedWork.set(lockDir, token); throw error; } + }, }; }, () => { - if (token && ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); + if (!token) return; + abandonedWork.set(lockDir, token); + if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); + abandonedWork.delete(lockDir); }); } diff --git a/test/helpers/lock-contenders.js b/test/helpers/lock-contenders.js new file mode 100644 index 0000000..1e412a7 --- /dev/null +++ b/test/helpers/lock-contenders.js @@ -0,0 +1,39 @@ +import { spawn } from 'node:child_process'; +import { rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export async function until(check) { + const deadline = Date.now() + 10_000; + while (!check()) { + if (Date.now() >= deadline) throw new Error('timed out waiting for a lock contender'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +export function lockContenders(t, dir, script, args, cleanupMarkers) { + const children = []; + const results = () => Promise.all(children); + t.after(async () => { + try { + for (const name of cleanupMarkers) writeFileSync(join(dir, name), ''); + } finally { + await results(); + rmSync(dir, { recursive: true, force: true }); + } + }); + return { + run(role) { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, ...args, role], { windowsHide: true }); + let stderr = ''; + const watchdog = setTimeout(() => child.kill('SIGKILL'), 15_000); + child.stderr.on('data', (data) => { stderr += data; }); + child.on('error', (error) => { stderr += String(error); }); + children.push(new Promise((resolve) => child.on('close', (code) => { + clearTimeout(watchdog); + resolve({ code, stderr }); + }))); + return child; + }, + results, + }; +} diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index 5e84cbd..d6b5261 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; -import { spawn, spawnSync } from 'node:child_process'; +import childProcess, { spawnSync } from 'node:child_process'; +import { syncBuiltinESMExports } from 'node:module'; import fs from 'node:fs'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, rmdirSync, statSync, unlinkSync, utimesSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -7,6 +8,7 @@ import { dirname, join } from 'node:path'; import { test } from 'node:test'; import { withLock } from '../../lib/runtime/store/file-lock.js'; +import { lockContenders, until } from '../helpers/lock-contenders.js'; const RUNTIME = new URL('../../lib/runtime/store/file-lock.js', import.meta.url).href; const STALE = new Date(0); @@ -22,14 +24,6 @@ function scratch(t, token = null) { return { dir, lock }; } -async function until(check) { - const deadline = Date.now() + 10_000; - while (!check()) { - if (Date.now() > deadline) throw new Error('timed out waiting for the lock contender'); - await new Promise((resolve) => setTimeout(resolve, 10)); - } -} - function holdAdmission(lock, pid = process.pid) { const admission = `${lock}.admission`; const owner = join(admission, `owner-${pid}-deadbeef`); @@ -38,6 +32,21 @@ function holdAdmission(lock, pid = process.pid) { return { admission, owner, release: () => { unlinkSync(owner); rmdirSync(admission); } }; } +test('taking an uncontended lock does not launch a process lookup', async (t) => { + const { lock } = scratch(t); + let launches = 0; + const spawn = childProcess.spawnSync; + childProcess.spawnSync = (...args) => { launches += 1; return spawn(...args); }; + syncBuiltinESMExports(); + try { + assert.equal(await withLock(lock, () => {}), true); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(launches, 0); +}); + test('stale legacy and dead-owner directories recover, but a fresh legacy lock stays held', async (t) => { const { lock } = scratch(t, 'legacy token'); let entered = 0; @@ -150,9 +159,35 @@ test('release refuses an unreadable owner explicitly without claiming cleanup', fs.readFileSync = read; } assert.equal(readFileSync(owner, 'utf8'), token, code); + assert.equal(await withLock(lock, () => {}), true, 'known abandoned work is recoverable after access returns'); } }); +test('a stale foreign admission is probed once within the release deadline', async (t) => { + const { lock } = scratch(t); + const spawn = childProcess.spawnSync; + const timeouts = []; + try { + await assert.rejects(withLock(lock, () => { + const guard = holdAdmission(lock, process.ppid); + unlinkSync(guard.owner); + writeFileSync(join(guard.admission, `owner-${process.ppid}-a1-${Date.now() - 30_000}`), ''); + utimesSync(guard.admission, STALE, STALE); + childProcess.spawnSync = (_command, _args, { timeout }) => { + timeouts.push(timeout); + return { status: null, stdout: '' }; + }; + syncBuiltinESMExports(); + }), { code: 'ELOCKED' }); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(timeouts.length, 1); + assert.ok(timeouts[0] > 0 && timeouts[0] <= 1_000); + assert.equal(existsSync(lock), true, 'an inconclusive identity probe does not reclaim a live guard'); +}); + test('a release failure preserves the original work error', async (t) => { const { lock } = scratch(t); const read = fs.readFileSync; @@ -289,6 +324,23 @@ test('an admission cleanup failure is surfaced instead of silently orphaning a l assert.equal(entered, true); }); +test('successful work is not reported as failed after its lock was removed', async (t) => { + const { lock } = scratch(t); + const unlink = fs.unlinkSync; + try { + assert.equal(await withLock(lock, () => { + fs.unlinkSync = (target) => { + if (dirname(target) === `${lock}.admission`) throw Object.assign(new Error('fixture cleanup refusal'), { code: 'EPERM' }); + return unlink(target); + }; + }), true); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(existsSync(lock), false); + assert.equal(await withLock(lock, () => {}), true, 'the stranded admission can be reclaimed'); +}); + test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { const { dir, lock } = scratch(t); const unlink = fs.unlinkSync; @@ -329,7 +381,7 @@ test('an old admission stays held by the same process, but a reused PID can be r } const [original] = readdirSync(admission); const birth = original.split('-')[3]; - assert.match(birth ?? '', /^[a-f0-9]{24}$/); + assert.match(birth ?? '', /^\d{13}$/); const replaceOwner = (name) => { unlinkSync(join(admission, readdirSync(admission)[0])); writeFileSync(join(admission, name), ''); @@ -337,7 +389,7 @@ test('an old admission stays held by the same process, but a reused PID can be r }; replaceOwner(`owner-${process.pid}-a1-${birth}`); assert.equal(await withLock(lock, async () => assert.fail('same process still owns the guard')), false); - const differentBirth = (birth[0] === '0' ? '1' : '0') + birth.slice(1); + const differentBirth = Number(birth) - 10_000; replaceOwner(`owner-${process.pid}-a2-${differentBirth}`); assert.equal(await withLock(lock, async () => {}), true); }); @@ -399,20 +451,7 @@ test('two contenders cannot replace the fresh lock while another reclaims a stal }); fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); `; - const children = []; - const run = (role) => { - const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, lock, role]); - let stderr = ''; - child.stderr.on('data', (data) => { stderr += data; }); - const done = new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr }))); - children.push({ child, done }); - }; - t.after(async () => { - writeFileSync(marker('resume'), ''); - writeFileSync(marker('release'), ''); - await Promise.all(children.map(({ done }) => done)); - rmSync(dir, { recursive: true, force: true }); - }); + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['resume', 'release']); run('A'); await until(() => existsSync(marker('stale-read'))); run('B'); @@ -422,8 +461,8 @@ test('two contenders cannot replace the fresh lock while another reclaims a stal await until(() => existsSync(marker('A-entered'))); const entered = ['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))); writeFileSync(marker('release'), ''); - const results = await Promise.all(children.map(({ done }) => done)); - assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); assert.deepEqual(entered, ['A'], 'only the process reclaiming the inspected stale directory may enter'); assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [true, false, false]); }); @@ -487,20 +526,7 @@ test('a delayed dead-admission reclaimer cannot unlink the fresh owner another c }); fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); `; - const children = []; - const run = (role) => { - const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, lock, role]); - let stderr = ''; - child.stderr.on('data', (data) => { stderr += data; }); - const done = new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr }))); - children.push({ child, done }); - return child; - }; - t.after(async () => { - for (const name of ['A-resume', 'B-resume', 'release']) writeFileSync(marker(name), ''); - await Promise.all(children.map(({ done }) => done)); - rmSync(dir, { recursive: true, force: true }); - }); + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['A-resume', 'B-resume', 'release']); run('A'); await until(() => existsSync(marker('dead-inspected'))); const fresh = run('B'); @@ -517,7 +543,7 @@ test('a delayed dead-admission reclaimer cannot unlink the fresh owner another c await until(() => existsSync(marker('B-entered'))); assert.deepEqual(['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))), ['B']); writeFileSync(marker('release'), ''); - const results = await Promise.all(children.map(({ done }) => done)); - assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [false, true, false]); }); From b815a6db297bd45db533616eb2efe3028b67ab93 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 11:01:01 +0700 Subject: [PATCH 11/13] Make abandoned admission guards recoverable across processes (cherry picked from commit a0e143e6362f2f0fb9777dd1164bb9bb77a6bfb1) --- lib/runtime/store/admission.js | 45 ++++++++++++++++++++++------------ test/runtime/file-lock.test.js | 39 +++++++++++++++++++++++------ 2 files changed, 62 insertions(+), 22 deletions(-) diff --git a/lib/runtime/store/admission.js b/lib/runtime/store/admission.js index 61f0e17..e9ec4ab 100644 --- a/lib/runtime/store/admission.js +++ b/lib/runtime/store/admission.js @@ -12,6 +12,7 @@ export const LOCK_RETRY_MS = 20; export const LOCK_ACCESS_ERRORS = new Set(['EPERM', 'EBUSY', 'EACCES']); const ADMISSION_IDENTITY_CHECK_MS = 10_000; const START_TIME_TOLERANCE_MS = 2_000; +const ABANDONED_PREFIX = 'abandoned-'; const abandonedAdmissions = new Map(); export function pause(ms) { @@ -22,15 +23,19 @@ function reclaimAdmission(admission, deadline, starts) { try { const info = fs.lstatSync(admission); if (!info.isDirectory()) return false; - const owners = fs.readdirSync(admission); - if (owners.length > 1) return false; - if (owners.length === 1) { - const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}|[0-9]{13}))?$/.exec(owners[0]); + const entries = fs.readdirSync(admission); + if (entries.length > 0) { + const marker = entries.find((name) => name.startsWith(ABANDONED_PREFIX)); + const ownerName = marker ? marker.slice(ABANDONED_PREFIX.length) : entries[0]; + if (entries.some((name) => name !== ownerName && name !== marker)) return false; + const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}|[0-9]{13}))?$/.exec(ownerName); if (!owner) return false; const pid = Number(owner[1]); if (!Number.isSafeInteger(pid) || pid <= 0) return false; - const abandonedHere = abandonedAdmissions.get(admission) === owners[0]; - if (!abandonedHere && !ownerIsGone(pid)) { + const abandonedHere = abandonedAdmissions.get(admission) === ownerName; + const markerPath = marker && path.join(admission, marker); + if (markerPath && !fs.lstatSync(markerPath).isFile()) return false; + if (!markerPath && !abandonedHere && !ownerIsGone(pid)) { const birth = Number(owner[2]); const remaining = deadline - Date.now(); if (owner[2]?.length !== 13 || !Number.isSafeInteger(birth) || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS || remaining <= 0) return false; @@ -39,7 +44,8 @@ function reclaimAdmission(admission, deadline, starts) { // Only a clearly newer process proves reuse. ps timestamps round to whole seconds. if (!actualBirth || actualBirth <= birth + START_TIME_TOLERANCE_MS) return false; } - fs.unlinkSync(path.join(admission, owners[0])); + if (entries.includes(ownerName)) fs.unlinkSync(path.join(admission, ownerName)); + if (markerPath) fs.unlinkSync(markerPath); if (abandonedHere) abandonedAdmissions.delete(admission); } // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot @@ -92,13 +98,23 @@ function releaseAdmission({ admission, owner }) { try { retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); } catch (cause) { - abandonedAdmissions.set(admission, owner); - throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + if (cause?.code !== 'ENOENT') { + throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + } } // The guard is released when its unique owner is gone. An empty directory can be reclaimed. try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} } +function abandonAdmission({ admission, owner }) { + // A separate marker can be written even when Windows denies access to the owner file. + try { + retryLockAccess(() => fs.writeFileSync(path.join(admission, ABANDONED_PREFIX + owner), '', { mode: 0o600, flag: 'wx' })); + } catch { + abandonedAdmissions.set(admission, owner); + } +} + // Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. export function admitted(lockDir, mutate, rollback, deadline = Date.now() + LOCK_WAIT_MS, starts = new Map()) { const admission = enterAdmission(lockDir, deadline, starts); @@ -107,17 +123,16 @@ export function admitted(lockDir, mutate, rollback, deadline = Date.now() + LOCK try { result = mutate(); } catch (error) { - try { releaseAdmission(admission); } catch {} + try { releaseAdmission(admission); } catch { abandonAdmission(admission); } throw error; } try { releaseAdmission(admission); } catch (error) { - // After an irreversible release, report its result. The stranded guard is tracked for recovery. - if (rollback) { - try { rollback(); } catch (rollbackError) { error.rollbackError = rollbackError; } - throw error; - } + // Publish only after rollback, so another process cannot enter while it runs. + try { rollback?.(); } catch (rollbackError) { error.rollbackError = rollbackError; } + abandonAdmission(admission); + if (rollback) throw error; } return result; } diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index d6b5261..efeba28 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -319,18 +319,19 @@ test('an admission cleanup failure is surfaced instead of silently orphaning a l } assert.equal(entered, false); assert.equal(existsSync(lock), false, 'the failed acquire must not leave a work lock'); - assert.match(readdirSync(admission)[0], new RegExp(`^owner-${process.pid}-`)); + assert.ok(readdirSync(admission).some((name) => name.startsWith(`owner-${process.pid}-`))); assert.equal(await withLock(lock, async () => { entered = true; }), true); assert.equal(entered, true); }); -test('successful work is not reported as failed after its lock was removed', async (t) => { - const { lock } = scratch(t); +test('another process reclaims a failed admission release while the original owner is alive', async (t) => { + const { dir, lock } = scratch(t); + const admission = `${lock}.admission`; const unlink = fs.unlinkSync; try { assert.equal(await withLock(lock, () => { fs.unlinkSync = (target) => { - if (dirname(target) === `${lock}.admission`) throw Object.assign(new Error('fixture cleanup refusal'), { code: 'EPERM' }); + if (dirname(target) === admission) throw Object.assign(new Error('fixture cleanup refusal'), { code: 'EPERM' }); return unlink(target); }; }), true); @@ -338,7 +339,31 @@ test('successful work is not reported as failed after its lock was removed', asy fs.unlinkSync = unlink; } assert.equal(existsSync(lock), false); - assert.equal(await withLock(lock, () => {}), true, 'the stranded admission can be reclaimed'); + assert.ok(readdirSync(admission).some((name) => name.startsWith(`owner-${process.pid}-`))); + + const other = spawnSync(process.execPath, ['--input-type=module', '-e', ` + import assert from 'node:assert/strict'; + const [runtime, lock, owner] = process.argv.slice(1); + const { withLock } = await import(runtime); + process.kill(Number(owner), 0); + assert.equal(await withLock(lock, () => {}), true, 'the stranded admission can be reclaimed'); + `, RUNTIME, lock, String(process.pid)], { encoding: 'utf8', timeout: 10_000, windowsHide: true }); + assert.ifError(other.error); + assert.equal(other.status, 0, other.stderr); + assert.deepEqual(readdirSync(dir), []); +}); + +test('admission recovery resumes after the abandoned owner has already been removed', async (t) => { + const { dir, lock } = scratch(t); + const guard = holdAdmission(lock); + const marker = join(guard.admission, `abandoned-owner-${process.pid}-abcd`); + writeFileSync(marker, ''); + assert.equal(await withLock(lock, () => assert.fail('a different owner is still active')), false); + assert.equal(existsSync(guard.owner), true); + + unlinkSync(guard.owner); + assert.equal(await withLock(lock, () => {}), true); + assert.deepEqual(readdirSync(dir), []); }); test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { @@ -379,11 +404,11 @@ test('an old admission stays held by the same process, but a reused PID can be r } finally { fs.unlinkSync = unlink; } - const [original] = readdirSync(admission); + const original = readdirSync(admission).find((name) => name.startsWith(`owner-${process.pid}-`)); const birth = original.split('-')[3]; assert.match(birth ?? '', /^\d{13}$/); const replaceOwner = (name) => { - unlinkSync(join(admission, readdirSync(admission)[0])); + for (const name of readdirSync(admission)) unlinkSync(join(admission, name)); writeFileSync(join(admission, name), ''); utimesSync(admission, STALE, STALE); }; From 41aee5e853d22ab82cf7ca5c4835a36ccc3f669a Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 12:23:26 +0700 Subject: [PATCH 12/13] Reclaim stale worker locks after confirmed PID reuse (cherry picked from commit 0490c99a60629c40f6a4be2d6ef70a580f772dd2) --- lib/runtime/process/owner.js | 12 +++++---- lib/runtime/store/admission.js | 7 +++-- lib/runtime/store/file-lock.js | 31 +++++++++++++++++----- test/runtime/file-lock.test.js | 47 ++++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 16 deletions(-) diff --git a/lib/runtime/process/owner.js b/lib/runtime/process/owner.js index c9ea00e..5ea48fa 100644 --- a/lib/runtime/process/owner.js +++ b/lib/runtime/process/owner.js @@ -6,12 +6,14 @@ import { readlinkSync } from 'node:fs'; import path from 'node:path'; const SHELLS = new Set(['sh', 'bash', 'zsh', 'dash', 'ksh', 'fish', 'cmd.exe', 'powershell', 'pwsh']); -const PS_TIMEOUT_MS = 1000; +export const PROCESS_START_TIMEOUT_MS = 1_000; +// POSIX ps reports start times to the nearest second. +export const PROCESS_START_TOLERANCE_MS = 2_000; const MAX_ANCESTORS = 16; const ownStartTime = Math.round(Date.now() - process.uptime() * 1000); -/** A bounded process start-time probe for admission recovery. The hot path needs no subprocess. */ -export function processStartTime(pid, timeout = PS_TIMEOUT_MS) { +/** A bounded process start-time probe for lock recovery. The hot path needs no subprocess. */ +export function processStartTime(pid, timeout = PROCESS_START_TIMEOUT_MS) { if (pid === process.pid) return ownStartTime; if (!Number.isSafeInteger(pid) || pid <= 0 || timeout <= 0) return null; const windows = process.platform === 'win32'; @@ -23,7 +25,7 @@ export function processStartTime(pid, timeout = PS_TIMEOUT_MS) { : ['-p', String(pid), '-o', 'lstart=']; const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; const result = spawnSync(executable, args, { - env, encoding: 'utf8', windowsHide: true, timeout: Math.min(timeout, PS_TIMEOUT_MS), killSignal: 'SIGKILL', + env, encoding: 'utf8', windowsHide: true, timeout, killSignal: 'SIGKILL', }); const started = result.status === 0 ? result.stdout.trim() : ''; if (!started) return null; @@ -57,7 +59,7 @@ function executablePathOf(pid, command) { } export function processInfo(pid) { - const probe = spawnSync('ps', ['-o', 'ppid=,comm=', '-p', String(pid)], { encoding: 'utf8', timeout: PS_TIMEOUT_MS }); + const probe = spawnSync('ps', ['-o', 'ppid=,comm=', '-p', String(pid)], { encoding: 'utf8', timeout: PROCESS_START_TIMEOUT_MS }); const match = /^\s*(\d+)\s+(.+?)\s*$/.exec(probe.stdout ?? ''); if (!match) return null; const command = match[2].replace(/^-/, ''); diff --git a/lib/runtime/store/admission.js b/lib/runtime/store/admission.js index e9ec4ab..b416d0b 100644 --- a/lib/runtime/store/admission.js +++ b/lib/runtime/store/admission.js @@ -5,13 +5,12 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import { ownerIsGone, processStartTime } from '../process/owner.js'; +import { ownerIsGone, processStartTime, PROCESS_START_TIMEOUT_MS, PROCESS_START_TOLERANCE_MS } from '../process/owner.js'; export const LOCK_WAIT_MS = 2_000; export const LOCK_RETRY_MS = 20; export const LOCK_ACCESS_ERRORS = new Set(['EPERM', 'EBUSY', 'EACCES']); const ADMISSION_IDENTITY_CHECK_MS = 10_000; -const START_TIME_TOLERANCE_MS = 2_000; const ABANDONED_PREFIX = 'abandoned-'; const abandonedAdmissions = new Map(); @@ -39,10 +38,10 @@ function reclaimAdmission(admission, deadline, starts) { const birth = Number(owner[2]); const remaining = deadline - Date.now(); if (owner[2]?.length !== 13 || !Number.isSafeInteger(birth) || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS || remaining <= 0) return false; - if (!starts.has(pid)) starts.set(pid, processStartTime(pid, remaining)); + if (!starts.has(pid)) starts.set(pid, processStartTime(pid, Math.min(remaining, PROCESS_START_TIMEOUT_MS))); const actualBirth = starts.get(pid); // Only a clearly newer process proves reuse. ps timestamps round to whole seconds. - if (!actualBirth || actualBirth <= birth + START_TIME_TOLERANCE_MS) return false; + if (!actualBirth || actualBirth <= birth + PROCESS_START_TOLERANCE_MS) return false; } if (entries.includes(ownerName)) fs.unlinkSync(path.join(admission, ownerName)); if (markerPath) fs.unlinkSync(markerPath); diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index 8b34c29..6763063 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -5,10 +5,12 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import { ownerIsGone } from '../process/owner.js'; +import { ownerIsGone, processStartTime, PROCESS_START_TIMEOUT_MS, PROCESS_START_TOLERANCE_MS } from '../process/owner.js'; import { admitted, LOCK_ACCESS_ERRORS, LOCK_WAIT_MS, LOCK_RETRY_MS, pause, retryLockAccess } from './admission.js'; const WORK_LOCK_STALE_MS = 2 * 60 * 1000; +const WORK_OWNER_PROBE_MS = process.platform === 'win32' ? 5_000 : PROCESS_START_TIMEOUT_MS; +const WORK_OWNER_WITH_START = /^\d+:[a-f0-9]+:([1-9]\d{12})$/; const CLAIM_STALE_MS = 60 * 1000; const abandonedWork = new Map(); const abandonedState = new Map(); @@ -214,15 +216,23 @@ async function releaseWorkLock(lockDir, token) { } function tryLock(lockDir, now) { - let token; - return admitted(lockDir, () => { + let token, suspect, reusedToken; + const take = () => admitted(lockDir, () => { const stale = isStale(lockDir, now); if (abandonedWork.has(lockDir) || stale) { let held; try { held = workOwnerOf(lockDir); } catch { return null; } - // Old versions wrote random tokens without a pid. Keep their lease recovery, but a known live owner - // cannot lose its lock just because a slow or paused worker missed a refresh. A reused pid stays busy. - if (held !== abandonedWork.get(lockDir) && (!stale || (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))))) return null; + if (held !== abandonedWork.get(lockDir)) { + if (!stale) return null; + if (held !== reusedToken && /^\d+:/.test(held ?? '')) { + const pid = ownerPidOf(held); + if (!ownerIsGone(pid)) { + const birth = WORK_OWNER_WITH_START.exec(held)?.[1]; + if (birth) suspect = { token: held, pid, birth: Number(birth) }; + return null; + } + } + } fs.rmSync(lockDir, { recursive: true, force: true }); abandonedWork.delete(lockDir); } @@ -231,7 +241,7 @@ function tryLock(lockDir, now) { } catch { return null; } - token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; + token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}:${processStartTime(process.pid)}`; fs.writeFileSync(path.join(lockDir, 'owner'), token); return { refresh: () => { @@ -255,6 +265,13 @@ function tryLock(lockDir, now) { if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); abandonedWork.delete(lockDir); }); + const lock = take(); + if (lock || !suspect) return lock; + // A foreign lookup can be slow. Release admission, then recheck staleness and the exact token. + const actualBirth = processStartTime(suspect.pid, WORK_OWNER_PROBE_MS); + if (!actualBirth || actualBirth <= suspect.birth + PROCESS_START_TOLERANCE_MS) return null; + reusedToken = suspect.token; + return take(); } /** Runs slow `work` under one owner's lock; dead or legacy directories recover after their lease expires. */ diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index efeba28..0e40c47 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -7,6 +7,7 @@ import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { test } from 'node:test'; +import { processStartTime } from '../../lib/runtime/process/owner.js'; import { withLock } from '../../lib/runtime/store/file-lock.js'; import { lockContenders, until } from '../helpers/lock-contenders.js'; @@ -79,6 +80,52 @@ test('an old mtime never evicts a live owner, which can refresh and release its assert.deepEqual(readdirSync(dir), []); }); +test('only a stale worker lock with proof of PID reuse can displace a live PID', async (t) => { + const birth = processStartTime(process.pid); + const prefix = `${process.pid}:deadbeef`; + for (const token of [prefix, `${prefix}:${birth}`, `${prefix}:${birth - 1_000}`, `${prefix}:invalid`]) { + const { lock } = scratch(t, token); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, () => assert.fail('owner identity is not disproved')), false); + assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), token); + } + + const { dir, lock } = scratch(t, `${prefix}:${birth - 10_000}`); + assert.equal(await withLock(lock, () => assert.fail('the lease is still fresh')), false); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, () => { + assert.match(readFileSync(join(lock, 'owner'), 'utf8'), new RegExp(`^${process.pid}:[a-f0-9]+:${birth}$`)); + }), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('a foreign PID probe leaves admission free and cannot remove a different worker token', async (t) => { + const pid = process.ppid; + const birth = Date.now(); + const old = `${pid}:aaaa:${birth - 10_000}`; + const replacement = `${pid}:bbbb:${birth - 10_000}`; + const { lock } = scratch(t, old); + utimesSync(lock, STALE, STALE); + const spawn = childProcess.spawnSync; + let probes = 0; + childProcess.spawnSync = () => { + probes += 1; + assert.equal(existsSync(`${lock}.admission`), false); + writeFileSync(join(lock, 'owner'), replacement); + const stdout = process.platform === 'win32' ? String(birth) : new Date(birth).toUTCString().replace(/ GMT$/, ''); + return { status: 0, stdout }; + }; + syncBuiltinESMExports(); + try { + assert.equal(await withLock(lock, () => assert.fail('the observed token was replaced')), false); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(probes, 1); + assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), replacement); +}); + test('a transient owner read error never turns a stale live lock into a legacy lock', async (t) => { for (const code of ['EBUSY', 'EPERM', 'EACCES', 'EIO']) { const { dir, lock } = scratch(t); From b66fdc4f294b10418ed9c697a590b1ed0a7436c9 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Thu, 5 Nov 2026 17:19:52 +0700 Subject: [PATCH 13/13] Refuse unlocked state updates and clean up owned partial replacements --- lib/runtime/store/file-lock.js | 24 +++-- test/runtime/file-stores.test.js | 156 ++++++++++++++++++++++++++++++- 2 files changed, 172 insertions(+), 8 deletions(-) diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index d591757..2463b2b 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -28,9 +28,17 @@ export function readState(target, fallback) { export function writeState(target, value) { fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 }); const temporary = `${target}.${process.pid}.${crypto.randomBytes(6).toString('hex')}.tmp`; - fs.writeFileSync(temporary, JSON.stringify(value), { mode: 0o600, flag: 'wx' }); const deadline = Date.now() + LOCK_WAIT_MS; + let descriptor; try { + descriptor = fs.openSync(temporary, 'wx', 0o600); + try { + fs.writeFileSync(descriptor, JSON.stringify(value)); + } catch (error) { + try { fs.closeSync(descriptor); } catch {} + throw error; + } + fs.closeSync(descriptor); for (;;) { try { fs.renameSync(temporary, target); @@ -41,7 +49,7 @@ export function writeState(target, value) { } } } finally { - removeState(temporary); + if (descriptor !== undefined) removeState(temporary); } } @@ -106,7 +114,7 @@ function release(lockPath, token) { const deadline = Date.now() + LOCK_WAIT_MS; for (;;) { try { - if (fs.readFileSync(lockPath, 'utf8') === token) fs.rmSync(lockPath, { force: true }); + if (fs.readFileSync(lockPath, 'utf8') === token) fs.unlinkSync(lockPath); return; } catch (error) { if (!['EACCES', 'EPERM', 'EBUSY'].includes(error?.code) || Date.now() >= deadline) return; @@ -117,18 +125,20 @@ function release(lockPath, token) { /** * Applies `mutate` to a state file under an exclusive lock, so hook processes sharing it cannot overwrite each - * other's entries. A dead owner's lock is taken over; one that cannot be had within LOCK_WAIT_MS is given up on - * and the update runs unguarded, which never blocks a session. `mutate` returns undefined to leave the file. + * other's entries. A dead owner's lock is taken over; an unavailable lock refuses the update before reading + * or mutating state. `mutate` returns undefined to leave the file. */ export function updateState(target, fallback, mutate) { const lockPath = `${target}.lock`; - const token = acquire(lockPath, Date.now() + LOCK_WAIT_MS); + let token, cause; + try { token = acquire(lockPath, Date.now() + LOCK_WAIT_MS); } catch (error) { cause = error; } + if (!token) throw Object.assign(new Error('State is busy or unavailable; no update was made', { cause }), { code: 'ELOCKED' }); try { const next = mutate(readState(target, fallback)); if (next !== undefined) writeState(target, next); return next; } finally { - if (token) release(lockPath, token); + release(lockPath, token); } } diff --git a/test/runtime/file-stores.test.js b/test/runtime/file-stores.test.js index b3e9219..0f56c33 100644 --- a/test/runtime/file-stores.test.js +++ b/test/runtime/file-stores.test.js @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; +import crypto from 'node:crypto'; import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import fs from 'node:fs'; import { tmpdir } from 'node:os'; @@ -78,6 +79,159 @@ test('a corrupt draft file is preserved for every packaged host', async () => { } }); +test('a refused state lock preserves drafts and never runs their mutator', async t => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + const lock = `${target}.lock`; + const original = '[{"title":"keep this draft"}]'; + writeFileSync(target, original); + const write = fs.writeFileSync; + t.mock.method(fs, 'writeFileSync', (file, ...args) => { + if (file === lock) throw Object.assign(new Error('fixture lock creation denial'), { code: 'EROFS' }); + return write(file, ...args); + }); + let mutations = 0; + try { + await assert.rejects(cursor.drafts.update(drafts => { mutations += 1; return [...drafts, { title: 'new draft' }]; }), { code: 'ELOCKED' }); + } finally { + t.mock.restoreAll(); + } + assert.equal(mutations, 0); + assert.equal(fs.readFileSync(target, 'utf8'), original); + assert.deepEqual(readdirSync(dir), ['cursor-distill-drafts.json']); +}); + +test('a held live state lock preserves drafts and never runs their mutator after waiting', async () => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + const lock = `${target}.lock`; + const original = '[{"title":"keep this draft"}]'; + const held = `${process.pid}:fixture-live-owner`; + writeFileSync(target, original); + writeFileSync(lock, held); + let mutations = 0; + await assert.rejects(cursor.drafts.update(drafts => { mutations += 1; return [...drafts, { title: 'new draft' }]; }), { code: 'ELOCKED' }); + assert.equal(mutations, 0); + assert.equal(fs.readFileSync(target, 'utf8'), original); + assert.equal(fs.readFileSync(lock, 'utf8'), held); +}); + +for (const closeFails of [false, true]) test(`a partial temporary state write preserves drafts and cleans up before a retry${closeFails ? ' even when close also fails' : ''}`, async t => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + const previous = [{ title: 'keep this draft' }]; + await cursor.drafts.update(() => previous); + const original = fs.readFileSync(target, 'utf8'); + const temporary = file => typeof file === 'string' && file.startsWith(target + '.') && file.endsWith('.tmp'); + const descriptors = new Set(); + const open = fs.openSync, write = fs.writeFileSync, close = fs.closeSync; + t.mock.method(fs, 'openSync', (file, ...args) => { + const descriptor = open(file, ...args); + if (temporary(file)) descriptors.add(descriptor); + return descriptor; + }); + let failed = false; + let closeFailed = false; + const writeFailure = Object.assign(new Error('fixture storage exhausted after a partial write'), { code: 'ENOSPC' }); + if (closeFails) t.mock.method(fs, 'closeSync', descriptor => { + const owned = descriptors.delete(descriptor); + const result = close(descriptor); + if (owned) { + closeFailed = true; + throw Object.assign(new Error('fixture close failure after closing the descriptor'), { code: 'EIO' }); + } + return result; + }); + t.mock.method(fs, 'writeFileSync', (file, ...args) => { + if (!failed && (temporary(file) || descriptors.has(file))) { + failed = true; + write(file, '[{"partial":', ...args.slice(1)); + throw writeFailure; + } + return write(file, ...args); + }); + try { + await assert.rejects(cursor.drafts.update(drafts => [...drafts, { title: 'new draft' }]), error => { + assert.equal(error, writeFailure); + assert.equal(error.code, 'ENOSPC'); + return true; + }); + } finally { + t.mock.restoreAll(); + } + assert.equal(failed, true); + assert.equal(closeFailed, closeFails); + assert.equal(fs.readFileSync(target, 'utf8'), original); + assert.deepEqual(readdirSync(dir), ['cursor-distill-drafts.json']); + await cursor.drafts.update(drafts => [...drafts, { title: 'retry succeeded' }]); + assert.deepEqual(await cursor.drafts.read(), [...previous, { title: 'retry succeeded' }]); +}); + +test('a temporary descriptor close failure preserves drafts without replacing the target and permits a retry', async t => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + const previous = [{ title: 'keep this draft' }]; + await cursor.drafts.update(() => previous); + const original = fs.readFileSync(target, 'utf8'); + const descriptors = new Set(); + const open = fs.openSync, close = fs.closeSync, rename = fs.renameSync; + t.mock.method(fs, 'openSync', (file, ...args) => { + const descriptor = open(file, ...args); + if (typeof file === 'string' && file.startsWith(target + '.') && file.endsWith('.tmp')) descriptors.add(descriptor); + return descriptor; + }); + let failed = false; + t.mock.method(fs, 'closeSync', descriptor => { + const owned = descriptors.delete(descriptor); + const result = close(descriptor); + if (owned) { + failed = true; + throw Object.assign(new Error('fixture close failure after closing the descriptor'), { code: 'EIO' }); + } + return result; + }); + let replacements = 0; + t.mock.method(fs, 'renameSync', (from, to) => { + if (to === target) replacements += 1; + return rename(from, to); + }); + try { + await assert.rejects(cursor.drafts.update(drafts => [...drafts, { title: 'new draft' }]), { code: 'EIO' }); + } finally { + t.mock.restoreAll(); + } + assert.equal(failed, true); + assert.equal(replacements, 0); + assert.equal(fs.readFileSync(target, 'utf8'), original); + assert.deepEqual(readdirSync(dir), ['cursor-distill-drafts.json']); + await cursor.drafts.update(drafts => [...drafts, { title: 'retry succeeded' }]); + assert.deepEqual(await cursor.drafts.read(), [...previous, { title: 'retry succeeded' }]); +}); + +test('a temporary file collision preserves the existing file and releases the state lock', async t => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + await cursor.drafts.update(() => [{ title: 'keep this draft' }]); + const original = fs.readFileSync(target, 'utf8'); + const suffix = '010203040506'; + const collision = `${target}.${process.pid}.${suffix}.tmp`; + const collisionContents = 'another invocation owns these bytes'; + writeFileSync(collision, collisionContents); + const random = crypto.randomBytes; + t.mock.method(crypto, 'randomBytes', (size, ...args) => size === 6 ? Buffer.from(suffix, 'hex') : random(size, ...args)); + try { + await assert.rejects(cursor.drafts.update(drafts => [...drafts, { title: 'new draft' }]), { code: 'EEXIST' }); + } finally { + t.mock.restoreAll(); + } + assert.equal(fs.readFileSync(target, 'utf8'), original); + assert.equal(fs.readFileSync(collision, 'utf8'), collisionContents); + assert.equal(existsSync(`${target}.lock`), false); + await cursor.drafts.update(drafts => [...drafts, { title: 'retry succeeded' }]); + assert.deepEqual(await cursor.drafts.read(), [{ title: 'keep this draft' }, { title: 'retry succeeded' }]); + assert.equal(fs.readFileSync(collision, 'utf8'), collisionContents); +}); + test('a failed draft replacement preserves the previous file, releases its lock and removes its temporary file', async t => { const { dir, stores: cursor } = stores(); const target = join(dir, 'cursor-distill-drafts.json'); @@ -160,7 +314,7 @@ for (let i = 0; i < 25; i += 1) updateState(${JSON.stringify(target)}, {}, (stat }); test('transient sharing denial while releasing a state lock leaves no orphaned lock', t => { - for (const operation of ['readFileSync', 'rmSync']) { + for (const operation of ['readFileSync', 'unlinkSync']) { const { dir } = stores(); const target = join(dir, 'counter.json'), lock = `${target}.lock`; const original = fs[operation];