From 05d14ec00aa9daa97af3c2614cd7c91dafb5d9d8 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Mon, 2 Nov 2026 11:47:32 +0700 Subject: [PATCH 1/7] fix: serialize stale worker-lock recovery through atomic admission --- lib/runtime/store/file-lock.js | 124 +++++++++++-- test/runtime/file-lock.test.js | 329 +++++++++++++++++++++++++++++++++ 2 files changed, 438 insertions(+), 15 deletions(-) create mode 100644 test/runtime/file-lock.test.js diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index d122790..fbc2d06 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -183,27 +183,121 @@ function ownsLock(lockDir, token) { return contentOf(path.join(lockDir, 'owner')) === token; } -function tryLock(lockDir, now) { - fs.mkdirSync(path.dirname(lockDir), { recursive: true, mode: 0o700 }); - if (isStale(lockDir, now)) fs.rmSync(lockDir, { recursive: true, force: true }); +function reclaimAdmission(admission) { try { - fs.mkdirSync(lockDir); + if (!fs.lstatSync(admission).isDirectory()) return false; + const owners = fs.readdirSync(admission); + if (owners.length > 1) return false; + if (owners.length === 1) { + const owner = /^owner-(\d+)-[a-f0-9]+$/.exec(owners[0]); + if (!owner || !ownerIsGone(Number(owner[1]))) return false; + fs.unlinkSync(path.join(admission, owners[0])); + } + // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot + // remove it even if another reclaimer replaced the directory after our read. + fs.rmdirSync(admission); + return true; } catch { + return false; + } +} + +function admissionOf(lockDir) { + const admission = `${lockDir}.admission`; + const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}`; + const staging = `${admission}.${owner}.claim`; + fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); + fs.mkdirSync(staging, { mode: 0o700 }); + try { + fs.writeFileSync(path.join(staging, owner), '', { mode: 0o600, flag: 'wx' }); + // Publishing a prepopulated directory never exposes an empty fresh guard to a stale reclaimer. + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + fs.renameSync(staging, admission); + return { admission, owner }; + } catch { + if (attempt > 0 || !reclaimAdmission(admission)) return null; + } + } return null; + } finally { + fs.rmSync(staging, { recursive: true, force: true }); } - const token = crypto.randomBytes(12).toString('hex'); - fs.writeFileSync(path.join(lockDir, 'owner'), token); - return { - refresh: () => { - if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); - }, - release: () => { +} + +function retryAdmissionCleanup(remove) { + for (let attempt = 0; ; attempt += 1) { + try { + return remove(); + } catch (error) { + if (attempt === 2 || !['EPERM', 'EBUSY', 'EACCES'].includes(error?.code)) throw error; + pause(LOCK_RETRY_MS); + } + } +} + +function releaseAdmission({ admission, owner }) { + try { + retryAdmissionCleanup(() => fs.unlinkSync(path.join(admission, owner))); + try { + retryAdmissionCleanup(() => fs.rmdirSync(admission)); + } catch (error) { + if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error?.code)) throw error; + } + } catch (cause) { + throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + } +} + +// Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. +function admitted(lockDir, mutate) { + const admission = admissionOf(lockDir); + if (!admission) return null; + try { + return mutate(); + } finally { + releaseAdmission(admission); + } +} + +async function releaseWorkLock(lockDir, token) { + const deadline = Date.now() + LOCK_WAIT_MS; + for (;;) { + if (admitted(lockDir, () => { if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); - }, - }; + return true; + })) return; + if (Date.now() >= deadline) throw Object.assign(new Error(`Could not release lock ${lockDir}`), { code: 'ELOCKED' }); + await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS)); + } +} + +function tryLock(lockDir, now) { + return admitted(lockDir, () => { + if (isStale(lockDir, now)) { + const held = contentOf(path.join(lockDir, 'owner')); + // Old versions wrote random tokens without a pid. Keep their lease recovery, but a known live owner + // cannot lose its lock just because a slow or paused worker missed a refresh. A reused pid stays busy. + if (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))) return null; + fs.rmSync(lockDir, { recursive: true, force: true }); + } + try { + fs.mkdirSync(lockDir); + } catch { + return null; + } + const token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; + fs.writeFileSync(path.join(lockDir, 'owner'), token); + return { + refresh: () => admitted(lockDir, () => { + if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); + }), + release: () => releaseWorkLock(lockDir, token), + }; + }); } -/** Runs slow `work` while holding a lock directory no other process gets meanwhile; false when it is held. */ +/** Runs slow `work` under one owner's lock; dead or legacy directories recover after their lease expires. */ export async function withLock(lockDir, work, now = Date.now()) { const lock = tryLock(lockDir, now); if (!lock) return false; @@ -211,6 +305,6 @@ export async function withLock(lockDir, work, now = Date.now()) { await work(lock); return true; } finally { - lock.release(); + await lock.release(); } } diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js new file mode 100644 index 0000000..d2b8bdb --- /dev/null +++ b/test/runtime/file-lock.test.js @@ -0,0 +1,329 @@ +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, rmdirSync, statSync, unlinkSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; + +import { withLock } from '../../lib/runtime/store/file-lock.js'; + +const RUNTIME = new URL('../../lib/runtime/store/file-lock.js', import.meta.url).href; +const STALE = new Date(0); + +function scratch(t, token = null) { + const dir = mkdtempSync(join(tmpdir(), 'evolver-work-lock-')); + const lock = join(dir, 'work.lock'); + if (token !== null) { + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), token); + } + t.after(() => rmSync(dir, { recursive: true, force: true })); + return { dir, lock }; +} + +async function until(check) { + const deadline = Date.now() + 10_000; + while (!check()) { + if (Date.now() > deadline) throw new Error('timed out waiting for the lock contender'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +function holdAdmission(lock, pid = process.pid) { + const admission = `${lock}.admission`; + const owner = join(admission, `owner-${pid}-deadbeef`); + mkdirSync(admission); + writeFileSync(owner, 'fixture owner'); + return { admission, owner, release: () => { unlinkSync(owner); rmdirSync(admission); } }; +} + +test('stale legacy and dead-owner directories recover, but a fresh legacy lock stays held', async (t) => { + const { lock } = scratch(t, 'legacy token'); + let entered = 0; + assert.equal(await withLock(lock, async () => { entered += 1; }), false); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), `${exited.pid}:deadbeef`); + holdAdmission(lock, exited.pid); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + assert.equal(entered, 2); + assert.equal(existsSync(lock), false); + assert.equal(existsSync(`${lock}.admission`), false); +}); + +test('an old mtime never evicts a live owner, which can refresh and release its lock', async (t) => { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + utimesSync(lock, STALE, STALE); + let entered = false; + assert.equal(await withLock(lock, async () => { entered = true; }), false); + assert.equal(entered, false); + held.refresh(); + assert.ok(statSync(lock).mtimeMs > STALE.getTime()); + }), true); + assert.equal(existsSync(lock), false); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release waits asynchronously for transient admission contention', async (t) => { + const { lock } = scratch(t); + let guard; + let finished = false; + let refreshedAt; + const running = withLock(lock, async (held) => { + guard = holdAdmission(lock); + utimesSync(lock, STALE, STALE); + held.refresh(); + refreshedAt = statSync(lock).mtimeMs; + }).then(() => { finished = true; }); + try { + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(refreshedAt, STALE.getTime(), 'refresh also respects admission'); + assert.equal(finished, false); + assert.equal(existsSync(lock), true); + } finally { + guard.release(); + await running; + } + assert.equal(existsSync(lock), false); +}); + +test('release refuses bounded admission contention without deleting the held directory', async (t) => { + const { lock } = scratch(t); + let guard; + const started = Date.now(); + await assert.rejects(withLock(lock, async () => { + guard = holdAdmission(lock); + }), { code: 'ELOCKED' }); + assert.ok(Date.now() - started < 5_000, 'cleanup cannot wait forever'); + assert.equal(existsSync(lock), true); + assert.equal(readFileSync(guard.owner, 'utf8'), 'fixture owner'); +}); + +test('an admission cleanup failure is surfaced instead of silently orphaning a live guard', async (t) => { + const { lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + return unlink(target); + }; + let entered = false; + try { + await assert.rejects(withLock(lock, async () => { entered = true; }), { code: 'ELOCKED' }); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(entered, false); + assert.match(readdirSync(admission)[0], new RegExp(`^owner-${process.pid}-`)); +}); + +test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { + const { dir, lock } = scratch(t); + const unlink = fs.unlinkSync; + let refused = false; + fs.unlinkSync = (target) => { + if (dirname(target) === `${lock}.admission` && !refused) { + refused = true; + throw Object.assign(new Error('temporarily refused by fixture'), { code: 'EPERM' }); + } + return unlink(target); + }; + let entered = 0; + try { + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(refused, true); + assert.equal(entered, 1); + assert.deepEqual(readdirSync(dir), []); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + assert.equal(entered, 2); + assert.deepEqual(readdirSync(dir), []); +}); + +test('busy and unrecognised admission guards stay intact and leave no staging directories', async (t) => { + const { dir, lock } = scratch(t); + const guard = holdAdmission(lock); + assert.equal(await withLock(lock, async () => assert.fail('a live guard is held')), false); + assert.deepEqual(readdirSync(dir), ['work.lock.admission']); + writeFileSync(join(guard.admission, 'unexpected'), 'keep'); + assert.equal(await withLock(lock, async () => assert.fail('multiple owners are unknown')), false); + assert.equal(readdirSync(guard.admission).length, 2); + unlinkSync(guard.owner); + assert.equal(await withLock(lock, async () => assert.fail('an unknown owner is held')), false); + assert.equal(readFileSync(join(guard.admission, 'unexpected'), 'utf8'), 'keep'); + assert.deepEqual(readdirSync(dir), ['work.lock.admission']); +}); + +test('two contenders cannot replace the fresh lock while another reclaims a stale directory', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-work-lock-')); + const lock = join(dir, 'work.lock'); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), 'legacy token'); + utimesSync(lock, STALE, STALE); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { withLock } from ${JSON.stringify(RUNTIME)}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const wait = async (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + }; + if (role === 'A') { + const stat = fs.statSync; + let paused = false; + fs.statSync = (file, ...args) => { + const result = stat(file, ...args); + if (file === lock && !paused) { + paused = true; + fs.writeFileSync(marker('stale-read'), ''); + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker('resume'))) { + if (Date.now() > deadline) throw new Error('timed out waiting to resume'); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + } + return result; + }; + } + const result = await withLock(lock, async () => { + fs.writeFileSync(marker(role + '-entered'), ''); + await wait('release'); + }); + fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); + `; + const children = []; + const run = (role) => { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, lock, role]); + let stderr = ''; + child.stderr.on('data', (data) => { stderr += data; }); + const done = new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr }))); + children.push({ child, done }); + }; + t.after(async () => { + writeFileSync(marker('resume'), ''); + writeFileSync(marker('release'), ''); + await Promise.all(children.map(({ done }) => done)); + rmSync(dir, { recursive: true, force: true }); + }); + run('A'); + await until(() => existsSync(marker('stale-read'))); + run('B'); + run('C'); + await until(() => ['B', 'C'].every((role) => existsSync(marker(role + '-entered')) || existsSync(marker(role + '-result')))); + writeFileSync(marker('resume'), ''); + await until(() => existsSync(marker('A-entered'))); + const entered = ['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))); + writeFileSync(marker('release'), ''); + const results = await Promise.all(children.map(({ done }) => done)); + assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); + assert.deepEqual(entered, ['A'], 'only the process reclaiming the inspected stale directory may enter'); + assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [true, false, false]); +}); + +test('a delayed dead-admission reclaimer cannot unlink the fresh owner another contender published', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-admission-race-')); + const lock = join(dir, 'work.lock'); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), 'legacy token'); + utimesSync(lock, STALE, STALE); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + holdAdmission(lock, exited.pid); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { withLock } from ${JSON.stringify(RUNTIME)}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const waitSync = (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + }; + if (role === 'A') { + const list = fs.readdirSync; + let paused = false; + fs.readdirSync = (file, ...args) => { + const result = list(file, ...args); + if (file === lock + '.admission' && !paused) { + paused = true; + fs.writeFileSync(marker('dead-inspected'), ''); + waitSync('A-resume'); + } + return result; + }; + } + if (role === 'B') { + const rename = fs.renameSync; + let paused = false; + fs.renameSync = (source, target) => { + const result = rename(source, target); + if (target === lock + '.admission' && !paused) { + paused = true; + fs.writeFileSync(marker('fresh-held'), ''); + waitSync('B-resume'); + } + return result; + }; + } + const result = await withLock(lock, async () => { + fs.writeFileSync(marker(role + '-entered'), ''); + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker('release'))) { + if (Date.now() > deadline) throw new Error('timed out waiting to release'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + }); + fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); + `; + const children = []; + const run = (role) => { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, lock, role]); + let stderr = ''; + child.stderr.on('data', (data) => { stderr += data; }); + const done = new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr }))); + children.push({ child, done }); + return child; + }; + t.after(async () => { + for (const name of ['A-resume', 'B-resume', 'release']) writeFileSync(marker(name), ''); + await Promise.all(children.map(({ done }) => done)); + rmSync(dir, { recursive: true, force: true }); + }); + run('A'); + await until(() => existsSync(marker('dead-inspected'))); + const fresh = run('B'); + await until(() => existsSync(marker('fresh-held'))); + run('C'); + await until(() => existsSync(marker('C-result')) || existsSync(marker('C-entered'))); + assert.equal(existsSync(marker('C-entered')), false, 'publishing a fresh guard never exposes an empty directory'); + writeFileSync(marker('A-resume'), ''); + await until(() => existsSync(marker('A-result')) || existsSync(marker('A-entered'))); + assert.equal(existsSync(marker('A-entered')), false, 'the delayed reclaimer cannot displace the fresh guard'); + assert.equal(readdirSync(`${lock}.admission`).length, 1); + assert.match(readdirSync(`${lock}.admission`)[0], new RegExp(`^owner-${fresh.pid}-`)); + writeFileSync(marker('B-resume'), ''); + await until(() => existsSync(marker('B-entered'))); + assert.deepEqual(['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))), ['B']); + writeFileSync(marker('release'), ''); + const results = await Promise.all(children.map(({ done }) => done)); + assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); + assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [false, true, false]); +}); From 71f98f860e4f938f777eb0d0caffd444024376b7 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Mon, 2 Nov 2026 12:05:22 +0700 Subject: [PATCH 2/7] fix: preserve worker ownership when lock metadata is unreadable --- lib/runtime/store/file-lock.js | 35 ++++++-- test/runtime/file-lock.test.js | 146 +++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 9 deletions(-) diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index fbc2d06..e8732b1 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -179,8 +179,17 @@ function isStale(lockDir, now) { } } +function workOwnerOf(lockDir) { + try { + return retryLockAccess(() => fs.readFileSync(path.join(lockDir, 'owner'), 'utf8')); + } catch (cause) { + if (cause?.code === 'ENOENT') return null; + throw Object.assign(new Error(`Could not read lock owner ${lockDir}`, { cause }), { code: 'ELOCKED' }); + } +} + function ownsLock(lockDir, token) { - return contentOf(path.join(lockDir, 'owner')) === token; + return workOwnerOf(lockDir) === token; } function reclaimAdmission(admission) { @@ -225,10 +234,10 @@ function admissionOf(lockDir) { } } -function retryAdmissionCleanup(remove) { +function retryLockAccess(access) { for (let attempt = 0; ; attempt += 1) { try { - return remove(); + return access(); } catch (error) { if (attempt === 2 || !['EPERM', 'EBUSY', 'EACCES'].includes(error?.code)) throw error; pause(LOCK_RETRY_MS); @@ -238,9 +247,9 @@ function retryAdmissionCleanup(remove) { function releaseAdmission({ admission, owner }) { try { - retryAdmissionCleanup(() => fs.unlinkSync(path.join(admission, owner))); + retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); try { - retryAdmissionCleanup(() => fs.rmdirSync(admission)); + retryLockAccess(() => fs.rmdirSync(admission)); } catch (error) { if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error?.code)) throw error; } @@ -275,7 +284,8 @@ async function releaseWorkLock(lockDir, token) { function tryLock(lockDir, now) { return admitted(lockDir, () => { if (isStale(lockDir, now)) { - const held = contentOf(path.join(lockDir, 'owner')); + let held; + try { held = workOwnerOf(lockDir); } catch { return null; } // Old versions wrote random tokens without a pid. Keep their lease recovery, but a known live owner // cannot lose its lock just because a slow or paused worker missed a refresh. A reused pid stays busy. if (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))) return null; @@ -289,9 +299,16 @@ function tryLock(lockDir, now) { const token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; fs.writeFileSync(path.join(lockDir, 'owner'), token); return { - refresh: () => admitted(lockDir, () => { - if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); - }), + refresh: () => { + try { + return admitted(lockDir, () => { + if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); + }); + } catch { + // Timer-driven refresh is best effort; a live pid stays protected and final release still surfaces errors. + return null; + } + }, release: () => releaseWorkLock(lockDir, token), }; }); diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index d2b8bdb..1156254 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -70,6 +70,152 @@ test('an old mtime never evicts a live owner, which can refresh and release its assert.deepEqual(readdirSync(dir), []); }); +test('a transient owner read error never turns a stale live lock into a legacy lock', async (t) => { + for (const code of ['EBUSY', 'EPERM', 'EACCES', 'EIO']) { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async () => { + const owner = join(lock, 'owner'); + const token = readFileSync(owner, 'utf8'); + utimesSync(lock, STALE, STALE); + const read = fs.readFileSync; + let refused = false; + let entered = false; + fs.readFileSync = (target, ...args) => { + if (target === owner && !refused) { + refused = true; + throw Object.assign(new Error('transient owner read refusal'), { code }); + } + return read(target, ...args); + }; + try { + assert.equal(await withLock(lock, async () => { entered = true; }), false, code); + } finally { + fs.readFileSync = read; + } + assert.equal(refused, true); + assert.equal(entered, false, code); + assert.equal(readFileSync(owner, 'utf8'), token, 'the live owner stays intact'); + }), true); + assert.deepEqual(readdirSync(dir), [], 'the original owner still releases normally'); + } +}); + +test('a stale directory left before its owner was written is recoverable', async (t) => { + const { dir, lock } = scratch(t); + mkdirSync(lock); + utimesSync(lock, STALE, STALE); + let entered = false; + assert.equal(await withLock(lock, async () => { entered = true; }), true); + assert.equal(entered, true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release retries a transient owner read failure and really removes its lock', async (t) => { + const { dir, lock } = scratch(t); + const owner = join(lock, 'owner'); + const read = fs.readFileSync; + let refused = false; + try { + assert.equal(await withLock(lock, async () => { + fs.readFileSync = (target, ...args) => { + if (target === owner && !refused) { + refused = true; + throw Object.assign(new Error('transient read refusal'), { code: 'EBUSY' }); + } + return read(target, ...args); + }; + }), true); + } finally { + fs.readFileSync = read; + } + assert.equal(refused, true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release refuses an unreadable owner explicitly without claiming cleanup', async (t) => { + for (const code of ['EBUSY', 'EPERM', 'EIO']) { + const { lock } = scratch(t); + const owner = join(lock, 'owner'); + const read = fs.readFileSync; + let token; + try { + await assert.rejects(withLock(lock, async () => { + token = read(owner, 'utf8'); + fs.readFileSync = (target, ...args) => { + if (target === owner) throw Object.assign(new Error('persistent read refusal'), { code }); + return read(target, ...args); + }; + }), { code: 'ELOCKED' }); + } finally { + fs.readFileSync = read; + } + assert.equal(readFileSync(owner, 'utf8'), token, code); + } +}); + +test('refresh leaves an unreadable live lock untouched without crashing its timer caller', async (t) => { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + const owner = join(lock, 'owner'); + utimesSync(lock, STALE, STALE); + const read = fs.readFileSync; + fs.readFileSync = (target, ...args) => { + if (target === owner) throw Object.assign(new Error('unreadable owner'), { code: 'EPERM' }); + return read(target, ...args); + }; + try { + assert.doesNotThrow(() => held.refresh()); + assert.equal(statSync(lock).mtimeMs, STALE.getTime()); + } finally { + fs.readFileSync = read; + } + }), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('refresh tolerates file access refusals and the next refresh and release still work', async (t) => { + for (const method of ['utimesSync', 'mkdirSync', 'readFileSync']) { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + utimesSync(lock, STALE, STALE); + const access = fs[method]; + let refused = false; + fs[method] = (target, ...args) => { + const matches = method === 'mkdirSync' ? String(target).startsWith(`${lock}.admission.`) + : target === (method === 'readFileSync' ? join(lock, 'owner') : lock); + if (matches && !refused) { + refused = true; + throw Object.assign(new Error('transient access refusal'), { code: 'EPERM' }); + } + return access(target, ...args); + }; + try { + assert.doesNotThrow(() => held.refresh(), method); + if (method !== 'readFileSync') assert.equal(statSync(lock).mtimeMs, STALE.getTime()); + } finally { + fs[method] = access; + } + assert.equal(refused, true, method); + held.refresh(); + assert.ok(statSync(lock).mtimeMs > STALE.getTime()); + }), true); + assert.deepEqual(readdirSync(dir), []); + } +}); + +test('release does not delete a missing or different owner', async (t) => { + for (const replacement of [null, `${process.pid}:another-owner`]) { + const { lock } = scratch(t); + assert.equal(await withLock(lock, async () => { + if (replacement === null) unlinkSync(join(lock, 'owner')); + else writeFileSync(join(lock, 'owner'), replacement); + }), true); + assert.equal(existsSync(lock), true); + assert.equal(existsSync(join(lock, 'owner')), replacement !== null); + if (replacement !== null) assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), replacement); + } +}); + test('release waits asynchronously for transient admission contention', async (t) => { const { lock } = scratch(t); let guard; From a638041595b46c05f827793a2fb6ca61a9bb3fc3 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 10:29:28 +0700 Subject: [PATCH 3/7] fix: recover stranded worker lock admission safely --- lib/runtime/store/file-lock.js | 61 +++++++++++++++++++++++++++------- test/runtime/file-lock.test.js | 31 +++++++++++++++++ 2 files changed, 80 insertions(+), 12 deletions(-) diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index e8732b1..8cd0c19 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -4,6 +4,7 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; +import { spawnSync } from 'node:child_process'; import { ownerIsGone } from '../process/owner.js'; @@ -11,6 +12,23 @@ const LOCK_WAIT_MS = 2_000; const LOCK_RETRY_MS = 20; const WORK_LOCK_STALE_MS = 2 * 60 * 1000; const CLAIM_STALE_MS = 60 * 1000; +const ADMISSION_IDENTITY_CHECK_MS = 10_000; +const abandonedAdmissions = new Map(); +let ownBirth; + +function processBirth(pid) { + const windows = process.platform === 'win32'; + const systemRoot = process.env.SystemRoot ?? 'C:\\Windows'; + if (windows && !path.isAbsolute(systemRoot)) return null; + const executable = windows ? path.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') : 'ps'; + const args = windows + ? ['-NoProfile', '-NonInteractive', '-Command', `(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToUniversalTime().Ticks`] + : ['-p', String(pid), '-o', 'lstart=']; + const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; + const result = spawnSync(executable, args, { env, encoding: 'utf8', windowsHide: true, timeout: windows ? 2500 : 1000 }); + const started = result.status === 0 ? result.stdout.trim() : ''; + return started ? crypto.createHash('sha256').update(started).digest('hex').slice(0, 24) : null; +} export function readState(target, fallback) { try { @@ -194,13 +212,23 @@ function ownsLock(lockDir, token) { function reclaimAdmission(admission) { try { - if (!fs.lstatSync(admission).isDirectory()) return false; + const info = fs.lstatSync(admission); + if (!info.isDirectory()) return false; const owners = fs.readdirSync(admission); if (owners.length > 1) return false; if (owners.length === 1) { - const owner = /^owner-(\d+)-[a-f0-9]+$/.exec(owners[0]); - if (!owner || !ownerIsGone(Number(owner[1]))) return false; + const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}))?$/.exec(owners[0]); + if (!owner) return false; + const pid = Number(owner[1]); + if (!Number.isSafeInteger(pid) || pid <= 0) return false; + const abandonedHere = abandonedAdmissions.get(admission) === owners[0]; + if (!abandonedHere && !ownerIsGone(pid)) { + if (!owner[2] || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS) return false; + const actualBirth = pid === process.pid ? ownBirth : processBirth(pid); + if (!actualBirth || actualBirth === owner[2]) return false; + } fs.unlinkSync(path.join(admission, owners[0])); + if (abandonedHere) abandonedAdmissions.delete(admission); } // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot // remove it even if another reclaimer replaced the directory after our read. @@ -213,7 +241,9 @@ function reclaimAdmission(admission) { function admissionOf(lockDir) { const admission = `${lockDir}.admission`; - const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}`; + if (ownBirth === undefined) ownBirth = processBirth(process.pid); + const birth = ownBirth ? `-${ownBirth}` : ''; + const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}${birth}`; const staging = `${admission}.${owner}.claim`; fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); fs.mkdirSync(staging, { mode: 0o700 }); @@ -223,6 +253,7 @@ function admissionOf(lockDir) { for (let attempt = 0; attempt < 2; attempt += 1) { try { fs.renameSync(staging, admission); + abandonedAdmissions.delete(admission); return { admission, owner }; } catch { if (attempt > 0 || !reclaimAdmission(admission)) return null; @@ -248,24 +279,27 @@ function retryLockAccess(access) { function releaseAdmission({ admission, owner }) { try { retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); - try { - retryLockAccess(() => fs.rmdirSync(admission)); - } catch (error) { - if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error?.code)) throw error; - } } catch (cause) { + abandonedAdmissions.set(admission, owner); throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); } + // The guard is released when its unique owner is gone. An empty directory can be reclaimed. + try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} } // Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. -function admitted(lockDir, mutate) { +function admitted(lockDir, mutate, rollback = () => {}) { const admission = admissionOf(lockDir); if (!admission) return null; try { return mutate(); } finally { - releaseAdmission(admission); + try { + releaseAdmission(admission); + } catch (error) { + rollback(); + throw error; + } } } @@ -282,6 +316,7 @@ async function releaseWorkLock(lockDir, token) { } function tryLock(lockDir, now) { + let token; return admitted(lockDir, () => { if (isStale(lockDir, now)) { let held; @@ -296,7 +331,7 @@ function tryLock(lockDir, now) { } catch { return null; } - const token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; + token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; fs.writeFileSync(path.join(lockDir, 'owner'), token); return { refresh: () => { @@ -311,6 +346,8 @@ function tryLock(lockDir, now) { }, release: () => releaseWorkLock(lockDir, token), }; + }, () => { + if (token && ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); }); } diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index 1156254..855aa91 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -266,7 +266,10 @@ test('an admission cleanup failure is surfaced instead of silently orphaning a l fs.unlinkSync = unlink; } assert.equal(entered, false); + assert.equal(existsSync(lock), false, 'the failed acquire must not leave a work lock'); assert.match(readdirSync(admission)[0], new RegExp(`^owner-${process.pid}-`)); + assert.equal(await withLock(lock, async () => { entered = true; }), true); + assert.equal(entered, true); }); test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { @@ -294,6 +297,34 @@ test('a transient admission cleanup refusal retries and leaves later work unbloc assert.deepEqual(readdirSync(dir), []); }); +test('an old admission stays held by the same process, but a reused PID can be recovered', async (t) => { + const { lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + return unlink(target); + }; + try { + await assert.rejects(withLock(lock, async () => assert.fail()), { code: 'ELOCKED' }); + } finally { + fs.unlinkSync = unlink; + } + const [original] = readdirSync(admission); + const birth = original.split('-')[3]; + assert.match(birth ?? '', /^[a-f0-9]{24}$/); + const replaceOwner = (name) => { + unlinkSync(join(admission, readdirSync(admission)[0])); + writeFileSync(join(admission, name), ''); + utimesSync(admission, STALE, STALE); + }; + replaceOwner(`owner-${process.pid}-a1-${birth}`); + assert.equal(await withLock(lock, async () => assert.fail('same process still owns the guard')), false); + const differentBirth = (birth[0] === '0' ? '1' : '0') + birth.slice(1); + replaceOwner(`owner-${process.pid}-a2-${differentBirth}`); + assert.equal(await withLock(lock, async () => {}), true); +}); + test('busy and unrecognised admission guards stay intact and leave no staging directories', async (t) => { const { dir, lock } = scratch(t); const guard = holdAdmission(lock); From 48a7f8312b0c4028cedc5aa9ea395e9dfc7a89ae Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 15:54:18 +0700 Subject: [PATCH 4/7] fix: preserve work errors when lock cleanup also fails --- lib/runtime/store/file-lock.js | 14 +++++++++++--- test/runtime/file-lock.test.js | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index 8cd0c19..2d2e91c 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -239,7 +239,7 @@ function reclaimAdmission(admission) { } } -function admissionOf(lockDir) { +function enterAdmission(lockDir) { const admission = `${lockDir}.admission`; if (ownBirth === undefined) ownBirth = processBirth(process.pid); const birth = ownBirth ? `-${ownBirth}` : ''; @@ -289,7 +289,7 @@ function releaseAdmission({ admission, owner }) { // Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. function admitted(lockDir, mutate, rollback = () => {}) { - const admission = admissionOf(lockDir); + const admission = enterAdmission(lockDir); if (!admission) return null; try { return mutate(); @@ -355,10 +355,18 @@ function tryLock(lockDir, now) { export async function withLock(lockDir, work, now = Date.now()) { const lock = tryLock(lockDir, now); if (!lock) return false; + let workFailed = false; try { await work(lock); return true; + } catch (error) { + workFailed = true; + throw error; } finally { - await lock.release(); + try { + await lock.release(); + } catch (error) { + if (!workFailed) throw error; + } } } diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index 855aa91..5e84cbd 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -153,6 +153,23 @@ test('release refuses an unreadable owner explicitly without claiming cleanup', } }); +test('a release failure preserves the original work error', async (t) => { + const { lock } = scratch(t); + const read = fs.readFileSync; + const workError = new Error('work failed'); + try { + await assert.rejects(withLock(lock, async () => { + fs.readFileSync = (file, ...args) => { + if (String(file) === join(lock, 'owner')) throw Object.assign(new Error('unreadable owner'), { code: 'EIO' }); + return read(file, ...args); + }; + throw workError; + }), (error) => error === workError); + } finally { + fs.readFileSync = read; + } +}); + test('refresh leaves an unreadable live lock untouched without crashing its timer caller', async (t) => { const { dir, lock } = scratch(t); assert.equal(await withLock(lock, async (held) => { From d1fc44ffe2c5a5ca0ab3d67aa5e064a91cb8a6c5 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 18:03:57 +0700 Subject: [PATCH 5/7] Recover admission without spawning on uncontended locks --- lib/runtime/process/owner.js | 22 +++++ lib/runtime/store/admission.js | 123 ++++++++++++++++++++++++++++ lib/runtime/store/file-lock.js | 140 ++++---------------------------- test/helpers/lock-contenders.js | 39 +++++++++ test/runtime/file-lock.test.js | 112 +++++++++++++++---------- 5 files changed, 270 insertions(+), 166 deletions(-) create mode 100644 lib/runtime/store/admission.js create mode 100644 test/helpers/lock-contenders.js diff --git a/lib/runtime/process/owner.js b/lib/runtime/process/owner.js index 8d39208..c9ea00e 100644 --- a/lib/runtime/process/owner.js +++ b/lib/runtime/process/owner.js @@ -8,6 +8,28 @@ import path from 'node:path'; const SHELLS = new Set(['sh', 'bash', 'zsh', 'dash', 'ksh', 'fish', 'cmd.exe', 'powershell', 'pwsh']); const PS_TIMEOUT_MS = 1000; const MAX_ANCESTORS = 16; +const ownStartTime = Math.round(Date.now() - process.uptime() * 1000); + +/** A bounded process start-time probe for admission recovery. The hot path needs no subprocess. */ +export function processStartTime(pid, timeout = PS_TIMEOUT_MS) { + if (pid === process.pid) return ownStartTime; + if (!Number.isSafeInteger(pid) || pid <= 0 || timeout <= 0) return null; + const windows = process.platform === 'win32'; + const systemRoot = process.env.SystemRoot ?? 'C:\\Windows'; + if (windows && !path.isAbsolute(systemRoot)) return null; + const executable = windows ? path.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') : '/bin/ps'; + const args = windows + ? ['-NoProfile', '-NonInteractive', '-Command', `([DateTimeOffset] (Get-Process -Id ${pid} -ErrorAction Stop).StartTime).ToUnixTimeMilliseconds()`] + : ['-p', String(pid), '-o', 'lstart=']; + const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; + const result = spawnSync(executable, args, { + env, encoding: 'utf8', windowsHide: true, timeout: Math.min(timeout, PS_TIMEOUT_MS), killSignal: 'SIGKILL', + }); + const started = result.status === 0 ? result.stdout.trim() : ''; + if (!started) return null; + const time = windows ? Number(started) : Date.parse(started + ' GMT'); + return Number.isSafeInteger(time) && time > 0 ? time : null; +} /** * Whether the process that wrote a state entry has exited. Signal 0 only probes: ESRCH means no such process; diff --git a/lib/runtime/store/admission.js b/lib/runtime/store/admission.js new file mode 100644 index 0000000..61f0e17 --- /dev/null +++ b/lib/runtime/store/admission.js @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 EvoMap + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { ownerIsGone, processStartTime } from '../process/owner.js'; + +export const LOCK_WAIT_MS = 2_000; +export const LOCK_RETRY_MS = 20; +export const LOCK_ACCESS_ERRORS = new Set(['EPERM', 'EBUSY', 'EACCES']); +const ADMISSION_IDENTITY_CHECK_MS = 10_000; +const START_TIME_TOLERANCE_MS = 2_000; +const abandonedAdmissions = new Map(); + +export function pause(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function reclaimAdmission(admission, deadline, starts) { + try { + const info = fs.lstatSync(admission); + if (!info.isDirectory()) return false; + const owners = fs.readdirSync(admission); + if (owners.length > 1) return false; + if (owners.length === 1) { + const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}|[0-9]{13}))?$/.exec(owners[0]); + if (!owner) return false; + const pid = Number(owner[1]); + if (!Number.isSafeInteger(pid) || pid <= 0) return false; + const abandonedHere = abandonedAdmissions.get(admission) === owners[0]; + if (!abandonedHere && !ownerIsGone(pid)) { + const birth = Number(owner[2]); + const remaining = deadline - Date.now(); + if (owner[2]?.length !== 13 || !Number.isSafeInteger(birth) || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS || remaining <= 0) return false; + if (!starts.has(pid)) starts.set(pid, processStartTime(pid, remaining)); + const actualBirth = starts.get(pid); + // Only a clearly newer process proves reuse. ps timestamps round to whole seconds. + if (!actualBirth || actualBirth <= birth + START_TIME_TOLERANCE_MS) return false; + } + fs.unlinkSync(path.join(admission, owners[0])); + if (abandonedHere) abandonedAdmissions.delete(admission); + } + // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot + // remove it even if another reclaimer replaced the directory after our read. + fs.rmdirSync(admission); + return true; + } catch { + return false; + } +} + +function enterAdmission(lockDir, deadline, starts) { + const admission = `${lockDir}.admission`; + const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}-${processStartTime(process.pid)}`; + const staging = `${admission}.${owner}.claim`; + fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); + fs.mkdirSync(staging, { mode: 0o700 }); + let published = false; + try { + fs.writeFileSync(path.join(staging, owner), '', { mode: 0o600, flag: 'wx' }); + // Publishing a prepopulated directory never exposes an empty fresh guard to a stale reclaimer. + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + fs.renameSync(staging, admission); + published = true; + abandonedAdmissions.delete(admission); + return { admission, owner }; + } catch { + if (attempt > 0 || !reclaimAdmission(admission, deadline, starts)) return null; + } + } + return null; + } finally { + if (!published) fs.rmSync(staging, { recursive: true, force: true }); + } +} + +export function retryLockAccess(access) { + for (let attempt = 0; ; attempt += 1) { + try { + return access(); + } catch (error) { + if (attempt === 2 || !LOCK_ACCESS_ERRORS.has(error?.code)) throw error; + pause(LOCK_RETRY_MS); + } + } +} + +function releaseAdmission({ admission, owner }) { + try { + retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); + } catch (cause) { + abandonedAdmissions.set(admission, owner); + throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + } + // The guard is released when its unique owner is gone. An empty directory can be reclaimed. + try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} +} + +// Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. +export function admitted(lockDir, mutate, rollback, deadline = Date.now() + LOCK_WAIT_MS, starts = new Map()) { + const admission = enterAdmission(lockDir, deadline, starts); + if (!admission) return null; + let result; + try { + result = mutate(); + } catch (error) { + try { releaseAdmission(admission); } catch {} + throw error; + } + try { + releaseAdmission(admission); + } catch (error) { + // After an irreversible release, report its result. The stranded guard is tracked for recovery. + if (rollback) { + try { rollback(); } catch (rollbackError) { error.rollbackError = rollbackError; } + throw error; + } + } + return result; +} diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index 2d2e91c..f28610a 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -4,31 +4,14 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import { spawnSync } from 'node:child_process'; import { ownerIsGone } from '../process/owner.js'; -const LOCK_WAIT_MS = 2_000; -const LOCK_RETRY_MS = 20; +import { admitted, LOCK_WAIT_MS, LOCK_RETRY_MS, pause, retryLockAccess } from './admission.js'; + const WORK_LOCK_STALE_MS = 2 * 60 * 1000; const CLAIM_STALE_MS = 60 * 1000; -const ADMISSION_IDENTITY_CHECK_MS = 10_000; -const abandonedAdmissions = new Map(); -let ownBirth; - -function processBirth(pid) { - const windows = process.platform === 'win32'; - const systemRoot = process.env.SystemRoot ?? 'C:\\Windows'; - if (windows && !path.isAbsolute(systemRoot)) return null; - const executable = windows ? path.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') : 'ps'; - const args = windows - ? ['-NoProfile', '-NonInteractive', '-Command', `(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToUniversalTime().Ticks`] - : ['-p', String(pid), '-o', 'lstart=']; - const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; - const result = spawnSync(executable, args, { env, encoding: 'utf8', windowsHide: true, timeout: windows ? 2500 : 1000 }); - const started = result.status === 0 ? result.stdout.trim() : ''; - return started ? crypto.createHash('sha256').update(started).digest('hex').slice(0, 24) : null; -} +const abandonedWork = new Map(); export function readState(target, fallback) { try { @@ -59,11 +42,6 @@ export function removeState(target) { } } -// Atomics.wait is the only sleep that holds a synchronous read-modify-write together. -function pause(ms) { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); -} - function contentOf(file) { try { return fs.readFileSync(file, 'utf8'); @@ -210,106 +188,14 @@ function ownsLock(lockDir, token) { return workOwnerOf(lockDir) === token; } -function reclaimAdmission(admission) { - try { - const info = fs.lstatSync(admission); - if (!info.isDirectory()) return false; - const owners = fs.readdirSync(admission); - if (owners.length > 1) return false; - if (owners.length === 1) { - const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}))?$/.exec(owners[0]); - if (!owner) return false; - const pid = Number(owner[1]); - if (!Number.isSafeInteger(pid) || pid <= 0) return false; - const abandonedHere = abandonedAdmissions.get(admission) === owners[0]; - if (!abandonedHere && !ownerIsGone(pid)) { - if (!owner[2] || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS) return false; - const actualBirth = pid === process.pid ? ownBirth : processBirth(pid); - if (!actualBirth || actualBirth === owner[2]) return false; - } - fs.unlinkSync(path.join(admission, owners[0])); - if (abandonedHere) abandonedAdmissions.delete(admission); - } - // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot - // remove it even if another reclaimer replaced the directory after our read. - fs.rmdirSync(admission); - return true; - } catch { - return false; - } -} - -function enterAdmission(lockDir) { - const admission = `${lockDir}.admission`; - if (ownBirth === undefined) ownBirth = processBirth(process.pid); - const birth = ownBirth ? `-${ownBirth}` : ''; - const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}${birth}`; - const staging = `${admission}.${owner}.claim`; - fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); - fs.mkdirSync(staging, { mode: 0o700 }); - try { - fs.writeFileSync(path.join(staging, owner), '', { mode: 0o600, flag: 'wx' }); - // Publishing a prepopulated directory never exposes an empty fresh guard to a stale reclaimer. - for (let attempt = 0; attempt < 2; attempt += 1) { - try { - fs.renameSync(staging, admission); - abandonedAdmissions.delete(admission); - return { admission, owner }; - } catch { - if (attempt > 0 || !reclaimAdmission(admission)) return null; - } - } - return null; - } finally { - fs.rmSync(staging, { recursive: true, force: true }); - } -} - -function retryLockAccess(access) { - for (let attempt = 0; ; attempt += 1) { - try { - return access(); - } catch (error) { - if (attempt === 2 || !['EPERM', 'EBUSY', 'EACCES'].includes(error?.code)) throw error; - pause(LOCK_RETRY_MS); - } - } -} - -function releaseAdmission({ admission, owner }) { - try { - retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); - } catch (cause) { - abandonedAdmissions.set(admission, owner); - throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); - } - // The guard is released when its unique owner is gone. An empty directory can be reclaimed. - try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} -} - -// Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. -function admitted(lockDir, mutate, rollback = () => {}) { - const admission = enterAdmission(lockDir); - if (!admission) return null; - try { - return mutate(); - } finally { - try { - releaseAdmission(admission); - } catch (error) { - rollback(); - throw error; - } - } -} - async function releaseWorkLock(lockDir, token) { const deadline = Date.now() + LOCK_WAIT_MS; + const starts = new Map(); for (;;) { if (admitted(lockDir, () => { if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); return true; - })) return; + }, null, deadline, starts)) return; if (Date.now() >= deadline) throw Object.assign(new Error(`Could not release lock ${lockDir}`), { code: 'ELOCKED' }); await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS)); } @@ -318,13 +204,15 @@ async function releaseWorkLock(lockDir, token) { function tryLock(lockDir, now) { let token; return admitted(lockDir, () => { - if (isStale(lockDir, now)) { + const stale = isStale(lockDir, now); + if (abandonedWork.has(lockDir) || stale) { let held; try { held = workOwnerOf(lockDir); } catch { return null; } // Old versions wrote random tokens without a pid. Keep their lease recovery, but a known live owner // cannot lose its lock just because a slow or paused worker missed a refresh. A reused pid stays busy. - if (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))) return null; + if (held !== abandonedWork.get(lockDir) && (!stale || (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))))) return null; fs.rmSync(lockDir, { recursive: true, force: true }); + abandonedWork.delete(lockDir); } try { fs.mkdirSync(lockDir); @@ -344,10 +232,16 @@ function tryLock(lockDir, now) { return null; } }, - release: () => releaseWorkLock(lockDir, token), + release: async () => { + try { await releaseWorkLock(lockDir, token); } + catch (error) { abandonedWork.set(lockDir, token); throw error; } + }, }; }, () => { - if (token && ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); + if (!token) return; + abandonedWork.set(lockDir, token); + if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); + abandonedWork.delete(lockDir); }); } diff --git a/test/helpers/lock-contenders.js b/test/helpers/lock-contenders.js new file mode 100644 index 0000000..1e412a7 --- /dev/null +++ b/test/helpers/lock-contenders.js @@ -0,0 +1,39 @@ +import { spawn } from 'node:child_process'; +import { rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export async function until(check) { + const deadline = Date.now() + 10_000; + while (!check()) { + if (Date.now() >= deadline) throw new Error('timed out waiting for a lock contender'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +export function lockContenders(t, dir, script, args, cleanupMarkers) { + const children = []; + const results = () => Promise.all(children); + t.after(async () => { + try { + for (const name of cleanupMarkers) writeFileSync(join(dir, name), ''); + } finally { + await results(); + rmSync(dir, { recursive: true, force: true }); + } + }); + return { + run(role) { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, ...args, role], { windowsHide: true }); + let stderr = ''; + const watchdog = setTimeout(() => child.kill('SIGKILL'), 15_000); + child.stderr.on('data', (data) => { stderr += data; }); + child.on('error', (error) => { stderr += String(error); }); + children.push(new Promise((resolve) => child.on('close', (code) => { + clearTimeout(watchdog); + resolve({ code, stderr }); + }))); + return child; + }, + results, + }; +} diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index 5e84cbd..d6b5261 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; -import { spawn, spawnSync } from 'node:child_process'; +import childProcess, { spawnSync } from 'node:child_process'; +import { syncBuiltinESMExports } from 'node:module'; import fs from 'node:fs'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, rmdirSync, statSync, unlinkSync, utimesSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -7,6 +8,7 @@ import { dirname, join } from 'node:path'; import { test } from 'node:test'; import { withLock } from '../../lib/runtime/store/file-lock.js'; +import { lockContenders, until } from '../helpers/lock-contenders.js'; const RUNTIME = new URL('../../lib/runtime/store/file-lock.js', import.meta.url).href; const STALE = new Date(0); @@ -22,14 +24,6 @@ function scratch(t, token = null) { return { dir, lock }; } -async function until(check) { - const deadline = Date.now() + 10_000; - while (!check()) { - if (Date.now() > deadline) throw new Error('timed out waiting for the lock contender'); - await new Promise((resolve) => setTimeout(resolve, 10)); - } -} - function holdAdmission(lock, pid = process.pid) { const admission = `${lock}.admission`; const owner = join(admission, `owner-${pid}-deadbeef`); @@ -38,6 +32,21 @@ function holdAdmission(lock, pid = process.pid) { return { admission, owner, release: () => { unlinkSync(owner); rmdirSync(admission); } }; } +test('taking an uncontended lock does not launch a process lookup', async (t) => { + const { lock } = scratch(t); + let launches = 0; + const spawn = childProcess.spawnSync; + childProcess.spawnSync = (...args) => { launches += 1; return spawn(...args); }; + syncBuiltinESMExports(); + try { + assert.equal(await withLock(lock, () => {}), true); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(launches, 0); +}); + test('stale legacy and dead-owner directories recover, but a fresh legacy lock stays held', async (t) => { const { lock } = scratch(t, 'legacy token'); let entered = 0; @@ -150,9 +159,35 @@ test('release refuses an unreadable owner explicitly without claiming cleanup', fs.readFileSync = read; } assert.equal(readFileSync(owner, 'utf8'), token, code); + assert.equal(await withLock(lock, () => {}), true, 'known abandoned work is recoverable after access returns'); } }); +test('a stale foreign admission is probed once within the release deadline', async (t) => { + const { lock } = scratch(t); + const spawn = childProcess.spawnSync; + const timeouts = []; + try { + await assert.rejects(withLock(lock, () => { + const guard = holdAdmission(lock, process.ppid); + unlinkSync(guard.owner); + writeFileSync(join(guard.admission, `owner-${process.ppid}-a1-${Date.now() - 30_000}`), ''); + utimesSync(guard.admission, STALE, STALE); + childProcess.spawnSync = (_command, _args, { timeout }) => { + timeouts.push(timeout); + return { status: null, stdout: '' }; + }; + syncBuiltinESMExports(); + }), { code: 'ELOCKED' }); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(timeouts.length, 1); + assert.ok(timeouts[0] > 0 && timeouts[0] <= 1_000); + assert.equal(existsSync(lock), true, 'an inconclusive identity probe does not reclaim a live guard'); +}); + test('a release failure preserves the original work error', async (t) => { const { lock } = scratch(t); const read = fs.readFileSync; @@ -289,6 +324,23 @@ test('an admission cleanup failure is surfaced instead of silently orphaning a l assert.equal(entered, true); }); +test('successful work is not reported as failed after its lock was removed', async (t) => { + const { lock } = scratch(t); + const unlink = fs.unlinkSync; + try { + assert.equal(await withLock(lock, () => { + fs.unlinkSync = (target) => { + if (dirname(target) === `${lock}.admission`) throw Object.assign(new Error('fixture cleanup refusal'), { code: 'EPERM' }); + return unlink(target); + }; + }), true); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(existsSync(lock), false); + assert.equal(await withLock(lock, () => {}), true, 'the stranded admission can be reclaimed'); +}); + test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { const { dir, lock } = scratch(t); const unlink = fs.unlinkSync; @@ -329,7 +381,7 @@ test('an old admission stays held by the same process, but a reused PID can be r } const [original] = readdirSync(admission); const birth = original.split('-')[3]; - assert.match(birth ?? '', /^[a-f0-9]{24}$/); + assert.match(birth ?? '', /^\d{13}$/); const replaceOwner = (name) => { unlinkSync(join(admission, readdirSync(admission)[0])); writeFileSync(join(admission, name), ''); @@ -337,7 +389,7 @@ test('an old admission stays held by the same process, but a reused PID can be r }; replaceOwner(`owner-${process.pid}-a1-${birth}`); assert.equal(await withLock(lock, async () => assert.fail('same process still owns the guard')), false); - const differentBirth = (birth[0] === '0' ? '1' : '0') + birth.slice(1); + const differentBirth = Number(birth) - 10_000; replaceOwner(`owner-${process.pid}-a2-${differentBirth}`); assert.equal(await withLock(lock, async () => {}), true); }); @@ -399,20 +451,7 @@ test('two contenders cannot replace the fresh lock while another reclaims a stal }); fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); `; - const children = []; - const run = (role) => { - const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, lock, role]); - let stderr = ''; - child.stderr.on('data', (data) => { stderr += data; }); - const done = new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr }))); - children.push({ child, done }); - }; - t.after(async () => { - writeFileSync(marker('resume'), ''); - writeFileSync(marker('release'), ''); - await Promise.all(children.map(({ done }) => done)); - rmSync(dir, { recursive: true, force: true }); - }); + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['resume', 'release']); run('A'); await until(() => existsSync(marker('stale-read'))); run('B'); @@ -422,8 +461,8 @@ test('two contenders cannot replace the fresh lock while another reclaims a stal await until(() => existsSync(marker('A-entered'))); const entered = ['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))); writeFileSync(marker('release'), ''); - const results = await Promise.all(children.map(({ done }) => done)); - assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); assert.deepEqual(entered, ['A'], 'only the process reclaiming the inspected stale directory may enter'); assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [true, false, false]); }); @@ -487,20 +526,7 @@ test('a delayed dead-admission reclaimer cannot unlink the fresh owner another c }); fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); `; - const children = []; - const run = (role) => { - const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, lock, role]); - let stderr = ''; - child.stderr.on('data', (data) => { stderr += data; }); - const done = new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr }))); - children.push({ child, done }); - return child; - }; - t.after(async () => { - for (const name of ['A-resume', 'B-resume', 'release']) writeFileSync(marker(name), ''); - await Promise.all(children.map(({ done }) => done)); - rmSync(dir, { recursive: true, force: true }); - }); + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['A-resume', 'B-resume', 'release']); run('A'); await until(() => existsSync(marker('dead-inspected'))); const fresh = run('B'); @@ -517,7 +543,7 @@ test('a delayed dead-admission reclaimer cannot unlink the fresh owner another c await until(() => existsSync(marker('B-entered'))); assert.deepEqual(['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))), ['B']); writeFileSync(marker('release'), ''); - const results = await Promise.all(children.map(({ done }) => done)); - assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [false, true, false]); }); From a0e143e6362f2f0fb9777dd1164bb9bb77a6bfb1 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 11:01:01 +0700 Subject: [PATCH 6/7] Make abandoned admission guards recoverable across processes --- lib/runtime/store/admission.js | 45 ++++++++++++++++++++++------------ test/runtime/file-lock.test.js | 39 +++++++++++++++++++++++------ 2 files changed, 62 insertions(+), 22 deletions(-) diff --git a/lib/runtime/store/admission.js b/lib/runtime/store/admission.js index 61f0e17..e9ec4ab 100644 --- a/lib/runtime/store/admission.js +++ b/lib/runtime/store/admission.js @@ -12,6 +12,7 @@ export const LOCK_RETRY_MS = 20; export const LOCK_ACCESS_ERRORS = new Set(['EPERM', 'EBUSY', 'EACCES']); const ADMISSION_IDENTITY_CHECK_MS = 10_000; const START_TIME_TOLERANCE_MS = 2_000; +const ABANDONED_PREFIX = 'abandoned-'; const abandonedAdmissions = new Map(); export function pause(ms) { @@ -22,15 +23,19 @@ function reclaimAdmission(admission, deadline, starts) { try { const info = fs.lstatSync(admission); if (!info.isDirectory()) return false; - const owners = fs.readdirSync(admission); - if (owners.length > 1) return false; - if (owners.length === 1) { - const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}|[0-9]{13}))?$/.exec(owners[0]); + const entries = fs.readdirSync(admission); + if (entries.length > 0) { + const marker = entries.find((name) => name.startsWith(ABANDONED_PREFIX)); + const ownerName = marker ? marker.slice(ABANDONED_PREFIX.length) : entries[0]; + if (entries.some((name) => name !== ownerName && name !== marker)) return false; + const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}|[0-9]{13}))?$/.exec(ownerName); if (!owner) return false; const pid = Number(owner[1]); if (!Number.isSafeInteger(pid) || pid <= 0) return false; - const abandonedHere = abandonedAdmissions.get(admission) === owners[0]; - if (!abandonedHere && !ownerIsGone(pid)) { + const abandonedHere = abandonedAdmissions.get(admission) === ownerName; + const markerPath = marker && path.join(admission, marker); + if (markerPath && !fs.lstatSync(markerPath).isFile()) return false; + if (!markerPath && !abandonedHere && !ownerIsGone(pid)) { const birth = Number(owner[2]); const remaining = deadline - Date.now(); if (owner[2]?.length !== 13 || !Number.isSafeInteger(birth) || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS || remaining <= 0) return false; @@ -39,7 +44,8 @@ function reclaimAdmission(admission, deadline, starts) { // Only a clearly newer process proves reuse. ps timestamps round to whole seconds. if (!actualBirth || actualBirth <= birth + START_TIME_TOLERANCE_MS) return false; } - fs.unlinkSync(path.join(admission, owners[0])); + if (entries.includes(ownerName)) fs.unlinkSync(path.join(admission, ownerName)); + if (markerPath) fs.unlinkSync(markerPath); if (abandonedHere) abandonedAdmissions.delete(admission); } // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot @@ -92,13 +98,23 @@ function releaseAdmission({ admission, owner }) { try { retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); } catch (cause) { - abandonedAdmissions.set(admission, owner); - throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + if (cause?.code !== 'ENOENT') { + throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + } } // The guard is released when its unique owner is gone. An empty directory can be reclaimed. try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} } +function abandonAdmission({ admission, owner }) { + // A separate marker can be written even when Windows denies access to the owner file. + try { + retryLockAccess(() => fs.writeFileSync(path.join(admission, ABANDONED_PREFIX + owner), '', { mode: 0o600, flag: 'wx' })); + } catch { + abandonedAdmissions.set(admission, owner); + } +} + // Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. export function admitted(lockDir, mutate, rollback, deadline = Date.now() + LOCK_WAIT_MS, starts = new Map()) { const admission = enterAdmission(lockDir, deadline, starts); @@ -107,17 +123,16 @@ export function admitted(lockDir, mutate, rollback, deadline = Date.now() + LOCK try { result = mutate(); } catch (error) { - try { releaseAdmission(admission); } catch {} + try { releaseAdmission(admission); } catch { abandonAdmission(admission); } throw error; } try { releaseAdmission(admission); } catch (error) { - // After an irreversible release, report its result. The stranded guard is tracked for recovery. - if (rollback) { - try { rollback(); } catch (rollbackError) { error.rollbackError = rollbackError; } - throw error; - } + // Publish only after rollback, so another process cannot enter while it runs. + try { rollback?.(); } catch (rollbackError) { error.rollbackError = rollbackError; } + abandonAdmission(admission); + if (rollback) throw error; } return result; } diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index d6b5261..efeba28 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -319,18 +319,19 @@ test('an admission cleanup failure is surfaced instead of silently orphaning a l } assert.equal(entered, false); assert.equal(existsSync(lock), false, 'the failed acquire must not leave a work lock'); - assert.match(readdirSync(admission)[0], new RegExp(`^owner-${process.pid}-`)); + assert.ok(readdirSync(admission).some((name) => name.startsWith(`owner-${process.pid}-`))); assert.equal(await withLock(lock, async () => { entered = true; }), true); assert.equal(entered, true); }); -test('successful work is not reported as failed after its lock was removed', async (t) => { - const { lock } = scratch(t); +test('another process reclaims a failed admission release while the original owner is alive', async (t) => { + const { dir, lock } = scratch(t); + const admission = `${lock}.admission`; const unlink = fs.unlinkSync; try { assert.equal(await withLock(lock, () => { fs.unlinkSync = (target) => { - if (dirname(target) === `${lock}.admission`) throw Object.assign(new Error('fixture cleanup refusal'), { code: 'EPERM' }); + if (dirname(target) === admission) throw Object.assign(new Error('fixture cleanup refusal'), { code: 'EPERM' }); return unlink(target); }; }), true); @@ -338,7 +339,31 @@ test('successful work is not reported as failed after its lock was removed', asy fs.unlinkSync = unlink; } assert.equal(existsSync(lock), false); - assert.equal(await withLock(lock, () => {}), true, 'the stranded admission can be reclaimed'); + assert.ok(readdirSync(admission).some((name) => name.startsWith(`owner-${process.pid}-`))); + + const other = spawnSync(process.execPath, ['--input-type=module', '-e', ` + import assert from 'node:assert/strict'; + const [runtime, lock, owner] = process.argv.slice(1); + const { withLock } = await import(runtime); + process.kill(Number(owner), 0); + assert.equal(await withLock(lock, () => {}), true, 'the stranded admission can be reclaimed'); + `, RUNTIME, lock, String(process.pid)], { encoding: 'utf8', timeout: 10_000, windowsHide: true }); + assert.ifError(other.error); + assert.equal(other.status, 0, other.stderr); + assert.deepEqual(readdirSync(dir), []); +}); + +test('admission recovery resumes after the abandoned owner has already been removed', async (t) => { + const { dir, lock } = scratch(t); + const guard = holdAdmission(lock); + const marker = join(guard.admission, `abandoned-owner-${process.pid}-abcd`); + writeFileSync(marker, ''); + assert.equal(await withLock(lock, () => assert.fail('a different owner is still active')), false); + assert.equal(existsSync(guard.owner), true); + + unlinkSync(guard.owner); + assert.equal(await withLock(lock, () => {}), true); + assert.deepEqual(readdirSync(dir), []); }); test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { @@ -379,11 +404,11 @@ test('an old admission stays held by the same process, but a reused PID can be r } finally { fs.unlinkSync = unlink; } - const [original] = readdirSync(admission); + const original = readdirSync(admission).find((name) => name.startsWith(`owner-${process.pid}-`)); const birth = original.split('-')[3]; assert.match(birth ?? '', /^\d{13}$/); const replaceOwner = (name) => { - unlinkSync(join(admission, readdirSync(admission)[0])); + for (const name of readdirSync(admission)) unlinkSync(join(admission, name)); writeFileSync(join(admission, name), ''); utimesSync(admission, STALE, STALE); }; From 0490c99a60629c40f6a4be2d6ef70a580f772dd2 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 12:23:26 +0700 Subject: [PATCH 7/7] Reclaim stale worker locks after confirmed PID reuse --- lib/runtime/process/owner.js | 12 +++++---- lib/runtime/store/admission.js | 7 +++-- lib/runtime/store/file-lock.js | 31 +++++++++++++++++----- test/runtime/file-lock.test.js | 47 ++++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 16 deletions(-) diff --git a/lib/runtime/process/owner.js b/lib/runtime/process/owner.js index c9ea00e..5ea48fa 100644 --- a/lib/runtime/process/owner.js +++ b/lib/runtime/process/owner.js @@ -6,12 +6,14 @@ import { readlinkSync } from 'node:fs'; import path from 'node:path'; const SHELLS = new Set(['sh', 'bash', 'zsh', 'dash', 'ksh', 'fish', 'cmd.exe', 'powershell', 'pwsh']); -const PS_TIMEOUT_MS = 1000; +export const PROCESS_START_TIMEOUT_MS = 1_000; +// POSIX ps reports start times to the nearest second. +export const PROCESS_START_TOLERANCE_MS = 2_000; const MAX_ANCESTORS = 16; const ownStartTime = Math.round(Date.now() - process.uptime() * 1000); -/** A bounded process start-time probe for admission recovery. The hot path needs no subprocess. */ -export function processStartTime(pid, timeout = PS_TIMEOUT_MS) { +/** A bounded process start-time probe for lock recovery. The hot path needs no subprocess. */ +export function processStartTime(pid, timeout = PROCESS_START_TIMEOUT_MS) { if (pid === process.pid) return ownStartTime; if (!Number.isSafeInteger(pid) || pid <= 0 || timeout <= 0) return null; const windows = process.platform === 'win32'; @@ -23,7 +25,7 @@ export function processStartTime(pid, timeout = PS_TIMEOUT_MS) { : ['-p', String(pid), '-o', 'lstart=']; const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; const result = spawnSync(executable, args, { - env, encoding: 'utf8', windowsHide: true, timeout: Math.min(timeout, PS_TIMEOUT_MS), killSignal: 'SIGKILL', + env, encoding: 'utf8', windowsHide: true, timeout, killSignal: 'SIGKILL', }); const started = result.status === 0 ? result.stdout.trim() : ''; if (!started) return null; @@ -57,7 +59,7 @@ function executablePathOf(pid, command) { } export function processInfo(pid) { - const probe = spawnSync('ps', ['-o', 'ppid=,comm=', '-p', String(pid)], { encoding: 'utf8', timeout: PS_TIMEOUT_MS }); + const probe = spawnSync('ps', ['-o', 'ppid=,comm=', '-p', String(pid)], { encoding: 'utf8', timeout: PROCESS_START_TIMEOUT_MS }); const match = /^\s*(\d+)\s+(.+?)\s*$/.exec(probe.stdout ?? ''); if (!match) return null; const command = match[2].replace(/^-/, ''); diff --git a/lib/runtime/store/admission.js b/lib/runtime/store/admission.js index e9ec4ab..b416d0b 100644 --- a/lib/runtime/store/admission.js +++ b/lib/runtime/store/admission.js @@ -5,13 +5,12 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import { ownerIsGone, processStartTime } from '../process/owner.js'; +import { ownerIsGone, processStartTime, PROCESS_START_TIMEOUT_MS, PROCESS_START_TOLERANCE_MS } from '../process/owner.js'; export const LOCK_WAIT_MS = 2_000; export const LOCK_RETRY_MS = 20; export const LOCK_ACCESS_ERRORS = new Set(['EPERM', 'EBUSY', 'EACCES']); const ADMISSION_IDENTITY_CHECK_MS = 10_000; -const START_TIME_TOLERANCE_MS = 2_000; const ABANDONED_PREFIX = 'abandoned-'; const abandonedAdmissions = new Map(); @@ -39,10 +38,10 @@ function reclaimAdmission(admission, deadline, starts) { const birth = Number(owner[2]); const remaining = deadline - Date.now(); if (owner[2]?.length !== 13 || !Number.isSafeInteger(birth) || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS || remaining <= 0) return false; - if (!starts.has(pid)) starts.set(pid, processStartTime(pid, remaining)); + if (!starts.has(pid)) starts.set(pid, processStartTime(pid, Math.min(remaining, PROCESS_START_TIMEOUT_MS))); const actualBirth = starts.get(pid); // Only a clearly newer process proves reuse. ps timestamps round to whole seconds. - if (!actualBirth || actualBirth <= birth + START_TIME_TOLERANCE_MS) return false; + if (!actualBirth || actualBirth <= birth + PROCESS_START_TOLERANCE_MS) return false; } if (entries.includes(ownerName)) fs.unlinkSync(path.join(admission, ownerName)); if (markerPath) fs.unlinkSync(markerPath); diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index f28610a..6a234ef 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -5,11 +5,13 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import { ownerIsGone } from '../process/owner.js'; +import { ownerIsGone, processStartTime, PROCESS_START_TIMEOUT_MS, PROCESS_START_TOLERANCE_MS } from '../process/owner.js'; import { admitted, LOCK_WAIT_MS, LOCK_RETRY_MS, pause, retryLockAccess } from './admission.js'; const WORK_LOCK_STALE_MS = 2 * 60 * 1000; +const WORK_OWNER_PROBE_MS = process.platform === 'win32' ? 5_000 : PROCESS_START_TIMEOUT_MS; +const WORK_OWNER_WITH_START = /^\d+:[a-f0-9]+:([1-9]\d{12})$/; const CLAIM_STALE_MS = 60 * 1000; const abandonedWork = new Map(); @@ -202,15 +204,23 @@ async function releaseWorkLock(lockDir, token) { } function tryLock(lockDir, now) { - let token; - return admitted(lockDir, () => { + let token, suspect, reusedToken; + const take = () => admitted(lockDir, () => { const stale = isStale(lockDir, now); if (abandonedWork.has(lockDir) || stale) { let held; try { held = workOwnerOf(lockDir); } catch { return null; } - // Old versions wrote random tokens without a pid. Keep their lease recovery, but a known live owner - // cannot lose its lock just because a slow or paused worker missed a refresh. A reused pid stays busy. - if (held !== abandonedWork.get(lockDir) && (!stale || (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))))) return null; + if (held !== abandonedWork.get(lockDir)) { + if (!stale) return null; + if (held !== reusedToken && /^\d+:/.test(held ?? '')) { + const pid = ownerPidOf(held); + if (!ownerIsGone(pid)) { + const birth = WORK_OWNER_WITH_START.exec(held)?.[1]; + if (birth) suspect = { token: held, pid, birth: Number(birth) }; + return null; + } + } + } fs.rmSync(lockDir, { recursive: true, force: true }); abandonedWork.delete(lockDir); } @@ -219,7 +229,7 @@ function tryLock(lockDir, now) { } catch { return null; } - token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; + token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}:${processStartTime(process.pid)}`; fs.writeFileSync(path.join(lockDir, 'owner'), token); return { refresh: () => { @@ -243,6 +253,13 @@ function tryLock(lockDir, now) { if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); abandonedWork.delete(lockDir); }); + const lock = take(); + if (lock || !suspect) return lock; + // A foreign lookup can be slow. Release admission, then recheck staleness and the exact token. + const actualBirth = processStartTime(suspect.pid, WORK_OWNER_PROBE_MS); + if (!actualBirth || actualBirth <= suspect.birth + PROCESS_START_TOLERANCE_MS) return null; + reusedToken = suspect.token; + return take(); } /** Runs slow `work` under one owner's lock; dead or legacy directories recover after their lease expires. */ diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index efeba28..0e40c47 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -7,6 +7,7 @@ import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { test } from 'node:test'; +import { processStartTime } from '../../lib/runtime/process/owner.js'; import { withLock } from '../../lib/runtime/store/file-lock.js'; import { lockContenders, until } from '../helpers/lock-contenders.js'; @@ -79,6 +80,52 @@ test('an old mtime never evicts a live owner, which can refresh and release its assert.deepEqual(readdirSync(dir), []); }); +test('only a stale worker lock with proof of PID reuse can displace a live PID', async (t) => { + const birth = processStartTime(process.pid); + const prefix = `${process.pid}:deadbeef`; + for (const token of [prefix, `${prefix}:${birth}`, `${prefix}:${birth - 1_000}`, `${prefix}:invalid`]) { + const { lock } = scratch(t, token); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, () => assert.fail('owner identity is not disproved')), false); + assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), token); + } + + const { dir, lock } = scratch(t, `${prefix}:${birth - 10_000}`); + assert.equal(await withLock(lock, () => assert.fail('the lease is still fresh')), false); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, () => { + assert.match(readFileSync(join(lock, 'owner'), 'utf8'), new RegExp(`^${process.pid}:[a-f0-9]+:${birth}$`)); + }), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('a foreign PID probe leaves admission free and cannot remove a different worker token', async (t) => { + const pid = process.ppid; + const birth = Date.now(); + const old = `${pid}:aaaa:${birth - 10_000}`; + const replacement = `${pid}:bbbb:${birth - 10_000}`; + const { lock } = scratch(t, old); + utimesSync(lock, STALE, STALE); + const spawn = childProcess.spawnSync; + let probes = 0; + childProcess.spawnSync = () => { + probes += 1; + assert.equal(existsSync(`${lock}.admission`), false); + writeFileSync(join(lock, 'owner'), replacement); + const stdout = process.platform === 'win32' ? String(birth) : new Date(birth).toUTCString().replace(/ GMT$/, ''); + return { status: 0, stdout }; + }; + syncBuiltinESMExports(); + try { + assert.equal(await withLock(lock, () => assert.fail('the observed token was replaced')), false); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(probes, 1); + assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), replacement); +}); + test('a transient owner read error never turns a stale live lock into a legacy lock', async (t) => { for (const code of ['EBUSY', 'EPERM', 'EACCES', 'EIO']) { const { dir, lock } = scratch(t);