diff --git a/lib/runtime/process/owner.js b/lib/runtime/process/owner.js index 8d39208..5ea48fa 100644 --- a/lib/runtime/process/owner.js +++ b/lib/runtime/process/owner.js @@ -6,8 +6,32 @@ import { readlinkSync } from 'node:fs'; import path from 'node:path'; const SHELLS = new Set(['sh', 'bash', 'zsh', 'dash', 'ksh', 'fish', 'cmd.exe', 'powershell', 'pwsh']); -const PS_TIMEOUT_MS = 1000; +export const PROCESS_START_TIMEOUT_MS = 1_000; +// POSIX ps reports start times to the nearest second. +export const PROCESS_START_TOLERANCE_MS = 2_000; const MAX_ANCESTORS = 16; +const ownStartTime = Math.round(Date.now() - process.uptime() * 1000); + +/** A bounded process start-time probe for lock recovery. The hot path needs no subprocess. */ +export function processStartTime(pid, timeout = PROCESS_START_TIMEOUT_MS) { + if (pid === process.pid) return ownStartTime; + if (!Number.isSafeInteger(pid) || pid <= 0 || timeout <= 0) return null; + const windows = process.platform === 'win32'; + const systemRoot = process.env.SystemRoot ?? 'C:\\Windows'; + if (windows && !path.isAbsolute(systemRoot)) return null; + const executable = windows ? path.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') : '/bin/ps'; + const args = windows + ? ['-NoProfile', '-NonInteractive', '-Command', `([DateTimeOffset] (Get-Process -Id ${pid} -ErrorAction Stop).StartTime).ToUnixTimeMilliseconds()`] + : ['-p', String(pid), '-o', 'lstart=']; + const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; + const result = spawnSync(executable, args, { + env, encoding: 'utf8', windowsHide: true, timeout, killSignal: 'SIGKILL', + }); + const started = result.status === 0 ? result.stdout.trim() : ''; + if (!started) return null; + const time = windows ? Number(started) : Date.parse(started + ' GMT'); + return Number.isSafeInteger(time) && time > 0 ? time : null; +} /** * Whether the process that wrote a state entry has exited. Signal 0 only probes: ESRCH means no such process; @@ -35,7 +59,7 @@ function executablePathOf(pid, command) { } export function processInfo(pid) { - const probe = spawnSync('ps', ['-o', 'ppid=,comm=', '-p', String(pid)], { encoding: 'utf8', timeout: PS_TIMEOUT_MS }); + const probe = spawnSync('ps', ['-o', 'ppid=,comm=', '-p', String(pid)], { encoding: 'utf8', timeout: PROCESS_START_TIMEOUT_MS }); const match = /^\s*(\d+)\s+(.+?)\s*$/.exec(probe.stdout ?? ''); if (!match) return null; const command = match[2].replace(/^-/, ''); diff --git a/lib/runtime/store/admission.js b/lib/runtime/store/admission.js new file mode 100644 index 0000000..b416d0b --- /dev/null +++ b/lib/runtime/store/admission.js @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 EvoMap + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { ownerIsGone, processStartTime, PROCESS_START_TIMEOUT_MS, PROCESS_START_TOLERANCE_MS } from '../process/owner.js'; + +export const LOCK_WAIT_MS = 2_000; +export const LOCK_RETRY_MS = 20; +export const LOCK_ACCESS_ERRORS = new Set(['EPERM', 'EBUSY', 'EACCES']); +const ADMISSION_IDENTITY_CHECK_MS = 10_000; +const ABANDONED_PREFIX = 'abandoned-'; +const abandonedAdmissions = new Map(); + +export function pause(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function reclaimAdmission(admission, deadline, starts) { + try { + const info = fs.lstatSync(admission); + if (!info.isDirectory()) return false; + const entries = fs.readdirSync(admission); + if (entries.length > 0) { + const marker = entries.find((name) => name.startsWith(ABANDONED_PREFIX)); + const ownerName = marker ? marker.slice(ABANDONED_PREFIX.length) : entries[0]; + if (entries.some((name) => name !== ownerName && name !== marker)) return false; + const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}|[0-9]{13}))?$/.exec(ownerName); + if (!owner) return false; + const pid = Number(owner[1]); + if (!Number.isSafeInteger(pid) || pid <= 0) return false; + const abandonedHere = abandonedAdmissions.get(admission) === ownerName; + const markerPath = marker && path.join(admission, marker); + if (markerPath && !fs.lstatSync(markerPath).isFile()) return false; + if (!markerPath && !abandonedHere && !ownerIsGone(pid)) { + const birth = Number(owner[2]); + const remaining = deadline - Date.now(); + if (owner[2]?.length !== 13 || !Number.isSafeInteger(birth) || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS || remaining <= 0) return false; + if (!starts.has(pid)) starts.set(pid, processStartTime(pid, Math.min(remaining, PROCESS_START_TIMEOUT_MS))); + const actualBirth = starts.get(pid); + // Only a clearly newer process proves reuse. ps timestamps round to whole seconds. + if (!actualBirth || actualBirth <= birth + PROCESS_START_TOLERANCE_MS) return false; + } + if (entries.includes(ownerName)) fs.unlinkSync(path.join(admission, ownerName)); + if (markerPath) fs.unlinkSync(markerPath); + if (abandonedHere) abandonedAdmissions.delete(admission); + } + // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot + // remove it even if another reclaimer replaced the directory after our read. + fs.rmdirSync(admission); + return true; + } catch { + return false; + } +} + +function enterAdmission(lockDir, deadline, starts) { + const admission = `${lockDir}.admission`; + const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}-${processStartTime(process.pid)}`; + const staging = `${admission}.${owner}.claim`; + fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); + fs.mkdirSync(staging, { mode: 0o700 }); + let published = false; + try { + fs.writeFileSync(path.join(staging, owner), '', { mode: 0o600, flag: 'wx' }); + // Publishing a prepopulated directory never exposes an empty fresh guard to a stale reclaimer. + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + fs.renameSync(staging, admission); + published = true; + abandonedAdmissions.delete(admission); + return { admission, owner }; + } catch { + if (attempt > 0 || !reclaimAdmission(admission, deadline, starts)) return null; + } + } + return null; + } finally { + if (!published) fs.rmSync(staging, { recursive: true, force: true }); + } +} + +export function retryLockAccess(access) { + for (let attempt = 0; ; attempt += 1) { + try { + return access(); + } catch (error) { + if (attempt === 2 || !LOCK_ACCESS_ERRORS.has(error?.code)) throw error; + pause(LOCK_RETRY_MS); + } + } +} + +function releaseAdmission({ admission, owner }) { + try { + retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); + } catch (cause) { + if (cause?.code !== 'ENOENT') { + throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + } + } + // The guard is released when its unique owner is gone. An empty directory can be reclaimed. + try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} +} + +function abandonAdmission({ admission, owner }) { + // A separate marker can be written even when Windows denies access to the owner file. + try { + retryLockAccess(() => fs.writeFileSync(path.join(admission, ABANDONED_PREFIX + owner), '', { mode: 0o600, flag: 'wx' })); + } catch { + abandonedAdmissions.set(admission, owner); + } +} + +// Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. +export function admitted(lockDir, mutate, rollback, deadline = Date.now() + LOCK_WAIT_MS, starts = new Map()) { + const admission = enterAdmission(lockDir, deadline, starts); + if (!admission) return null; + let result; + try { + result = mutate(); + } catch (error) { + try { releaseAdmission(admission); } catch { abandonAdmission(admission); } + throw error; + } + try { + releaseAdmission(admission); + } catch (error) { + // Publish only after rollback, so another process cannot enter while it runs. + try { rollback?.(); } catch (rollbackError) { error.rollbackError = rollbackError; } + abandonAdmission(admission); + if (rollback) throw error; + } + return result; +} diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index d122790..6a234ef 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -5,12 +5,15 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import { ownerIsGone } from '../process/owner.js'; +import { ownerIsGone, processStartTime, PROCESS_START_TIMEOUT_MS, PROCESS_START_TOLERANCE_MS } from '../process/owner.js'; + +import { admitted, LOCK_WAIT_MS, LOCK_RETRY_MS, pause, retryLockAccess } from './admission.js'; -const LOCK_WAIT_MS = 2_000; -const LOCK_RETRY_MS = 20; const WORK_LOCK_STALE_MS = 2 * 60 * 1000; +const WORK_OWNER_PROBE_MS = process.platform === 'win32' ? 5_000 : PROCESS_START_TIMEOUT_MS; +const WORK_OWNER_WITH_START = /^\d+:[a-f0-9]+:([1-9]\d{12})$/; const CLAIM_STALE_MS = 60 * 1000; +const abandonedWork = new Map(); export function readState(target, fallback) { try { @@ -41,11 +44,6 @@ export function removeState(target) { } } -// Atomics.wait is the only sleep that holds a synchronous read-modify-write together. -function pause(ms) { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); -} - function contentOf(file) { try { return fs.readFileSync(file, 'utf8'); @@ -179,38 +177,107 @@ function isStale(lockDir, now) { } } +function workOwnerOf(lockDir) { + try { + return retryLockAccess(() => fs.readFileSync(path.join(lockDir, 'owner'), 'utf8')); + } catch (cause) { + if (cause?.code === 'ENOENT') return null; + throw Object.assign(new Error(`Could not read lock owner ${lockDir}`, { cause }), { code: 'ELOCKED' }); + } +} + function ownsLock(lockDir, token) { - return contentOf(path.join(lockDir, 'owner')) === token; + return workOwnerOf(lockDir) === token; } -function tryLock(lockDir, now) { - fs.mkdirSync(path.dirname(lockDir), { recursive: true, mode: 0o700 }); - if (isStale(lockDir, now)) fs.rmSync(lockDir, { recursive: true, force: true }); - try { - fs.mkdirSync(lockDir); - } catch { - return null; - } - const token = crypto.randomBytes(12).toString('hex'); - fs.writeFileSync(path.join(lockDir, 'owner'), token); - return { - refresh: () => { - if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); - }, - release: () => { +async function releaseWorkLock(lockDir, token) { + const deadline = Date.now() + LOCK_WAIT_MS; + const starts = new Map(); + for (;;) { + if (admitted(lockDir, () => { if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); - }, - }; + return true; + }, null, deadline, starts)) return; + if (Date.now() >= deadline) throw Object.assign(new Error(`Could not release lock ${lockDir}`), { code: 'ELOCKED' }); + await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS)); + } } -/** Runs slow `work` while holding a lock directory no other process gets meanwhile; false when it is held. */ +function tryLock(lockDir, now) { + let token, suspect, reusedToken; + const take = () => admitted(lockDir, () => { + const stale = isStale(lockDir, now); + if (abandonedWork.has(lockDir) || stale) { + let held; + try { held = workOwnerOf(lockDir); } catch { return null; } + if (held !== abandonedWork.get(lockDir)) { + if (!stale) return null; + if (held !== reusedToken && /^\d+:/.test(held ?? '')) { + const pid = ownerPidOf(held); + if (!ownerIsGone(pid)) { + const birth = WORK_OWNER_WITH_START.exec(held)?.[1]; + if (birth) suspect = { token: held, pid, birth: Number(birth) }; + return null; + } + } + } + fs.rmSync(lockDir, { recursive: true, force: true }); + abandonedWork.delete(lockDir); + } + try { + fs.mkdirSync(lockDir); + } catch { + return null; + } + token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}:${processStartTime(process.pid)}`; + fs.writeFileSync(path.join(lockDir, 'owner'), token); + return { + refresh: () => { + try { + return admitted(lockDir, () => { + if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); + }); + } catch { + // Timer-driven refresh is best effort; a live pid stays protected and final release still surfaces errors. + return null; + } + }, + release: async () => { + try { await releaseWorkLock(lockDir, token); } + catch (error) { abandonedWork.set(lockDir, token); throw error; } + }, + }; + }, () => { + if (!token) return; + abandonedWork.set(lockDir, token); + if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); + abandonedWork.delete(lockDir); + }); + const lock = take(); + if (lock || !suspect) return lock; + // A foreign lookup can be slow. Release admission, then recheck staleness and the exact token. + const actualBirth = processStartTime(suspect.pid, WORK_OWNER_PROBE_MS); + if (!actualBirth || actualBirth <= suspect.birth + PROCESS_START_TOLERANCE_MS) return null; + reusedToken = suspect.token; + return take(); +} + +/** Runs slow `work` under one owner's lock; dead or legacy directories recover after their lease expires. */ export async function withLock(lockDir, work, now = Date.now()) { const lock = tryLock(lockDir, now); if (!lock) return false; + let workFailed = false; try { await work(lock); return true; + } catch (error) { + workFailed = true; + throw error; } finally { - lock.release(); + try { + await lock.release(); + } catch (error) { + if (!workFailed) throw error; + } } } diff --git a/test/helpers/lock-contenders.js b/test/helpers/lock-contenders.js new file mode 100644 index 0000000..1e412a7 --- /dev/null +++ b/test/helpers/lock-contenders.js @@ -0,0 +1,39 @@ +import { spawn } from 'node:child_process'; +import { rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export async function until(check) { + const deadline = Date.now() + 10_000; + while (!check()) { + if (Date.now() >= deadline) throw new Error('timed out waiting for a lock contender'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +export function lockContenders(t, dir, script, args, cleanupMarkers) { + const children = []; + const results = () => Promise.all(children); + t.after(async () => { + try { + for (const name of cleanupMarkers) writeFileSync(join(dir, name), ''); + } finally { + await results(); + rmSync(dir, { recursive: true, force: true }); + } + }); + return { + run(role) { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, ...args, role], { windowsHide: true }); + let stderr = ''; + const watchdog = setTimeout(() => child.kill('SIGKILL'), 15_000); + child.stderr.on('data', (data) => { stderr += data; }); + child.on('error', (error) => { stderr += String(error); }); + children.push(new Promise((resolve) => child.on('close', (code) => { + clearTimeout(watchdog); + resolve({ code, stderr }); + }))); + return child; + }, + results, + }; +} diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js new file mode 100644 index 0000000..0e40c47 --- /dev/null +++ b/test/runtime/file-lock.test.js @@ -0,0 +1,621 @@ +import assert from 'node:assert/strict'; +import childProcess, { spawnSync } from 'node:child_process'; +import { syncBuiltinESMExports } from 'node:module'; +import fs from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, rmdirSync, statSync, unlinkSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; + +import { processStartTime } from '../../lib/runtime/process/owner.js'; +import { withLock } from '../../lib/runtime/store/file-lock.js'; +import { lockContenders, until } from '../helpers/lock-contenders.js'; + +const RUNTIME = new URL('../../lib/runtime/store/file-lock.js', import.meta.url).href; +const STALE = new Date(0); + +function scratch(t, token = null) { + const dir = mkdtempSync(join(tmpdir(), 'evolver-work-lock-')); + const lock = join(dir, 'work.lock'); + if (token !== null) { + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), token); + } + t.after(() => rmSync(dir, { recursive: true, force: true })); + return { dir, lock }; +} + +function holdAdmission(lock, pid = process.pid) { + const admission = `${lock}.admission`; + const owner = join(admission, `owner-${pid}-deadbeef`); + mkdirSync(admission); + writeFileSync(owner, 'fixture owner'); + return { admission, owner, release: () => { unlinkSync(owner); rmdirSync(admission); } }; +} + +test('taking an uncontended lock does not launch a process lookup', async (t) => { + const { lock } = scratch(t); + let launches = 0; + const spawn = childProcess.spawnSync; + childProcess.spawnSync = (...args) => { launches += 1; return spawn(...args); }; + syncBuiltinESMExports(); + try { + assert.equal(await withLock(lock, () => {}), true); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(launches, 0); +}); + +test('stale legacy and dead-owner directories recover, but a fresh legacy lock stays held', async (t) => { + const { lock } = scratch(t, 'legacy token'); + let entered = 0; + assert.equal(await withLock(lock, async () => { entered += 1; }), false); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), `${exited.pid}:deadbeef`); + holdAdmission(lock, exited.pid); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + assert.equal(entered, 2); + assert.equal(existsSync(lock), false); + assert.equal(existsSync(`${lock}.admission`), false); +}); + +test('an old mtime never evicts a live owner, which can refresh and release its lock', async (t) => { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + utimesSync(lock, STALE, STALE); + let entered = false; + assert.equal(await withLock(lock, async () => { entered = true; }), false); + assert.equal(entered, false); + held.refresh(); + assert.ok(statSync(lock).mtimeMs > STALE.getTime()); + }), true); + assert.equal(existsSync(lock), false); + assert.deepEqual(readdirSync(dir), []); +}); + +test('only a stale worker lock with proof of PID reuse can displace a live PID', async (t) => { + const birth = processStartTime(process.pid); + const prefix = `${process.pid}:deadbeef`; + for (const token of [prefix, `${prefix}:${birth}`, `${prefix}:${birth - 1_000}`, `${prefix}:invalid`]) { + const { lock } = scratch(t, token); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, () => assert.fail('owner identity is not disproved')), false); + assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), token); + } + + const { dir, lock } = scratch(t, `${prefix}:${birth - 10_000}`); + assert.equal(await withLock(lock, () => assert.fail('the lease is still fresh')), false); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, () => { + assert.match(readFileSync(join(lock, 'owner'), 'utf8'), new RegExp(`^${process.pid}:[a-f0-9]+:${birth}$`)); + }), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('a foreign PID probe leaves admission free and cannot remove a different worker token', async (t) => { + const pid = process.ppid; + const birth = Date.now(); + const old = `${pid}:aaaa:${birth - 10_000}`; + const replacement = `${pid}:bbbb:${birth - 10_000}`; + const { lock } = scratch(t, old); + utimesSync(lock, STALE, STALE); + const spawn = childProcess.spawnSync; + let probes = 0; + childProcess.spawnSync = () => { + probes += 1; + assert.equal(existsSync(`${lock}.admission`), false); + writeFileSync(join(lock, 'owner'), replacement); + const stdout = process.platform === 'win32' ? String(birth) : new Date(birth).toUTCString().replace(/ GMT$/, ''); + return { status: 0, stdout }; + }; + syncBuiltinESMExports(); + try { + assert.equal(await withLock(lock, () => assert.fail('the observed token was replaced')), false); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(probes, 1); + assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), replacement); +}); + +test('a transient owner read error never turns a stale live lock into a legacy lock', async (t) => { + for (const code of ['EBUSY', 'EPERM', 'EACCES', 'EIO']) { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async () => { + const owner = join(lock, 'owner'); + const token = readFileSync(owner, 'utf8'); + utimesSync(lock, STALE, STALE); + const read = fs.readFileSync; + let refused = false; + let entered = false; + fs.readFileSync = (target, ...args) => { + if (target === owner && !refused) { + refused = true; + throw Object.assign(new Error('transient owner read refusal'), { code }); + } + return read(target, ...args); + }; + try { + assert.equal(await withLock(lock, async () => { entered = true; }), false, code); + } finally { + fs.readFileSync = read; + } + assert.equal(refused, true); + assert.equal(entered, false, code); + assert.equal(readFileSync(owner, 'utf8'), token, 'the live owner stays intact'); + }), true); + assert.deepEqual(readdirSync(dir), [], 'the original owner still releases normally'); + } +}); + +test('a stale directory left before its owner was written is recoverable', async (t) => { + const { dir, lock } = scratch(t); + mkdirSync(lock); + utimesSync(lock, STALE, STALE); + let entered = false; + assert.equal(await withLock(lock, async () => { entered = true; }), true); + assert.equal(entered, true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release retries a transient owner read failure and really removes its lock', async (t) => { + const { dir, lock } = scratch(t); + const owner = join(lock, 'owner'); + const read = fs.readFileSync; + let refused = false; + try { + assert.equal(await withLock(lock, async () => { + fs.readFileSync = (target, ...args) => { + if (target === owner && !refused) { + refused = true; + throw Object.assign(new Error('transient read refusal'), { code: 'EBUSY' }); + } + return read(target, ...args); + }; + }), true); + } finally { + fs.readFileSync = read; + } + assert.equal(refused, true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release refuses an unreadable owner explicitly without claiming cleanup', async (t) => { + for (const code of ['EBUSY', 'EPERM', 'EIO']) { + const { lock } = scratch(t); + const owner = join(lock, 'owner'); + const read = fs.readFileSync; + let token; + try { + await assert.rejects(withLock(lock, async () => { + token = read(owner, 'utf8'); + fs.readFileSync = (target, ...args) => { + if (target === owner) throw Object.assign(new Error('persistent read refusal'), { code }); + return read(target, ...args); + }; + }), { code: 'ELOCKED' }); + } finally { + fs.readFileSync = read; + } + assert.equal(readFileSync(owner, 'utf8'), token, code); + assert.equal(await withLock(lock, () => {}), true, 'known abandoned work is recoverable after access returns'); + } +}); + +test('a stale foreign admission is probed once within the release deadline', async (t) => { + const { lock } = scratch(t); + const spawn = childProcess.spawnSync; + const timeouts = []; + try { + await assert.rejects(withLock(lock, () => { + const guard = holdAdmission(lock, process.ppid); + unlinkSync(guard.owner); + writeFileSync(join(guard.admission, `owner-${process.ppid}-a1-${Date.now() - 30_000}`), ''); + utimesSync(guard.admission, STALE, STALE); + childProcess.spawnSync = (_command, _args, { timeout }) => { + timeouts.push(timeout); + return { status: null, stdout: '' }; + }; + syncBuiltinESMExports(); + }), { code: 'ELOCKED' }); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(timeouts.length, 1); + assert.ok(timeouts[0] > 0 && timeouts[0] <= 1_000); + assert.equal(existsSync(lock), true, 'an inconclusive identity probe does not reclaim a live guard'); +}); + +test('a release failure preserves the original work error', async (t) => { + const { lock } = scratch(t); + const read = fs.readFileSync; + const workError = new Error('work failed'); + try { + await assert.rejects(withLock(lock, async () => { + fs.readFileSync = (file, ...args) => { + if (String(file) === join(lock, 'owner')) throw Object.assign(new Error('unreadable owner'), { code: 'EIO' }); + return read(file, ...args); + }; + throw workError; + }), (error) => error === workError); + } finally { + fs.readFileSync = read; + } +}); + +test('refresh leaves an unreadable live lock untouched without crashing its timer caller', async (t) => { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + const owner = join(lock, 'owner'); + utimesSync(lock, STALE, STALE); + const read = fs.readFileSync; + fs.readFileSync = (target, ...args) => { + if (target === owner) throw Object.assign(new Error('unreadable owner'), { code: 'EPERM' }); + return read(target, ...args); + }; + try { + assert.doesNotThrow(() => held.refresh()); + assert.equal(statSync(lock).mtimeMs, STALE.getTime()); + } finally { + fs.readFileSync = read; + } + }), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('refresh tolerates file access refusals and the next refresh and release still work', async (t) => { + for (const method of ['utimesSync', 'mkdirSync', 'readFileSync']) { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + utimesSync(lock, STALE, STALE); + const access = fs[method]; + let refused = false; + fs[method] = (target, ...args) => { + const matches = method === 'mkdirSync' ? String(target).startsWith(`${lock}.admission.`) + : target === (method === 'readFileSync' ? join(lock, 'owner') : lock); + if (matches && !refused) { + refused = true; + throw Object.assign(new Error('transient access refusal'), { code: 'EPERM' }); + } + return access(target, ...args); + }; + try { + assert.doesNotThrow(() => held.refresh(), method); + if (method !== 'readFileSync') assert.equal(statSync(lock).mtimeMs, STALE.getTime()); + } finally { + fs[method] = access; + } + assert.equal(refused, true, method); + held.refresh(); + assert.ok(statSync(lock).mtimeMs > STALE.getTime()); + }), true); + assert.deepEqual(readdirSync(dir), []); + } +}); + +test('release does not delete a missing or different owner', async (t) => { + for (const replacement of [null, `${process.pid}:another-owner`]) { + const { lock } = scratch(t); + assert.equal(await withLock(lock, async () => { + if (replacement === null) unlinkSync(join(lock, 'owner')); + else writeFileSync(join(lock, 'owner'), replacement); + }), true); + assert.equal(existsSync(lock), true); + assert.equal(existsSync(join(lock, 'owner')), replacement !== null); + if (replacement !== null) assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), replacement); + } +}); + +test('release waits asynchronously for transient admission contention', async (t) => { + const { lock } = scratch(t); + let guard; + let finished = false; + let refreshedAt; + const running = withLock(lock, async (held) => { + guard = holdAdmission(lock); + utimesSync(lock, STALE, STALE); + held.refresh(); + refreshedAt = statSync(lock).mtimeMs; + }).then(() => { finished = true; }); + try { + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(refreshedAt, STALE.getTime(), 'refresh also respects admission'); + assert.equal(finished, false); + assert.equal(existsSync(lock), true); + } finally { + guard.release(); + await running; + } + assert.equal(existsSync(lock), false); +}); + +test('release refuses bounded admission contention without deleting the held directory', async (t) => { + const { lock } = scratch(t); + let guard; + const started = Date.now(); + await assert.rejects(withLock(lock, async () => { + guard = holdAdmission(lock); + }), { code: 'ELOCKED' }); + assert.ok(Date.now() - started < 5_000, 'cleanup cannot wait forever'); + assert.equal(existsSync(lock), true); + assert.equal(readFileSync(guard.owner, 'utf8'), 'fixture owner'); +}); + +test('an admission cleanup failure is surfaced instead of silently orphaning a live guard', async (t) => { + const { lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + return unlink(target); + }; + let entered = false; + try { + await assert.rejects(withLock(lock, async () => { entered = true; }), { code: 'ELOCKED' }); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(entered, false); + assert.equal(existsSync(lock), false, 'the failed acquire must not leave a work lock'); + assert.ok(readdirSync(admission).some((name) => name.startsWith(`owner-${process.pid}-`))); + assert.equal(await withLock(lock, async () => { entered = true; }), true); + assert.equal(entered, true); +}); + +test('another process reclaims a failed admission release while the original owner is alive', async (t) => { + const { dir, lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + try { + assert.equal(await withLock(lock, () => { + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('fixture cleanup refusal'), { code: 'EPERM' }); + return unlink(target); + }; + }), true); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(existsSync(lock), false); + assert.ok(readdirSync(admission).some((name) => name.startsWith(`owner-${process.pid}-`))); + + const other = spawnSync(process.execPath, ['--input-type=module', '-e', ` + import assert from 'node:assert/strict'; + const [runtime, lock, owner] = process.argv.slice(1); + const { withLock } = await import(runtime); + process.kill(Number(owner), 0); + assert.equal(await withLock(lock, () => {}), true, 'the stranded admission can be reclaimed'); + `, RUNTIME, lock, String(process.pid)], { encoding: 'utf8', timeout: 10_000, windowsHide: true }); + assert.ifError(other.error); + assert.equal(other.status, 0, other.stderr); + assert.deepEqual(readdirSync(dir), []); +}); + +test('admission recovery resumes after the abandoned owner has already been removed', async (t) => { + const { dir, lock } = scratch(t); + const guard = holdAdmission(lock); + const marker = join(guard.admission, `abandoned-owner-${process.pid}-abcd`); + writeFileSync(marker, ''); + assert.equal(await withLock(lock, () => assert.fail('a different owner is still active')), false); + assert.equal(existsSync(guard.owner), true); + + unlinkSync(guard.owner); + assert.equal(await withLock(lock, () => {}), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { + const { dir, lock } = scratch(t); + const unlink = fs.unlinkSync; + let refused = false; + fs.unlinkSync = (target) => { + if (dirname(target) === `${lock}.admission` && !refused) { + refused = true; + throw Object.assign(new Error('temporarily refused by fixture'), { code: 'EPERM' }); + } + return unlink(target); + }; + let entered = 0; + try { + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(refused, true); + assert.equal(entered, 1); + assert.deepEqual(readdirSync(dir), []); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + assert.equal(entered, 2); + assert.deepEqual(readdirSync(dir), []); +}); + +test('an old admission stays held by the same process, but a reused PID can be recovered', async (t) => { + const { lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + return unlink(target); + }; + try { + await assert.rejects(withLock(lock, async () => assert.fail()), { code: 'ELOCKED' }); + } finally { + fs.unlinkSync = unlink; + } + const original = readdirSync(admission).find((name) => name.startsWith(`owner-${process.pid}-`)); + const birth = original.split('-')[3]; + assert.match(birth ?? '', /^\d{13}$/); + const replaceOwner = (name) => { + for (const name of readdirSync(admission)) unlinkSync(join(admission, name)); + writeFileSync(join(admission, name), ''); + utimesSync(admission, STALE, STALE); + }; + replaceOwner(`owner-${process.pid}-a1-${birth}`); + assert.equal(await withLock(lock, async () => assert.fail('same process still owns the guard')), false); + const differentBirth = Number(birth) - 10_000; + replaceOwner(`owner-${process.pid}-a2-${differentBirth}`); + assert.equal(await withLock(lock, async () => {}), true); +}); + +test('busy and unrecognised admission guards stay intact and leave no staging directories', async (t) => { + const { dir, lock } = scratch(t); + const guard = holdAdmission(lock); + assert.equal(await withLock(lock, async () => assert.fail('a live guard is held')), false); + assert.deepEqual(readdirSync(dir), ['work.lock.admission']); + writeFileSync(join(guard.admission, 'unexpected'), 'keep'); + assert.equal(await withLock(lock, async () => assert.fail('multiple owners are unknown')), false); + assert.equal(readdirSync(guard.admission).length, 2); + unlinkSync(guard.owner); + assert.equal(await withLock(lock, async () => assert.fail('an unknown owner is held')), false); + assert.equal(readFileSync(join(guard.admission, 'unexpected'), 'utf8'), 'keep'); + assert.deepEqual(readdirSync(dir), ['work.lock.admission']); +}); + +test('two contenders cannot replace the fresh lock while another reclaims a stale directory', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-work-lock-')); + const lock = join(dir, 'work.lock'); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), 'legacy token'); + utimesSync(lock, STALE, STALE); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { withLock } from ${JSON.stringify(RUNTIME)}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const wait = async (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + }; + if (role === 'A') { + const stat = fs.statSync; + let paused = false; + fs.statSync = (file, ...args) => { + const result = stat(file, ...args); + if (file === lock && !paused) { + paused = true; + fs.writeFileSync(marker('stale-read'), ''); + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker('resume'))) { + if (Date.now() > deadline) throw new Error('timed out waiting to resume'); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + } + return result; + }; + } + const result = await withLock(lock, async () => { + fs.writeFileSync(marker(role + '-entered'), ''); + await wait('release'); + }); + fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); + `; + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['resume', 'release']); + run('A'); + await until(() => existsSync(marker('stale-read'))); + run('B'); + run('C'); + await until(() => ['B', 'C'].every((role) => existsSync(marker(role + '-entered')) || existsSync(marker(role + '-result')))); + writeFileSync(marker('resume'), ''); + await until(() => existsSync(marker('A-entered'))); + const entered = ['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))); + writeFileSync(marker('release'), ''); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); + assert.deepEqual(entered, ['A'], 'only the process reclaiming the inspected stale directory may enter'); + assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [true, false, false]); +}); + +test('a delayed dead-admission reclaimer cannot unlink the fresh owner another contender published', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-admission-race-')); + const lock = join(dir, 'work.lock'); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), 'legacy token'); + utimesSync(lock, STALE, STALE); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + holdAdmission(lock, exited.pid); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { withLock } from ${JSON.stringify(RUNTIME)}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const waitSync = (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + }; + if (role === 'A') { + const list = fs.readdirSync; + let paused = false; + fs.readdirSync = (file, ...args) => { + const result = list(file, ...args); + if (file === lock + '.admission' && !paused) { + paused = true; + fs.writeFileSync(marker('dead-inspected'), ''); + waitSync('A-resume'); + } + return result; + }; + } + if (role === 'B') { + const rename = fs.renameSync; + let paused = false; + fs.renameSync = (source, target) => { + const result = rename(source, target); + if (target === lock + '.admission' && !paused) { + paused = true; + fs.writeFileSync(marker('fresh-held'), ''); + waitSync('B-resume'); + } + return result; + }; + } + const result = await withLock(lock, async () => { + fs.writeFileSync(marker(role + '-entered'), ''); + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker('release'))) { + if (Date.now() > deadline) throw new Error('timed out waiting to release'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + }); + fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); + `; + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['A-resume', 'B-resume', 'release']); + run('A'); + await until(() => existsSync(marker('dead-inspected'))); + const fresh = run('B'); + await until(() => existsSync(marker('fresh-held'))); + run('C'); + await until(() => existsSync(marker('C-result')) || existsSync(marker('C-entered'))); + assert.equal(existsSync(marker('C-entered')), false, 'publishing a fresh guard never exposes an empty directory'); + writeFileSync(marker('A-resume'), ''); + await until(() => existsSync(marker('A-result')) || existsSync(marker('A-entered'))); + assert.equal(existsSync(marker('A-entered')), false, 'the delayed reclaimer cannot displace the fresh guard'); + assert.equal(readdirSync(`${lock}.admission`).length, 1); + assert.match(readdirSync(`${lock}.admission`)[0], new RegExp(`^owner-${fresh.pid}-`)); + writeFileSync(marker('B-resume'), ''); + await until(() => existsSync(marker('B-entered'))); + assert.deepEqual(['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))), ['B']); + writeFileSync(marker('release'), ''); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); + assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [false, true, false]); +});