diff --git a/lib/agent/driver.js b/lib/agent/driver.js index 8ebe3e8..54e6e16 100644 --- a/lib/agent/driver.js +++ b/lib/agent/driver.js @@ -171,7 +171,8 @@ export function createDriver({ profile, ports, transcripts, env = process.env, l const at = { sessionId, projectDir: profile.payload.projectDir(payload, context) }; const waitLeft = (budgetOf = 'prompt') => waitBudgetOf(profile.budgets, budgetOf, { lifetime, startedAt }); return await HANDLERS[event]({ profile, ports, hostEvent, payload, context, at, waitLeft, startedAt, options }); - } catch { + } catch (error) { + options.onError?.(error); return profile.reply.empty(hostEvent, payload); } }, diff --git a/lib/channels/hook/runner.js b/lib/channels/hook/runner.js index a591080..fca96d4 100644 --- a/lib/channels/hook/runner.js +++ b/lib/channels/hook/runner.js @@ -13,7 +13,7 @@ import { readPayload } from './stdin.js'; * One command-hook invocation: the host's payload on stdin, the agent's answer as JSON on stdout, exit 0. Inside * this plugin's own headless distillation the hook answers the agent's empty reply and does nothing else. */ -export async function runHookProcess(agent, hostEvent, { stdin = process.stdin, stdout = process.stdout, env = process.env, exit = process.exit, profileOf = profileNamed } = {}) { +export async function runHookProcess(agent, hostEvent, { stdin = process.stdin, stdout = process.stdout, stderr = process.stderr, env = process.env, exit = process.exit, profileOf = profileNamed } = {}) { const startedAt = Date.now(); const write = (answer) => stdout.write(JSON.stringify(answer ?? {}), () => exit(0)); const profile = await profileOf(agent).catch(() => null); @@ -22,5 +22,14 @@ export async function runHookProcess(agent, hostEvent, { stdin = process.stdin, if (isHeadlessRun(env)) return write(profile.reply.empty(hostEvent)); const { ports } = environmentFor({ profile, env, lifetime: EPHEMERAL, runBackground: runBackgroundTasks }); const driver = createDriver({ profile, ports, transcripts, env, lifetime: EPHEMERAL, startedAt }); - return write(await driver.handle(hostEvent, payload)); + let diagnostic; + const answer = await driver.handle(hostEvent, payload, { + onError(error) { + if (['EINVALIDSTATE', 'ESTATEIO', 'ELOCKED'].includes(error?.code)) { + diagnostic = `Evolver hook skipped a state operation (${error.code}). Check the plugin state directory and retry.\n`; + } + }, + }); + if (diagnostic) return stderr.write(diagnostic, () => write(answer)); + return write(answer); } diff --git a/lib/runtime/process/owner.js b/lib/runtime/process/owner.js index 8d39208..5ea48fa 100644 --- a/lib/runtime/process/owner.js +++ b/lib/runtime/process/owner.js @@ -6,8 +6,32 @@ import { readlinkSync } from 'node:fs'; import path from 'node:path'; const SHELLS = new Set(['sh', 'bash', 'zsh', 'dash', 'ksh', 'fish', 'cmd.exe', 'powershell', 'pwsh']); -const PS_TIMEOUT_MS = 1000; +export const PROCESS_START_TIMEOUT_MS = 1_000; +// POSIX ps reports start times to the nearest second. +export const PROCESS_START_TOLERANCE_MS = 2_000; const MAX_ANCESTORS = 16; +const ownStartTime = Math.round(Date.now() - process.uptime() * 1000); + +/** A bounded process start-time probe for lock recovery. The hot path needs no subprocess. */ +export function processStartTime(pid, timeout = PROCESS_START_TIMEOUT_MS) { + if (pid === process.pid) return ownStartTime; + if (!Number.isSafeInteger(pid) || pid <= 0 || timeout <= 0) return null; + const windows = process.platform === 'win32'; + const systemRoot = process.env.SystemRoot ?? 'C:\\Windows'; + if (windows && !path.isAbsolute(systemRoot)) return null; + const executable = windows ? path.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') : '/bin/ps'; + const args = windows + ? ['-NoProfile', '-NonInteractive', '-Command', `([DateTimeOffset] (Get-Process -Id ${pid} -ErrorAction Stop).StartTime).ToUnixTimeMilliseconds()`] + : ['-p', String(pid), '-o', 'lstart=']; + const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; + const result = spawnSync(executable, args, { + env, encoding: 'utf8', windowsHide: true, timeout, killSignal: 'SIGKILL', + }); + const started = result.status === 0 ? result.stdout.trim() : ''; + if (!started) return null; + const time = windows ? Number(started) : Date.parse(started + ' GMT'); + return Number.isSafeInteger(time) && time > 0 ? time : null; +} /** * Whether the process that wrote a state entry has exited. Signal 0 only probes: ESRCH means no such process; @@ -35,7 +59,7 @@ function executablePathOf(pid, command) { } export function processInfo(pid) { - const probe = spawnSync('ps', ['-o', 'ppid=,comm=', '-p', String(pid)], { encoding: 'utf8', timeout: PS_TIMEOUT_MS }); + const probe = spawnSync('ps', ['-o', 'ppid=,comm=', '-p', String(pid)], { encoding: 'utf8', timeout: PROCESS_START_TIMEOUT_MS }); const match = /^\s*(\d+)\s+(.+?)\s*$/.exec(probe.stdout ?? ''); if (!match) return null; const command = match[2].replace(/^-/, ''); diff --git a/lib/runtime/store/admission.js b/lib/runtime/store/admission.js new file mode 100644 index 0000000..b416d0b --- /dev/null +++ b/lib/runtime/store/admission.js @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 EvoMap + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { ownerIsGone, processStartTime, PROCESS_START_TIMEOUT_MS, PROCESS_START_TOLERANCE_MS } from '../process/owner.js'; + +export const LOCK_WAIT_MS = 2_000; +export const LOCK_RETRY_MS = 20; +export const LOCK_ACCESS_ERRORS = new Set(['EPERM', 'EBUSY', 'EACCES']); +const ADMISSION_IDENTITY_CHECK_MS = 10_000; +const ABANDONED_PREFIX = 'abandoned-'; +const abandonedAdmissions = new Map(); + +export function pause(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function reclaimAdmission(admission, deadline, starts) { + try { + const info = fs.lstatSync(admission); + if (!info.isDirectory()) return false; + const entries = fs.readdirSync(admission); + if (entries.length > 0) { + const marker = entries.find((name) => name.startsWith(ABANDONED_PREFIX)); + const ownerName = marker ? marker.slice(ABANDONED_PREFIX.length) : entries[0]; + if (entries.some((name) => name !== ownerName && name !== marker)) return false; + const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}|[0-9]{13}))?$/.exec(ownerName); + if (!owner) return false; + const pid = Number(owner[1]); + if (!Number.isSafeInteger(pid) || pid <= 0) return false; + const abandonedHere = abandonedAdmissions.get(admission) === ownerName; + const markerPath = marker && path.join(admission, marker); + if (markerPath && !fs.lstatSync(markerPath).isFile()) return false; + if (!markerPath && !abandonedHere && !ownerIsGone(pid)) { + const birth = Number(owner[2]); + const remaining = deadline - Date.now(); + if (owner[2]?.length !== 13 || !Number.isSafeInteger(birth) || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS || remaining <= 0) return false; + if (!starts.has(pid)) starts.set(pid, processStartTime(pid, Math.min(remaining, PROCESS_START_TIMEOUT_MS))); + const actualBirth = starts.get(pid); + // Only a clearly newer process proves reuse. ps timestamps round to whole seconds. + if (!actualBirth || actualBirth <= birth + PROCESS_START_TOLERANCE_MS) return false; + } + if (entries.includes(ownerName)) fs.unlinkSync(path.join(admission, ownerName)); + if (markerPath) fs.unlinkSync(markerPath); + if (abandonedHere) abandonedAdmissions.delete(admission); + } + // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot + // remove it even if another reclaimer replaced the directory after our read. + fs.rmdirSync(admission); + return true; + } catch { + return false; + } +} + +function enterAdmission(lockDir, deadline, starts) { + const admission = `${lockDir}.admission`; + const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}-${processStartTime(process.pid)}`; + const staging = `${admission}.${owner}.claim`; + fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); + fs.mkdirSync(staging, { mode: 0o700 }); + let published = false; + try { + fs.writeFileSync(path.join(staging, owner), '', { mode: 0o600, flag: 'wx' }); + // Publishing a prepopulated directory never exposes an empty fresh guard to a stale reclaimer. + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + fs.renameSync(staging, admission); + published = true; + abandonedAdmissions.delete(admission); + return { admission, owner }; + } catch { + if (attempt > 0 || !reclaimAdmission(admission, deadline, starts)) return null; + } + } + return null; + } finally { + if (!published) fs.rmSync(staging, { recursive: true, force: true }); + } +} + +export function retryLockAccess(access) { + for (let attempt = 0; ; attempt += 1) { + try { + return access(); + } catch (error) { + if (attempt === 2 || !LOCK_ACCESS_ERRORS.has(error?.code)) throw error; + pause(LOCK_RETRY_MS); + } + } +} + +function releaseAdmission({ admission, owner }) { + try { + retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); + } catch (cause) { + if (cause?.code !== 'ENOENT') { + throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + } + } + // The guard is released when its unique owner is gone. An empty directory can be reclaimed. + try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} +} + +function abandonAdmission({ admission, owner }) { + // A separate marker can be written even when Windows denies access to the owner file. + try { + retryLockAccess(() => fs.writeFileSync(path.join(admission, ABANDONED_PREFIX + owner), '', { mode: 0o600, flag: 'wx' })); + } catch { + abandonedAdmissions.set(admission, owner); + } +} + +// Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. +export function admitted(lockDir, mutate, rollback, deadline = Date.now() + LOCK_WAIT_MS, starts = new Map()) { + const admission = enterAdmission(lockDir, deadline, starts); + if (!admission) return null; + let result; + try { + result = mutate(); + } catch (error) { + try { releaseAdmission(admission); } catch { abandonAdmission(admission); } + throw error; + } + try { + releaseAdmission(admission); + } catch (error) { + // Publish only after rollback, so another process cannot enter while it runs. + try { rollback?.(); } catch (rollbackError) { error.rollbackError = rollbackError; } + abandonAdmission(admission); + if (rollback) throw error; + } + return result; +} diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index f730098..e1d0888 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -5,32 +5,54 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import { ownerIsGone } from '../process/owner.js'; +import { ownerIsGone, processStartTime, PROCESS_START_TIMEOUT_MS, PROCESS_START_TOLERANCE_MS } from '../process/owner.js'; +import { admitted, LOCK_ACCESS_ERRORS, LOCK_WAIT_MS, LOCK_RETRY_MS, pause, retryLockAccess } from './admission.js'; -const LOCK_WAIT_MS = 2_000; -const LOCK_RETRY_MS = 20; const WORK_LOCK_STALE_MS = 2 * 60 * 1000; +const WORK_OWNER_PROBE_MS = process.platform === 'win32' ? 5_000 : PROCESS_START_TIMEOUT_MS; +const WORK_OWNER_WITH_START = /^\d+:[a-f0-9]+:([1-9]\d{12})$/; const CLAIM_STALE_MS = 60 * 1000; +const abandonedWork = new Map(); +const abandonedState = new Map(); export function readState(target, fallback) { try { const parsed = JSON.parse(fs.readFileSync(target, 'utf8')); - return parsed && typeof parsed === 'object' && Array.isArray(parsed) === Array.isArray(fallback) ? parsed : fallback; - } catch { - return fallback; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed) !== Array.isArray(fallback)) throw Object.assign(new Error(`Invalid state shape in ${target}; repair it before retrying.`), { code: 'EINVALIDSTATE' }); + return parsed; + } catch (error) { + if (error?.code === 'ENOENT') return fallback; + if (error instanceof SyntaxError) throw Object.assign(new Error(`Invalid JSON in ${target}; repair it before retrying.`), { code: 'EINVALIDSTATE', cause: error }); + if (error?.code === 'EINVALIDSTATE') throw error; + throw Object.assign(new Error(`Could not read state in ${target}; check access and retry.`), { code: 'ESTATEIO', cause: error }); } } export function writeState(target, value) { fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 }); const temporary = `${target}.${process.pid}.${crypto.randomBytes(6).toString('hex')}.tmp`; - fs.writeFileSync(temporary, JSON.stringify(value), { mode: 0o600, flag: 'wx' }); + const deadline = Date.now() + LOCK_WAIT_MS; + let descriptor; try { - fs.renameSync(temporary, target); - } catch (error) { - if (!['EEXIST', 'EPERM'].includes(error?.code)) throw error; - fs.rmSync(target, { force: true }); - fs.renameSync(temporary, target); + descriptor = fs.openSync(temporary, 'wx', 0o600); + try { + fs.writeFileSync(descriptor, JSON.stringify(value)); + } catch (error) { + try { fs.closeSync(descriptor); } catch {} + throw error; + } + fs.closeSync(descriptor); + for (;;) { + try { + fs.renameSync(temporary, target); + return; + } catch (error) { + if (!['EEXIST', 'EACCES', 'EPERM', 'EBUSY'].includes(error?.code) || Date.now() >= deadline) throw error; + pause(LOCK_RETRY_MS); + } + } + } finally { + if (descriptor !== undefined) removeState(temporary); } } @@ -41,74 +63,81 @@ export function removeState(target) { } } -// Atomics.wait is the only sleep that holds a synchronous read-modify-write together. -function pause(ms) { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); -} - -function contentOf(file) { +function stateOwnerOf(file) { try { - return fs.readFileSync(file, 'utf8'); - } catch { - return null; + return retryLockAccess(() => fs.readFileSync(file, 'utf8')); + } catch (error) { + if (error?.code === 'ENOENT') return null; + throw error; } } const ownerPidOf = (token) => Number(token.split(':')[0]); -// Another process may reclaim the same dead lock and a third take a fresh one before this removal happens, so the -// lock is first moved aside, and only what was inspected as dead is deleted; anything else is put back. -function reclaim(lockPath, deadToken) { - const aside = `${lockPath}.${crypto.randomBytes(6).toString('hex')}.reclaimed`; - try { - fs.renameSync(lockPath, aside); - } catch { - return; - } - if (contentOf(aside) !== deadToken) { - try { - fs.linkSync(aside, lockPath); - } catch { - } - } - removeState(aside); -} - function acquire(lockPath, deadline) { const token = `${process.pid}:${crypto.randomBytes(8).toString('hex')}`; - for (;;) { + const starts = new Map(); + while (Date.now() < deadline) { + let acquired = false; try { - fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 }); - fs.writeFileSync(lockPath, token, { mode: 0o600, flag: 'wx' }); - return token; + const result = admitted(lockPath, () => { + try { + fs.writeFileSync(lockPath, token, { mode: 0o600, flag: 'wx' }); + acquired = true; + return token; + } catch (error) { + if (error?.code !== 'EEXIST') throw error; + // Admission serializes inspection and removal with every other acquisition. + const held = stateOwnerOf(lockPath); + if (held !== null && (abandonedState.get(lockPath) === held || ownerIsGone(ownerPidOf(held)))) { + fs.unlinkSync(lockPath); + abandonedState.delete(lockPath); + } + return null; + } + }, () => { + if (!acquired) return; + abandonedState.set(lockPath, token); + if (stateOwnerOf(lockPath) === token) retryLockAccess(() => fs.unlinkSync(lockPath)); + abandonedState.delete(lockPath); + }, deadline, starts); + if (result) { + abandonedState.delete(lockPath); + return result; + } } catch (error) { - if (error?.code !== 'EEXIST') return null; - const held = contentOf(lockPath); - if (held !== null && ownerIsGone(ownerPidOf(held))) reclaim(lockPath, held); - else if (Date.now() >= deadline) return null; - else pause(LOCK_RETRY_MS); + if (!LOCK_ACCESS_ERRORS.has(error?.code)) throw error; } + if (Date.now() < deadline) pause(LOCK_RETRY_MS); } + return null; } function release(lockPath, token) { - if (contentOf(lockPath) === token) removeState(lockPath); + try { + if (stateOwnerOf(lockPath) === token) retryLockAccess(() => fs.unlinkSync(lockPath)); + } catch { + // The state write is already settled. Keep the exact token for the next admitted acquire. + abandonedState.set(lockPath, token); + } } /** * Applies `mutate` to a state file under an exclusive lock, so hook processes sharing it cannot overwrite each - * other's entries. A dead owner's lock is taken over; one that cannot be had within LOCK_WAIT_MS is given up on - * and the update runs unguarded, which never blocks a session. `mutate` returns undefined to leave the file. + * other's entries. A dead owner's lock is taken over; failed admission never permits an unlocked write. + * `mutate` returns undefined to leave the file. */ export function updateState(target, fallback, mutate) { const lockPath = `${target}.lock`; - const token = acquire(lockPath, Date.now() + LOCK_WAIT_MS); + let token, cause; + try { token = acquire(lockPath, Date.now() + LOCK_WAIT_MS); } catch (error) { cause = error; } + if (!token) throw Object.assign(new Error('State is busy or unavailable; no update was made', { cause }), { code: 'ELOCKED' }); try { const next = mutate(readState(target, fallback)); if (next !== undefined) writeState(target, next); return next; } finally { - if (token) release(lockPath, token); + release(lockPath, token); } } @@ -180,38 +209,107 @@ function isStale(lockDir, now) { } } +function workOwnerOf(lockDir) { + try { + return retryLockAccess(() => fs.readFileSync(path.join(lockDir, 'owner'), 'utf8')); + } catch (cause) { + if (cause?.code === 'ENOENT') return null; + throw Object.assign(new Error(`Could not read lock owner ${lockDir}`, { cause }), { code: 'ELOCKED' }); + } +} + function ownsLock(lockDir, token) { - return contentOf(path.join(lockDir, 'owner')) === token; + return workOwnerOf(lockDir) === token; } -function tryLock(lockDir, now) { - fs.mkdirSync(path.dirname(lockDir), { recursive: true, mode: 0o700 }); - if (isStale(lockDir, now)) fs.rmSync(lockDir, { recursive: true, force: true }); - try { - fs.mkdirSync(lockDir); - } catch { - return null; - } - const token = crypto.randomBytes(12).toString('hex'); - fs.writeFileSync(path.join(lockDir, 'owner'), token); - return { - refresh: () => { - if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); - }, - release: () => { +async function releaseWorkLock(lockDir, token) { + const deadline = Date.now() + LOCK_WAIT_MS; + const starts = new Map(); + for (;;) { + if (admitted(lockDir, () => { if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); - }, - }; + return true; + }, null, deadline, starts)) return; + if (Date.now() >= deadline) throw Object.assign(new Error(`Could not release lock ${lockDir}`), { code: 'ELOCKED' }); + await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS)); + } } -/** Runs slow `work` while holding a lock directory no other process gets meanwhile; false when it is held. */ +function tryLock(lockDir, now) { + let token, suspect, reusedToken; + const take = () => admitted(lockDir, () => { + const stale = isStale(lockDir, now); + if (abandonedWork.has(lockDir) || stale) { + let held; + try { held = workOwnerOf(lockDir); } catch { return null; } + if (held !== abandonedWork.get(lockDir)) { + if (!stale) return null; + if (held !== reusedToken && /^\d+:/.test(held ?? '')) { + const pid = ownerPidOf(held); + if (!ownerIsGone(pid)) { + const birth = WORK_OWNER_WITH_START.exec(held)?.[1]; + if (birth) suspect = { token: held, pid, birth: Number(birth) }; + return null; + } + } + } + fs.rmSync(lockDir, { recursive: true, force: true }); + abandonedWork.delete(lockDir); + } + try { + fs.mkdirSync(lockDir); + } catch { + return null; + } + token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}:${processStartTime(process.pid)}`; + fs.writeFileSync(path.join(lockDir, 'owner'), token); + return { + refresh: () => { + try { + return admitted(lockDir, () => { + if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); + }); + } catch { + // Timer-driven refresh is best effort; a live pid stays protected and final release still surfaces errors. + return null; + } + }, + release: async () => { + try { await releaseWorkLock(lockDir, token); } + catch (error) { abandonedWork.set(lockDir, token); throw error; } + }, + }; + }, () => { + if (!token) return; + abandonedWork.set(lockDir, token); + if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); + abandonedWork.delete(lockDir); + }); + const lock = take(); + if (lock || !suspect) return lock; + // A foreign lookup can be slow. Release admission, then recheck staleness and the exact token. + const actualBirth = processStartTime(suspect.pid, WORK_OWNER_PROBE_MS); + if (!actualBirth || actualBirth <= suspect.birth + PROCESS_START_TOLERANCE_MS) return null; + reusedToken = suspect.token; + return take(); +} + +/** Runs slow `work` under one owner's lock; dead or legacy directories recover after their lease expires. */ export async function withLock(lockDir, work, now = Date.now()) { const lock = tryLock(lockDir, now); if (!lock) return false; + let workFailed = false; try { await work(lock); return true; + } catch (error) { + workFailed = true; + throw error; } finally { - lock.release(); + try { + await lock.release(); + } catch (error) { + if (!workFailed) throw error; + } } } diff --git a/test/channels/hook.test.js b/test/channels/hook.test.js index ac20184..29cf3b2 100644 --- a/test/channels/hook.test.js +++ b/test/channels/hook.test.js @@ -1,4 +1,7 @@ import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; import { Readable } from 'node:stream'; import { after, before, test } from 'node:test'; @@ -19,12 +22,12 @@ let proxy; before(async () => { proxy = await fakeProxy(); }); after(() => proxy.close()); -function invoke(hostEvent, payload, { env = proxy.env, raw } = {}) { +function invoke(hostEvent, payload, { env = proxy.env, raw, stderr } = {}) { return new Promise((resolve) => { let written = ''; const stdout = { write: (text, done) => { written += text; done(); } }; const stdin = Readable.from([raw ?? JSON.stringify(payload)]); - runHookProcess('testhook', hostEvent, { stdin, stdout, env, exit: () => resolve(JSON.parse(written)), profileOf: async () => profile }); + runHookProcess('testhook', hostEvent, { stdin, stdout, stderr, env, exit: () => resolve(JSON.parse(written)), profileOf: async () => profile }); }); } @@ -44,3 +47,18 @@ test('inside the plugin\'s own headless run, a hook answers the empty reply and test('a payload that is not JSON still gets an answer', async () => { assert.deepEqual(await invoke('UserPromptSubmit', null, { raw: '{not json' }), { empty: 'UserPromptSubmit' }); }); + +test('corrupt hook state stays intact and emits a diagnostic alongside the neutral reply', async () => { + const dir = profile.stateDir(proxy.env); + const key = createHash('sha256').update('s-corrupt').digest('hex').slice(0, 16); + const file = join(dir, `testhook-session-${key}.json`); + mkdirSync(dir, { recursive: true }); + writeFileSync(file, '{"private text":'); + let diagnostic = ''; + const stderr = { write: (text, done) => { diagnostic += text; done(); } }; + + assert.deepEqual(await invoke('UserPromptSubmit', { session_id: 's-corrupt', cwd: '/proj', prompt: PROMPT }, { stderr }), { empty: 'UserPromptSubmit' }); + assert.match(diagnostic, /EINVALIDSTATE/); + assert.doesNotMatch(diagnostic, /private text/); + assert.equal(readFileSync(file, 'utf8'), '{"private text":'); +}); diff --git a/test/helpers/lock-contenders.js b/test/helpers/lock-contenders.js new file mode 100644 index 0000000..1e412a7 --- /dev/null +++ b/test/helpers/lock-contenders.js @@ -0,0 +1,39 @@ +import { spawn } from 'node:child_process'; +import { rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export async function until(check) { + const deadline = Date.now() + 10_000; + while (!check()) { + if (Date.now() >= deadline) throw new Error('timed out waiting for a lock contender'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +export function lockContenders(t, dir, script, args, cleanupMarkers) { + const children = []; + const results = () => Promise.all(children); + t.after(async () => { + try { + for (const name of cleanupMarkers) writeFileSync(join(dir, name), ''); + } finally { + await results(); + rmSync(dir, { recursive: true, force: true }); + } + }); + return { + run(role) { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, ...args, role], { windowsHide: true }); + let stderr = ''; + const watchdog = setTimeout(() => child.kill('SIGKILL'), 15_000); + child.stderr.on('data', (data) => { stderr += data; }); + child.on('error', (error) => { stderr += String(error); }); + children.push(new Promise((resolve) => child.on('close', (code) => { + clearTimeout(watchdog); + resolve({ code, stderr }); + }))); + return child; + }, + results, + }; +} diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js new file mode 100644 index 0000000..0e40c47 --- /dev/null +++ b/test/runtime/file-lock.test.js @@ -0,0 +1,621 @@ +import assert from 'node:assert/strict'; +import childProcess, { spawnSync } from 'node:child_process'; +import { syncBuiltinESMExports } from 'node:module'; +import fs from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, rmdirSync, statSync, unlinkSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; + +import { processStartTime } from '../../lib/runtime/process/owner.js'; +import { withLock } from '../../lib/runtime/store/file-lock.js'; +import { lockContenders, until } from '../helpers/lock-contenders.js'; + +const RUNTIME = new URL('../../lib/runtime/store/file-lock.js', import.meta.url).href; +const STALE = new Date(0); + +function scratch(t, token = null) { + const dir = mkdtempSync(join(tmpdir(), 'evolver-work-lock-')); + const lock = join(dir, 'work.lock'); + if (token !== null) { + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), token); + } + t.after(() => rmSync(dir, { recursive: true, force: true })); + return { dir, lock }; +} + +function holdAdmission(lock, pid = process.pid) { + const admission = `${lock}.admission`; + const owner = join(admission, `owner-${pid}-deadbeef`); + mkdirSync(admission); + writeFileSync(owner, 'fixture owner'); + return { admission, owner, release: () => { unlinkSync(owner); rmdirSync(admission); } }; +} + +test('taking an uncontended lock does not launch a process lookup', async (t) => { + const { lock } = scratch(t); + let launches = 0; + const spawn = childProcess.spawnSync; + childProcess.spawnSync = (...args) => { launches += 1; return spawn(...args); }; + syncBuiltinESMExports(); + try { + assert.equal(await withLock(lock, () => {}), true); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(launches, 0); +}); + +test('stale legacy and dead-owner directories recover, but a fresh legacy lock stays held', async (t) => { + const { lock } = scratch(t, 'legacy token'); + let entered = 0; + assert.equal(await withLock(lock, async () => { entered += 1; }), false); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), `${exited.pid}:deadbeef`); + holdAdmission(lock, exited.pid); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + assert.equal(entered, 2); + assert.equal(existsSync(lock), false); + assert.equal(existsSync(`${lock}.admission`), false); +}); + +test('an old mtime never evicts a live owner, which can refresh and release its lock', async (t) => { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + utimesSync(lock, STALE, STALE); + let entered = false; + assert.equal(await withLock(lock, async () => { entered = true; }), false); + assert.equal(entered, false); + held.refresh(); + assert.ok(statSync(lock).mtimeMs > STALE.getTime()); + }), true); + assert.equal(existsSync(lock), false); + assert.deepEqual(readdirSync(dir), []); +}); + +test('only a stale worker lock with proof of PID reuse can displace a live PID', async (t) => { + const birth = processStartTime(process.pid); + const prefix = `${process.pid}:deadbeef`; + for (const token of [prefix, `${prefix}:${birth}`, `${prefix}:${birth - 1_000}`, `${prefix}:invalid`]) { + const { lock } = scratch(t, token); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, () => assert.fail('owner identity is not disproved')), false); + assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), token); + } + + const { dir, lock } = scratch(t, `${prefix}:${birth - 10_000}`); + assert.equal(await withLock(lock, () => assert.fail('the lease is still fresh')), false); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, () => { + assert.match(readFileSync(join(lock, 'owner'), 'utf8'), new RegExp(`^${process.pid}:[a-f0-9]+:${birth}$`)); + }), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('a foreign PID probe leaves admission free and cannot remove a different worker token', async (t) => { + const pid = process.ppid; + const birth = Date.now(); + const old = `${pid}:aaaa:${birth - 10_000}`; + const replacement = `${pid}:bbbb:${birth - 10_000}`; + const { lock } = scratch(t, old); + utimesSync(lock, STALE, STALE); + const spawn = childProcess.spawnSync; + let probes = 0; + childProcess.spawnSync = () => { + probes += 1; + assert.equal(existsSync(`${lock}.admission`), false); + writeFileSync(join(lock, 'owner'), replacement); + const stdout = process.platform === 'win32' ? String(birth) : new Date(birth).toUTCString().replace(/ GMT$/, ''); + return { status: 0, stdout }; + }; + syncBuiltinESMExports(); + try { + assert.equal(await withLock(lock, () => assert.fail('the observed token was replaced')), false); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(probes, 1); + assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), replacement); +}); + +test('a transient owner read error never turns a stale live lock into a legacy lock', async (t) => { + for (const code of ['EBUSY', 'EPERM', 'EACCES', 'EIO']) { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async () => { + const owner = join(lock, 'owner'); + const token = readFileSync(owner, 'utf8'); + utimesSync(lock, STALE, STALE); + const read = fs.readFileSync; + let refused = false; + let entered = false; + fs.readFileSync = (target, ...args) => { + if (target === owner && !refused) { + refused = true; + throw Object.assign(new Error('transient owner read refusal'), { code }); + } + return read(target, ...args); + }; + try { + assert.equal(await withLock(lock, async () => { entered = true; }), false, code); + } finally { + fs.readFileSync = read; + } + assert.equal(refused, true); + assert.equal(entered, false, code); + assert.equal(readFileSync(owner, 'utf8'), token, 'the live owner stays intact'); + }), true); + assert.deepEqual(readdirSync(dir), [], 'the original owner still releases normally'); + } +}); + +test('a stale directory left before its owner was written is recoverable', async (t) => { + const { dir, lock } = scratch(t); + mkdirSync(lock); + utimesSync(lock, STALE, STALE); + let entered = false; + assert.equal(await withLock(lock, async () => { entered = true; }), true); + assert.equal(entered, true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release retries a transient owner read failure and really removes its lock', async (t) => { + const { dir, lock } = scratch(t); + const owner = join(lock, 'owner'); + const read = fs.readFileSync; + let refused = false; + try { + assert.equal(await withLock(lock, async () => { + fs.readFileSync = (target, ...args) => { + if (target === owner && !refused) { + refused = true; + throw Object.assign(new Error('transient read refusal'), { code: 'EBUSY' }); + } + return read(target, ...args); + }; + }), true); + } finally { + fs.readFileSync = read; + } + assert.equal(refused, true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release refuses an unreadable owner explicitly without claiming cleanup', async (t) => { + for (const code of ['EBUSY', 'EPERM', 'EIO']) { + const { lock } = scratch(t); + const owner = join(lock, 'owner'); + const read = fs.readFileSync; + let token; + try { + await assert.rejects(withLock(lock, async () => { + token = read(owner, 'utf8'); + fs.readFileSync = (target, ...args) => { + if (target === owner) throw Object.assign(new Error('persistent read refusal'), { code }); + return read(target, ...args); + }; + }), { code: 'ELOCKED' }); + } finally { + fs.readFileSync = read; + } + assert.equal(readFileSync(owner, 'utf8'), token, code); + assert.equal(await withLock(lock, () => {}), true, 'known abandoned work is recoverable after access returns'); + } +}); + +test('a stale foreign admission is probed once within the release deadline', async (t) => { + const { lock } = scratch(t); + const spawn = childProcess.spawnSync; + const timeouts = []; + try { + await assert.rejects(withLock(lock, () => { + const guard = holdAdmission(lock, process.ppid); + unlinkSync(guard.owner); + writeFileSync(join(guard.admission, `owner-${process.ppid}-a1-${Date.now() - 30_000}`), ''); + utimesSync(guard.admission, STALE, STALE); + childProcess.spawnSync = (_command, _args, { timeout }) => { + timeouts.push(timeout); + return { status: null, stdout: '' }; + }; + syncBuiltinESMExports(); + }), { code: 'ELOCKED' }); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(timeouts.length, 1); + assert.ok(timeouts[0] > 0 && timeouts[0] <= 1_000); + assert.equal(existsSync(lock), true, 'an inconclusive identity probe does not reclaim a live guard'); +}); + +test('a release failure preserves the original work error', async (t) => { + const { lock } = scratch(t); + const read = fs.readFileSync; + const workError = new Error('work failed'); + try { + await assert.rejects(withLock(lock, async () => { + fs.readFileSync = (file, ...args) => { + if (String(file) === join(lock, 'owner')) throw Object.assign(new Error('unreadable owner'), { code: 'EIO' }); + return read(file, ...args); + }; + throw workError; + }), (error) => error === workError); + } finally { + fs.readFileSync = read; + } +}); + +test('refresh leaves an unreadable live lock untouched without crashing its timer caller', async (t) => { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + const owner = join(lock, 'owner'); + utimesSync(lock, STALE, STALE); + const read = fs.readFileSync; + fs.readFileSync = (target, ...args) => { + if (target === owner) throw Object.assign(new Error('unreadable owner'), { code: 'EPERM' }); + return read(target, ...args); + }; + try { + assert.doesNotThrow(() => held.refresh()); + assert.equal(statSync(lock).mtimeMs, STALE.getTime()); + } finally { + fs.readFileSync = read; + } + }), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('refresh tolerates file access refusals and the next refresh and release still work', async (t) => { + for (const method of ['utimesSync', 'mkdirSync', 'readFileSync']) { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + utimesSync(lock, STALE, STALE); + const access = fs[method]; + let refused = false; + fs[method] = (target, ...args) => { + const matches = method === 'mkdirSync' ? String(target).startsWith(`${lock}.admission.`) + : target === (method === 'readFileSync' ? join(lock, 'owner') : lock); + if (matches && !refused) { + refused = true; + throw Object.assign(new Error('transient access refusal'), { code: 'EPERM' }); + } + return access(target, ...args); + }; + try { + assert.doesNotThrow(() => held.refresh(), method); + if (method !== 'readFileSync') assert.equal(statSync(lock).mtimeMs, STALE.getTime()); + } finally { + fs[method] = access; + } + assert.equal(refused, true, method); + held.refresh(); + assert.ok(statSync(lock).mtimeMs > STALE.getTime()); + }), true); + assert.deepEqual(readdirSync(dir), []); + } +}); + +test('release does not delete a missing or different owner', async (t) => { + for (const replacement of [null, `${process.pid}:another-owner`]) { + const { lock } = scratch(t); + assert.equal(await withLock(lock, async () => { + if (replacement === null) unlinkSync(join(lock, 'owner')); + else writeFileSync(join(lock, 'owner'), replacement); + }), true); + assert.equal(existsSync(lock), true); + assert.equal(existsSync(join(lock, 'owner')), replacement !== null); + if (replacement !== null) assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), replacement); + } +}); + +test('release waits asynchronously for transient admission contention', async (t) => { + const { lock } = scratch(t); + let guard; + let finished = false; + let refreshedAt; + const running = withLock(lock, async (held) => { + guard = holdAdmission(lock); + utimesSync(lock, STALE, STALE); + held.refresh(); + refreshedAt = statSync(lock).mtimeMs; + }).then(() => { finished = true; }); + try { + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(refreshedAt, STALE.getTime(), 'refresh also respects admission'); + assert.equal(finished, false); + assert.equal(existsSync(lock), true); + } finally { + guard.release(); + await running; + } + assert.equal(existsSync(lock), false); +}); + +test('release refuses bounded admission contention without deleting the held directory', async (t) => { + const { lock } = scratch(t); + let guard; + const started = Date.now(); + await assert.rejects(withLock(lock, async () => { + guard = holdAdmission(lock); + }), { code: 'ELOCKED' }); + assert.ok(Date.now() - started < 5_000, 'cleanup cannot wait forever'); + assert.equal(existsSync(lock), true); + assert.equal(readFileSync(guard.owner, 'utf8'), 'fixture owner'); +}); + +test('an admission cleanup failure is surfaced instead of silently orphaning a live guard', async (t) => { + const { lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + return unlink(target); + }; + let entered = false; + try { + await assert.rejects(withLock(lock, async () => { entered = true; }), { code: 'ELOCKED' }); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(entered, false); + assert.equal(existsSync(lock), false, 'the failed acquire must not leave a work lock'); + assert.ok(readdirSync(admission).some((name) => name.startsWith(`owner-${process.pid}-`))); + assert.equal(await withLock(lock, async () => { entered = true; }), true); + assert.equal(entered, true); +}); + +test('another process reclaims a failed admission release while the original owner is alive', async (t) => { + const { dir, lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + try { + assert.equal(await withLock(lock, () => { + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('fixture cleanup refusal'), { code: 'EPERM' }); + return unlink(target); + }; + }), true); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(existsSync(lock), false); + assert.ok(readdirSync(admission).some((name) => name.startsWith(`owner-${process.pid}-`))); + + const other = spawnSync(process.execPath, ['--input-type=module', '-e', ` + import assert from 'node:assert/strict'; + const [runtime, lock, owner] = process.argv.slice(1); + const { withLock } = await import(runtime); + process.kill(Number(owner), 0); + assert.equal(await withLock(lock, () => {}), true, 'the stranded admission can be reclaimed'); + `, RUNTIME, lock, String(process.pid)], { encoding: 'utf8', timeout: 10_000, windowsHide: true }); + assert.ifError(other.error); + assert.equal(other.status, 0, other.stderr); + assert.deepEqual(readdirSync(dir), []); +}); + +test('admission recovery resumes after the abandoned owner has already been removed', async (t) => { + const { dir, lock } = scratch(t); + const guard = holdAdmission(lock); + const marker = join(guard.admission, `abandoned-owner-${process.pid}-abcd`); + writeFileSync(marker, ''); + assert.equal(await withLock(lock, () => assert.fail('a different owner is still active')), false); + assert.equal(existsSync(guard.owner), true); + + unlinkSync(guard.owner); + assert.equal(await withLock(lock, () => {}), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { + const { dir, lock } = scratch(t); + const unlink = fs.unlinkSync; + let refused = false; + fs.unlinkSync = (target) => { + if (dirname(target) === `${lock}.admission` && !refused) { + refused = true; + throw Object.assign(new Error('temporarily refused by fixture'), { code: 'EPERM' }); + } + return unlink(target); + }; + let entered = 0; + try { + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(refused, true); + assert.equal(entered, 1); + assert.deepEqual(readdirSync(dir), []); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + assert.equal(entered, 2); + assert.deepEqual(readdirSync(dir), []); +}); + +test('an old admission stays held by the same process, but a reused PID can be recovered', async (t) => { + const { lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + return unlink(target); + }; + try { + await assert.rejects(withLock(lock, async () => assert.fail()), { code: 'ELOCKED' }); + } finally { + fs.unlinkSync = unlink; + } + const original = readdirSync(admission).find((name) => name.startsWith(`owner-${process.pid}-`)); + const birth = original.split('-')[3]; + assert.match(birth ?? '', /^\d{13}$/); + const replaceOwner = (name) => { + for (const name of readdirSync(admission)) unlinkSync(join(admission, name)); + writeFileSync(join(admission, name), ''); + utimesSync(admission, STALE, STALE); + }; + replaceOwner(`owner-${process.pid}-a1-${birth}`); + assert.equal(await withLock(lock, async () => assert.fail('same process still owns the guard')), false); + const differentBirth = Number(birth) - 10_000; + replaceOwner(`owner-${process.pid}-a2-${differentBirth}`); + assert.equal(await withLock(lock, async () => {}), true); +}); + +test('busy and unrecognised admission guards stay intact and leave no staging directories', async (t) => { + const { dir, lock } = scratch(t); + const guard = holdAdmission(lock); + assert.equal(await withLock(lock, async () => assert.fail('a live guard is held')), false); + assert.deepEqual(readdirSync(dir), ['work.lock.admission']); + writeFileSync(join(guard.admission, 'unexpected'), 'keep'); + assert.equal(await withLock(lock, async () => assert.fail('multiple owners are unknown')), false); + assert.equal(readdirSync(guard.admission).length, 2); + unlinkSync(guard.owner); + assert.equal(await withLock(lock, async () => assert.fail('an unknown owner is held')), false); + assert.equal(readFileSync(join(guard.admission, 'unexpected'), 'utf8'), 'keep'); + assert.deepEqual(readdirSync(dir), ['work.lock.admission']); +}); + +test('two contenders cannot replace the fresh lock while another reclaims a stale directory', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-work-lock-')); + const lock = join(dir, 'work.lock'); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), 'legacy token'); + utimesSync(lock, STALE, STALE); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { withLock } from ${JSON.stringify(RUNTIME)}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const wait = async (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + }; + if (role === 'A') { + const stat = fs.statSync; + let paused = false; + fs.statSync = (file, ...args) => { + const result = stat(file, ...args); + if (file === lock && !paused) { + paused = true; + fs.writeFileSync(marker('stale-read'), ''); + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker('resume'))) { + if (Date.now() > deadline) throw new Error('timed out waiting to resume'); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + } + return result; + }; + } + const result = await withLock(lock, async () => { + fs.writeFileSync(marker(role + '-entered'), ''); + await wait('release'); + }); + fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); + `; + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['resume', 'release']); + run('A'); + await until(() => existsSync(marker('stale-read'))); + run('B'); + run('C'); + await until(() => ['B', 'C'].every((role) => existsSync(marker(role + '-entered')) || existsSync(marker(role + '-result')))); + writeFileSync(marker('resume'), ''); + await until(() => existsSync(marker('A-entered'))); + const entered = ['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))); + writeFileSync(marker('release'), ''); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); + assert.deepEqual(entered, ['A'], 'only the process reclaiming the inspected stale directory may enter'); + assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [true, false, false]); +}); + +test('a delayed dead-admission reclaimer cannot unlink the fresh owner another contender published', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-admission-race-')); + const lock = join(dir, 'work.lock'); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), 'legacy token'); + utimesSync(lock, STALE, STALE); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + holdAdmission(lock, exited.pid); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { withLock } from ${JSON.stringify(RUNTIME)}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const waitSync = (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + }; + if (role === 'A') { + const list = fs.readdirSync; + let paused = false; + fs.readdirSync = (file, ...args) => { + const result = list(file, ...args); + if (file === lock + '.admission' && !paused) { + paused = true; + fs.writeFileSync(marker('dead-inspected'), ''); + waitSync('A-resume'); + } + return result; + }; + } + if (role === 'B') { + const rename = fs.renameSync; + let paused = false; + fs.renameSync = (source, target) => { + const result = rename(source, target); + if (target === lock + '.admission' && !paused) { + paused = true; + fs.writeFileSync(marker('fresh-held'), ''); + waitSync('B-resume'); + } + return result; + }; + } + const result = await withLock(lock, async () => { + fs.writeFileSync(marker(role + '-entered'), ''); + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker('release'))) { + if (Date.now() > deadline) throw new Error('timed out waiting to release'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + }); + fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); + `; + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['A-resume', 'B-resume', 'release']); + run('A'); + await until(() => existsSync(marker('dead-inspected'))); + const fresh = run('B'); + await until(() => existsSync(marker('fresh-held'))); + run('C'); + await until(() => existsSync(marker('C-result')) || existsSync(marker('C-entered'))); + assert.equal(existsSync(marker('C-entered')), false, 'publishing a fresh guard never exposes an empty directory'); + writeFileSync(marker('A-resume'), ''); + await until(() => existsSync(marker('A-result')) || existsSync(marker('A-entered'))); + assert.equal(existsSync(marker('A-entered')), false, 'the delayed reclaimer cannot displace the fresh guard'); + assert.equal(readdirSync(`${lock}.admission`).length, 1); + assert.match(readdirSync(`${lock}.admission`)[0], new RegExp(`^owner-${fresh.pid}-`)); + writeFileSync(marker('B-resume'), ''); + await until(() => existsSync(marker('B-entered'))); + assert.deepEqual(['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))), ['B']); + writeFileSync(marker('release'), ''); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); + assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [false, true, false]); +}); diff --git a/test/runtime/file-stores.test.js b/test/runtime/file-stores.test.js index 7f52193..0f56c33 100644 --- a/test/runtime/file-stores.test.js +++ b/test/runtime/file-stores.test.js @@ -1,11 +1,13 @@ import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; -import { mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import crypto from 'node:crypto'; +import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import fs from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { after, test } from 'node:test'; -import { readState } from '../../lib/runtime/store/file-lock.js'; +import { readState, updateState } from '../../lib/runtime/store/file-lock.js'; import { fileStoresOf } from '../../lib/runtime/store/file-stores.js'; const dirs = []; @@ -42,6 +44,232 @@ test('the outbox takes queued reports and keeps what each delivery attempt settl assert.equal(seen[0].sentAt, 5); }); +test('an unreadable or invalid draft table is preserved and never passed to an update as empty', async t => { + for (const failure of ['EBUSY', 'EACCES', 'invalid-json', 'wrong-shape']) { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + const original = failure === 'invalid-json' ? '[unfinished' : failure === 'wrong-shape' ? '{}' : '[{"title":"keep this draft"}]'; + writeFileSync(target, original); + const originalRead = fs.readFileSync; + t.mock.method(fs, 'readFileSync', (file, ...args) => { + if (file === target && ['EBUSY', 'EACCES'].includes(failure)) throw Object.assign(new Error('fixture read denial'), { code: failure }); + return originalRead(file, ...args); + }); + let mutated = false; + await assert.rejects(cursor.drafts.update(drafts => { mutated = true; return [...drafts, { title: 'new draft' }]; }), failure); + assert.equal(mutated, false, failure); + t.mock.restoreAll(); + assert.equal(fs.readFileSync(target, 'utf8'), original, failure); + assert.equal(existsSync(`${target}.lock`), false, failure); + } + const { stores: cursor } = stores(); + assert.deepEqual(await cursor.drafts.read(), [], 'an absent table is still a fresh store'); + await cursor.drafts.update(() => [{ title: 'first draft' }]); + assert.deepEqual(await cursor.drafts.read(), [{ title: 'first draft' }]); +}); + +test('a corrupt draft file is preserved for every packaged host', async () => { + for (const agent of readdirSync(new URL('../../hosts/', import.meta.url))) { + if (!existsSync(new URL(`../../hosts/${agent}/profile.js`, import.meta.url))) continue; + const { dir, stores: host } = stores(agent); + const target = join(dir, `${agent}-distill-drafts.json`); + writeFileSync(target, '[unfinished'); + await assert.rejects(host.drafts.update(() => assert.fail('mutator must not run')), { code: 'EINVALIDSTATE' }); + assert.equal(fs.readFileSync(target, 'utf8'), '[unfinished', agent); + } +}); + +test('a refused state lock preserves drafts and never runs their mutator', async t => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + const lock = `${target}.lock`; + const original = '[{"title":"keep this draft"}]'; + writeFileSync(target, original); + const write = fs.writeFileSync; + t.mock.method(fs, 'writeFileSync', (file, ...args) => { + if (file === lock) throw Object.assign(new Error('fixture lock creation denial'), { code: 'EROFS' }); + return write(file, ...args); + }); + let mutations = 0; + try { + await assert.rejects(cursor.drafts.update(drafts => { mutations += 1; return [...drafts, { title: 'new draft' }]; }), { code: 'ELOCKED' }); + } finally { + t.mock.restoreAll(); + } + assert.equal(mutations, 0); + assert.equal(fs.readFileSync(target, 'utf8'), original); + assert.deepEqual(readdirSync(dir), ['cursor-distill-drafts.json']); +}); + +test('a held live state lock preserves drafts and never runs their mutator after waiting', async () => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + const lock = `${target}.lock`; + const original = '[{"title":"keep this draft"}]'; + const held = `${process.pid}:fixture-live-owner`; + writeFileSync(target, original); + writeFileSync(lock, held); + let mutations = 0; + await assert.rejects(cursor.drafts.update(drafts => { mutations += 1; return [...drafts, { title: 'new draft' }]; }), { code: 'ELOCKED' }); + assert.equal(mutations, 0); + assert.equal(fs.readFileSync(target, 'utf8'), original); + assert.equal(fs.readFileSync(lock, 'utf8'), held); +}); + +for (const closeFails of [false, true]) test(`a partial temporary state write preserves drafts and cleans up before a retry${closeFails ? ' even when close also fails' : ''}`, async t => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + const previous = [{ title: 'keep this draft' }]; + await cursor.drafts.update(() => previous); + const original = fs.readFileSync(target, 'utf8'); + const temporary = file => typeof file === 'string' && file.startsWith(target + '.') && file.endsWith('.tmp'); + const descriptors = new Set(); + const open = fs.openSync, write = fs.writeFileSync, close = fs.closeSync; + t.mock.method(fs, 'openSync', (file, ...args) => { + const descriptor = open(file, ...args); + if (temporary(file)) descriptors.add(descriptor); + return descriptor; + }); + let failed = false; + let closeFailed = false; + const writeFailure = Object.assign(new Error('fixture storage exhausted after a partial write'), { code: 'ENOSPC' }); + if (closeFails) t.mock.method(fs, 'closeSync', descriptor => { + const owned = descriptors.delete(descriptor); + const result = close(descriptor); + if (owned) { + closeFailed = true; + throw Object.assign(new Error('fixture close failure after closing the descriptor'), { code: 'EIO' }); + } + return result; + }); + t.mock.method(fs, 'writeFileSync', (file, ...args) => { + if (!failed && (temporary(file) || descriptors.has(file))) { + failed = true; + write(file, '[{"partial":', ...args.slice(1)); + throw writeFailure; + } + return write(file, ...args); + }); + try { + await assert.rejects(cursor.drafts.update(drafts => [...drafts, { title: 'new draft' }]), error => { + assert.equal(error, writeFailure); + assert.equal(error.code, 'ENOSPC'); + return true; + }); + } finally { + t.mock.restoreAll(); + } + assert.equal(failed, true); + assert.equal(closeFailed, closeFails); + assert.equal(fs.readFileSync(target, 'utf8'), original); + assert.deepEqual(readdirSync(dir), ['cursor-distill-drafts.json']); + await cursor.drafts.update(drafts => [...drafts, { title: 'retry succeeded' }]); + assert.deepEqual(await cursor.drafts.read(), [...previous, { title: 'retry succeeded' }]); +}); + +test('a temporary descriptor close failure preserves drafts without replacing the target and permits a retry', async t => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + const previous = [{ title: 'keep this draft' }]; + await cursor.drafts.update(() => previous); + const original = fs.readFileSync(target, 'utf8'); + const descriptors = new Set(); + const open = fs.openSync, close = fs.closeSync, rename = fs.renameSync; + t.mock.method(fs, 'openSync', (file, ...args) => { + const descriptor = open(file, ...args); + if (typeof file === 'string' && file.startsWith(target + '.') && file.endsWith('.tmp')) descriptors.add(descriptor); + return descriptor; + }); + let failed = false; + t.mock.method(fs, 'closeSync', descriptor => { + const owned = descriptors.delete(descriptor); + const result = close(descriptor); + if (owned) { + failed = true; + throw Object.assign(new Error('fixture close failure after closing the descriptor'), { code: 'EIO' }); + } + return result; + }); + let replacements = 0; + t.mock.method(fs, 'renameSync', (from, to) => { + if (to === target) replacements += 1; + return rename(from, to); + }); + try { + await assert.rejects(cursor.drafts.update(drafts => [...drafts, { title: 'new draft' }]), { code: 'EIO' }); + } finally { + t.mock.restoreAll(); + } + assert.equal(failed, true); + assert.equal(replacements, 0); + assert.equal(fs.readFileSync(target, 'utf8'), original); + assert.deepEqual(readdirSync(dir), ['cursor-distill-drafts.json']); + await cursor.drafts.update(drafts => [...drafts, { title: 'retry succeeded' }]); + assert.deepEqual(await cursor.drafts.read(), [...previous, { title: 'retry succeeded' }]); +}); + +test('a temporary file collision preserves the existing file and releases the state lock', async t => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + await cursor.drafts.update(() => [{ title: 'keep this draft' }]); + const original = fs.readFileSync(target, 'utf8'); + const suffix = '010203040506'; + const collision = `${target}.${process.pid}.${suffix}.tmp`; + const collisionContents = 'another invocation owns these bytes'; + writeFileSync(collision, collisionContents); + const random = crypto.randomBytes; + t.mock.method(crypto, 'randomBytes', (size, ...args) => size === 6 ? Buffer.from(suffix, 'hex') : random(size, ...args)); + try { + await assert.rejects(cursor.drafts.update(drafts => [...drafts, { title: 'new draft' }]), { code: 'EEXIST' }); + } finally { + t.mock.restoreAll(); + } + assert.equal(fs.readFileSync(target, 'utf8'), original); + assert.equal(fs.readFileSync(collision, 'utf8'), collisionContents); + assert.equal(existsSync(`${target}.lock`), false); + await cursor.drafts.update(drafts => [...drafts, { title: 'retry succeeded' }]); + assert.deepEqual(await cursor.drafts.read(), [{ title: 'keep this draft' }, { title: 'retry succeeded' }]); + assert.equal(fs.readFileSync(collision, 'utf8'), collisionContents); +}); + +test('a failed draft replacement preserves the previous file, releases its lock and removes its temporary file', async t => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + await cursor.drafts.update(() => [{ title: 'keep this draft' }]); + const before = fs.readFileSync(target, 'utf8'); + const rename = fs.renameSync; + t.mock.method(fs, 'renameSync', (from, to) => { + if (to === target) throw Object.assign(new Error('fixture replacement denial'), { code: 'EPERM' }); + return rename(from, to); + }); + await assert.rejects(cursor.drafts.update(drafts => [...drafts, { title: 'new draft' }]), { code: 'EPERM' }); + t.mock.restoreAll(); + assert.equal(fs.readFileSync(target, 'utf8'), before); + assert.deepEqual(readdirSync(dir), ['cursor-distill-drafts.json']); + await cursor.drafts.update(drafts => [...drafts, { title: 'retry succeeded' }]); + assert.deepEqual(await cursor.drafts.read(), [{ title: 'keep this draft' }, { title: 'retry succeeded' }]); +}); + +test('a transient replacement denial retries while readers can still see the previous drafts', async t => { + const { dir, stores: cursor } = stores(); + const target = join(dir, 'cursor-distill-drafts.json'); + const previous = [{ title: 'keep this draft' }]; + await cursor.drafts.update(() => previous); + const rename = fs.renameSync; + let attempts = 0; + t.mock.method(fs, 'renameSync', (from, to) => { + if (to === target) { + assert.deepEqual(JSON.parse(fs.readFileSync(target, 'utf8')), previous); + if (++attempts === 1) throw Object.assign(new Error('fixture sharing denial'), { code: 'EBUSY' }); + } + return rename(from, to); + }); + await cursor.drafts.update(drafts => [...drafts, { title: 'new draft' }]); + assert.equal(attempts, 2); + assert.deepEqual(await cursor.drafts.read(), [...previous, { title: 'new draft' }]); + assert.deepEqual(readdirSync(dir), ['cursor-distill-drafts.json']); +}); + test('a report appended after an interrupted queue write survives draining and commit', async () => { const { dir, stores: cursor } = stores(); const earlier = { report: { task_id: 'earlier', asset_id: 'a' }, sessionId: 's1' }; @@ -74,11 +302,36 @@ test('parallel processes updating one state file through the lock lose no update const lock = new URL('../../lib/runtime/store/file-lock.js', import.meta.url).href; const script = `import { updateState } from ${JSON.stringify(lock)}; for (let i = 0; i < 25; i += 1) updateState(${JSON.stringify(target)}, {}, (state) => ({ count: (state.count ?? 0) + 1 }));`; - const run = () => new Promise((resolve) => spawn(process.execPath, ['--input-type=module', '-e', script]).on('close', resolve)); - await Promise.all([run(), run(), run(), run()]); + const run = () => new Promise((resolve) => { + const child = spawn(process.execPath, ['--input-type=module', '-e', script]); + let stderr = ''; + child.stderr.on('data', chunk => { stderr += chunk; }); + child.on('close', code => resolve({ code, stderr })); + }); + const results = await Promise.all([run(), run(), run(), run()]); + assert.deepEqual(results.map(result => result.code), [0, 0, 0, 0], results.map(result => result.stderr).join('\n')); assert.equal(readState(target, {}).count, 100); }); +test('transient sharing denial while releasing a state lock leaves no orphaned lock', t => { + for (const operation of ['readFileSync', 'unlinkSync']) { + const { dir } = stores(); + const target = join(dir, 'counter.json'), lock = `${target}.lock`; + const original = fs[operation]; + let denied = false; + t.mock.method(fs, operation, (file, ...args) => { + if (file === lock && !denied) { denied = true; throw Object.assign(new Error('sharing violation'), { code: 'EPERM' }); } + return original(file, ...args); + }); + updateState(target, {}, state => ({ count: (state.count ?? 0) + 1 })); + assert.equal(denied, true); + assert.equal(existsSync(lock), false, operation); + updateState(target, {}, state => ({ count: state.count + 1 })); + assert.equal(readState(target, {}).count, 2); + t.mock.restoreAll(); + } +}); + test('a session file outlives the outbox: a session resumed within a month still counts its turns on', async () => { const dir = mkdtempSync(join(tmpdir(), 'evolver-prune-')); dirs.push(dir); diff --git a/test/runtime/state-lock.test.js b/test/runtime/state-lock.test.js new file mode 100644 index 0000000..b1f39f3 --- /dev/null +++ b/test/runtime/state-lock.test.js @@ -0,0 +1,323 @@ +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; + +import { updateState } from '../../lib/runtime/store/file-lock.js'; +import { lockContenders, until } from '../helpers/lock-contenders.js'; + +const RUNTIME = new URL('../../lib/runtime/store/file-lock.js', import.meta.url).href; +const BUSY = { code: 'ELOCKED', message: 'State is busy or unavailable; no update was made' }; + +function scratch(t) { + const dir = mkdtempSync(join(tmpdir(), 'evolver-state-lock-')); + const target = join(dir, 'state.json'); + writeFileSync(target, '{"count":0}'); + t.after(() => rmSync(dir, { recursive: true, force: true })); + return { dir, target, lock: `${target}.lock` }; +} + +test('a failed admission release after acquiring a state token lets the next update proceed', (t) => { + const { target, lock } = scratch(t); + const unlink = fs.unlinkSync; + fs.unlinkSync = (file) => { + if (typeof file === 'string' && dirname(file) === `${lock}.admission`) { + throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + } + return unlink(file); + }; + try { + assert.throws(() => updateState(target, {}, () => assert.fail('mutator must not run')), BUSY); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(existsSync(lock), false); + assert.deepEqual(updateState(target, {}, (state) => ({ count: state.count + 1 })), { count: 1 }); +}); + +test('a failed acquire rollback keeps its own token recoverable', (t) => { + const { target, lock } = scratch(t); + const unlink = fs.unlinkSync; + fs.unlinkSync = (file) => { + if (String(file) === lock || dirname(String(file)) === `${lock}.admission`) { + throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + } + return unlink(file); + }; + try { + assert.throws(() => updateState(target, {}, () => assert.fail('mutator must not run')), BUSY); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(existsSync(lock), true); + assert.deepEqual(updateState(target, {}, (state) => ({ count: state.count + 1 })), { count: 1 }); + assert.equal(existsSync(lock), false); +}); + +test('a settled state write stays successful and its stranded token can be reclaimed', (t) => { + const { target, lock } = scratch(t); + const unlink = fs.unlinkSync; + try { + assert.deepEqual(updateState(target, {}, (state) => { + fs.unlinkSync = (file) => { + if (String(file) === lock) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + return unlink(file); + }; + return { count: state.count + 1 }; + }), { count: 1 }); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(existsSync(lock), true); + assert.deepEqual(updateState(target, {}, (state) => ({ count: state.count + 1 })), { count: 2 }); + assert.equal(existsSync(lock), false); +}); + +for (const stage of ['mutator', 'read', 'replacement']) { + test(`an original ${stage} failure survives a release denial and its stranded state token is reclaimed on retry`, t => { + const { dir, target, lock } = scratch(t); + const original = readFileSync(target, 'utf8'); + const failure = Object.assign(new Error(`fixture ${stage} failure`), { code: stage === 'read' ? 'EACCES' : 'EIO' }); + const read = fs.readFileSync, rename = fs.renameSync, unlink = fs.unlinkSync; + if (stage === 'read') t.mock.method(fs, 'readFileSync', (file, ...args) => { + if (file === target) throw failure; + return read(file, ...args); + }); + if (stage === 'replacement') t.mock.method(fs, 'renameSync', (from, to) => { + if (to === target) throw failure; + return rename(from, to); + }); + let releaseDenials = 0; + t.mock.method(fs, 'unlinkSync', (file, ...args) => { + if (file === lock) { + releaseDenials += 1; + throw Object.assign(new Error('fixture state token release denial'), { code: 'EPERM' }); + } + return unlink(file, ...args); + }); + let mutations = 0; + try { + assert.throws(() => updateState(target, {}, state => { + mutations += 1; + assert.deepEqual(state, { count: 0 }); + if (stage === 'mutator') throw failure; + return { count: state.count + 1 }; + }), error => { + if (stage === 'read') { + assert.equal(error.code, 'ESTATEIO'); + assert.equal(error.cause, failure); + } else assert.equal(error, failure); + return true; + }); + } finally { + t.mock.restoreAll(); + } + assert.equal(mutations, stage === 'read' ? 0 : 1); + assert.ok(releaseDenials > 0, 'the state token release was actually refused'); + assert.equal(readFileSync(target, 'utf8'), original); + assert.equal(existsSync(lock), true); + assert.match(readFileSync(lock, 'utf8'), new RegExp(`^${process.pid}:`)); + assert.deepEqual(fs.readdirSync(dir).sort(), ['state.json', 'state.json.lock']); + let retries = 0; + assert.deepEqual(updateState(target, {}, state => { retries += 1; return { count: state.count + 1 }; }), { count: 1 }); + assert.equal(retries, 1); + assert.deepEqual(JSON.parse(readFileSync(target, 'utf8')), { count: 1 }); + assert.deepEqual(fs.readdirSync(dir), ['state.json']); + }); +} + +test('a held state lock never runs an unguarded mutator after its wait expires', (t) => { + const { target, lock } = scratch(t); + const held = `${process.pid}:alive`; + writeFileSync(lock, held); + let called = false; + const started = Date.now(); + assert.throws(() => updateState(target, {}, () => { called = true; return { count: 1 }; }), BUSY); + assert.ok(Date.now() - started < 5_000); + assert.equal(called, false); + assert.equal(readFileSync(target, 'utf8'), '{"count":0}'); + assert.equal(readFileSync(lock, 'utf8'), held); +}); + +test('a legacy dead state owner is recovered, and file permission errors stop before mutation', (t) => { + const { target, lock } = scratch(t); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + writeFileSync(lock, `${exited.pid}:deadbeef`); + assert.deepEqual(updateState(target, {}, (stored) => ({ count: stored.count + 1 })), { count: 1 }); + assert.equal(existsSync(lock), false); + const write = fs.writeFileSync; + fs.writeFileSync = (file, ...args) => { + if (file === lock) throw Object.assign(new Error('refused by fixture'), { code: 'EROFS' }); + return write(file, ...args); + }; + let called = false; + try { + assert.throws(() => updateState(target, {}, () => { called = true; return {}; }), BUSY); + } finally { + fs.writeFileSync = write; + } + assert.equal(called, false); + assert.deepEqual(JSON.parse(readFileSync(target, 'utf8')), { count: 1 }); + assert.deepEqual(updateState(target, {}, (stored) => ({ count: stored.count + 1 })), { count: 2 }); +}); + +test('a transient sharing denial retries outside admission and runs the mutator only once', (t) => { + const { target, lock } = scratch(t); + const write = fs.writeFileSync; + let attempts = 0; + fs.writeFileSync = (file, ...args) => { + if (file === lock && ++attempts === 1) throw Object.assign(new Error('temporarily refused by fixture'), { code: 'EPERM' }); + return write(file, ...args); + }; + let called = 0; + try { + assert.deepEqual(updateState(target, {}, (stored) => { called += 1; return { count: stored.count + 1 }; }), { count: 1 }); + } finally { + fs.writeFileSync = write; + } + assert.equal(attempts, 2); + assert.equal(called, 1); + assert.equal(existsSync(lock), false); +}); + +test('an admission setup refusal leaves state unchanged and never calls the mutator', (t) => { + const { target, lock } = scratch(t); + const mkdir = fs.mkdirSync; + fs.mkdirSync = (file, ...args) => { + if (String(file).startsWith(`${lock}.admission.`)) throw Object.assign(new Error('read-only fixture'), { code: 'EROFS' }); + return mkdir(file, ...args); + }; + let called = false; + try { + assert.throws(() => updateState(target, {}, () => { called = true; return {}; }), BUSY); + } finally { + fs.mkdirSync = mkdir; + } + assert.equal(called, false); + assert.equal(readFileSync(target, 'utf8'), '{"count":0}'); + assert.deepEqual(updateState(target, {}, (stored) => ({ count: stored.count + 1 })), { count: 1 }); +}); + +test('a refused dead-owner reclaim obeys the same bounded wait without calling the mutator', async (t) => { + const { target, lock } = scratch(t); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + const held = `${exited.pid}:deadbeef`; + writeFileSync(lock, held); + const script = ` + import fs from 'node:fs'; + import { updateState } from ${JSON.stringify(RUNTIME)}; + const [target, lock] = process.argv.slice(1); + const unlink = fs.unlinkSync; + let reclaims = 0; + fs.unlinkSync = (file) => { + if (String(file) === lock) { + reclaims += 1; + throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + } + return unlink(file); + }; + let called = false; + try { updateState(target, {}, () => { called = true; return {}; }); } + catch (error) { process.stdout.write(JSON.stringify({ code: error.code, called, reclaims })); } + `; + const child = spawn(process.execPath, ['--input-type=module', '-e', script, target, lock]); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (data) => { stdout += data; }); + child.stderr.on('data', (data) => { stderr += data; }); + const watchdog = setTimeout(() => child.kill('SIGKILL'), 5_000); + const code = await new Promise((resolve) => child.on('close', resolve)); + clearTimeout(watchdog); + assert.equal(code, 0, stderr || 'a failed reclaim must not spin forever'); + const result = JSON.parse(stdout); + assert.equal(result.code, 'ELOCKED'); + assert.equal(result.called, false); + assert.ok(result.reclaims > 0); + assert.equal(readFileSync(target, 'utf8'), '{"count":0}'); + assert.equal(readFileSync(lock, 'utf8'), held); + assert.deepEqual(updateState(target, {}, (stored) => ({ count: stored.count + 1 })), { count: 1 }); +}); + +test('three state updaters preserve every increment while recovering one dead file lock', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-state-lock-race-')); + const target = join(dir, 'state.json'); + const lock = `${target}.lock`; + writeFileSync(target, '{"count":0}'); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + const dead = `${exited.pid}:deadbeef`; + writeFileSync(lock, dead); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { updateState } from ${JSON.stringify(RUNTIME)}; + const [dir, target, lock, role] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const wait = (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + }; + // Start all contenders before orchestrating the lock interleaving. + updateState(marker(role + '-startup.json'), {}, () => undefined); + fs.writeFileSync(marker(role + '-ready'), ''); + wait(role + '-start'); + const read = fs.readFileSync; + let paused = false; + fs.readFileSync = (file, ...args) => { + const result = read(file, ...args); + if (role === 'A' && file === lock && !paused) { + paused = true; + fs.writeFileSync(marker('A-read-dead'), ''); + wait('A-resume'); + } + return result; + }; + const rename = fs.renameSync; + fs.renameSync = (source, destination) => { + try { return rename(source, destination); } + catch (error) { + if (destination === lock + '.admission') fs.writeFileSync(marker(role + '-guard-busy'), ''); + throw error; + } + }; + const write = fs.writeFileSync; + fs.writeFileSync = (file, ...args) => { + if (file === lock) write(marker(role + '-attempted'), ''); + return write(file, ...args); + }; + updateState(target, {}, (stored) => { + try { fs.writeFileSync(marker('active'), role, { flag: 'wx' }); } + catch { fs.writeFileSync(marker('overlap'), ''); } + fs.writeFileSync(marker(role + '-entered'), ''); + wait('release'); + if (fs.readFileSync(marker('active'), 'utf8') === role) fs.unlinkSync(marker('active')); + return { count: stored.count + 1 }; + }); + `; + const { run, results } = lockContenders(t, dir, script, [dir, target, lock], ['A-start', 'B-start', 'C-start', 'A-resume', 'release']); + for (const role of ['A', 'B', 'C']) run(role); + await until(() => ['A', 'B', 'C'].every((role) => existsSync(marker(role + '-ready')))); + writeFileSync(marker('A-start'), ''); + await until(() => existsSync(marker('A-read-dead'))); + writeFileSync(marker('B-start'), ''); + await until(() => existsSync(marker('B-entered')) || existsSync(marker('B-guard-busy'))); + assert.equal(existsSync(marker('B-entered')), false, 'the competing process waits for admission'); + writeFileSync(marker('A-resume'), ''); + await until(() => ['A', 'B'].some((role) => existsSync(marker(role + '-entered')))); + writeFileSync(marker('C-start'), ''); + await until(() => existsSync(marker('C-attempted')) || existsSync(marker('C-guard-busy'))); + writeFileSync(marker('release'), ''); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); + assert.equal(existsSync(marker('overlap')), false, 'state mutators never overlap during dead-owner recovery'); + assert.deepEqual(JSON.parse(readFileSync(target, 'utf8')), { count: 3 }); +});