From b20e3ca6d0f58d83d141d86f6d112c468a75d4ff Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Mon, 2 Nov 2026 11:19:32 +0700 Subject: [PATCH 01/22] fix: discard distillation replies invalidated by a correction --- lib/app/distill-session.js | 22 +++++++- test/app/distill-session.test.js | 96 ++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/lib/app/distill-session.js b/lib/app/distill-session.js index 81e7617..55208a0 100644 --- a/lib/app/distill-session.js +++ b/lib/app/distill-session.js @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 EvoMap -import { withDraft } from '../distill.js'; +import { withDraft, withoutDraft } from '../distill.js'; import { candidateSessionOf, claimedLeftovers, claimedOwn, isCurrentCandidate, settledCandidate, withoutExpiredLeftovers } from '../distill/candidate.js'; import { argumentsOf, declineOf, distillBody, distillPrompt } from '../distill/prompt.js'; import { distillOutcomeOf, isSettled, needsAgentModel } from '../distill/result.js'; @@ -26,8 +26,26 @@ const settle = ({ candidates }, { sessionId, candidate }, outcome, countsFailure async function post(ports, { sessionId, candidate }, args, usage, signal) { const body = distillBody({ platform: ports.identity.agent, sessionId: taskSessionId(ports.identity, sessionId), candidate, args, usage }); const result = await ports.proxy.request('POST', '/conversation/distill', body, { timeoutMs: DISTILL_TIMEOUT_MS, signal }); + if (!isCurrentCandidate(await ports.candidates.read(), sessionId, candidate)) return { status: 'dropped' }; const outcome = distillOutcomeOf(result, { title: args.title, sessionId }, ports.clock.now()); - if (outcome.draft) await ports.drafts.update((drafts) => withDraft(drafts, outcome.draft, ports.clock.now())); + if (outcome.draft) { + outcome.draft = { ...outcome.draft, sourceCandidate: { at: candidate.at, turn: candidate.turn } }; + const fromCandidate = (draft) => draft?.sessionId === sessionId + && draft.sourceCandidate?.at === candidate.at && draft.sourceCandidate?.turn === candidate.turn; + let previous; + await ports.drafts.update((drafts) => { + previous = withoutDraft(drafts, null, ports.clock.now()).find((draft) => draft.id === outcome.draft.id); + return withDraft(drafts, outcome.draft, ports.clock.now()); + }); + if (!isCurrentCandidate(await ports.candidates.read(), sessionId, candidate)) { + await ports.drafts.update((drafts) => { + if (!drafts.some((draft) => draft.id === outcome.draft.id && fromCandidate(draft))) return undefined; + const kept = withoutDraft(drafts, outcome.draft.id, ports.clock.now()); + return previous && !fromCandidate(previous) ? withDraft(kept, previous, ports.clock.now()) : kept; + }); + return { status: 'dropped' }; + } + } return { ...outcome, ...(usage && { usage }) }; } diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index 35f2a2d..63465e4 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -107,6 +107,102 @@ test('a candidate dropped by a correction while the model answers is not posted' assert.equal(requests.some((request) => request.path === '/conversation/distill'), false); }); +test('a correction while a draft request is pending discards its late reply', async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const started = Promise.withResolvers(); + const finish = Promise.withResolvers(); + const request = ports.proxy.request; + ports.proxy.request = async (...args) => { + const reply = await request(...args); + if (args[1] === '/conversation/distill') { + started.resolve(); + await finish.promise; + } + return reply; + }; + const running = distillSession(ports, { sessionId: 's1' }); + await started.promise; + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + finish.resolve(); + assert.equal((await running).status, 'dropped'); + assert.deepEqual(await ports.drafts.read(), []); + assert.equal((await ports.candidates.read()).s1, undefined); +}); + +test('a correction while an agent Gene request is pending discards its late draft', async () => { + const { ports } = await withCandidate(); + const writtenFor = (await ports.candidates.read()).s1.at; + const started = Promise.withResolvers(); + const finish = Promise.withResolvers(); + const request = ports.proxy.request; + ports.proxy.request = async (...args) => { + const reply = await request(...args); + if (args[1] === '/conversation/distill') { + started.resolve(); + await finish.promise; + } + return reply; + }; + const running = postAgentGene(ports, { sessionId: 's1', text: JSON.stringify(GENE), writtenFor }); + await started.promise; + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + finish.resolve(); + assert.equal((await running).status, 'dropped'); + assert.deepEqual(await ports.drafts.read(), []); + assert.equal((await ports.candidates.read()).s1, undefined); +}); + +test('a correction crossing a draft write removes that draft and keeps earlier drafts', async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const kept = { id: 'earlier-draft', title: 'Keep this method', createdAt: ports.clock.now() }; + await ports.drafts.update(() => [kept]); + const started = Promise.withResolvers(); + const finish = Promise.withResolvers(); + const update = ports.drafts.update; + let firstWrite = true; + ports.drafts.update = async (mutate) => { + const written = await update(mutate); + if (firstWrite) { + firstWrite = false; + started.resolve(); + await finish.promise; + } + return written; + }; + const running = distillSession(ports, { sessionId: 's1' }); + await started.promise; + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + finish.resolve(); + assert.equal((await running).status, 'dropped'); + assert.deepEqual(await ports.drafts.read(), [kept]); +}); + +for (const other of ['legacy draft', 'same candidate', 'newer turn', 'other session']) { + test(`draft compensation preserves a same-id ${other} only when it is not the corrected candidate`, async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const candidate = (await ports.candidates.read()).s1; + const previous = { id: DRAFTED.distill_id, title: 'Earlier method', createdAt: ports.clock.now(), sessionId: 's1' }; + if (other === 'same candidate') previous.sourceCandidate = { at: candidate.at, turn: candidate.turn }; + const replacement = { ...previous, title: 'Concurrent method', sourceCandidate: { at: candidate.at, turn: candidate.turn + 1 } }; + if (other === 'other session') Object.assign(replacement, { sessionId: 's2', sourceCandidate: { at: candidate.at, turn: candidate.turn } }); + await ports.drafts.update(() => [previous]); + const update = ports.drafts.update; + let firstWrite = true; + ports.drafts.update = async (mutate) => { + const written = await update(mutate); + if (firstWrite) { + firstWrite = false; + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + if (other === 'newer turn' || other === 'other session') await update(() => [replacement]); + } + return written; + }; + assert.equal((await distillSession(ports, { sessionId: 's1' })).status, 'dropped'); + const expected = other === 'same candidate' ? [] : [other === 'legacy draft' ? previous : replacement]; + assert.deepEqual(await ports.drafts.read(), expected); + }); +} + test('a failing headless model keeps the candidate, and three failed tries give it up', async () => { const { ports } = await withCandidate({ model: { complete: async () => { throw new Error('Authentication required'); } } }); await sessionEnded(ports, { sessionId: 's1', projectDir: '/proj' }); From 67191f56d55b5ea985d00ad9a8fcaafc50f371c3 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Mon, 2 Nov 2026 11:26:55 +0700 Subject: [PATCH 02/22] fix: restore drafts displaced by a corrected distillation write --- lib/app/distill-session.js | 18 ++++++++++++------ test/app/distill-session.test.js | 19 +++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/app/distill-session.js b/lib/app/distill-session.js index 55208a0..eedb44b 100644 --- a/lib/app/distill-session.js +++ b/lib/app/distill-session.js @@ -32,16 +32,22 @@ async function post(ports, { sessionId, candidate }, args, usage, signal) { outcome.draft = { ...outcome.draft, sourceCandidate: { at: candidate.at, turn: candidate.turn } }; const fromCandidate = (draft) => draft?.sessionId === sessionId && draft.sourceCandidate?.at === candidate.at && draft.sourceCandidate?.turn === candidate.turn; - let previous; + let displaced; await ports.drafts.update((drafts) => { - previous = withoutDraft(drafts, null, ports.clock.now()).find((draft) => draft.id === outcome.draft.id); - return withDraft(drafts, outcome.draft, ports.clock.now()); + const live = withoutDraft(drafts, null, ports.clock.now()); + const next = withDraft(live, outcome.draft, ports.clock.now()); + displaced = live.filter((draft) => draft.id === outcome.draft.id || !next.some((kept) => kept.id === draft.id)); + return next; }); if (!isCurrentCandidate(await ports.candidates.read(), sessionId, candidate)) { await ports.drafts.update((drafts) => { - if (!drafts.some((draft) => draft.id === outcome.draft.id && fromCandidate(draft))) return undefined; - const kept = withoutDraft(drafts, outcome.draft.id, ports.clock.now()); - return previous && !fromCandidate(previous) ? withDraft(kept, previous, ports.clock.now()) : kept; + const stillOurs = drafts.some((draft) => draft.id === outcome.draft.id && fromCandidate(draft)); + const kept = withoutDraft(drafts, null, ports.clock.now()).filter((draft) => draft.id !== outcome.draft.id || !fromCandidate(draft)); + const restore = withoutDraft(displaced, null, ports.clock.now()).filter((draft) => !fromCandidate(draft) + && (draft.id !== outcome.draft.id || stillOurs) && !kept.some((existing) => existing.id === draft.id)); + if (!stillOurs && restore.length === 0) return undefined; + // Undo this insertion's losses without rolling back concurrent records or applying the cap a second time. + return [...restore, ...kept]; }); return { status: 'dropped' }; } diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index 63465e4..5ba0df7 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -152,10 +152,10 @@ test('a correction while an agent Gene request is pending discards its late draf assert.equal((await ports.candidates.read()).s1, undefined); }); -test('a correction crossing a draft write removes that draft and keeps earlier drafts', async () => { +for (const [count, replace] of [[1, false], [20, false], [20, true]]) test(`a correction crossing a draft write keeps ${count} earlier drafts${replace ? ' and a concurrent same-id replacement' : ''}`, async () => { const { ports } = await withCandidate({ model: modelAnswering(GENE) }); - const kept = { id: 'earlier-draft', title: 'Keep this method', createdAt: ports.clock.now() }; - await ports.drafts.update(() => [kept]); + const kept = Array.from({ length: count }, (_, index) => ({ id: `earlier-${index}`, title: 'Keep this method', createdAt: ports.clock.now() })); + await ports.drafts.update(() => kept); const started = Promise.withResolvers(); const finish = Promise.withResolvers(); const update = ports.drafts.update; @@ -172,12 +172,18 @@ test('a correction crossing a draft write removes that draft and keeps earlier d const running = distillSession(ports, { sessionId: 's1' }); await started.promise; await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + let replacement; + if (replace) await update((drafts) => drafts.map((draft) => { + if (draft.id !== DRAFTED.distill_id) return draft; + replacement = { ...draft, title: 'Concurrent method', sourceCandidate: { ...draft.sourceCandidate, turn: draft.sourceCandidate.turn + 1 } }; + return replacement; + })); finish.resolve(); assert.equal((await running).status, 'dropped'); - assert.deepEqual(await ports.drafts.read(), [kept]); + assert.deepEqual(await ports.drafts.read(), replace ? [...kept, replacement] : kept); }); -for (const other of ['legacy draft', 'same candidate', 'newer turn', 'other session']) { +for (const other of ['legacy draft', 'same candidate', 'newer turn', 'other session', 'discarded draft']) { test(`draft compensation preserves a same-id ${other} only when it is not the corrected candidate`, async () => { const { ports } = await withCandidate({ model: modelAnswering(GENE) }); const candidate = (await ports.candidates.read()).s1; @@ -194,11 +200,12 @@ for (const other of ['legacy draft', 'same candidate', 'newer turn', 'other sess firstWrite = false; await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); if (other === 'newer turn' || other === 'other session') await update(() => [replacement]); + if (other === 'discarded draft') await update(() => []); } return written; }; assert.equal((await distillSession(ports, { sessionId: 's1' })).status, 'dropped'); - const expected = other === 'same candidate' ? [] : [other === 'legacy draft' ? previous : replacement]; + const expected = other === 'same candidate' || other === 'discarded draft' ? [] : [other === 'legacy draft' ? previous : replacement]; assert.deepEqual(await ports.drafts.read(), expected); }); } From 05d14ec00aa9daa97af3c2614cd7c91dafb5d9d8 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Mon, 2 Nov 2026 11:47:32 +0700 Subject: [PATCH 03/22] fix: serialize stale worker-lock recovery through atomic admission --- lib/runtime/store/file-lock.js | 124 +++++++++++-- test/runtime/file-lock.test.js | 329 +++++++++++++++++++++++++++++++++ 2 files changed, 438 insertions(+), 15 deletions(-) create mode 100644 test/runtime/file-lock.test.js diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index d122790..fbc2d06 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -183,27 +183,121 @@ function ownsLock(lockDir, token) { return contentOf(path.join(lockDir, 'owner')) === token; } -function tryLock(lockDir, now) { - fs.mkdirSync(path.dirname(lockDir), { recursive: true, mode: 0o700 }); - if (isStale(lockDir, now)) fs.rmSync(lockDir, { recursive: true, force: true }); +function reclaimAdmission(admission) { try { - fs.mkdirSync(lockDir); + if (!fs.lstatSync(admission).isDirectory()) return false; + const owners = fs.readdirSync(admission); + if (owners.length > 1) return false; + if (owners.length === 1) { + const owner = /^owner-(\d+)-[a-f0-9]+$/.exec(owners[0]); + if (!owner || !ownerIsGone(Number(owner[1]))) return false; + fs.unlinkSync(path.join(admission, owners[0])); + } + // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot + // remove it even if another reclaimer replaced the directory after our read. + fs.rmdirSync(admission); + return true; } catch { + return false; + } +} + +function admissionOf(lockDir) { + const admission = `${lockDir}.admission`; + const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}`; + const staging = `${admission}.${owner}.claim`; + fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); + fs.mkdirSync(staging, { mode: 0o700 }); + try { + fs.writeFileSync(path.join(staging, owner), '', { mode: 0o600, flag: 'wx' }); + // Publishing a prepopulated directory never exposes an empty fresh guard to a stale reclaimer. + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + fs.renameSync(staging, admission); + return { admission, owner }; + } catch { + if (attempt > 0 || !reclaimAdmission(admission)) return null; + } + } return null; + } finally { + fs.rmSync(staging, { recursive: true, force: true }); } - const token = crypto.randomBytes(12).toString('hex'); - fs.writeFileSync(path.join(lockDir, 'owner'), token); - return { - refresh: () => { - if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); - }, - release: () => { +} + +function retryAdmissionCleanup(remove) { + for (let attempt = 0; ; attempt += 1) { + try { + return remove(); + } catch (error) { + if (attempt === 2 || !['EPERM', 'EBUSY', 'EACCES'].includes(error?.code)) throw error; + pause(LOCK_RETRY_MS); + } + } +} + +function releaseAdmission({ admission, owner }) { + try { + retryAdmissionCleanup(() => fs.unlinkSync(path.join(admission, owner))); + try { + retryAdmissionCleanup(() => fs.rmdirSync(admission)); + } catch (error) { + if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error?.code)) throw error; + } + } catch (cause) { + throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + } +} + +// Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. +function admitted(lockDir, mutate) { + const admission = admissionOf(lockDir); + if (!admission) return null; + try { + return mutate(); + } finally { + releaseAdmission(admission); + } +} + +async function releaseWorkLock(lockDir, token) { + const deadline = Date.now() + LOCK_WAIT_MS; + for (;;) { + if (admitted(lockDir, () => { if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); - }, - }; + return true; + })) return; + if (Date.now() >= deadline) throw Object.assign(new Error(`Could not release lock ${lockDir}`), { code: 'ELOCKED' }); + await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS)); + } +} + +function tryLock(lockDir, now) { + return admitted(lockDir, () => { + if (isStale(lockDir, now)) { + const held = contentOf(path.join(lockDir, 'owner')); + // Old versions wrote random tokens without a pid. Keep their lease recovery, but a known live owner + // cannot lose its lock just because a slow or paused worker missed a refresh. A reused pid stays busy. + if (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))) return null; + fs.rmSync(lockDir, { recursive: true, force: true }); + } + try { + fs.mkdirSync(lockDir); + } catch { + return null; + } + const token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; + fs.writeFileSync(path.join(lockDir, 'owner'), token); + return { + refresh: () => admitted(lockDir, () => { + if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); + }), + release: () => releaseWorkLock(lockDir, token), + }; + }); } -/** Runs slow `work` while holding a lock directory no other process gets meanwhile; false when it is held. */ +/** Runs slow `work` under one owner's lock; dead or legacy directories recover after their lease expires. */ export async function withLock(lockDir, work, now = Date.now()) { const lock = tryLock(lockDir, now); if (!lock) return false; @@ -211,6 +305,6 @@ export async function withLock(lockDir, work, now = Date.now()) { await work(lock); return true; } finally { - lock.release(); + await lock.release(); } } diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js new file mode 100644 index 0000000..d2b8bdb --- /dev/null +++ b/test/runtime/file-lock.test.js @@ -0,0 +1,329 @@ +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, rmdirSync, statSync, unlinkSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; + +import { withLock } from '../../lib/runtime/store/file-lock.js'; + +const RUNTIME = new URL('../../lib/runtime/store/file-lock.js', import.meta.url).href; +const STALE = new Date(0); + +function scratch(t, token = null) { + const dir = mkdtempSync(join(tmpdir(), 'evolver-work-lock-')); + const lock = join(dir, 'work.lock'); + if (token !== null) { + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), token); + } + t.after(() => rmSync(dir, { recursive: true, force: true })); + return { dir, lock }; +} + +async function until(check) { + const deadline = Date.now() + 10_000; + while (!check()) { + if (Date.now() > deadline) throw new Error('timed out waiting for the lock contender'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +function holdAdmission(lock, pid = process.pid) { + const admission = `${lock}.admission`; + const owner = join(admission, `owner-${pid}-deadbeef`); + mkdirSync(admission); + writeFileSync(owner, 'fixture owner'); + return { admission, owner, release: () => { unlinkSync(owner); rmdirSync(admission); } }; +} + +test('stale legacy and dead-owner directories recover, but a fresh legacy lock stays held', async (t) => { + const { lock } = scratch(t, 'legacy token'); + let entered = 0; + assert.equal(await withLock(lock, async () => { entered += 1; }), false); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), `${exited.pid}:deadbeef`); + holdAdmission(lock, exited.pid); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + assert.equal(entered, 2); + assert.equal(existsSync(lock), false); + assert.equal(existsSync(`${lock}.admission`), false); +}); + +test('an old mtime never evicts a live owner, which can refresh and release its lock', async (t) => { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + utimesSync(lock, STALE, STALE); + let entered = false; + assert.equal(await withLock(lock, async () => { entered = true; }), false); + assert.equal(entered, false); + held.refresh(); + assert.ok(statSync(lock).mtimeMs > STALE.getTime()); + }), true); + assert.equal(existsSync(lock), false); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release waits asynchronously for transient admission contention', async (t) => { + const { lock } = scratch(t); + let guard; + let finished = false; + let refreshedAt; + const running = withLock(lock, async (held) => { + guard = holdAdmission(lock); + utimesSync(lock, STALE, STALE); + held.refresh(); + refreshedAt = statSync(lock).mtimeMs; + }).then(() => { finished = true; }); + try { + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(refreshedAt, STALE.getTime(), 'refresh also respects admission'); + assert.equal(finished, false); + assert.equal(existsSync(lock), true); + } finally { + guard.release(); + await running; + } + assert.equal(existsSync(lock), false); +}); + +test('release refuses bounded admission contention without deleting the held directory', async (t) => { + const { lock } = scratch(t); + let guard; + const started = Date.now(); + await assert.rejects(withLock(lock, async () => { + guard = holdAdmission(lock); + }), { code: 'ELOCKED' }); + assert.ok(Date.now() - started < 5_000, 'cleanup cannot wait forever'); + assert.equal(existsSync(lock), true); + assert.equal(readFileSync(guard.owner, 'utf8'), 'fixture owner'); +}); + +test('an admission cleanup failure is surfaced instead of silently orphaning a live guard', async (t) => { + const { lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + return unlink(target); + }; + let entered = false; + try { + await assert.rejects(withLock(lock, async () => { entered = true; }), { code: 'ELOCKED' }); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(entered, false); + assert.match(readdirSync(admission)[0], new RegExp(`^owner-${process.pid}-`)); +}); + +test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { + const { dir, lock } = scratch(t); + const unlink = fs.unlinkSync; + let refused = false; + fs.unlinkSync = (target) => { + if (dirname(target) === `${lock}.admission` && !refused) { + refused = true; + throw Object.assign(new Error('temporarily refused by fixture'), { code: 'EPERM' }); + } + return unlink(target); + }; + let entered = 0; + try { + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(refused, true); + assert.equal(entered, 1); + assert.deepEqual(readdirSync(dir), []); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + assert.equal(entered, 2); + assert.deepEqual(readdirSync(dir), []); +}); + +test('busy and unrecognised admission guards stay intact and leave no staging directories', async (t) => { + const { dir, lock } = scratch(t); + const guard = holdAdmission(lock); + assert.equal(await withLock(lock, async () => assert.fail('a live guard is held')), false); + assert.deepEqual(readdirSync(dir), ['work.lock.admission']); + writeFileSync(join(guard.admission, 'unexpected'), 'keep'); + assert.equal(await withLock(lock, async () => assert.fail('multiple owners are unknown')), false); + assert.equal(readdirSync(guard.admission).length, 2); + unlinkSync(guard.owner); + assert.equal(await withLock(lock, async () => assert.fail('an unknown owner is held')), false); + assert.equal(readFileSync(join(guard.admission, 'unexpected'), 'utf8'), 'keep'); + assert.deepEqual(readdirSync(dir), ['work.lock.admission']); +}); + +test('two contenders cannot replace the fresh lock while another reclaims a stale directory', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-work-lock-')); + const lock = join(dir, 'work.lock'); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), 'legacy token'); + utimesSync(lock, STALE, STALE); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { withLock } from ${JSON.stringify(RUNTIME)}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const wait = async (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + }; + if (role === 'A') { + const stat = fs.statSync; + let paused = false; + fs.statSync = (file, ...args) => { + const result = stat(file, ...args); + if (file === lock && !paused) { + paused = true; + fs.writeFileSync(marker('stale-read'), ''); + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker('resume'))) { + if (Date.now() > deadline) throw new Error('timed out waiting to resume'); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + } + return result; + }; + } + const result = await withLock(lock, async () => { + fs.writeFileSync(marker(role + '-entered'), ''); + await wait('release'); + }); + fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); + `; + const children = []; + const run = (role) => { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, lock, role]); + let stderr = ''; + child.stderr.on('data', (data) => { stderr += data; }); + const done = new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr }))); + children.push({ child, done }); + }; + t.after(async () => { + writeFileSync(marker('resume'), ''); + writeFileSync(marker('release'), ''); + await Promise.all(children.map(({ done }) => done)); + rmSync(dir, { recursive: true, force: true }); + }); + run('A'); + await until(() => existsSync(marker('stale-read'))); + run('B'); + run('C'); + await until(() => ['B', 'C'].every((role) => existsSync(marker(role + '-entered')) || existsSync(marker(role + '-result')))); + writeFileSync(marker('resume'), ''); + await until(() => existsSync(marker('A-entered'))); + const entered = ['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))); + writeFileSync(marker('release'), ''); + const results = await Promise.all(children.map(({ done }) => done)); + assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); + assert.deepEqual(entered, ['A'], 'only the process reclaiming the inspected stale directory may enter'); + assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [true, false, false]); +}); + +test('a delayed dead-admission reclaimer cannot unlink the fresh owner another contender published', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-admission-race-')); + const lock = join(dir, 'work.lock'); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), 'legacy token'); + utimesSync(lock, STALE, STALE); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + holdAdmission(lock, exited.pid); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { withLock } from ${JSON.stringify(RUNTIME)}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const waitSync = (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + }; + if (role === 'A') { + const list = fs.readdirSync; + let paused = false; + fs.readdirSync = (file, ...args) => { + const result = list(file, ...args); + if (file === lock + '.admission' && !paused) { + paused = true; + fs.writeFileSync(marker('dead-inspected'), ''); + waitSync('A-resume'); + } + return result; + }; + } + if (role === 'B') { + const rename = fs.renameSync; + let paused = false; + fs.renameSync = (source, target) => { + const result = rename(source, target); + if (target === lock + '.admission' && !paused) { + paused = true; + fs.writeFileSync(marker('fresh-held'), ''); + waitSync('B-resume'); + } + return result; + }; + } + const result = await withLock(lock, async () => { + fs.writeFileSync(marker(role + '-entered'), ''); + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker('release'))) { + if (Date.now() > deadline) throw new Error('timed out waiting to release'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + }); + fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); + `; + const children = []; + const run = (role) => { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, lock, role]); + let stderr = ''; + child.stderr.on('data', (data) => { stderr += data; }); + const done = new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr }))); + children.push({ child, done }); + return child; + }; + t.after(async () => { + for (const name of ['A-resume', 'B-resume', 'release']) writeFileSync(marker(name), ''); + await Promise.all(children.map(({ done }) => done)); + rmSync(dir, { recursive: true, force: true }); + }); + run('A'); + await until(() => existsSync(marker('dead-inspected'))); + const fresh = run('B'); + await until(() => existsSync(marker('fresh-held'))); + run('C'); + await until(() => existsSync(marker('C-result')) || existsSync(marker('C-entered'))); + assert.equal(existsSync(marker('C-entered')), false, 'publishing a fresh guard never exposes an empty directory'); + writeFileSync(marker('A-resume'), ''); + await until(() => existsSync(marker('A-result')) || existsSync(marker('A-entered'))); + assert.equal(existsSync(marker('A-entered')), false, 'the delayed reclaimer cannot displace the fresh guard'); + assert.equal(readdirSync(`${lock}.admission`).length, 1); + assert.match(readdirSync(`${lock}.admission`)[0], new RegExp(`^owner-${fresh.pid}-`)); + writeFileSync(marker('B-resume'), ''); + await until(() => existsSync(marker('B-entered'))); + assert.deepEqual(['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))), ['B']); + writeFileSync(marker('release'), ''); + const results = await Promise.all(children.map(({ done }) => done)); + assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); + assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [false, true, false]); +}); From 71f98f860e4f938f777eb0d0caffd444024376b7 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Mon, 2 Nov 2026 12:05:22 +0700 Subject: [PATCH 04/22] fix: preserve worker ownership when lock metadata is unreadable --- lib/runtime/store/file-lock.js | 35 ++++++-- test/runtime/file-lock.test.js | 146 +++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 9 deletions(-) diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index fbc2d06..e8732b1 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -179,8 +179,17 @@ function isStale(lockDir, now) { } } +function workOwnerOf(lockDir) { + try { + return retryLockAccess(() => fs.readFileSync(path.join(lockDir, 'owner'), 'utf8')); + } catch (cause) { + if (cause?.code === 'ENOENT') return null; + throw Object.assign(new Error(`Could not read lock owner ${lockDir}`, { cause }), { code: 'ELOCKED' }); + } +} + function ownsLock(lockDir, token) { - return contentOf(path.join(lockDir, 'owner')) === token; + return workOwnerOf(lockDir) === token; } function reclaimAdmission(admission) { @@ -225,10 +234,10 @@ function admissionOf(lockDir) { } } -function retryAdmissionCleanup(remove) { +function retryLockAccess(access) { for (let attempt = 0; ; attempt += 1) { try { - return remove(); + return access(); } catch (error) { if (attempt === 2 || !['EPERM', 'EBUSY', 'EACCES'].includes(error?.code)) throw error; pause(LOCK_RETRY_MS); @@ -238,9 +247,9 @@ function retryAdmissionCleanup(remove) { function releaseAdmission({ admission, owner }) { try { - retryAdmissionCleanup(() => fs.unlinkSync(path.join(admission, owner))); + retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); try { - retryAdmissionCleanup(() => fs.rmdirSync(admission)); + retryLockAccess(() => fs.rmdirSync(admission)); } catch (error) { if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error?.code)) throw error; } @@ -275,7 +284,8 @@ async function releaseWorkLock(lockDir, token) { function tryLock(lockDir, now) { return admitted(lockDir, () => { if (isStale(lockDir, now)) { - const held = contentOf(path.join(lockDir, 'owner')); + let held; + try { held = workOwnerOf(lockDir); } catch { return null; } // Old versions wrote random tokens without a pid. Keep their lease recovery, but a known live owner // cannot lose its lock just because a slow or paused worker missed a refresh. A reused pid stays busy. if (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))) return null; @@ -289,9 +299,16 @@ function tryLock(lockDir, now) { const token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; fs.writeFileSync(path.join(lockDir, 'owner'), token); return { - refresh: () => admitted(lockDir, () => { - if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); - }), + refresh: () => { + try { + return admitted(lockDir, () => { + if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); + }); + } catch { + // Timer-driven refresh is best effort; a live pid stays protected and final release still surfaces errors. + return null; + } + }, release: () => releaseWorkLock(lockDir, token), }; }); diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index d2b8bdb..1156254 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -70,6 +70,152 @@ test('an old mtime never evicts a live owner, which can refresh and release its assert.deepEqual(readdirSync(dir), []); }); +test('a transient owner read error never turns a stale live lock into a legacy lock', async (t) => { + for (const code of ['EBUSY', 'EPERM', 'EACCES', 'EIO']) { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async () => { + const owner = join(lock, 'owner'); + const token = readFileSync(owner, 'utf8'); + utimesSync(lock, STALE, STALE); + const read = fs.readFileSync; + let refused = false; + let entered = false; + fs.readFileSync = (target, ...args) => { + if (target === owner && !refused) { + refused = true; + throw Object.assign(new Error('transient owner read refusal'), { code }); + } + return read(target, ...args); + }; + try { + assert.equal(await withLock(lock, async () => { entered = true; }), false, code); + } finally { + fs.readFileSync = read; + } + assert.equal(refused, true); + assert.equal(entered, false, code); + assert.equal(readFileSync(owner, 'utf8'), token, 'the live owner stays intact'); + }), true); + assert.deepEqual(readdirSync(dir), [], 'the original owner still releases normally'); + } +}); + +test('a stale directory left before its owner was written is recoverable', async (t) => { + const { dir, lock } = scratch(t); + mkdirSync(lock); + utimesSync(lock, STALE, STALE); + let entered = false; + assert.equal(await withLock(lock, async () => { entered = true; }), true); + assert.equal(entered, true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release retries a transient owner read failure and really removes its lock', async (t) => { + const { dir, lock } = scratch(t); + const owner = join(lock, 'owner'); + const read = fs.readFileSync; + let refused = false; + try { + assert.equal(await withLock(lock, async () => { + fs.readFileSync = (target, ...args) => { + if (target === owner && !refused) { + refused = true; + throw Object.assign(new Error('transient read refusal'), { code: 'EBUSY' }); + } + return read(target, ...args); + }; + }), true); + } finally { + fs.readFileSync = read; + } + assert.equal(refused, true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release refuses an unreadable owner explicitly without claiming cleanup', async (t) => { + for (const code of ['EBUSY', 'EPERM', 'EIO']) { + const { lock } = scratch(t); + const owner = join(lock, 'owner'); + const read = fs.readFileSync; + let token; + try { + await assert.rejects(withLock(lock, async () => { + token = read(owner, 'utf8'); + fs.readFileSync = (target, ...args) => { + if (target === owner) throw Object.assign(new Error('persistent read refusal'), { code }); + return read(target, ...args); + }; + }), { code: 'ELOCKED' }); + } finally { + fs.readFileSync = read; + } + assert.equal(readFileSync(owner, 'utf8'), token, code); + } +}); + +test('refresh leaves an unreadable live lock untouched without crashing its timer caller', async (t) => { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + const owner = join(lock, 'owner'); + utimesSync(lock, STALE, STALE); + const read = fs.readFileSync; + fs.readFileSync = (target, ...args) => { + if (target === owner) throw Object.assign(new Error('unreadable owner'), { code: 'EPERM' }); + return read(target, ...args); + }; + try { + assert.doesNotThrow(() => held.refresh()); + assert.equal(statSync(lock).mtimeMs, STALE.getTime()); + } finally { + fs.readFileSync = read; + } + }), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('refresh tolerates file access refusals and the next refresh and release still work', async (t) => { + for (const method of ['utimesSync', 'mkdirSync', 'readFileSync']) { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + utimesSync(lock, STALE, STALE); + const access = fs[method]; + let refused = false; + fs[method] = (target, ...args) => { + const matches = method === 'mkdirSync' ? String(target).startsWith(`${lock}.admission.`) + : target === (method === 'readFileSync' ? join(lock, 'owner') : lock); + if (matches && !refused) { + refused = true; + throw Object.assign(new Error('transient access refusal'), { code: 'EPERM' }); + } + return access(target, ...args); + }; + try { + assert.doesNotThrow(() => held.refresh(), method); + if (method !== 'readFileSync') assert.equal(statSync(lock).mtimeMs, STALE.getTime()); + } finally { + fs[method] = access; + } + assert.equal(refused, true, method); + held.refresh(); + assert.ok(statSync(lock).mtimeMs > STALE.getTime()); + }), true); + assert.deepEqual(readdirSync(dir), []); + } +}); + +test('release does not delete a missing or different owner', async (t) => { + for (const replacement of [null, `${process.pid}:another-owner`]) { + const { lock } = scratch(t); + assert.equal(await withLock(lock, async () => { + if (replacement === null) unlinkSync(join(lock, 'owner')); + else writeFileSync(join(lock, 'owner'), replacement); + }), true); + assert.equal(existsSync(lock), true); + assert.equal(existsSync(join(lock, 'owner')), replacement !== null); + if (replacement !== null) assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), replacement); + } +}); + test('release waits asynchronously for transient admission contention', async (t) => { const { lock } = scratch(t); let guard; From a638041595b46c05f827793a2fb6ca61a9bb3fc3 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 10:29:28 +0700 Subject: [PATCH 05/22] fix: recover stranded worker lock admission safely --- lib/runtime/store/file-lock.js | 61 +++++++++++++++++++++++++++------- test/runtime/file-lock.test.js | 31 +++++++++++++++++ 2 files changed, 80 insertions(+), 12 deletions(-) diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index e8732b1..8cd0c19 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -4,6 +4,7 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; +import { spawnSync } from 'node:child_process'; import { ownerIsGone } from '../process/owner.js'; @@ -11,6 +12,23 @@ const LOCK_WAIT_MS = 2_000; const LOCK_RETRY_MS = 20; const WORK_LOCK_STALE_MS = 2 * 60 * 1000; const CLAIM_STALE_MS = 60 * 1000; +const ADMISSION_IDENTITY_CHECK_MS = 10_000; +const abandonedAdmissions = new Map(); +let ownBirth; + +function processBirth(pid) { + const windows = process.platform === 'win32'; + const systemRoot = process.env.SystemRoot ?? 'C:\\Windows'; + if (windows && !path.isAbsolute(systemRoot)) return null; + const executable = windows ? path.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') : 'ps'; + const args = windows + ? ['-NoProfile', '-NonInteractive', '-Command', `(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToUniversalTime().Ticks`] + : ['-p', String(pid), '-o', 'lstart=']; + const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; + const result = spawnSync(executable, args, { env, encoding: 'utf8', windowsHide: true, timeout: windows ? 2500 : 1000 }); + const started = result.status === 0 ? result.stdout.trim() : ''; + return started ? crypto.createHash('sha256').update(started).digest('hex').slice(0, 24) : null; +} export function readState(target, fallback) { try { @@ -194,13 +212,23 @@ function ownsLock(lockDir, token) { function reclaimAdmission(admission) { try { - if (!fs.lstatSync(admission).isDirectory()) return false; + const info = fs.lstatSync(admission); + if (!info.isDirectory()) return false; const owners = fs.readdirSync(admission); if (owners.length > 1) return false; if (owners.length === 1) { - const owner = /^owner-(\d+)-[a-f0-9]+$/.exec(owners[0]); - if (!owner || !ownerIsGone(Number(owner[1]))) return false; + const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}))?$/.exec(owners[0]); + if (!owner) return false; + const pid = Number(owner[1]); + if (!Number.isSafeInteger(pid) || pid <= 0) return false; + const abandonedHere = abandonedAdmissions.get(admission) === owners[0]; + if (!abandonedHere && !ownerIsGone(pid)) { + if (!owner[2] || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS) return false; + const actualBirth = pid === process.pid ? ownBirth : processBirth(pid); + if (!actualBirth || actualBirth === owner[2]) return false; + } fs.unlinkSync(path.join(admission, owners[0])); + if (abandonedHere) abandonedAdmissions.delete(admission); } // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot // remove it even if another reclaimer replaced the directory after our read. @@ -213,7 +241,9 @@ function reclaimAdmission(admission) { function admissionOf(lockDir) { const admission = `${lockDir}.admission`; - const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}`; + if (ownBirth === undefined) ownBirth = processBirth(process.pid); + const birth = ownBirth ? `-${ownBirth}` : ''; + const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}${birth}`; const staging = `${admission}.${owner}.claim`; fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); fs.mkdirSync(staging, { mode: 0o700 }); @@ -223,6 +253,7 @@ function admissionOf(lockDir) { for (let attempt = 0; attempt < 2; attempt += 1) { try { fs.renameSync(staging, admission); + abandonedAdmissions.delete(admission); return { admission, owner }; } catch { if (attempt > 0 || !reclaimAdmission(admission)) return null; @@ -248,24 +279,27 @@ function retryLockAccess(access) { function releaseAdmission({ admission, owner }) { try { retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); - try { - retryLockAccess(() => fs.rmdirSync(admission)); - } catch (error) { - if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error?.code)) throw error; - } } catch (cause) { + abandonedAdmissions.set(admission, owner); throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); } + // The guard is released when its unique owner is gone. An empty directory can be reclaimed. + try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} } // Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. -function admitted(lockDir, mutate) { +function admitted(lockDir, mutate, rollback = () => {}) { const admission = admissionOf(lockDir); if (!admission) return null; try { return mutate(); } finally { - releaseAdmission(admission); + try { + releaseAdmission(admission); + } catch (error) { + rollback(); + throw error; + } } } @@ -282,6 +316,7 @@ async function releaseWorkLock(lockDir, token) { } function tryLock(lockDir, now) { + let token; return admitted(lockDir, () => { if (isStale(lockDir, now)) { let held; @@ -296,7 +331,7 @@ function tryLock(lockDir, now) { } catch { return null; } - const token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; + token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; fs.writeFileSync(path.join(lockDir, 'owner'), token); return { refresh: () => { @@ -311,6 +346,8 @@ function tryLock(lockDir, now) { }, release: () => releaseWorkLock(lockDir, token), }; + }, () => { + if (token && ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); }); } diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index 1156254..855aa91 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -266,7 +266,10 @@ test('an admission cleanup failure is surfaced instead of silently orphaning a l fs.unlinkSync = unlink; } assert.equal(entered, false); + assert.equal(existsSync(lock), false, 'the failed acquire must not leave a work lock'); assert.match(readdirSync(admission)[0], new RegExp(`^owner-${process.pid}-`)); + assert.equal(await withLock(lock, async () => { entered = true; }), true); + assert.equal(entered, true); }); test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { @@ -294,6 +297,34 @@ test('a transient admission cleanup refusal retries and leaves later work unbloc assert.deepEqual(readdirSync(dir), []); }); +test('an old admission stays held by the same process, but a reused PID can be recovered', async (t) => { + const { lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + return unlink(target); + }; + try { + await assert.rejects(withLock(lock, async () => assert.fail()), { code: 'ELOCKED' }); + } finally { + fs.unlinkSync = unlink; + } + const [original] = readdirSync(admission); + const birth = original.split('-')[3]; + assert.match(birth ?? '', /^[a-f0-9]{24}$/); + const replaceOwner = (name) => { + unlinkSync(join(admission, readdirSync(admission)[0])); + writeFileSync(join(admission, name), ''); + utimesSync(admission, STALE, STALE); + }; + replaceOwner(`owner-${process.pid}-a1-${birth}`); + assert.equal(await withLock(lock, async () => assert.fail('same process still owns the guard')), false); + const differentBirth = (birth[0] === '0' ? '1' : '0') + birth.slice(1); + replaceOwner(`owner-${process.pid}-a2-${differentBirth}`); + assert.equal(await withLock(lock, async () => {}), true); +}); + test('busy and unrecognised admission guards stay intact and leave no staging directories', async (t) => { const { dir, lock } = scratch(t); const guard = holdAdmission(lock); From 48a7f8312b0c4028cedc5aa9ea395e9dfc7a89ae Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 15:54:18 +0700 Subject: [PATCH 06/22] fix: preserve work errors when lock cleanup also fails --- lib/runtime/store/file-lock.js | 14 +++++++++++--- test/runtime/file-lock.test.js | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index 8cd0c19..2d2e91c 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -239,7 +239,7 @@ function reclaimAdmission(admission) { } } -function admissionOf(lockDir) { +function enterAdmission(lockDir) { const admission = `${lockDir}.admission`; if (ownBirth === undefined) ownBirth = processBirth(process.pid); const birth = ownBirth ? `-${ownBirth}` : ''; @@ -289,7 +289,7 @@ function releaseAdmission({ admission, owner }) { // Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. function admitted(lockDir, mutate, rollback = () => {}) { - const admission = admissionOf(lockDir); + const admission = enterAdmission(lockDir); if (!admission) return null; try { return mutate(); @@ -355,10 +355,18 @@ function tryLock(lockDir, now) { export async function withLock(lockDir, work, now = Date.now()) { const lock = tryLock(lockDir, now); if (!lock) return false; + let workFailed = false; try { await work(lock); return true; + } catch (error) { + workFailed = true; + throw error; } finally { - await lock.release(); + try { + await lock.release(); + } catch (error) { + if (!workFailed) throw error; + } } } diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index 855aa91..5e84cbd 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -153,6 +153,23 @@ test('release refuses an unreadable owner explicitly without claiming cleanup', } }); +test('a release failure preserves the original work error', async (t) => { + const { lock } = scratch(t); + const read = fs.readFileSync; + const workError = new Error('work failed'); + try { + await assert.rejects(withLock(lock, async () => { + fs.readFileSync = (file, ...args) => { + if (String(file) === join(lock, 'owner')) throw Object.assign(new Error('unreadable owner'), { code: 'EIO' }); + return read(file, ...args); + }; + throw workError; + }), (error) => error === workError); + } finally { + fs.readFileSync = read; + } +}); + test('refresh leaves an unreadable live lock untouched without crashing its timer caller', async (t) => { const { dir, lock } = scratch(t); assert.equal(await withLock(lock, async (held) => { From d1fc44ffe2c5a5ca0ab3d67aa5e064a91cb8a6c5 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 18:03:57 +0700 Subject: [PATCH 07/22] Recover admission without spawning on uncontended locks --- lib/runtime/process/owner.js | 22 +++++ lib/runtime/store/admission.js | 123 ++++++++++++++++++++++++++++ lib/runtime/store/file-lock.js | 140 ++++---------------------------- test/helpers/lock-contenders.js | 39 +++++++++ test/runtime/file-lock.test.js | 112 +++++++++++++++---------- 5 files changed, 270 insertions(+), 166 deletions(-) create mode 100644 lib/runtime/store/admission.js create mode 100644 test/helpers/lock-contenders.js diff --git a/lib/runtime/process/owner.js b/lib/runtime/process/owner.js index 8d39208..c9ea00e 100644 --- a/lib/runtime/process/owner.js +++ b/lib/runtime/process/owner.js @@ -8,6 +8,28 @@ import path from 'node:path'; const SHELLS = new Set(['sh', 'bash', 'zsh', 'dash', 'ksh', 'fish', 'cmd.exe', 'powershell', 'pwsh']); const PS_TIMEOUT_MS = 1000; const MAX_ANCESTORS = 16; +const ownStartTime = Math.round(Date.now() - process.uptime() * 1000); + +/** A bounded process start-time probe for admission recovery. The hot path needs no subprocess. */ +export function processStartTime(pid, timeout = PS_TIMEOUT_MS) { + if (pid === process.pid) return ownStartTime; + if (!Number.isSafeInteger(pid) || pid <= 0 || timeout <= 0) return null; + const windows = process.platform === 'win32'; + const systemRoot = process.env.SystemRoot ?? 'C:\\Windows'; + if (windows && !path.isAbsolute(systemRoot)) return null; + const executable = windows ? path.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') : '/bin/ps'; + const args = windows + ? ['-NoProfile', '-NonInteractive', '-Command', `([DateTimeOffset] (Get-Process -Id ${pid} -ErrorAction Stop).StartTime).ToUnixTimeMilliseconds()`] + : ['-p', String(pid), '-o', 'lstart=']; + const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; + const result = spawnSync(executable, args, { + env, encoding: 'utf8', windowsHide: true, timeout: Math.min(timeout, PS_TIMEOUT_MS), killSignal: 'SIGKILL', + }); + const started = result.status === 0 ? result.stdout.trim() : ''; + if (!started) return null; + const time = windows ? Number(started) : Date.parse(started + ' GMT'); + return Number.isSafeInteger(time) && time > 0 ? time : null; +} /** * Whether the process that wrote a state entry has exited. Signal 0 only probes: ESRCH means no such process; diff --git a/lib/runtime/store/admission.js b/lib/runtime/store/admission.js new file mode 100644 index 0000000..61f0e17 --- /dev/null +++ b/lib/runtime/store/admission.js @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 EvoMap + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { ownerIsGone, processStartTime } from '../process/owner.js'; + +export const LOCK_WAIT_MS = 2_000; +export const LOCK_RETRY_MS = 20; +export const LOCK_ACCESS_ERRORS = new Set(['EPERM', 'EBUSY', 'EACCES']); +const ADMISSION_IDENTITY_CHECK_MS = 10_000; +const START_TIME_TOLERANCE_MS = 2_000; +const abandonedAdmissions = new Map(); + +export function pause(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function reclaimAdmission(admission, deadline, starts) { + try { + const info = fs.lstatSync(admission); + if (!info.isDirectory()) return false; + const owners = fs.readdirSync(admission); + if (owners.length > 1) return false; + if (owners.length === 1) { + const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}|[0-9]{13}))?$/.exec(owners[0]); + if (!owner) return false; + const pid = Number(owner[1]); + if (!Number.isSafeInteger(pid) || pid <= 0) return false; + const abandonedHere = abandonedAdmissions.get(admission) === owners[0]; + if (!abandonedHere && !ownerIsGone(pid)) { + const birth = Number(owner[2]); + const remaining = deadline - Date.now(); + if (owner[2]?.length !== 13 || !Number.isSafeInteger(birth) || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS || remaining <= 0) return false; + if (!starts.has(pid)) starts.set(pid, processStartTime(pid, remaining)); + const actualBirth = starts.get(pid); + // Only a clearly newer process proves reuse. ps timestamps round to whole seconds. + if (!actualBirth || actualBirth <= birth + START_TIME_TOLERANCE_MS) return false; + } + fs.unlinkSync(path.join(admission, owners[0])); + if (abandonedHere) abandonedAdmissions.delete(admission); + } + // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot + // remove it even if another reclaimer replaced the directory after our read. + fs.rmdirSync(admission); + return true; + } catch { + return false; + } +} + +function enterAdmission(lockDir, deadline, starts) { + const admission = `${lockDir}.admission`; + const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}-${processStartTime(process.pid)}`; + const staging = `${admission}.${owner}.claim`; + fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); + fs.mkdirSync(staging, { mode: 0o700 }); + let published = false; + try { + fs.writeFileSync(path.join(staging, owner), '', { mode: 0o600, flag: 'wx' }); + // Publishing a prepopulated directory never exposes an empty fresh guard to a stale reclaimer. + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + fs.renameSync(staging, admission); + published = true; + abandonedAdmissions.delete(admission); + return { admission, owner }; + } catch { + if (attempt > 0 || !reclaimAdmission(admission, deadline, starts)) return null; + } + } + return null; + } finally { + if (!published) fs.rmSync(staging, { recursive: true, force: true }); + } +} + +export function retryLockAccess(access) { + for (let attempt = 0; ; attempt += 1) { + try { + return access(); + } catch (error) { + if (attempt === 2 || !LOCK_ACCESS_ERRORS.has(error?.code)) throw error; + pause(LOCK_RETRY_MS); + } + } +} + +function releaseAdmission({ admission, owner }) { + try { + retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); + } catch (cause) { + abandonedAdmissions.set(admission, owner); + throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + } + // The guard is released when its unique owner is gone. An empty directory can be reclaimed. + try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} +} + +// Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. +export function admitted(lockDir, mutate, rollback, deadline = Date.now() + LOCK_WAIT_MS, starts = new Map()) { + const admission = enterAdmission(lockDir, deadline, starts); + if (!admission) return null; + let result; + try { + result = mutate(); + } catch (error) { + try { releaseAdmission(admission); } catch {} + throw error; + } + try { + releaseAdmission(admission); + } catch (error) { + // After an irreversible release, report its result. The stranded guard is tracked for recovery. + if (rollback) { + try { rollback(); } catch (rollbackError) { error.rollbackError = rollbackError; } + throw error; + } + } + return result; +} diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index 2d2e91c..f28610a 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -4,31 +4,14 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import { spawnSync } from 'node:child_process'; import { ownerIsGone } from '../process/owner.js'; -const LOCK_WAIT_MS = 2_000; -const LOCK_RETRY_MS = 20; +import { admitted, LOCK_WAIT_MS, LOCK_RETRY_MS, pause, retryLockAccess } from './admission.js'; + const WORK_LOCK_STALE_MS = 2 * 60 * 1000; const CLAIM_STALE_MS = 60 * 1000; -const ADMISSION_IDENTITY_CHECK_MS = 10_000; -const abandonedAdmissions = new Map(); -let ownBirth; - -function processBirth(pid) { - const windows = process.platform === 'win32'; - const systemRoot = process.env.SystemRoot ?? 'C:\\Windows'; - if (windows && !path.isAbsolute(systemRoot)) return null; - const executable = windows ? path.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') : 'ps'; - const args = windows - ? ['-NoProfile', '-NonInteractive', '-Command', `(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToUniversalTime().Ticks`] - : ['-p', String(pid), '-o', 'lstart=']; - const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; - const result = spawnSync(executable, args, { env, encoding: 'utf8', windowsHide: true, timeout: windows ? 2500 : 1000 }); - const started = result.status === 0 ? result.stdout.trim() : ''; - return started ? crypto.createHash('sha256').update(started).digest('hex').slice(0, 24) : null; -} +const abandonedWork = new Map(); export function readState(target, fallback) { try { @@ -59,11 +42,6 @@ export function removeState(target) { } } -// Atomics.wait is the only sleep that holds a synchronous read-modify-write together. -function pause(ms) { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); -} - function contentOf(file) { try { return fs.readFileSync(file, 'utf8'); @@ -210,106 +188,14 @@ function ownsLock(lockDir, token) { return workOwnerOf(lockDir) === token; } -function reclaimAdmission(admission) { - try { - const info = fs.lstatSync(admission); - if (!info.isDirectory()) return false; - const owners = fs.readdirSync(admission); - if (owners.length > 1) return false; - if (owners.length === 1) { - const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}))?$/.exec(owners[0]); - if (!owner) return false; - const pid = Number(owner[1]); - if (!Number.isSafeInteger(pid) || pid <= 0) return false; - const abandonedHere = abandonedAdmissions.get(admission) === owners[0]; - if (!abandonedHere && !ownerIsGone(pid)) { - if (!owner[2] || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS) return false; - const actualBirth = pid === process.pid ? ownBirth : processBirth(pid); - if (!actualBirth || actualBirth === owner[2]) return false; - } - fs.unlinkSync(path.join(admission, owners[0])); - if (abandonedHere) abandonedAdmissions.delete(admission); - } - // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot - // remove it even if another reclaimer replaced the directory after our read. - fs.rmdirSync(admission); - return true; - } catch { - return false; - } -} - -function enterAdmission(lockDir) { - const admission = `${lockDir}.admission`; - if (ownBirth === undefined) ownBirth = processBirth(process.pid); - const birth = ownBirth ? `-${ownBirth}` : ''; - const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}${birth}`; - const staging = `${admission}.${owner}.claim`; - fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); - fs.mkdirSync(staging, { mode: 0o700 }); - try { - fs.writeFileSync(path.join(staging, owner), '', { mode: 0o600, flag: 'wx' }); - // Publishing a prepopulated directory never exposes an empty fresh guard to a stale reclaimer. - for (let attempt = 0; attempt < 2; attempt += 1) { - try { - fs.renameSync(staging, admission); - abandonedAdmissions.delete(admission); - return { admission, owner }; - } catch { - if (attempt > 0 || !reclaimAdmission(admission)) return null; - } - } - return null; - } finally { - fs.rmSync(staging, { recursive: true, force: true }); - } -} - -function retryLockAccess(access) { - for (let attempt = 0; ; attempt += 1) { - try { - return access(); - } catch (error) { - if (attempt === 2 || !['EPERM', 'EBUSY', 'EACCES'].includes(error?.code)) throw error; - pause(LOCK_RETRY_MS); - } - } -} - -function releaseAdmission({ admission, owner }) { - try { - retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); - } catch (cause) { - abandonedAdmissions.set(admission, owner); - throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); - } - // The guard is released when its unique owner is gone. An empty directory can be reclaimed. - try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} -} - -// Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. -function admitted(lockDir, mutate, rollback = () => {}) { - const admission = enterAdmission(lockDir); - if (!admission) return null; - try { - return mutate(); - } finally { - try { - releaseAdmission(admission); - } catch (error) { - rollback(); - throw error; - } - } -} - async function releaseWorkLock(lockDir, token) { const deadline = Date.now() + LOCK_WAIT_MS; + const starts = new Map(); for (;;) { if (admitted(lockDir, () => { if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); return true; - })) return; + }, null, deadline, starts)) return; if (Date.now() >= deadline) throw Object.assign(new Error(`Could not release lock ${lockDir}`), { code: 'ELOCKED' }); await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS)); } @@ -318,13 +204,15 @@ async function releaseWorkLock(lockDir, token) { function tryLock(lockDir, now) { let token; return admitted(lockDir, () => { - if (isStale(lockDir, now)) { + const stale = isStale(lockDir, now); + if (abandonedWork.has(lockDir) || stale) { let held; try { held = workOwnerOf(lockDir); } catch { return null; } // Old versions wrote random tokens without a pid. Keep their lease recovery, but a known live owner // cannot lose its lock just because a slow or paused worker missed a refresh. A reused pid stays busy. - if (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))) return null; + if (held !== abandonedWork.get(lockDir) && (!stale || (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))))) return null; fs.rmSync(lockDir, { recursive: true, force: true }); + abandonedWork.delete(lockDir); } try { fs.mkdirSync(lockDir); @@ -344,10 +232,16 @@ function tryLock(lockDir, now) { return null; } }, - release: () => releaseWorkLock(lockDir, token), + release: async () => { + try { await releaseWorkLock(lockDir, token); } + catch (error) { abandonedWork.set(lockDir, token); throw error; } + }, }; }, () => { - if (token && ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); + if (!token) return; + abandonedWork.set(lockDir, token); + if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); + abandonedWork.delete(lockDir); }); } diff --git a/test/helpers/lock-contenders.js b/test/helpers/lock-contenders.js new file mode 100644 index 0000000..1e412a7 --- /dev/null +++ b/test/helpers/lock-contenders.js @@ -0,0 +1,39 @@ +import { spawn } from 'node:child_process'; +import { rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export async function until(check) { + const deadline = Date.now() + 10_000; + while (!check()) { + if (Date.now() >= deadline) throw new Error('timed out waiting for a lock contender'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +export function lockContenders(t, dir, script, args, cleanupMarkers) { + const children = []; + const results = () => Promise.all(children); + t.after(async () => { + try { + for (const name of cleanupMarkers) writeFileSync(join(dir, name), ''); + } finally { + await results(); + rmSync(dir, { recursive: true, force: true }); + } + }); + return { + run(role) { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, ...args, role], { windowsHide: true }); + let stderr = ''; + const watchdog = setTimeout(() => child.kill('SIGKILL'), 15_000); + child.stderr.on('data', (data) => { stderr += data; }); + child.on('error', (error) => { stderr += String(error); }); + children.push(new Promise((resolve) => child.on('close', (code) => { + clearTimeout(watchdog); + resolve({ code, stderr }); + }))); + return child; + }, + results, + }; +} diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index 5e84cbd..d6b5261 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; -import { spawn, spawnSync } from 'node:child_process'; +import childProcess, { spawnSync } from 'node:child_process'; +import { syncBuiltinESMExports } from 'node:module'; import fs from 'node:fs'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, rmdirSync, statSync, unlinkSync, utimesSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -7,6 +8,7 @@ import { dirname, join } from 'node:path'; import { test } from 'node:test'; import { withLock } from '../../lib/runtime/store/file-lock.js'; +import { lockContenders, until } from '../helpers/lock-contenders.js'; const RUNTIME = new URL('../../lib/runtime/store/file-lock.js', import.meta.url).href; const STALE = new Date(0); @@ -22,14 +24,6 @@ function scratch(t, token = null) { return { dir, lock }; } -async function until(check) { - const deadline = Date.now() + 10_000; - while (!check()) { - if (Date.now() > deadline) throw new Error('timed out waiting for the lock contender'); - await new Promise((resolve) => setTimeout(resolve, 10)); - } -} - function holdAdmission(lock, pid = process.pid) { const admission = `${lock}.admission`; const owner = join(admission, `owner-${pid}-deadbeef`); @@ -38,6 +32,21 @@ function holdAdmission(lock, pid = process.pid) { return { admission, owner, release: () => { unlinkSync(owner); rmdirSync(admission); } }; } +test('taking an uncontended lock does not launch a process lookup', async (t) => { + const { lock } = scratch(t); + let launches = 0; + const spawn = childProcess.spawnSync; + childProcess.spawnSync = (...args) => { launches += 1; return spawn(...args); }; + syncBuiltinESMExports(); + try { + assert.equal(await withLock(lock, () => {}), true); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(launches, 0); +}); + test('stale legacy and dead-owner directories recover, but a fresh legacy lock stays held', async (t) => { const { lock } = scratch(t, 'legacy token'); let entered = 0; @@ -150,9 +159,35 @@ test('release refuses an unreadable owner explicitly without claiming cleanup', fs.readFileSync = read; } assert.equal(readFileSync(owner, 'utf8'), token, code); + assert.equal(await withLock(lock, () => {}), true, 'known abandoned work is recoverable after access returns'); } }); +test('a stale foreign admission is probed once within the release deadline', async (t) => { + const { lock } = scratch(t); + const spawn = childProcess.spawnSync; + const timeouts = []; + try { + await assert.rejects(withLock(lock, () => { + const guard = holdAdmission(lock, process.ppid); + unlinkSync(guard.owner); + writeFileSync(join(guard.admission, `owner-${process.ppid}-a1-${Date.now() - 30_000}`), ''); + utimesSync(guard.admission, STALE, STALE); + childProcess.spawnSync = (_command, _args, { timeout }) => { + timeouts.push(timeout); + return { status: null, stdout: '' }; + }; + syncBuiltinESMExports(); + }), { code: 'ELOCKED' }); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(timeouts.length, 1); + assert.ok(timeouts[0] > 0 && timeouts[0] <= 1_000); + assert.equal(existsSync(lock), true, 'an inconclusive identity probe does not reclaim a live guard'); +}); + test('a release failure preserves the original work error', async (t) => { const { lock } = scratch(t); const read = fs.readFileSync; @@ -289,6 +324,23 @@ test('an admission cleanup failure is surfaced instead of silently orphaning a l assert.equal(entered, true); }); +test('successful work is not reported as failed after its lock was removed', async (t) => { + const { lock } = scratch(t); + const unlink = fs.unlinkSync; + try { + assert.equal(await withLock(lock, () => { + fs.unlinkSync = (target) => { + if (dirname(target) === `${lock}.admission`) throw Object.assign(new Error('fixture cleanup refusal'), { code: 'EPERM' }); + return unlink(target); + }; + }), true); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(existsSync(lock), false); + assert.equal(await withLock(lock, () => {}), true, 'the stranded admission can be reclaimed'); +}); + test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { const { dir, lock } = scratch(t); const unlink = fs.unlinkSync; @@ -329,7 +381,7 @@ test('an old admission stays held by the same process, but a reused PID can be r } const [original] = readdirSync(admission); const birth = original.split('-')[3]; - assert.match(birth ?? '', /^[a-f0-9]{24}$/); + assert.match(birth ?? '', /^\d{13}$/); const replaceOwner = (name) => { unlinkSync(join(admission, readdirSync(admission)[0])); writeFileSync(join(admission, name), ''); @@ -337,7 +389,7 @@ test('an old admission stays held by the same process, but a reused PID can be r }; replaceOwner(`owner-${process.pid}-a1-${birth}`); assert.equal(await withLock(lock, async () => assert.fail('same process still owns the guard')), false); - const differentBirth = (birth[0] === '0' ? '1' : '0') + birth.slice(1); + const differentBirth = Number(birth) - 10_000; replaceOwner(`owner-${process.pid}-a2-${differentBirth}`); assert.equal(await withLock(lock, async () => {}), true); }); @@ -399,20 +451,7 @@ test('two contenders cannot replace the fresh lock while another reclaims a stal }); fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); `; - const children = []; - const run = (role) => { - const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, lock, role]); - let stderr = ''; - child.stderr.on('data', (data) => { stderr += data; }); - const done = new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr }))); - children.push({ child, done }); - }; - t.after(async () => { - writeFileSync(marker('resume'), ''); - writeFileSync(marker('release'), ''); - await Promise.all(children.map(({ done }) => done)); - rmSync(dir, { recursive: true, force: true }); - }); + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['resume', 'release']); run('A'); await until(() => existsSync(marker('stale-read'))); run('B'); @@ -422,8 +461,8 @@ test('two contenders cannot replace the fresh lock while another reclaims a stal await until(() => existsSync(marker('A-entered'))); const entered = ['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))); writeFileSync(marker('release'), ''); - const results = await Promise.all(children.map(({ done }) => done)); - assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); assert.deepEqual(entered, ['A'], 'only the process reclaiming the inspected stale directory may enter'); assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [true, false, false]); }); @@ -487,20 +526,7 @@ test('a delayed dead-admission reclaimer cannot unlink the fresh owner another c }); fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); `; - const children = []; - const run = (role) => { - const child = spawn(process.execPath, ['--input-type=module', '-e', script, dir, lock, role]); - let stderr = ''; - child.stderr.on('data', (data) => { stderr += data; }); - const done = new Promise((resolve) => child.on('close', (code) => resolve({ code, stderr }))); - children.push({ child, done }); - return child; - }; - t.after(async () => { - for (const name of ['A-resume', 'B-resume', 'release']) writeFileSync(marker(name), ''); - await Promise.all(children.map(({ done }) => done)); - rmSync(dir, { recursive: true, force: true }); - }); + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['A-resume', 'B-resume', 'release']); run('A'); await until(() => existsSync(marker('dead-inspected'))); const fresh = run('B'); @@ -517,7 +543,7 @@ test('a delayed dead-admission reclaimer cannot unlink the fresh owner another c await until(() => existsSync(marker('B-entered'))); assert.deepEqual(['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))), ['B']); writeFileSync(marker('release'), ''); - const results = await Promise.all(children.map(({ done }) => done)); - assert.ok(results.every(({ code }) => code === 0), JSON.stringify(results)); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [false, true, false]); }); From 60a6b2b90871fc54f255539b15b8c4bd9f58767b Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Tue, 3 Nov 2026 18:04:17 +0700 Subject: [PATCH 08/22] Remove corrected drafts without restoring older drafts --- lib/app/distill-session.js | 19 +++++++++---------- lib/distill.js | 6 +++--- test/app/distill-session.test.js | 4 ++-- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/lib/app/distill-session.js b/lib/app/distill-session.js index 2bd9f7e..0c260ab 100644 --- a/lib/app/distill-session.js +++ b/lib/app/distill-session.js @@ -23,6 +23,13 @@ async function claim({ candidates, owner }, claimOf) { const settle = ({ candidates }, { sessionId, candidate }, outcome, countsFailure) => candidates.update((pending) => settledCandidate(pending, sessionId, candidate, { done: isSettled(outcome), counted: countsFailure })); +async function saveDraft({ drafts, candidates }, { sessionId, candidate }, draft) { + await drafts.update((list) => withDraft(list, draft)); + if (isCurrentCandidate(await candidates.read(), sessionId, candidate)) return true; + await drafts.update((list) => withoutCorrectedDraft(list, draft)); + return false; +} + async function post(ports, { sessionId, candidate }, args, usage, signal) { const body = distillBody({ platform: ports.identity.agent, sessionId: taskSessionId(ports.identity, sessionId), candidate, args, usage }); const result = await ports.proxy.request('POST', '/conversation/distill', body, { timeoutMs: DISTILL_TIMEOUT_MS, signal }); @@ -30,16 +37,8 @@ async function post(ports, { sessionId, candidate }, args, usage, signal) { const outcome = distillOutcomeOf(result, { title: args.title, sessionId }, ports.clock.now()); if (outcome.draft) { const draft = { ...outcome.draft, sourceCandidate: { at: candidate.at, turn: candidate.turn } }; - outcome.draft = draft; - let previous; - await ports.drafts.update((drafts) => { - previous = drafts.find((existing) => existing.id === draft.id); - return withDraft(drafts, draft); - }); - if (!isCurrentCandidate(await ports.candidates.read(), sessionId, candidate)) { - await ports.drafts.update((drafts) => withoutCorrectedDraft(drafts, draft, previous)); - return { status: 'dropped' }; - } + if (!await saveDraft(ports, { sessionId, candidate }, draft)) return { status: 'dropped' }; + return { ...outcome, draft, ...(usage && { usage }) }; } return { ...outcome, ...(usage && { usage }) }; } diff --git a/lib/distill.js b/lib/distill.js index cb88e21..cc90666 100644 --- a/lib/distill.js +++ b/lib/distill.js @@ -151,13 +151,13 @@ export function withDraft(drafts, draft) { return kept; } -export function withoutCorrectedDraft(drafts, corrected, previous) { +export function withoutCorrectedDraft(drafts, corrected) { + if (!Array.isArray(drafts)) return undefined; const isCorrected = (draft) => draft.id === corrected.id && draft.sessionId === corrected.sessionId && draft.sourceCandidate?.at === corrected.sourceCandidate.at && draft.sourceCandidate?.turn === corrected.sourceCandidate.turn; if (!drafts.some(isCorrected)) return undefined; - const restore = previous && !isCorrected(previous) ? [previous] : []; - return drafts.flatMap((draft) => isCorrected(draft) ? restore : [draft]); + return drafts.filter((draft) => !isCorrected(draft)); } export function withoutDraft(drafts, id) { diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index 8f279c3..0f49073 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -207,7 +207,7 @@ for (const [count, replace] of [[1, false], [25, false], [25, true]]) test(`a co }); for (const other of ['legacy draft', 'same candidate', 'newer turn', 'other session', 'discarded draft']) { - test(`draft compensation preserves a same-id ${other} only when it is not the corrected candidate`, async () => { + test(`draft compensation never restores a displaced draft and preserves concurrent replacements (${other})`, async () => { const { ports } = await withCandidate({ model: modelAnswering(GENE) }); const candidate = (await ports.candidates.read()).s1; const previous = { id: DRAFTED.distill_id, title: 'Earlier method', createdAt: ports.clock.now(), sessionId: 's1' }; @@ -230,7 +230,7 @@ for (const other of ['legacy draft', 'same candidate', 'newer turn', 'other sess return written; }; assert.equal((await distillSession(ports, { sessionId: 's1' })).status, 'dropped'); - const expected = other === 'same candidate' || other === 'discarded draft' ? [] : [other === 'legacy draft' ? previous : replacement]; + const expected = other === 'newer turn' || other === 'other session' ? [replacement] : []; assert.deepEqual(await ports.drafts.read(), [before, ...expected, after]); }); } From 5167732b8088b237c196c78e5da8e5c9d02fbcd5 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 09:53:52 +0700 Subject: [PATCH 09/22] Report failed cleanup of corrected drafts --- lib/app/commands/distill.js | 1 + lib/app/distill-session.js | 21 ++++++++++++------ test/app/distill-session.test.js | 37 ++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/lib/app/commands/distill.js b/lib/app/commands/distill.js index 089fcd3..3916d1e 100644 --- a/lib/app/commands/distill.js +++ b/lib/app/commands/distill.js @@ -14,6 +14,7 @@ export function distillText(outcome) { case 'skipped': return failure(`The Proxy kept nothing: ${outcome.reason}.`); case 'unparsed': return failure('The Gene must be one JSON object with at least a "summary".'); case 'dropped': return failure('A newer turn replaced the one this Gene was written for; nothing was sent.'); + case 'cleanup_failed': return failure('A correction invalidated the draft. Evolver could not remove it. Check /evolver-drafts and discard it if it is still listed.'); default: return failure(`Distillation ${outcome.status}${outcome.error ? `: ${outcome.error}` : ''}.`); } } diff --git a/lib/app/distill-session.js b/lib/app/distill-session.js index 0c260ab..bf65b89 100644 --- a/lib/app/distill-session.js +++ b/lib/app/distill-session.js @@ -20,14 +20,20 @@ async function claim({ candidates, owner }, claimOf) { return claimed; } -const settle = ({ candidates }, { sessionId, candidate }, outcome, countsFailure) => - candidates.update((pending) => settledCandidate(pending, sessionId, candidate, { done: isSettled(outcome), counted: countsFailure })); +function settle({ candidates }, { sessionId, candidate }, outcome, countsFailure) { + if (outcome?.status === 'dropped' || outcome?.status === 'cleanup_failed') return; + return candidates.update((pending) => settledCandidate(pending, sessionId, candidate, { done: isSettled(outcome), counted: countsFailure })); +} async function saveDraft({ drafts, candidates }, { sessionId, candidate }, draft) { await drafts.update((list) => withDraft(list, draft)); - if (isCurrentCandidate(await candidates.read(), sessionId, candidate)) return true; - await drafts.update((list) => withoutCorrectedDraft(list, draft)); - return false; + if (isCurrentCandidate(await candidates.read(), sessionId, candidate)) return null; + try { + await drafts.update((list) => withoutCorrectedDraft(list, draft)); + return { status: 'dropped' }; + } catch { + return { status: 'cleanup_failed' }; + } } async function post(ports, { sessionId, candidate }, args, usage, signal) { @@ -37,7 +43,8 @@ async function post(ports, { sessionId, candidate }, args, usage, signal) { const outcome = distillOutcomeOf(result, { title: args.title, sessionId }, ports.clock.now()); if (outcome.draft) { const draft = { ...outcome.draft, sourceCandidate: { at: candidate.at, turn: candidate.turn } }; - if (!await saveDraft(ports, { sessionId, candidate }, draft)) return { status: 'dropped' }; + const correction = await saveDraft(ports, { sessionId, candidate }, draft); + if (correction) return correction; return { ...outcome, draft, ...(usage && { usage }) }; } return { ...outcome, ...(usage && { usage }) }; @@ -78,7 +85,7 @@ async function distillClaimed(ports, claimed, focus) { async function run(ports, claimed, { focus = '', countModelFailures = true } = {}) { const outcome = await distillClaimed(ports, claimed, focus); const counts = outcome.status !== 'stopped' && (countModelFailures || !needsAgentModel(outcome)); - if (outcome.status !== 'dropped') await settle(ports, claimed, outcome, counts); + await settle(ports, claimed, outcome, counts); return outcome; } diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index 0f49073..799e46a 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import { runBackgroundTasks } from '../../lib/app/background.js'; +import { distillText } from '../../lib/app/commands/distill.js'; import { runCommand } from '../../lib/app/commands/index.js'; import { distillLeftovers, distillSession, postAgentGene } from '../../lib/app/distill-session.js'; import { adoptStrategy, promptSubmitted } from '../../lib/app/prompt.js'; @@ -235,6 +236,42 @@ for (const other of ['legacy draft', 'same candidate', 'newer turn', 'other sess }); } +for (const entry of ['headless model', 'agent Gene']) { + test(`a failed correction cleanup is reported for a draft from the ${entry}`, async (t) => { + const { ports, requests } = await withCandidate({ model: modelAnswering(GENE) }); + const writtenFor = (await ports.candidates.read()).s1.at; + const update = ports.drafts.update; + let writes = 0; + let signal; + if (entry === 'headless model') { + t.mock.timers.enable({ apis: ['setInterval'] }); + const request = ports.proxy.request; + ports.proxy.request = (...args) => { + if (args[1] === '/conversation/distill') signal = args[3].signal; + return request(...args); + }; + } + ports.drafts.update = async (mutate) => { + if (++writes === 2) throw Object.assign(new Error('fixture cleanup denied'), { code: 'EACCES' }); + await update(mutate); + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + if (signal) t.mock.timers.tick(1_000); + }; + + const outcome = entry === 'headless model' + ? await distillSession(ports, { sessionId: 's1' }) + : await postAgentGene(ports, { sessionId: 's1', text: JSON.stringify(GENE), writtenFor }); + if (entry === 'headless model') assert.equal(signal.aborted, true); + assert.equal(writes, 2); + assert.equal(requests.filter(({ path }) => path === '/conversation/distill').length, 1); + assert.equal((await ports.candidates.read()).s1, undefined); + assert.deepEqual((await ports.drafts.read()).map(({ id }) => id), [DRAFTED.distill_id]); + assert.equal(outcome.status, 'cleanup_failed'); + assert.equal(distillText(outcome).ok, false); + assert.match(distillText(outcome).text, /could not remove it/i); + }); +} + test('a failing headless model keeps the candidate, and three failed tries give it up', async () => { const { ports } = await withCandidate({ model: { complete: async () => { throw new Error('Authentication required'); } } }); await sessionEnded(ports, { sessionId: 's1', projectDir: '/proj' }); From 5dd1271619935796a1731109bd15921448547ba3 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 10:53:39 +0700 Subject: [PATCH 10/22] Keep the Proxy result when a correction invalidates a candidate --- lib/app/commands/distill.js | 2 +- lib/app/distill-session.js | 2 +- test/app/distill-session.test.js | 31 ++++++++++++++++++++++++++++++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/lib/app/commands/distill.js b/lib/app/commands/distill.js index 3916d1e..9fa9b45 100644 --- a/lib/app/commands/distill.js +++ b/lib/app/commands/distill.js @@ -13,7 +13,7 @@ export function distillText(outcome) { case 'stored': return success(shown(outcome.data ?? 'distilled')); case 'skipped': return failure(`The Proxy kept nothing: ${outcome.reason}.`); case 'unparsed': return failure('The Gene must be one JSON object with at least a "summary".'); - case 'dropped': return failure('A newer turn replaced the one this Gene was written for; nothing was sent.'); + case 'dropped': return failure('This Gene was written for a turn that is no longer current.'); case 'cleanup_failed': return failure('A correction invalidated the draft. Evolver could not remove it. Check /evolver-drafts and discard it if it is still listed.'); default: return failure(`Distillation ${outcome.status}${outcome.error ? `: ${outcome.error}` : ''}.`); } diff --git a/lib/app/distill-session.js b/lib/app/distill-session.js index bf65b89..48bc294 100644 --- a/lib/app/distill-session.js +++ b/lib/app/distill-session.js @@ -39,9 +39,9 @@ async function saveDraft({ drafts, candidates }, { sessionId, candidate }, draft async function post(ports, { sessionId, candidate }, args, usage, signal) { const body = distillBody({ platform: ports.identity.agent, sessionId: taskSessionId(ports.identity, sessionId), candidate, args, usage }); const result = await ports.proxy.request('POST', '/conversation/distill', body, { timeoutMs: DISTILL_TIMEOUT_MS, signal }); - if (!isCurrentCandidate(await ports.candidates.read(), sessionId, candidate)) return { status: 'dropped' }; const outcome = distillOutcomeOf(result, { title: args.title, sessionId }, ports.clock.now()); if (outcome.draft) { + if (!isCurrentCandidate(await ports.candidates.read(), sessionId, candidate)) return { status: 'dropped' }; const draft = { ...outcome.draft, sourceCandidate: { at: candidate.at, turn: candidate.turn } }; const correction = await saveDraft(ports, { sessionId, candidate }, draft); if (correction) return correction; diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index 799e46a..db42860 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -148,7 +148,36 @@ test('a correction while a draft request is pending discards its late reply', as await started.promise; await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); finish.resolve(); - assert.equal((await running).status, 'dropped'); + const outcome = await running; + assert.equal(outcome.status, 'dropped'); + assert.match(distillText(outcome).text, /turn that is no longer current/i); + assert.deepEqual(await ports.drafts.read(), []); + assert.equal((await ports.candidates.read()).s1, undefined); +}); + +test('a correction while a request is pending still reports a Gene the Proxy stored', async () => { + const stored = { status: 'stored', gene: DRAFTED.gene }; + const { ports, requests } = await withCandidate({ model: modelAnswering(GENE), distillAnswer: stored }); + const started = Promise.withResolvers(); + const finish = Promise.withResolvers(); + const request = ports.proxy.request; + ports.proxy.request = async (...args) => { + const reply = await request(...args); + if (args[1] === '/conversation/distill') { + started.resolve(); + await finish.promise; + } + return reply; + }; + + const running = runCommand(ports, { sessionId: 's1', name: 'evolver-distill' }); + await started.promise; + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + finish.resolve(); + const answer = await running; + assert.equal(answer.ok, true); + assert.deepEqual(JSON.parse(answer.text), stored); + assert.equal(requests.filter(({ path }) => path === '/conversation/distill').length, 1); assert.deepEqual(await ports.drafts.read(), []); assert.equal((await ports.candidates.read()).s1, undefined); }); From 58f0c14d6f7ca93f3c707c3de65c30a12254ca82 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 10:56:22 +0700 Subject: [PATCH 11/22] Preserve the MCP wording for replaced turns --- lib/app/commands/distill.js | 2 +- test/app/distill-session.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/app/commands/distill.js b/lib/app/commands/distill.js index 9fa9b45..2168158 100644 --- a/lib/app/commands/distill.js +++ b/lib/app/commands/distill.js @@ -13,7 +13,7 @@ export function distillText(outcome) { case 'stored': return success(shown(outcome.data ?? 'distilled')); case 'skipped': return failure(`The Proxy kept nothing: ${outcome.reason}.`); case 'unparsed': return failure('The Gene must be one JSON object with at least a "summary".'); - case 'dropped': return failure('This Gene was written for a turn that is no longer current.'); + case 'dropped': return failure('A newer turn replaced the one this Gene was written for.'); case 'cleanup_failed': return failure('A correction invalidated the draft. Evolver could not remove it. Check /evolver-drafts and discard it if it is still listed.'); default: return failure(`Distillation ${outcome.status}${outcome.error ? `: ${outcome.error}` : ''}.`); } diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index db42860..cae23bf 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -150,7 +150,7 @@ test('a correction while a draft request is pending discards its late reply', as finish.resolve(); const outcome = await running; assert.equal(outcome.status, 'dropped'); - assert.match(distillText(outcome).text, /turn that is no longer current/i); + assert.equal(distillText(outcome).text, 'A newer turn replaced the one this Gene was written for.'); assert.deepEqual(await ports.drafts.read(), []); assert.equal((await ports.candidates.read()).s1, undefined); }); From a0e143e6362f2f0fb9777dd1164bb9bb77a6bfb1 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 11:01:01 +0700 Subject: [PATCH 12/22] Make abandoned admission guards recoverable across processes --- lib/runtime/store/admission.js | 45 ++++++++++++++++++++++------------ test/runtime/file-lock.test.js | 39 +++++++++++++++++++++++------ 2 files changed, 62 insertions(+), 22 deletions(-) diff --git a/lib/runtime/store/admission.js b/lib/runtime/store/admission.js index 61f0e17..e9ec4ab 100644 --- a/lib/runtime/store/admission.js +++ b/lib/runtime/store/admission.js @@ -12,6 +12,7 @@ export const LOCK_RETRY_MS = 20; export const LOCK_ACCESS_ERRORS = new Set(['EPERM', 'EBUSY', 'EACCES']); const ADMISSION_IDENTITY_CHECK_MS = 10_000; const START_TIME_TOLERANCE_MS = 2_000; +const ABANDONED_PREFIX = 'abandoned-'; const abandonedAdmissions = new Map(); export function pause(ms) { @@ -22,15 +23,19 @@ function reclaimAdmission(admission, deadline, starts) { try { const info = fs.lstatSync(admission); if (!info.isDirectory()) return false; - const owners = fs.readdirSync(admission); - if (owners.length > 1) return false; - if (owners.length === 1) { - const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}|[0-9]{13}))?$/.exec(owners[0]); + const entries = fs.readdirSync(admission); + if (entries.length > 0) { + const marker = entries.find((name) => name.startsWith(ABANDONED_PREFIX)); + const ownerName = marker ? marker.slice(ABANDONED_PREFIX.length) : entries[0]; + if (entries.some((name) => name !== ownerName && name !== marker)) return false; + const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}|[0-9]{13}))?$/.exec(ownerName); if (!owner) return false; const pid = Number(owner[1]); if (!Number.isSafeInteger(pid) || pid <= 0) return false; - const abandonedHere = abandonedAdmissions.get(admission) === owners[0]; - if (!abandonedHere && !ownerIsGone(pid)) { + const abandonedHere = abandonedAdmissions.get(admission) === ownerName; + const markerPath = marker && path.join(admission, marker); + if (markerPath && !fs.lstatSync(markerPath).isFile()) return false; + if (!markerPath && !abandonedHere && !ownerIsGone(pid)) { const birth = Number(owner[2]); const remaining = deadline - Date.now(); if (owner[2]?.length !== 13 || !Number.isSafeInteger(birth) || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS || remaining <= 0) return false; @@ -39,7 +44,8 @@ function reclaimAdmission(admission, deadline, starts) { // Only a clearly newer process proves reuse. ps timestamps round to whole seconds. if (!actualBirth || actualBirth <= birth + START_TIME_TOLERANCE_MS) return false; } - fs.unlinkSync(path.join(admission, owners[0])); + if (entries.includes(ownerName)) fs.unlinkSync(path.join(admission, ownerName)); + if (markerPath) fs.unlinkSync(markerPath); if (abandonedHere) abandonedAdmissions.delete(admission); } // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot @@ -92,13 +98,23 @@ function releaseAdmission({ admission, owner }) { try { retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); } catch (cause) { - abandonedAdmissions.set(admission, owner); - throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + if (cause?.code !== 'ENOENT') { + throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + } } // The guard is released when its unique owner is gone. An empty directory can be reclaimed. try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} } +function abandonAdmission({ admission, owner }) { + // A separate marker can be written even when Windows denies access to the owner file. + try { + retryLockAccess(() => fs.writeFileSync(path.join(admission, ABANDONED_PREFIX + owner), '', { mode: 0o600, flag: 'wx' })); + } catch { + abandonedAdmissions.set(admission, owner); + } +} + // Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. export function admitted(lockDir, mutate, rollback, deadline = Date.now() + LOCK_WAIT_MS, starts = new Map()) { const admission = enterAdmission(lockDir, deadline, starts); @@ -107,17 +123,16 @@ export function admitted(lockDir, mutate, rollback, deadline = Date.now() + LOCK try { result = mutate(); } catch (error) { - try { releaseAdmission(admission); } catch {} + try { releaseAdmission(admission); } catch { abandonAdmission(admission); } throw error; } try { releaseAdmission(admission); } catch (error) { - // After an irreversible release, report its result. The stranded guard is tracked for recovery. - if (rollback) { - try { rollback(); } catch (rollbackError) { error.rollbackError = rollbackError; } - throw error; - } + // Publish only after rollback, so another process cannot enter while it runs. + try { rollback?.(); } catch (rollbackError) { error.rollbackError = rollbackError; } + abandonAdmission(admission); + if (rollback) throw error; } return result; } diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index d6b5261..efeba28 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -319,18 +319,19 @@ test('an admission cleanup failure is surfaced instead of silently orphaning a l } assert.equal(entered, false); assert.equal(existsSync(lock), false, 'the failed acquire must not leave a work lock'); - assert.match(readdirSync(admission)[0], new RegExp(`^owner-${process.pid}-`)); + assert.ok(readdirSync(admission).some((name) => name.startsWith(`owner-${process.pid}-`))); assert.equal(await withLock(lock, async () => { entered = true; }), true); assert.equal(entered, true); }); -test('successful work is not reported as failed after its lock was removed', async (t) => { - const { lock } = scratch(t); +test('another process reclaims a failed admission release while the original owner is alive', async (t) => { + const { dir, lock } = scratch(t); + const admission = `${lock}.admission`; const unlink = fs.unlinkSync; try { assert.equal(await withLock(lock, () => { fs.unlinkSync = (target) => { - if (dirname(target) === `${lock}.admission`) throw Object.assign(new Error('fixture cleanup refusal'), { code: 'EPERM' }); + if (dirname(target) === admission) throw Object.assign(new Error('fixture cleanup refusal'), { code: 'EPERM' }); return unlink(target); }; }), true); @@ -338,7 +339,31 @@ test('successful work is not reported as failed after its lock was removed', asy fs.unlinkSync = unlink; } assert.equal(existsSync(lock), false); - assert.equal(await withLock(lock, () => {}), true, 'the stranded admission can be reclaimed'); + assert.ok(readdirSync(admission).some((name) => name.startsWith(`owner-${process.pid}-`))); + + const other = spawnSync(process.execPath, ['--input-type=module', '-e', ` + import assert from 'node:assert/strict'; + const [runtime, lock, owner] = process.argv.slice(1); + const { withLock } = await import(runtime); + process.kill(Number(owner), 0); + assert.equal(await withLock(lock, () => {}), true, 'the stranded admission can be reclaimed'); + `, RUNTIME, lock, String(process.pid)], { encoding: 'utf8', timeout: 10_000, windowsHide: true }); + assert.ifError(other.error); + assert.equal(other.status, 0, other.stderr); + assert.deepEqual(readdirSync(dir), []); +}); + +test('admission recovery resumes after the abandoned owner has already been removed', async (t) => { + const { dir, lock } = scratch(t); + const guard = holdAdmission(lock); + const marker = join(guard.admission, `abandoned-owner-${process.pid}-abcd`); + writeFileSync(marker, ''); + assert.equal(await withLock(lock, () => assert.fail('a different owner is still active')), false); + assert.equal(existsSync(guard.owner), true); + + unlinkSync(guard.owner); + assert.equal(await withLock(lock, () => {}), true); + assert.deepEqual(readdirSync(dir), []); }); test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { @@ -379,11 +404,11 @@ test('an old admission stays held by the same process, but a reused PID can be r } finally { fs.unlinkSync = unlink; } - const [original] = readdirSync(admission); + const original = readdirSync(admission).find((name) => name.startsWith(`owner-${process.pid}-`)); const birth = original.split('-')[3]; assert.match(birth ?? '', /^\d{13}$/); const replaceOwner = (name) => { - unlinkSync(join(admission, readdirSync(admission)[0])); + for (const name of readdirSync(admission)) unlinkSync(join(admission, name)); writeFileSync(join(admission, name), ''); utimesSync(admission, STALE, STALE); }; From 0490c99a60629c40f6a4be2d6ef70a580f772dd2 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 12:23:26 +0700 Subject: [PATCH 13/22] Reclaim stale worker locks after confirmed PID reuse --- lib/runtime/process/owner.js | 12 +++++---- lib/runtime/store/admission.js | 7 +++-- lib/runtime/store/file-lock.js | 31 +++++++++++++++++----- test/runtime/file-lock.test.js | 47 ++++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 16 deletions(-) diff --git a/lib/runtime/process/owner.js b/lib/runtime/process/owner.js index c9ea00e..5ea48fa 100644 --- a/lib/runtime/process/owner.js +++ b/lib/runtime/process/owner.js @@ -6,12 +6,14 @@ import { readlinkSync } from 'node:fs'; import path from 'node:path'; const SHELLS = new Set(['sh', 'bash', 'zsh', 'dash', 'ksh', 'fish', 'cmd.exe', 'powershell', 'pwsh']); -const PS_TIMEOUT_MS = 1000; +export const PROCESS_START_TIMEOUT_MS = 1_000; +// POSIX ps reports start times to the nearest second. +export const PROCESS_START_TOLERANCE_MS = 2_000; const MAX_ANCESTORS = 16; const ownStartTime = Math.round(Date.now() - process.uptime() * 1000); -/** A bounded process start-time probe for admission recovery. The hot path needs no subprocess. */ -export function processStartTime(pid, timeout = PS_TIMEOUT_MS) { +/** A bounded process start-time probe for lock recovery. The hot path needs no subprocess. */ +export function processStartTime(pid, timeout = PROCESS_START_TIMEOUT_MS) { if (pid === process.pid) return ownStartTime; if (!Number.isSafeInteger(pid) || pid <= 0 || timeout <= 0) return null; const windows = process.platform === 'win32'; @@ -23,7 +25,7 @@ export function processStartTime(pid, timeout = PS_TIMEOUT_MS) { : ['-p', String(pid), '-o', 'lstart=']; const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; const result = spawnSync(executable, args, { - env, encoding: 'utf8', windowsHide: true, timeout: Math.min(timeout, PS_TIMEOUT_MS), killSignal: 'SIGKILL', + env, encoding: 'utf8', windowsHide: true, timeout, killSignal: 'SIGKILL', }); const started = result.status === 0 ? result.stdout.trim() : ''; if (!started) return null; @@ -57,7 +59,7 @@ function executablePathOf(pid, command) { } export function processInfo(pid) { - const probe = spawnSync('ps', ['-o', 'ppid=,comm=', '-p', String(pid)], { encoding: 'utf8', timeout: PS_TIMEOUT_MS }); + const probe = spawnSync('ps', ['-o', 'ppid=,comm=', '-p', String(pid)], { encoding: 'utf8', timeout: PROCESS_START_TIMEOUT_MS }); const match = /^\s*(\d+)\s+(.+?)\s*$/.exec(probe.stdout ?? ''); if (!match) return null; const command = match[2].replace(/^-/, ''); diff --git a/lib/runtime/store/admission.js b/lib/runtime/store/admission.js index e9ec4ab..b416d0b 100644 --- a/lib/runtime/store/admission.js +++ b/lib/runtime/store/admission.js @@ -5,13 +5,12 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import { ownerIsGone, processStartTime } from '../process/owner.js'; +import { ownerIsGone, processStartTime, PROCESS_START_TIMEOUT_MS, PROCESS_START_TOLERANCE_MS } from '../process/owner.js'; export const LOCK_WAIT_MS = 2_000; export const LOCK_RETRY_MS = 20; export const LOCK_ACCESS_ERRORS = new Set(['EPERM', 'EBUSY', 'EACCES']); const ADMISSION_IDENTITY_CHECK_MS = 10_000; -const START_TIME_TOLERANCE_MS = 2_000; const ABANDONED_PREFIX = 'abandoned-'; const abandonedAdmissions = new Map(); @@ -39,10 +38,10 @@ function reclaimAdmission(admission, deadline, starts) { const birth = Number(owner[2]); const remaining = deadline - Date.now(); if (owner[2]?.length !== 13 || !Number.isSafeInteger(birth) || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS || remaining <= 0) return false; - if (!starts.has(pid)) starts.set(pid, processStartTime(pid, remaining)); + if (!starts.has(pid)) starts.set(pid, processStartTime(pid, Math.min(remaining, PROCESS_START_TIMEOUT_MS))); const actualBirth = starts.get(pid); // Only a clearly newer process proves reuse. ps timestamps round to whole seconds. - if (!actualBirth || actualBirth <= birth + START_TIME_TOLERANCE_MS) return false; + if (!actualBirth || actualBirth <= birth + PROCESS_START_TOLERANCE_MS) return false; } if (entries.includes(ownerName)) fs.unlinkSync(path.join(admission, ownerName)); if (markerPath) fs.unlinkSync(markerPath); diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index f28610a..6a234ef 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -5,11 +5,13 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import { ownerIsGone } from '../process/owner.js'; +import { ownerIsGone, processStartTime, PROCESS_START_TIMEOUT_MS, PROCESS_START_TOLERANCE_MS } from '../process/owner.js'; import { admitted, LOCK_WAIT_MS, LOCK_RETRY_MS, pause, retryLockAccess } from './admission.js'; const WORK_LOCK_STALE_MS = 2 * 60 * 1000; +const WORK_OWNER_PROBE_MS = process.platform === 'win32' ? 5_000 : PROCESS_START_TIMEOUT_MS; +const WORK_OWNER_WITH_START = /^\d+:[a-f0-9]+:([1-9]\d{12})$/; const CLAIM_STALE_MS = 60 * 1000; const abandonedWork = new Map(); @@ -202,15 +204,23 @@ async function releaseWorkLock(lockDir, token) { } function tryLock(lockDir, now) { - let token; - return admitted(lockDir, () => { + let token, suspect, reusedToken; + const take = () => admitted(lockDir, () => { const stale = isStale(lockDir, now); if (abandonedWork.has(lockDir) || stale) { let held; try { held = workOwnerOf(lockDir); } catch { return null; } - // Old versions wrote random tokens without a pid. Keep their lease recovery, but a known live owner - // cannot lose its lock just because a slow or paused worker missed a refresh. A reused pid stays busy. - if (held !== abandonedWork.get(lockDir) && (!stale || (/^\d+:/.test(held ?? '') && !ownerIsGone(ownerPidOf(held))))) return null; + if (held !== abandonedWork.get(lockDir)) { + if (!stale) return null; + if (held !== reusedToken && /^\d+:/.test(held ?? '')) { + const pid = ownerPidOf(held); + if (!ownerIsGone(pid)) { + const birth = WORK_OWNER_WITH_START.exec(held)?.[1]; + if (birth) suspect = { token: held, pid, birth: Number(birth) }; + return null; + } + } + } fs.rmSync(lockDir, { recursive: true, force: true }); abandonedWork.delete(lockDir); } @@ -219,7 +229,7 @@ function tryLock(lockDir, now) { } catch { return null; } - token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}`; + token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}:${processStartTime(process.pid)}`; fs.writeFileSync(path.join(lockDir, 'owner'), token); return { refresh: () => { @@ -243,6 +253,13 @@ function tryLock(lockDir, now) { if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); abandonedWork.delete(lockDir); }); + const lock = take(); + if (lock || !suspect) return lock; + // A foreign lookup can be slow. Release admission, then recheck staleness and the exact token. + const actualBirth = processStartTime(suspect.pid, WORK_OWNER_PROBE_MS); + if (!actualBirth || actualBirth <= suspect.birth + PROCESS_START_TOLERANCE_MS) return null; + reusedToken = suspect.token; + return take(); } /** Runs slow `work` under one owner's lock; dead or legacy directories recover after their lease expires. */ diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js index efeba28..0e40c47 100644 --- a/test/runtime/file-lock.test.js +++ b/test/runtime/file-lock.test.js @@ -7,6 +7,7 @@ import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { test } from 'node:test'; +import { processStartTime } from '../../lib/runtime/process/owner.js'; import { withLock } from '../../lib/runtime/store/file-lock.js'; import { lockContenders, until } from '../helpers/lock-contenders.js'; @@ -79,6 +80,52 @@ test('an old mtime never evicts a live owner, which can refresh and release its assert.deepEqual(readdirSync(dir), []); }); +test('only a stale worker lock with proof of PID reuse can displace a live PID', async (t) => { + const birth = processStartTime(process.pid); + const prefix = `${process.pid}:deadbeef`; + for (const token of [prefix, `${prefix}:${birth}`, `${prefix}:${birth - 1_000}`, `${prefix}:invalid`]) { + const { lock } = scratch(t, token); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, () => assert.fail('owner identity is not disproved')), false); + assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), token); + } + + const { dir, lock } = scratch(t, `${prefix}:${birth - 10_000}`); + assert.equal(await withLock(lock, () => assert.fail('the lease is still fresh')), false); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, () => { + assert.match(readFileSync(join(lock, 'owner'), 'utf8'), new RegExp(`^${process.pid}:[a-f0-9]+:${birth}$`)); + }), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('a foreign PID probe leaves admission free and cannot remove a different worker token', async (t) => { + const pid = process.ppid; + const birth = Date.now(); + const old = `${pid}:aaaa:${birth - 10_000}`; + const replacement = `${pid}:bbbb:${birth - 10_000}`; + const { lock } = scratch(t, old); + utimesSync(lock, STALE, STALE); + const spawn = childProcess.spawnSync; + let probes = 0; + childProcess.spawnSync = () => { + probes += 1; + assert.equal(existsSync(`${lock}.admission`), false); + writeFileSync(join(lock, 'owner'), replacement); + const stdout = process.platform === 'win32' ? String(birth) : new Date(birth).toUTCString().replace(/ GMT$/, ''); + return { status: 0, stdout }; + }; + syncBuiltinESMExports(); + try { + assert.equal(await withLock(lock, () => assert.fail('the observed token was replaced')), false); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(probes, 1); + assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), replacement); +}); + test('a transient owner read error never turns a stale live lock into a legacy lock', async (t) => { for (const code of ['EBUSY', 'EPERM', 'EACCES', 'EIO']) { const { dir, lock } = scratch(t); From d00d39a91268cf8d6b531fdc57dbd8781b96904a Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 12:49:12 +0700 Subject: [PATCH 14/22] Keep newer drafts when corrected distill replies arrive late --- lib/app/commands/distill.js | 1 + lib/app/distill-session.js | 5 ++- lib/distill.js | 11 ++++++ lib/distill/candidate.js | 7 +++- lib/distill/result.js | 2 +- test/app/distill-session.test.js | 66 +++++++++++++++++++++++++++++--- test/distill.test.js | 10 +++++ 7 files changed, 91 insertions(+), 11 deletions(-) diff --git a/lib/app/commands/distill.js b/lib/app/commands/distill.js index 2168158..3d063a9 100644 --- a/lib/app/commands/distill.js +++ b/lib/app/commands/distill.js @@ -14,6 +14,7 @@ export function distillText(outcome) { case 'skipped': return failure(`The Proxy kept nothing: ${outcome.reason}.`); case 'unparsed': return failure('The Gene must be one JSON object with at least a "summary".'); case 'dropped': return failure('A newer turn replaced the one this Gene was written for.'); + case 'superseded': return failure('A draft with the same ID is already saved.'); case 'cleanup_failed': return failure('A correction invalidated the draft. Evolver could not remove it. Check /evolver-drafts and discard it if it is still listed.'); default: return failure(`Distillation ${outcome.status}${outcome.error ? `: ${outcome.error}` : ''}.`); } diff --git a/lib/app/distill-session.js b/lib/app/distill-session.js index 48bc294..76dca05 100644 --- a/lib/app/distill-session.js +++ b/lib/app/distill-session.js @@ -26,8 +26,9 @@ function settle({ candidates }, { sessionId, candidate }, outcome, countsFailure } async function saveDraft({ drafts, candidates }, { sessionId, candidate }, draft) { - await drafts.update((list) => withDraft(list, draft)); - if (isCurrentCandidate(await candidates.read(), sessionId, candidate)) return null; + const written = await drafts.update((list) => withDraft(list, draft)); + if (isCurrentCandidate(await candidates.read(), sessionId, candidate)) return written ? null : { status: 'superseded' }; + if (!written) return { status: 'dropped' }; try { await drafts.update((list) => withoutCorrectedDraft(list, draft)); return { status: 'dropped' }; diff --git a/lib/distill.js b/lib/distill.js index cc90666..348edb0 100644 --- a/lib/distill.js +++ b/lib/distill.js @@ -146,6 +146,17 @@ export function skipReasonOf(result) { export function withDraft(drafts, draft) { const list = Array.isArray(drafts) ? drafts : []; const index = list.findIndex((existing) => existing.id === draft.id); + const existing = list[index]; + const saved = existing?.sourceCandidate; + const incoming = draft.sourceCandidate; + if (saved && incoming) { + // Candidate order, not reply time, decides which draft can replace the other. + const sameSession = existing.sessionId === draft.sessionId; + const newer = sameSession && saved.turn !== incoming.turn + ? saved.turn > incoming.turn + : saved.at > incoming.at || (!sameSession && saved.at === incoming.at); + if (newer) return undefined; + } const kept = list.filter((existing) => existing.id !== draft.id); kept.splice(index < 0 ? kept.length : index, 0, draft); return kept; diff --git a/lib/distill/candidate.js b/lib/distill/candidate.js index c7cc757..caedf6a 100644 --- a/lib/distill/candidate.js +++ b/lib/distill/candidate.js @@ -83,7 +83,10 @@ export const withClosedCandidate = (pending, sessionId) => (pending[sessionId] ? { ...pending, [sessionId]: { ...pending[sessionId], closed: true } } : undefined); /** Whether `candidate` is still the session's: not dropped by a correction, nor replaced by a newer turn. */ -export const isCurrentCandidate = (pending, sessionId, candidate) => pending[sessionId]?.at === candidate.at; +export function isCurrentCandidate(pending, sessionId, candidate) { + const current = pending[sessionId]; + return Boolean(current) && current.at === candidate.at && current.turn === candidate.turn; +} const isHeld = (candidate, isGone) => Boolean(candidate.claimedBy) && !isGone(candidate.claimedBy); @@ -126,7 +129,7 @@ export function withoutExpiredLeftovers(pending, { now, isGone }) { */ export function settledCandidate(pending, sessionId, candidate, { done, counted }) { const current = pending[sessionId]; - if (!current || current.at !== candidate.at) return undefined; + if (!isCurrentCandidate(pending, sessionId, candidate)) return undefined; const attempts = (current.attempts ?? 0) + (counted ? 1 : 0); if (done || attempts >= MAX_ATTEMPTS) return without(pending, sessionId); return { ...pending, [sessionId]: { ...current, attempts, claimedBy: null } }; diff --git a/lib/distill/result.js b/lib/distill/result.js index 406cfac..47f67be 100644 --- a/lib/distill/result.js +++ b/lib/distill/result.js @@ -3,7 +3,7 @@ import { draftOf, skipReasonOf } from '../distill.js'; -const SETTLED = new Set(['stored', 'drafted', 'skipped', 'declined']); +const SETTLED = new Set(['stored', 'drafted', 'skipped', 'declined', 'superseded']); const MODEL_FAILURES = new Set(['no_model', 'model_failed', 'unparsed']); /** diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index cae23bf..3a8a8f9 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -29,14 +29,17 @@ const modelAnswering = (answer, onCall = () => {}) => ({ async function withCandidate(options = {}) { const made = memoryPorts({ distillAnswer: DRAFTED, ...options }); - const { ports } = made; - await promptSubmitted(ports, { sessionId: 's1', prompt: 'fix the flaky redis retries', waitMs: 1_000 }); - await toolFinished(ports, { sessionId: 's1', note: { kind: 'edit', paths: ['/proj/lib/redis.js'] } }); - await toolFinished(ports, { sessionId: 's1', note: { kind: 'shell', command: 'npm test', exit: 0, output: 'ok' } }); - await turnEnded(ports, { sessionId: 's1', reason: 'answer', answer: 'Fixed the retries.', model: 'model-x', projectDir: '/proj' }); + await verifiedTurn(made.ports, 's1'); return made; } +async function verifiedTurn(ports, sessionId, prompt = 'fix the flaky redis retries') { + await promptSubmitted(ports, { sessionId, prompt, waitMs: 1_000 }); + await toolFinished(ports, { sessionId, note: { kind: 'edit', paths: ['/proj/lib/redis.js'] } }); + await toolFinished(ports, { sessionId, note: { kind: 'shell', command: 'npm test', exit: 0, output: 'ok' } }); + await turnEnded(ports, { sessionId, reason: 'answer', answer: 'Fixed the retries.', model: 'model-x', projectDir: '/proj' }); +} + test('a session-end distillation the Proxy answers as a draft is kept as a draft and settles the candidate', async () => { const { ports, requests } = await withCandidate({ model: modelAnswering(GENE) }); await sessionEnded(ports, { sessionId: 's1', projectDir: '/proj' }); @@ -205,6 +208,56 @@ test('a correction while an agent Gene request is pending discards its late draf assert.equal((await ports.candidates.read()).s1, undefined); }); +test('a corrected request reaching the draft store late preserves a newer draft with the same ID', async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const earlier = (await ports.candidates.read()).s1; + const started = Promise.withResolvers(); + const finish = Promise.withResolvers(); + const update = ports.drafts.update; + let pauseFirst = true; + ports.drafts.update = async (mutate) => { + if (pauseFirst) { + pauseFirst = false; + started.resolve(); + await finish.promise; + } + return update(mutate); + }; + + const late = distillSession(ports, { sessionId: 's1' }); + await started.promise; + let newer; + try { + await verifiedTurn(ports, 's1', 'still failing'); + ports.model = modelAnswering({ ...GENE, title: 'Newer verified retries' }); + newer = await distillSession(ports, { sessionId: 's1' }); + assert.equal(newer.status, 'drafted'); + assert.equal(newer.draft.id, DRAFTED.distill_id); + assert.equal(newer.draft.sourceCandidate.at, earlier.at); + assert.equal(newer.draft.sourceCandidate.turn, earlier.turn + 1); + assert.deepEqual(await ports.drafts.read(), [newer.draft]); + } finally { + finish.resolve(); + } + + assert.equal((await late).status, 'dropped'); + assert.deepEqual(await ports.drafts.read(), [newer.draft]); +}); + +test('an older candidate from another session settles without replacing a newer same-ID draft', async () => { + const { ports, advance } = await withCandidate({ model: modelAnswering(GENE) }); + advance(1); + await verifiedTurn(ports, 's2'); + const newer = await distillSession(ports, { sessionId: 's2' }); + assert.equal(newer.status, 'drafted'); + + const outcome = await distillSession(ports, { sessionId: 's1' }); + assert.equal(outcome.status, 'superseded'); + assert.match(distillText(outcome).text, /same ID is already saved/); + assert.equal((await ports.candidates.read()).s1, undefined); + assert.deepEqual(await ports.drafts.read(), [newer.draft]); +}); + for (const [count, replace] of [[1, false], [25, false], [25, true]]) test(`a correction crossing a draft write keeps ${count} earlier drafts${replace ? ' and a concurrent same-id replacement' : ''}`, async () => { const { ports } = await withCandidate({ model: modelAnswering(GENE) }); const kept = Array.from({ length: count }, (_, index) => ({ id: `earlier-${index}`, title: 'Keep this method', createdAt: ports.clock.now() })); @@ -282,9 +335,10 @@ for (const entry of ['headless model', 'agent Gene']) { } ports.drafts.update = async (mutate) => { if (++writes === 2) throw Object.assign(new Error('fixture cleanup denied'), { code: 'EACCES' }); - await update(mutate); + const written = await update(mutate); await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); if (signal) t.mock.timers.tick(1_000); + return written; }; const outcome = entry === 'headless model' diff --git a/test/distill.test.js b/test/distill.test.js index b64af7a..a50be9a 100644 --- a/test/distill.test.js +++ b/test/distill.test.js @@ -5,6 +5,7 @@ import { distillPromptOf, distillRequestOf, draftNamed, draftOf, draftsListText, isDistillableTurn, lessonOf, skipReasonOf, submitRequestOf, withDraft, withoutDraft, } from '../lib/distill.js'; +import { isCurrentCandidate, settledCandidate } from '../lib/distill/candidate.js'; const LESSON = { reusable: true, @@ -14,6 +15,15 @@ const LESSON = { signals: ['timeout'], }; +test('settling an earlier turn cannot remove a candidate with the same timestamp', () => { + const earlier = { at: 1_000, turn: 1 }; + const current = { at: 1_000, turn: 2 }; + const pending = { session: current }; + assert.equal(isCurrentCandidate(pending, 'session', earlier), false); + assert.equal(settledCandidate(pending, 'session', earlier, { done: true }), undefined); + assert.equal(isCurrentCandidate(pending, 'session', current), true); +}); + test('only an answered person turn that edited and ended on a passing check is distillable', () => { const turn = { reason: 'answer', agentId: undefined, isPersonTurn: true, changedLines: 3, checks: [{ command: 'npm test', failed: false }] }; assert.equal(isDistillableTurn(turn), true); From d4952d908b77808468fcaaf7c69062a87a26c544 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 13:04:11 +0700 Subject: [PATCH 15/22] Remove saved drafts when their turn is corrected --- lib/app/prompt.js | 11 +++--- lib/distill.js | 6 ++++ test/app/distill-session.test.js | 61 ++++++++++++++++++++++++++++---- 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/lib/app/prompt.js b/lib/app/prompt.js index c49da4d..861db06 100644 --- a/lib/app/prompt.js +++ b/lib/app/prompt.js @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 EvoMap +import { withoutDraftsForTurn } from '../distill.js'; import { withoutCandidate } from '../distill/candidate.js'; import { promptTextOf, recallStrategy, taskTextOf } from '../recall.js'; import { correctionReportsOf, followUpLabelOf } from '../session/follow-ups.js'; @@ -14,14 +15,16 @@ const LATE_RECIPE_WAIT_MS = 6_000; /** * The next prompt is the only verdict on a turn available without asking anyone: a correction drops the - * session's candidate first, then the last turn's reuses are reported again under `:correction`. The session is - * marked corrected before the drop, so a re-judge of that turn writing meanwhile sees the mark and takes its write back. + * session's candidate and that turn's drafts, then its reuses are reported again under `:correction`. Mark the + * turn first so a re-judge sees the correction, and drop the candidate before removing drafts so a late write + * after the removal still sees the candidate is gone. */ -async function labelFollowUp({ sessions, candidates, reports, scheduler }, sessionId, prompt) { +async function labelFollowUp({ sessions, candidates, drafts, reports, scheduler }, sessionId, prompt) { const label = followUpLabelOf(prompt); if (label === 'correction') { - await sessions.update(sessionId, (stored) => correctedSession(sessionOf(stored))); + const { correctedTurn } = await sessions.update(sessionId, (stored) => correctedSession(sessionOf(stored))); await candidates.update((pending) => withoutCandidate(pending, sessionId)); + if (correctedTurn > 0) await drafts.update((list) => withoutDraftsForTurn(list, sessionId, correctedTurn)); } const queued = correctionReportsOf(sessionOf(await sessions.read(sessionId)).labelable, label, sessionId); if (queued.length === 0) return; diff --git a/lib/distill.js b/lib/distill.js index 348edb0..7287c34 100644 --- a/lib/distill.js +++ b/lib/distill.js @@ -171,6 +171,12 @@ export function withoutCorrectedDraft(drafts, corrected) { return drafts.filter((draft) => !isCorrected(draft)); } +export function withoutDraftsForTurn(drafts, sessionId, turn) { + if (!Array.isArray(drafts)) return undefined; + const kept = drafts.filter((draft) => draft.sessionId !== sessionId || draft.sourceCandidate?.turn !== turn); + return kept.length === drafts.length ? undefined : kept; +} + export function withoutDraft(drafts, id) { return (Array.isArray(drafts) ? drafts : []).filter((draft) => draft.id !== id); } diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index 3a8a8f9..8ea86ed 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -258,6 +258,51 @@ test('an older candidate from another session settles without replacing a newer assert.deepEqual(await ports.drafts.read(), [newer.draft]); }); +test('a correction after the final candidate snapshot removes only the corrected turn draft', async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const candidate = (await ports.candidates.read()).s1; + const kept = [ + { id: 'earlier', sessionId: 's1', sourceCandidate: { at: candidate.at - 1, turn: candidate.turn - 1 } }, + { id: 'other-session', sessionId: 's2', sourceCandidate: { at: candidate.at, turn: candidate.turn } }, + { id: 'legacy', sessionId: 's1' }, + ]; + await ports.drafts.update(() => kept); + const captured = Promise.withResolvers(); + const resume = Promise.withResolvers(); + const read = ports.candidates.read; + let paused = false; + ports.candidates.read = async () => { + const snapshot = await read(); + if (!paused && (await ports.drafts.read()).some(({ id }) => id === DRAFTED.distill_id)) { + paused = true; + captured.resolve(); + await resume.promise; + } + return snapshot; + }; + + const running = distillSession(ports, { sessionId: 's1' }); + await captured.promise; + try { + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + } finally { + resume.resolve(); + } + await running; + assert.equal((await ports.candidates.read()).s1, undefined); + assert.deepEqual(await ports.drafts.read(), kept); +}); + +test('a correction removes the saved draft even after its candidate has settled', async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const saved = await distillSession(ports, { sessionId: 's1' }); + assert.equal(saved.status, 'drafted'); + assert.equal((await ports.candidates.read()).s1, undefined); + + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + assert.deepEqual(await ports.drafts.read(), []); +}); + for (const [count, replace] of [[1, false], [25, false], [25, true]]) test(`a correction crossing a draft write keeps ${count} earlier drafts${replace ? ' and a concurrent same-id replacement' : ''}`, async () => { const { ports } = await withCandidate({ model: modelAnswering(GENE) }); const kept = Array.from({ length: count }, (_, index) => ({ id: `earlier-${index}`, title: 'Keep this method', createdAt: ports.clock.now() })); @@ -277,13 +322,13 @@ for (const [count, replace] of [[1, false], [25, false], [25, true]]) test(`a co }; const running = distillSession(ports, { sessionId: 's1' }); await started.promise; - await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); let replacement; if (replace) await update((drafts) => drafts.map((draft) => { if (draft.id !== DRAFTED.distill_id) return draft; replacement = { ...draft, title: 'Concurrent method', sourceCandidate: { ...draft.sourceCandidate, turn: draft.sourceCandidate.turn + 1 } }; return replacement; })); + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); finish.resolve(); assert.equal((await running).status, 'dropped'); assert.deepEqual(await ports.drafts.read(), replace ? [...kept, replacement] : kept); @@ -297,8 +342,8 @@ for (const other of ['legacy draft', 'same candidate', 'newer turn', 'other sess if (other === 'same candidate') previous.sourceCandidate = { at: candidate.at, turn: candidate.turn }; const replacement = { ...previous, title: 'Concurrent method', sourceCandidate: { at: candidate.at, turn: candidate.turn + 1 } }; if (other === 'other session') Object.assign(replacement, { sessionId: 's2', sourceCandidate: { at: candidate.at, turn: candidate.turn } }); - const before = { ...previous, id: 'before' }; - const after = { ...previous, id: 'after' }; + const before = { ...previous, id: 'before', sourceCandidate: { at: candidate.at - 1, turn: candidate.turn - 1 } }; + const after = { ...previous, id: 'after', sessionId: 's2' }; await ports.drafts.update(() => [before, previous, after]); const update = ports.drafts.update; let firstWrite = true; @@ -334,9 +379,9 @@ for (const entry of ['headless model', 'agent Gene']) { }; } ports.drafts.update = async (mutate) => { - if (++writes === 2) throw Object.assign(new Error('fixture cleanup denied'), { code: 'EACCES' }); + if (++writes > 1) throw Object.assign(new Error('fixture cleanup denied'), { code: 'EACCES' }); const written = await update(mutate); - await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + await assert.rejects(promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }), { code: 'EACCES' }); if (signal) t.mock.timers.tick(1_000); return written; }; @@ -345,13 +390,17 @@ for (const entry of ['headless model', 'agent Gene']) { ? await distillSession(ports, { sessionId: 's1' }) : await postAgentGene(ports, { sessionId: 's1', text: JSON.stringify(GENE), writtenFor }); if (entry === 'headless model') assert.equal(signal.aborted, true); - assert.equal(writes, 2); + assert.equal(writes, 3); assert.equal(requests.filter(({ path }) => path === '/conversation/distill').length, 1); assert.equal((await ports.candidates.read()).s1, undefined); assert.deepEqual((await ports.drafts.read()).map(({ id }) => id), [DRAFTED.distill_id]); assert.equal(outcome.status, 'cleanup_failed'); assert.equal(distillText(outcome).ok, false); assert.match(distillText(outcome).text, /could not remove it/i); + + ports.drafts.update = update; + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + assert.deepEqual(await ports.drafts.read(), []); }); } From d6f4cd2007aec69a65f13d46f179473c420db663 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 13:09:32 +0700 Subject: [PATCH 16/22] Confirm saved drafts before reporting distillation success --- lib/app/commands/distill.js | 1 + lib/app/distill-session.js | 19 +++++++++++--- lib/distill.js | 10 +++++--- test/app/distill-session.test.js | 44 +++++++++++++++++++++++++++++++- 4 files changed, 66 insertions(+), 8 deletions(-) diff --git a/lib/app/commands/distill.js b/lib/app/commands/distill.js index 3d063a9..b004865 100644 --- a/lib/app/commands/distill.js +++ b/lib/app/commands/distill.js @@ -10,6 +10,7 @@ export function distillText(outcome) { case 'busy': return failure('Another process is distilling this session right now.'); case 'declined': return success(`No Gene: the turn holds no method worth reusing (${outcome.reason}).`); case 'drafted': return success(`Distilled "${outcome.draft.title}" into a draft. Publish it with /evolver-drafts publish 1, or list drafts with /evolver-drafts.`); + case 'draft_missing': return failure('The distilled draft is no longer available.'); case 'stored': return success(shown(outcome.data ?? 'distilled')); case 'skipped': return failure(`The Proxy kept nothing: ${outcome.reason}.`); case 'unparsed': return failure('The Gene must be one JSON object with at least a "summary".'); diff --git a/lib/app/distill-session.js b/lib/app/distill-session.js index 76dca05..ac57772 100644 --- a/lib/app/distill-session.js +++ b/lib/app/distill-session.js @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 EvoMap -import { withDraft, withoutCorrectedDraft } from '../distill.js'; +import { sameDraftSource, withDraft, withoutCorrectedDraft } from '../distill.js'; import { candidateSessionOf, claimedLeftovers, claimedOwn, isCurrentCandidate, settledCandidate, withoutExpiredLeftovers } from '../distill/candidate.js'; import { argumentsOf, declineOf, distillBody, distillPrompt } from '../distill/prompt.js'; import { distillOutcomeOf, isSettled, needsAgentModel } from '../distill/result.js'; @@ -25,6 +25,16 @@ function settle({ candidates }, { sessionId, candidate }, outcome, countsFailure return candidates.update((pending) => settledCandidate(pending, sessionId, candidate, { done: isSettled(outcome), counted: countsFailure })); } +async function confirmDraftOutcomes({ drafts }, outcomes) { + if (!outcomes.some((outcome) => outcome.status === 'drafted')) return outcomes; + // Correction can remove a draft during settlement or a later run in the same batch. + const list = await drafts.read(); + return outcomes.map((outcome) => { + if (outcome.status !== 'drafted' || (Array.isArray(list) && list.some((draft) => sameDraftSource(draft, outcome.draft)))) return outcome; + return { status: 'draft_missing', ...(outcome.usage && { usage: outcome.usage }) }; + }); +} + async function saveDraft({ drafts, candidates }, { sessionId, candidate }, draft) { const written = await drafts.update((list) => withDraft(list, draft)); if (isCurrentCandidate(await candidates.read(), sessionId, candidate)) return written ? null : { status: 'superseded' }; @@ -108,15 +118,16 @@ export async function distillSession(ports, { sessionId: asked, focus = '', anyS const claimed = await claim(ports, (pending, owner) => claimedOwn(pending, sessionId, owner)); if (!claimed) return { status: 'busy' }; const outcome = await run(ports, claimed, { focus, countModelFailures: false }); + if (!needsAgentModel(outcome)) return (await confirmDraftOutcomes(ports, [outcome]))[0]; const due = (await ports.candidates.read())[sessionId]; - return needsAgentModel(outcome) && due ? { ...outcome, agentGene: { writtenFor: due.at, prompt: distillPrompt(due, focus) } } : outcome; + return due ? { ...outcome, agentGene: { writtenFor: due.at, prompt: distillPrompt(due, focus) } } : outcome; } /** Distills the candidates of sessions that are over: ended, or killed with their owner. */ export async function distillLeftovers(ports) { const outcomes = []; for (const claimed of (await claim(ports, claimedLeftovers)) ?? []) outcomes.push(await run(ports, claimed)); - return outcomes; + return confirmDraftOutcomes(ports, outcomes); } /** Drops the leftovers that waited too long for a Gene: without a model here, nothing else would take them. */ @@ -139,8 +150,8 @@ export async function postAgentGene(ports, { sessionId: asked, text, writtenFor, try { if (claimed.candidate.at !== writtenFor) return { status: 'dropped' }; outcome = declined ? { status: 'declined', reason: declined } : await post(ports, claimed, args, null, AbortSignal.timeout(DISTILL_TIMEOUT_MS)); - return outcome; } finally { await settle(ports, claimed, outcome, false); } + return (await confirmDraftOutcomes(ports, [outcome]))[0]; } diff --git a/lib/distill.js b/lib/distill.js index 7287c34..7be1418 100644 --- a/lib/distill.js +++ b/lib/distill.js @@ -162,11 +162,15 @@ export function withDraft(drafts, draft) { return kept; } +export function sameDraftSource(draft, reference) { + return draft.id === reference.id && draft.sessionId === reference.sessionId + && draft.sourceCandidate?.at === reference.sourceCandidate.at + && draft.sourceCandidate?.turn === reference.sourceCandidate.turn; +} + export function withoutCorrectedDraft(drafts, corrected) { if (!Array.isArray(drafts)) return undefined; - const isCorrected = (draft) => draft.id === corrected.id && draft.sessionId === corrected.sessionId - && draft.sourceCandidate?.at === corrected.sourceCandidate.at - && draft.sourceCandidate?.turn === corrected.sourceCandidate.turn; + const isCorrected = (draft) => sameDraftSource(draft, corrected); if (!drafts.some(isCorrected)) return undefined; return drafts.filter((draft) => !isCorrected(draft)); } diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index 8ea86ed..ab97a1e 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -288,11 +288,37 @@ test('a correction after the final candidate snapshot removes only the corrected } finally { resume.resolve(); } - await running; + const outcome = await running; + assert.equal(outcome.status, 'draft_missing'); + assert.equal(distillText(outcome).ok, false); + assert.match(distillText(outcome).text, /no longer available/); assert.equal((await ports.candidates.read()).s1, undefined); assert.deepEqual(await ports.drafts.read(), kept); }); +test('an agent Gene does not report a draft replaced with the same ID during settlement', async () => { + const { ports } = await withCandidate(); + const writtenFor = (await ports.candidates.read()).s1.at; + const update = ports.candidates.update; + let replacement; + ports.candidates.update = async (mutate) => { + const next = await update(mutate); + if (!replacement && next && !next.s1) { + const [draft] = await ports.drafts.read(); + replacement = { ...draft, title: 'Draft from another session', sessionId: 's2' }; + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + await ports.drafts.update(() => [replacement]); + } + return next; + }; + + const outcome = await postAgentGene(ports, { sessionId: 's1', text: JSON.stringify(GENE), writtenFor }); + assert.equal(outcome.status, 'draft_missing'); + assert.equal(distillText(outcome).ok, false); + assert.equal(replacement.id, DRAFTED.distill_id); + assert.deepEqual(await ports.drafts.read(), [replacement]); +}); + test('a correction removes the saved draft even after its candidate has settled', async () => { const { ports } = await withCandidate({ model: modelAnswering(GENE) }); const saved = await distillSession(ports, { sessionId: 's1' }); @@ -303,6 +329,22 @@ test('a correction removes the saved draft even after its candidate has settled' assert.deepEqual(await ports.drafts.read(), []); }); +test('leftover distillation checks earlier drafts again before returning the batch', async () => { + const { ports, advance } = await withCandidate(); + await sessionEnded(ports, { sessionId: 's1', projectDir: '/proj' }); + advance(1); + await verifiedTurn(ports, 's2'); + await sessionEnded(ports, { sessionId: 's2', projectDir: '/proj' }); + let completions = 0; + ports.model = modelAnswering(GENE, async () => { + if (++completions === 2) await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + }); + + const outcomes = await distillLeftovers(ports); + assert.deepEqual(outcomes.map(({ status }) => status), ['draft_missing', 'drafted']); + assert.deepEqual(await ports.drafts.read(), [outcomes[1].draft]); +}); + for (const [count, replace] of [[1, false], [25, false], [25, true]]) test(`a correction crossing a draft write keeps ${count} earlier drafts${replace ? ' and a concurrent same-id replacement' : ''}`, async () => { const { ports } = await withCandidate({ model: modelAnswering(GENE) }); const kept = Array.from({ length: count }, (_, index) => ({ id: `earlier-${index}`, title: 'Keep this method', createdAt: ports.clock.now() })); From 32a5c7c659a7c74d7017dd7cf3628553392fd4de Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 13:21:09 +0700 Subject: [PATCH 17/22] Bind submitted Genes to their exact distillation candidate --- hosts/workbuddy/commands.js | 2 +- lib/agent/defaults/mcp.js | 2 +- lib/agent/profile.js | 2 +- lib/app/distill-session.js | 17 ++++++------ lib/channels/mcp/tools.js | 2 +- lib/channels/script/runner.js | 2 +- lib/distill/candidate.js | 12 ++++++-- test/agent/driver.test.js | 2 +- test/app/distill-session.test.js | 40 ++++++++++++++++++++++++--- test/channels/mcp.test.js | 7 +++-- test/channels/script.test.js | 18 +++++++++++- test/hosts/workbuddy/commands.test.js | 4 +-- 12 files changed, 83 insertions(+), 27 deletions(-) diff --git a/hosts/workbuddy/commands.js b/hosts/workbuddy/commands.js index 06914ec..4d4901e 100644 --- a/hosts/workbuddy/commands.js +++ b/hosts/workbuddy/commands.js @@ -48,7 +48,7 @@ function agentGeneRequest({ text, agentGene }, payload) { 'Write the Gene with your own model: answer the request below with the JSON object only, put it between the', 'two marker lines of this command, run the command with your shell tool, and show the user what it prints.', '', - `${env.join(' ')} node ${shellQuoted(payload.script)} ${AGENT_NAME} evolver-distill --post ${agentGene.writtenFor} <<'${end}'`, + `${env.join(' ')} node ${shellQuoted(payload.script)} ${AGENT_NAME} evolver-distill --post ${shellQuoted(agentGene.writtenFor)} <<'${end}'`, '', end, '', diff --git a/lib/agent/defaults/mcp.js b/lib/agent/defaults/mcp.js index 382d925..19a591b 100644 --- a/lib/agent/defaults/mcp.js +++ b/lib/agent/defaults/mcp.js @@ -76,7 +76,7 @@ export const mcpReply = Object.freeze({ empty: (hostEvent) => asContent(EMPTY[hostEvent] ?? ''), command: (result) => asContent(result.text, result.ok === false), turnEnd: (ended) => asContent(ended?.agentGene - ? `Reported. This task qualifies for distillation: call ${MCP_TOOLS.DISTILL} with written_for ${ended.agentGene.writtenFor} and the JSON the request below asks for, or with {"skip": ""} if it holds no method worth reusing.\n\n${ended.agentGene.prompt}` + ? `Reported. This task qualifies for distillation: call ${MCP_TOOLS.DISTILL} with written_for "${ended.agentGene.writtenFor}" and the JSON the request below asks for, or with {"skip": ""} if it holds no method worth reusing.\n\n${ended.agentGene.prompt}` : 'Reported.'), lifetime: 'turn', }); diff --git a/lib/agent/profile.js b/lib/agent/profile.js index e743ab2..f7994dd 100644 --- a/lib/agent/profile.js +++ b/lib/agent/profile.js @@ -25,7 +25,7 @@ const COMMON = Object.freeze({ stateDir: defaultStateDir, legacy: Object.freeze([]), reply: Object.freeze({ strategyAt: STRATEGY_AT.PROMPT, toolResultContext: false }), - payload: Object.freeze({ transcriptPath: (payload) => payload.transcript_path ?? null, gene: (payload) => payload.gene, writtenFor: (payload) => Number(payload.written_for), hostTurn: () => null }), + payload: Object.freeze({ transcriptPath: (payload) => payload.transcript_path ?? null, gene: (payload) => payload.gene, writtenFor: (payload) => payload.written_for, hostTurn: () => null }), commands: Object.freeze({ entry: COMMAND_ENTRIES.CLI_ONLY, parse: typedFromPrompt, scriptArgs: (text) => text, scriptAnswer: (result) => result.text }), }); diff --git a/lib/app/distill-session.js b/lib/app/distill-session.js index ac57772..8ee7d84 100644 --- a/lib/app/distill-session.js +++ b/lib/app/distill-session.js @@ -2,7 +2,7 @@ // Copyright (c) 2026 EvoMap import { sameDraftSource, withDraft, withoutCorrectedDraft } from '../distill.js'; -import { candidateSessionOf, claimedLeftovers, claimedOwn, isCurrentCandidate, settledCandidate, withoutExpiredLeftovers } from '../distill/candidate.js'; +import { candidateReferenceOf, candidateSessionOf, claimedLeftovers, claimedOwn, isCurrentCandidate, settledCandidate, withoutExpiredLeftovers } from '../distill/candidate.js'; import { argumentsOf, declineOf, distillBody, distillPrompt } from '../distill/prompt.js'; import { distillOutcomeOf, isSettled, needsAgentModel } from '../distill/result.js'; import { taskSessionId } from './turn-reports.js'; @@ -101,10 +101,11 @@ async function run(ports, claimed, { focus = '', countModelFailures = true } = { } // A command run outside any session (a terminal) acts on the newest candidate, or the one a Gene was written for. -async function sessionToDistill({ candidates }, { sessionId, anySession, at }) { +async function sessionToDistill({ candidates }, { sessionId, anySession, writtenFor }) { + if (!anySession) return sessionId; const pending = await candidates.read(); - if (pending[sessionId] || !anySession) return sessionId; - return candidateSessionOf(pending, { at }) ?? sessionId; + if (writtenFor === undefined && pending[sessionId]) return sessionId; + return candidateSessionOf(pending, { writtenFor }) ?? sessionId; } /** @@ -120,7 +121,7 @@ export async function distillSession(ports, { sessionId: asked, focus = '', anyS const outcome = await run(ports, claimed, { focus, countModelFailures: false }); if (!needsAgentModel(outcome)) return (await confirmDraftOutcomes(ports, [outcome]))[0]; const due = (await ports.candidates.read())[sessionId]; - return due ? { ...outcome, agentGene: { writtenFor: due.at, prompt: distillPrompt(due, focus) } } : outcome; + return due ? { ...outcome, agentGene: { writtenFor: candidateReferenceOf(sessionId, due), prompt: distillPrompt(due, focus) } } : outcome; } /** Distills the candidates of sessions that are over: ended, or killed with their owner. */ @@ -136,11 +137,11 @@ export async function dropExpiredLeftovers({ candidates, owner, clock }) { } /** - * Posts the Gene the agent's own model wrote for the candidate stamped `writtenFor`, or settles it when the agent + * Posts the Gene the agent's own model wrote for the candidate identified by `writtenFor`, or settles it when the agent * judged it not worth one. A Gene written for a candidate since replaced is not sent. */ export async function postAgentGene(ports, { sessionId: asked, text, writtenFor, anySession = false }) { - const sessionId = await sessionToDistill(ports, { sessionId: asked, anySession, at: writtenFor }); + const sessionId = await sessionToDistill(ports, { sessionId: asked, anySession, writtenFor }); const declined = declineOf(text); const args = declined ? null : argumentsOf(text); if (!declined && !args) return { status: 'unparsed' }; @@ -148,7 +149,7 @@ export async function postAgentGene(ports, { sessionId: asked, text, writtenFor, if (!claimed) return (await ports.candidates.read())[sessionId] ? { status: 'busy' } : { status: 'nothing' }; let outcome; try { - if (claimed.candidate.at !== writtenFor) return { status: 'dropped' }; + if (candidateReferenceOf(sessionId, claimed.candidate) !== writtenFor) return { status: 'dropped' }; outcome = declined ? { status: 'declined', reason: declined } : await post(ports, claimed, args, null, AbortSignal.timeout(DISTILL_TIMEOUT_MS)); } finally { await settle(ports, claimed, outcome, false); diff --git a/lib/channels/mcp/tools.js b/lib/channels/mcp/tools.js index 5fa1646..46932c8 100644 --- a/lib/channels/mcp/tools.js +++ b/lib/channels/mcp/tools.js @@ -28,7 +28,7 @@ export const TOOL_DEFINITIONS = Object.freeze([ { name: MCP_TOOLS.DISTILL, description: 'Only when a report asked for it: the reusable Gene you wrote for that turn, or {"skip": ""}.', - input: { written_for: z.number().int(), gene: z.union([z.string(), z.record(z.string(), z.unknown())]) }, + input: { written_for: z.union([z.string().max(64), z.number().int()]).describe('Copy the candidate reference from the report exactly.'), gene: z.union([z.string(), z.record(z.string(), z.unknown())]) }, }, { name: MCP_TOOLS.STATUS, description: 'Evolver health: Proxy, engine setup, Hub sign-in, distillation candidate and drafts.', input: {} }, { name: MCP_TOOLS.SEARCH, description: 'Find the EvoMap strategy that best fits a task.', input: { query: z.string().min(1) } }, diff --git a/lib/channels/script/runner.js b/lib/channels/script/runner.js index 25f3a78..7eaf5d5 100644 --- a/lib/channels/script/runner.js +++ b/lib/channels/script/runner.js @@ -60,7 +60,7 @@ export async function runCommandScript(agent, name, { argv = [], stdin = process const { ports } = environmentFor({ profile, env, lifetime: EPHEMERAL, runBackground: runBackgroundTasks }); const driver = createDriver({ profile, ports, transcripts, env, lifetime: EPHEMERAL }); const result = postAt >= 0 - ? await driver.direct(at, { gene: input, writtenFor: Number(argv[postAt + 1]) }) + ? await driver.direct(at, { gene: input, writtenFor: argv[postAt + 1] }) : await driver.direct(at, { command: { name, args: input } }); return write(profile.commands.scriptAnswer(result, script)); } diff --git a/lib/distill/candidate.js b/lib/distill/candidate.js index caedf6a..f49412c 100644 --- a/lib/distill/candidate.js +++ b/lib/distill/candidate.js @@ -1,6 +1,8 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 EvoMap +import { createHash } from 'node:crypto'; + import { verdictOf } from '../checks.js'; import { withoutSecrets } from '../redact.js'; @@ -62,10 +64,14 @@ export function withRejudgedCandidate(pending, sessionId, candidate) { return { ...pending, [sessionId]: { ...current, ...candidate, at: current.at, attempts: current.attempts } }; } -/** The session of the newest candidate, or of the one stamped `at`; null when there is none. */ -export function candidateSessionOf(pending, { at } = {}) { +export function candidateReferenceOf(sessionId, { at, turn }) { + return `v1-${createHash('sha256').update(JSON.stringify([sessionId, at, turn])).digest('base64url')}`; +} + +/** The session of the newest candidate, or the one matching a Gene reference. */ +export function candidateSessionOf(pending, { writtenFor } = {}) { const entries = Object.entries(pending ?? {}); - if (at !== undefined) return entries.find(([, candidate]) => candidate?.at === at)?.[0] ?? null; + if (writtenFor !== undefined) return entries.find(([id, candidate]) => candidate && candidateReferenceOf(id, candidate) === writtenFor)?.[0] ?? null; return entries.sort(([, left], [, right]) => (right?.at ?? 0) - (left?.at ?? 0))[0]?.[0] ?? null; } diff --git a/test/agent/driver.test.js b/test/agent/driver.test.js index 884f8c4..fbc374f 100644 --- a/test/agent/driver.test.js +++ b/test/agent/driver.test.js @@ -100,7 +100,7 @@ test('the MCP profile turns tool calls into a turn whose report is agent-reporte const recalled = await driver.handle('evolver_recall', { session_id: 'm1', turn_id: 't1', task: PROMPT }); assert.match(recalled.content[0].text, /Apply them where they fit\.\n\nturn_id: t1 /); const reported = await driver.handle('evolver_report', { session_id: 'm1', turn_id: 't1', outcome: 'success', summary: 'Fixed the retries.', checks: [{ command: 'npm test', exit: 0 }], files: ['lib/redis.js'] }); - assert.match(reported.content[0].text, /call evolver_distill with written_for \d+/); + assert.match(reported.content[0].text, /call evolver_distill with written_for "v1-[A-Za-z0-9_-]{43}"/); assert.equal(queued[0].report.evidence.reported_by, 'agent'); assert.equal(queued[0].report.evidence.tier, 'none'); const status = await driver.handle('evolver_status', { session_id: 'm1' }); diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index ab97a1e..265f577 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -10,7 +10,7 @@ import { sessionEnded } from '../../lib/app/session-end.js'; import { toolFinished } from '../../lib/app/tool-result.js'; import { TASKS } from '../../lib/app/ports.js'; import { turnEnded } from '../../lib/app/turn-end.js'; -import { UNDISTILLED_LEFTOVER_TTL_MS } from '../../lib/distill/candidate.js'; +import { candidateReferenceOf, UNDISTILLED_LEFTOVER_TTL_MS } from '../../lib/distill/candidate.js'; import { memoryPorts } from './memory-ports.js'; const GENE = { title: 'Stabilise Redis retries', summary: 'Flaky Redis-backed tests stop failing once retries are unbounded and the client is shared.', strategy: ['Set maxRetriesPerRequest to null', 'Share one client', 'Close it in teardown'] }; @@ -187,7 +187,7 @@ test('a correction while a request is pending still reports a Gene the Proxy sto test('a correction while an agent Gene request is pending discards its late draft', async () => { const { ports } = await withCandidate(); - const writtenFor = (await ports.candidates.read()).s1.at; + const writtenFor = candidateReferenceOf('s1', (await ports.candidates.read()).s1); const started = Promise.withResolvers(); const finish = Promise.withResolvers(); const request = ports.proxy.request; @@ -298,7 +298,7 @@ test('a correction after the final candidate snapshot removes only the corrected test('an agent Gene does not report a draft replaced with the same ID during settlement', async () => { const { ports } = await withCandidate(); - const writtenFor = (await ports.candidates.read()).s1.at; + const writtenFor = candidateReferenceOf('s1', (await ports.candidates.read()).s1); const update = ports.candidates.update; let replacement; ports.candidates.update = async (mutate) => { @@ -408,7 +408,7 @@ for (const other of ['legacy draft', 'same candidate', 'newer turn', 'other sess for (const entry of ['headless model', 'agent Gene']) { test(`a failed correction cleanup is reported for a draft from the ${entry}`, async (t) => { const { ports, requests } = await withCandidate({ model: modelAnswering(GENE) }); - const writtenFor = (await ports.candidates.read()).s1.at; + const writtenFor = candidateReferenceOf('s1', (await ports.candidates.read()).s1); const update = ports.drafts.update; let writes = 0; let signal; @@ -489,6 +489,38 @@ test('a Gene written for a candidate since replaced is not sent', async () => { assert.equal(requests.some((request) => request.path === '/conversation/distill'), false); }); +test('a Gene from an older turn cannot post against a new candidate with the same timestamp', async () => { + const { ports, requests } = await withCandidate(); + const earlier = (await ports.candidates.read()).s1; + const oldRequest = await distillSession(ports, { sessionId: 's1' }); + await verifiedTurn(ports, 's1', 'still failing'); + const newer = (await ports.candidates.read()).s1; + assert.equal(newer.at, earlier.at); + assert.equal(newer.turn, earlier.turn + 1); + + const gene = { sessionId: 's1', text: JSON.stringify(GENE) }; + assert.equal((await postAgentGene(ports, { ...gene, writtenFor: oldRequest.agentGene.writtenFor })).status, 'dropped'); + assert.equal((await postAgentGene(ports, { ...gene, writtenFor: earlier.at })).status, 'dropped'); + assert.equal(requests.some(({ path }) => path === '/conversation/distill'), false); + const fresh = await distillSession(ports, { sessionId: 's1' }); + assert.notEqual(fresh.agentGene.writtenFor, oldRequest.agentGene.writtenFor); + assert.equal((await postAgentGene(ports, { ...gene, writtenFor: fresh.agentGene.writtenFor })).status, 'drafted'); +}); + +test('a terminal Gene selects its session even when another candidate has the same timestamp and turn', async () => { + const { ports, requests } = await withCandidate(); + await verifiedTurn(ports, 's2'); + const { s1, s2 } = await ports.candidates.read(); + assert.deepEqual([s1.at, s1.turn], [s2.at, s2.turn]); + const { agentGene } = await distillSession(ports, { sessionId: 's2' }); + const gene = { text: JSON.stringify(GENE), writtenFor: agentGene.writtenFor }; + + assert.equal((await postAgentGene(ports, { ...gene, sessionId: 's1' })).status, 'dropped'); + assert.equal((await postAgentGene(ports, { ...gene, sessionId: 'terminal', anySession: true })).status, 'drafted'); + assert.deepEqual(requests.filter(({ path }) => path === '/conversation/distill').map(({ body }) => body.session_id), ['testagent:s2']); + assert.equal((await ports.candidates.read()).s1.turn, s1.turn); +}); + test('every command answers from the plugin; evolve is the one prompt for the agent', async () => { const { ports } = memoryPorts(); assert.equal((await runCommand(ports, { sessionId: 's1', name: 'evolver-sync', args: '--dry-run "two words"' })).text, 'ran sync --dry-run two words'); diff --git a/test/channels/mcp.test.js b/test/channels/mcp.test.js index 9253eb0..b65d5ce 100644 --- a/test/channels/mcp.test.js +++ b/test/channels/mcp.test.js @@ -46,9 +46,10 @@ test('a recalled turn is reported under its turn_id, agent-reported, and its Gen assert.match(recalled, /Apply them where they fit\./); const turnId = turnIdOf(recalled); const reported = textOf(await call(client, 'evolver_report', { turn_id: turnId, outcome: 'success', summary: 'Fixed the retries.', checks: [{ command: 'npm test', exit: 0 }], files: ['lib/redis.js'] })); - const writtenFor = Number(/written_for (\d+)/.exec(reported)?.[1]); - assert.ok(writtenFor > 0); - assert.match(textOf(await call(client, 'evolver_distill', { written_for: writtenFor + 1, gene: GENE })), /newer turn replaced/); + const writtenFor = /written_for "(v1-[A-Za-z0-9_-]{43})"/.exec(reported)?.[1]; + assert.ok(writtenFor); + assert.match(textOf(await call(client, 'evolver_distill', { written_for: 'wrong-reference', gene: GENE })), /newer turn replaced/); + assert.match(textOf(await call(client, 'evolver_distill', { written_for: 123, gene: GENE })), /newer turn replaced/); assert.match(textOf(await call(client, 'evolver_distill', { written_for: writtenFor, gene: GENE })), /Distilled "Stabilise Redis retries" into a draft/); assert.match(textOf(await call(client, 'evolver_drafts')), /Stabilise Redis retries/); const delivered = () => proxy.requests.find((request) => request.path === '/asset/reuse-result')?.body; diff --git a/test/channels/script.test.js b/test/channels/script.test.js index f176e80..b1fd144 100644 --- a/test/channels/script.test.js +++ b/test/channels/script.test.js @@ -1,9 +1,12 @@ import assert from 'node:assert/strict'; +import { join } from 'node:path'; import { Readable } from 'node:stream'; import { after, before, test } from 'node:test'; import { defineProfile } from '../../lib/agent/profile.js'; import { runCommandScript } from '../../lib/channels/script/runner.js'; +import { candidateReferenceOf } from '../../lib/distill/candidate.js'; +import { fileStoresOf } from '../../lib/runtime/store/file-stores.js'; import { fakeProxy } from '../helpers/fake-proxy.js'; // A host whose own readers know nothing of a script: the channel must not need them. @@ -16,7 +19,7 @@ const profile = defineProfile({ commands: { entry: 'inline-script', sessionOf: (env) => env.TEST_SESSION_ID ?? null, - scriptAnswer: (result, script) => `${result.text}\n[${script.session_id} script ${script.script.endsWith('bin/command.mjs')} nonce ${script.nonce.length}]`, + scriptAnswer: (result, script) => `${result.text}\n[${script.session_id} script ${script.script.endsWith(join('bin', 'command.mjs'))} nonce ${script.nonce.length}]`, }, }); @@ -57,6 +60,19 @@ test('a Gene posted for no candidate says there is nothing to distill', async () assert.match(await run('evolver-distill', { input: '{"summary":"s"}', argv: ['--post', '123'] }), /^Nothing to distill yet/); }); +test('the terminal sends a Gene reference unchanged to the candidate it names', async () => { + const candidate = { at: 1_000, turn: 1, attempts: 0, prompt: 'Repair retries', answer: 'Done', paths: [], commands: [], checks: [] }; + const { candidates } = fileStoresOf({ dir: profile.stateDir(proxy.env), agent: profile.name }); + await candidates.update(() => ({ s1: candidate })); + const input = '{"summary":"Retry safely","strategy":["Retry once"]}'; + const posted = () => proxy.requests.filter(({ path }) => path === '/conversation/distill'); + + assert.match(await run('evolver-distill', { input, argv: ['--post', String(candidate.at)], env: { TEST_SESSION_ID: 's1' } }), /^A newer turn replaced/); + assert.equal(posted().length, 0); + assert.match(await run('evolver-distill', { input, argv: ['--post', candidateReferenceOf('s1', candidate)] }), /not_in_this_test/); + assert.deepEqual(posted().map(({ body }) => body.session_id), ['testscript:s1']); +}); + test('a host whose commands run only from a terminal never waits on a stdin held open', async () => { const terminalOnly = defineProfile({ ...profile, commands: { entry: 'cli-only' } }); const heldOpen = () => new Readable({ read() {} }); diff --git a/test/hosts/workbuddy/commands.test.js b/test/hosts/workbuddy/commands.test.js index 56475a5..9a3a930 100644 --- a/test/hosts/workbuddy/commands.test.js +++ b/test/hosts/workbuddy/commands.test.js @@ -46,11 +46,11 @@ test('a command script prints the answer\'s text; a hook with nothing to say ans test('a Gene the headless model could not write is handed to the agent with a command whose heredoc ends on the nonce', () => { const payload = { session_id: "s'1", script: '/plugin/bin/command.mjs', nonce: '0123456789abcdef', state_dir: '/data/evolver' }; - const result = { ok: false, text: 'Distillation model_failed: spawn codebuddy ENOENT.', agentGene: { writtenFor: 1700, prompt: 'Below is the record…' } }; + const result = { ok: false, text: 'Distillation model_failed: spawn codebuddy ENOENT.', agentGene: { writtenFor: 'v1-fixture', prompt: 'Below is the record…' } }; const lines = profile.commands.scriptAnswer(result, payload).split('\n'); assert.equal(lines[0], 'EVOLVER: WRITE THE GENE YOURSELF'); assert.match(lines[1], /\(Distillation model_failed: spawn codebuddy ENOENT\)\.$/); - assert.equal(lines[5], "CODEBUDDY_SESSION_ID='s'\\''1' CODEBUDDY_PLUGIN_DATA='/data/evolver' node '/plugin/bin/command.mjs' workbuddy evolver-distill --post 1700 <<'EVOLVER_GENE_0123456789abcdef'"); + assert.equal(lines[5], "CODEBUDDY_SESSION_ID='s'\\''1' CODEBUDDY_PLUGIN_DATA='/data/evolver' node '/plugin/bin/command.mjs' workbuddy evolver-distill --post 'v1-fixture' <<'EVOLVER_GENE_0123456789abcdef'"); assert.deepEqual(lines.slice(6, 8), ['', 'EVOLVER_GENE_0123456789abcdef']); assert.equal(lines.at(-1), 'Below is the record…'); }); From 0eb59d35d0894cf37273b5d1374f9b5173dbda6a Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 13:25:02 +0700 Subject: [PATCH 18/22] Generate Gene references through the candidate store --- lib/app/distill-session.js | 11 ++++++----- lib/app/ports.js | 2 +- lib/distill/candidate.js | 11 ++--------- lib/runtime/store/file-stores.js | 6 +++++- test/app/distill-session.test.js | 8 ++++---- test/app/memory-ports.js | 3 ++- test/channels/script.test.js | 3 +-- 7 files changed, 21 insertions(+), 23 deletions(-) diff --git a/lib/app/distill-session.js b/lib/app/distill-session.js index 8ee7d84..0b12749 100644 --- a/lib/app/distill-session.js +++ b/lib/app/distill-session.js @@ -2,7 +2,7 @@ // Copyright (c) 2026 EvoMap import { sameDraftSource, withDraft, withoutCorrectedDraft } from '../distill.js'; -import { candidateReferenceOf, candidateSessionOf, claimedLeftovers, claimedOwn, isCurrentCandidate, settledCandidate, withoutExpiredLeftovers } from '../distill/candidate.js'; +import { candidateSessionOf, claimedLeftovers, claimedOwn, isCurrentCandidate, settledCandidate, withoutExpiredLeftovers } from '../distill/candidate.js'; import { argumentsOf, declineOf, distillBody, distillPrompt } from '../distill/prompt.js'; import { distillOutcomeOf, isSettled, needsAgentModel } from '../distill/result.js'; import { taskSessionId } from './turn-reports.js'; @@ -104,8 +104,9 @@ async function run(ports, claimed, { focus = '', countModelFailures = true } = { async function sessionToDistill({ candidates }, { sessionId, anySession, writtenFor }) { if (!anySession) return sessionId; const pending = await candidates.read(); - if (writtenFor === undefined && pending[sessionId]) return sessionId; - return candidateSessionOf(pending, { writtenFor }) ?? sessionId; + if (writtenFor === undefined) return pending[sessionId] ? sessionId : candidateSessionOf(pending) ?? sessionId; + const match = Object.entries(pending).find(([id, candidate]) => candidate && candidates.referenceOf(id, candidate) === writtenFor); + return match?.[0] ?? sessionId; } /** @@ -121,7 +122,7 @@ export async function distillSession(ports, { sessionId: asked, focus = '', anyS const outcome = await run(ports, claimed, { focus, countModelFailures: false }); if (!needsAgentModel(outcome)) return (await confirmDraftOutcomes(ports, [outcome]))[0]; const due = (await ports.candidates.read())[sessionId]; - return due ? { ...outcome, agentGene: { writtenFor: candidateReferenceOf(sessionId, due), prompt: distillPrompt(due, focus) } } : outcome; + return due ? { ...outcome, agentGene: { writtenFor: ports.candidates.referenceOf(sessionId, due), prompt: distillPrompt(due, focus) } } : outcome; } /** Distills the candidates of sessions that are over: ended, or killed with their owner. */ @@ -149,7 +150,7 @@ export async function postAgentGene(ports, { sessionId: asked, text, writtenFor, if (!claimed) return (await ports.candidates.read())[sessionId] ? { status: 'busy' } : { status: 'nothing' }; let outcome; try { - if (candidateReferenceOf(sessionId, claimed.candidate) !== writtenFor) return { status: 'dropped' }; + if (ports.candidates.referenceOf(sessionId, claimed.candidate) !== writtenFor) return { status: 'dropped' }; outcome = declined ? { status: 'declined', reason: declined } : await post(ports, claimed, args, null, AbortSignal.timeout(DISTILL_TIMEOUT_MS)); } finally { await settle(ports, claimed, outcome, false); diff --git a/lib/app/ports.js b/lib/app/ports.js index 9d9658e..c7b0367 100644 --- a/lib/app/ports.js +++ b/lib/app/ports.js @@ -99,7 +99,7 @@ * @property {SessionStore} sessions * @property {TurnNotes} notes * @property {EditBaseline} baseline - * @property {TableStore} candidates + * @property {TableStore & { referenceOf: (sessionId: string, candidate: object) => string }} candidates * @property {TableStore} drafts * @property {ReportQueue} reports * @property {Outbox} outbox diff --git a/lib/distill/candidate.js b/lib/distill/candidate.js index f49412c..cb19b48 100644 --- a/lib/distill/candidate.js +++ b/lib/distill/candidate.js @@ -1,8 +1,6 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 EvoMap -import { createHash } from 'node:crypto'; - import { verdictOf } from '../checks.js'; import { withoutSecrets } from '../redact.js'; @@ -64,14 +62,9 @@ export function withRejudgedCandidate(pending, sessionId, candidate) { return { ...pending, [sessionId]: { ...current, ...candidate, at: current.at, attempts: current.attempts } }; } -export function candidateReferenceOf(sessionId, { at, turn }) { - return `v1-${createHash('sha256').update(JSON.stringify([sessionId, at, turn])).digest('base64url')}`; -} - -/** The session of the newest candidate, or the one matching a Gene reference. */ -export function candidateSessionOf(pending, { writtenFor } = {}) { +/** The session of the newest candidate, or null when there is none. */ +export function candidateSessionOf(pending) { const entries = Object.entries(pending ?? {}); - if (writtenFor !== undefined) return entries.find(([id, candidate]) => candidate && candidateReferenceOf(id, candidate) === writtenFor)?.[0] ?? null; return entries.sort(([, left], [, right]) => (right?.at ?? 0) - (left?.at ?? 0))[0]?.[0] ?? null; } diff --git a/lib/runtime/store/file-stores.js b/lib/runtime/store/file-stores.js index df32350..d638713 100644 --- a/lib/runtime/store/file-stores.js +++ b/lib/runtime/store/file-stores.js @@ -19,6 +19,10 @@ const MAX_DRAIN_ROUNDS = 5; const SESSION_TTL_MS = 30 * 24 * 60 * 60 * 1000; const digestOf = (key) => crypto.createHash('sha256').update(String(key)).digest('hex').slice(0, 16); +export function candidateReferenceOf(sessionId, { at, turn }) { + return `v1-${crypto.createHash('sha256').update(JSON.stringify([sessionId, at, turn])).digest('base64url')}`; +} + export function contentHashOf(file) { try { return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); @@ -157,7 +161,7 @@ export function fileStoresOf({ dir, agent }) { sessions, notes, baseline, - candidates: tableStore(file('distill-pending'), {}), + candidates: { ...tableStore(file('distill-pending'), {}), referenceOf: candidateReferenceOf }, drafts: tableStore(file('distill-drafts'), []), reports, outbox, diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index 265f577..2bef602 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -10,7 +10,7 @@ import { sessionEnded } from '../../lib/app/session-end.js'; import { toolFinished } from '../../lib/app/tool-result.js'; import { TASKS } from '../../lib/app/ports.js'; import { turnEnded } from '../../lib/app/turn-end.js'; -import { candidateReferenceOf, UNDISTILLED_LEFTOVER_TTL_MS } from '../../lib/distill/candidate.js'; +import { UNDISTILLED_LEFTOVER_TTL_MS } from '../../lib/distill/candidate.js'; import { memoryPorts } from './memory-ports.js'; const GENE = { title: 'Stabilise Redis retries', summary: 'Flaky Redis-backed tests stop failing once retries are unbounded and the client is shared.', strategy: ['Set maxRetriesPerRequest to null', 'Share one client', 'Close it in teardown'] }; @@ -187,7 +187,7 @@ test('a correction while a request is pending still reports a Gene the Proxy sto test('a correction while an agent Gene request is pending discards its late draft', async () => { const { ports } = await withCandidate(); - const writtenFor = candidateReferenceOf('s1', (await ports.candidates.read()).s1); + const writtenFor = ports.candidates.referenceOf('s1', (await ports.candidates.read()).s1); const started = Promise.withResolvers(); const finish = Promise.withResolvers(); const request = ports.proxy.request; @@ -298,7 +298,7 @@ test('a correction after the final candidate snapshot removes only the corrected test('an agent Gene does not report a draft replaced with the same ID during settlement', async () => { const { ports } = await withCandidate(); - const writtenFor = candidateReferenceOf('s1', (await ports.candidates.read()).s1); + const writtenFor = ports.candidates.referenceOf('s1', (await ports.candidates.read()).s1); const update = ports.candidates.update; let replacement; ports.candidates.update = async (mutate) => { @@ -408,7 +408,7 @@ for (const other of ['legacy draft', 'same candidate', 'newer turn', 'other sess for (const entry of ['headless model', 'agent Gene']) { test(`a failed correction cleanup is reported for a draft from the ${entry}`, async (t) => { const { ports, requests } = await withCandidate({ model: modelAnswering(GENE) }); - const writtenFor = candidateReferenceOf('s1', (await ports.candidates.read()).s1); + const writtenFor = ports.candidates.referenceOf('s1', (await ports.candidates.read()).s1); const update = ports.drafts.update; let writes = 0; let signal; diff --git a/test/app/memory-ports.js b/test/app/memory-ports.js index a314516..ba973df 100644 --- a/test/app/memory-ports.js +++ b/test/app/memory-ports.js @@ -1,5 +1,6 @@ // In-memory ports: the use cases run against these without a filesystem, a process or a network. +import { candidateReferenceOf } from '../../lib/runtime/store/file-stores.js'; import { recipeSearchAnswer } from '../helpers/recipe-search.js'; export const ASSET = { @@ -106,7 +107,7 @@ export function memoryPorts({ assets = [ASSET], recipes = [], stepAssets = [], d return { before: new Map(taken.filter(ofTurn).map(({ file, before }) => [file, before])), commit }; }, }, - candidates: table({}), + candidates: { ...table({}), referenceOf: candidateReferenceOf }, drafts: table([]), reports: { enqueue: async (entries) => queued.push(...entries) }, outbox: { drain: async () => true }, diff --git a/test/channels/script.test.js b/test/channels/script.test.js index b1fd144..9d7c72e 100644 --- a/test/channels/script.test.js +++ b/test/channels/script.test.js @@ -5,7 +5,6 @@ import { after, before, test } from 'node:test'; import { defineProfile } from '../../lib/agent/profile.js'; import { runCommandScript } from '../../lib/channels/script/runner.js'; -import { candidateReferenceOf } from '../../lib/distill/candidate.js'; import { fileStoresOf } from '../../lib/runtime/store/file-stores.js'; import { fakeProxy } from '../helpers/fake-proxy.js'; @@ -69,7 +68,7 @@ test('the terminal sends a Gene reference unchanged to the candidate it names', assert.match(await run('evolver-distill', { input, argv: ['--post', String(candidate.at)], env: { TEST_SESSION_ID: 's1' } }), /^A newer turn replaced/); assert.equal(posted().length, 0); - assert.match(await run('evolver-distill', { input, argv: ['--post', candidateReferenceOf('s1', candidate)] }), /not_in_this_test/); + assert.match(await run('evolver-distill', { input, argv: ['--post', candidates.referenceOf('s1', candidate)] }), /not_in_this_test/); assert.deepEqual(posted().map(({ body }) => body.session_id), ['testscript:s1']); }); From f070a4918f1cf93e29a7f78fe7c9528b8da08aa8 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 14:08:12 +0700 Subject: [PATCH 19/22] Block publication of invalidated distillation drafts --- lib/app/commands/drafts.js | 18 +++++++++- lib/app/distill-session.js | 12 ++++--- lib/app/ports.js | 1 + lib/app/prompt.js | 9 ++--- lib/app/turn-end.js | 2 +- lib/distill.js | 16 ++++++--- lib/distill/candidate.js | 7 ++-- lib/runtime/environment.js | 1 + lib/runtime/store/file-stores.js | 12 +++++++ lib/session/state.js | 6 ++-- test/app/distill-session.test.js | 23 ++++++++++++- test/app/draft-publication.test.js | 53 ++++++++++++++++++++++++++++++ test/app/memory-ports.js | 5 +++ test/runtime/file-stores.test.js | 15 +++++++++ 14 files changed, 160 insertions(+), 20 deletions(-) diff --git a/lib/app/commands/drafts.js b/lib/app/commands/drafts.js index 895a8a3..82a1a0b 100644 --- a/lib/app/commands/drafts.js +++ b/lib/app/commands/drafts.js @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 EvoMap -import { draftNamed, draftsListText, reverifiedOf, submitRequestOf, withoutDraft } from '../../distill.js'; +import { draftNamed, draftsListText, reverifiedOf, sameDraftSource, submitRequestOf, withoutDraft } from '../../distill.js'; import { DISTILL_TIMEOUT_MS } from '../distill-session.js'; import { failure, success } from './result.js'; @@ -12,9 +12,22 @@ async function discard({ drafts }, draft) { return success(`Discarded "${draft.title}".`); } +async function publicationBlock({ draftCorrections }, draft) { + const source = draft.sourceCandidate; + if (source && await draftCorrections.has(draft.sessionId, source.turn, source.sessionEpoch)) { + return failure(`"${draft.title}" was invalidated by a correction and cannot be published. You can discard it.`); + } + if (draft.pendingCandidateCheck) return failure(`"${draft.title}" was not confirmed by distillation and cannot be published. You can discard it.`); + return null; +} + async function publish(ports, draft, signal) { if (draft.publishable === false || !draft.publish_receipt) { + const blocked = await publicationBlock(ports, draft); + if (blocked) return blocked; const verified = await ports.proxy.request('POST', '/asset/reverify', { assets: draft.assets, persist: false }, { timeoutMs: DISTILL_TIMEOUT_MS, signal }); + const invalidated = await publicationBlock(ports, draft); + if (invalidated) return invalidated; const promoted = verified.ok && reverifiedOf(verified.data); if (!promoted) { return failure(`"${draft.title}" needs trusted Proxy verification before publication: ${verified.error ?? verified.data?.error ?? 'invalid reverification response'}. The draft was kept.`); @@ -24,6 +37,7 @@ async function publish(ports, draft, signal) { await ports.drafts.update((list) => { const current = list.find((existing) => existing.id === draft.id); if (!current || current.createdAt !== draft.createdAt + || (draft.sourceCandidate ? !sameDraftSource(current, draft) : current.sourceCandidate) || current.publishable !== draft.publishable || Boolean(current.publish_receipt) !== Boolean(draft.publish_receipt)) return undefined; saved = true; @@ -32,6 +46,8 @@ async function publish(ports, draft, signal) { if (!saved) return failure(`"${draft.title}" changed or was discarded during verification; it was not submitted.`); draft = next; } + const blocked = await publicationBlock(ports, draft); + if (blocked) return blocked; const request = submitRequestOf(draft); const answered = await ports.proxy.request(request.method, request.path, request.body, { signal }); if (!answered.ok) return failure(`"${draft.title}" was not submitted: ${answered.error}`); diff --git a/lib/app/distill-session.js b/lib/app/distill-session.js index 0b12749..37a5436 100644 --- a/lib/app/distill-session.js +++ b/lib/app/distill-session.js @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 EvoMap -import { sameDraftSource, withDraft, withoutCorrectedDraft } from '../distill.js'; +import { sameDraftSource, withConfirmedDraft, withDraft, withoutCorrectedDraft } from '../distill.js'; import { candidateSessionOf, claimedLeftovers, claimedOwn, isCurrentCandidate, settledCandidate, withoutExpiredLeftovers } from '../distill/candidate.js'; import { argumentsOf, declineOf, distillBody, distillPrompt } from '../distill/prompt.js'; import { distillOutcomeOf, isSettled, needsAgentModel } from '../distill/result.js'; @@ -36,8 +36,12 @@ async function confirmDraftOutcomes({ drafts }, outcomes) { } async function saveDraft({ drafts, candidates }, { sessionId, candidate }, draft) { - const written = await drafts.update((list) => withDraft(list, draft)); - if (isCurrentCandidate(await candidates.read(), sessionId, candidate)) return written ? null : { status: 'superseded' }; + const written = await drafts.update((list) => withDraft(list, { ...draft, pendingCandidateCheck: true })); + if (isCurrentCandidate(await candidates.read(), sessionId, candidate)) { + if (!written) return { status: 'superseded' }; + const confirmed = await drafts.update((list) => withConfirmedDraft(list, draft)); + return confirmed ? null : { status: 'draft_missing' }; + } if (!written) return { status: 'dropped' }; try { await drafts.update((list) => withoutCorrectedDraft(list, draft)); @@ -53,7 +57,7 @@ async function post(ports, { sessionId, candidate }, args, usage, signal) { const outcome = distillOutcomeOf(result, { title: args.title, sessionId }, ports.clock.now()); if (outcome.draft) { if (!isCurrentCandidate(await ports.candidates.read(), sessionId, candidate)) return { status: 'dropped' }; - const draft = { ...outcome.draft, sourceCandidate: { at: candidate.at, turn: candidate.turn } }; + const draft = { ...outcome.draft, sourceCandidate: { at: candidate.at, turn: candidate.turn, sessionEpoch: candidate.sessionEpoch } }; const correction = await saveDraft(ports, { sessionId, candidate }, draft); if (correction) return correction; return { ...outcome, draft, ...(usage && { usage }) }; diff --git a/lib/app/ports.js b/lib/app/ports.js index c7b0367..2d64d1b 100644 --- a/lib/app/ports.js +++ b/lib/app/ports.js @@ -101,6 +101,7 @@ * @property {EditBaseline} baseline * @property {TableStore & { referenceOf: (sessionId: string, candidate: object) => string }} candidates * @property {TableStore} drafts + * @property {{ mark: (sessionId: string, turn: number, sessionEpoch?: number) => Promise, has: (sessionId: string, turn: number, sessionEpoch?: number) => Promise }} draftCorrections * @property {ReportQueue} reports * @property {Outbox} outbox * @property {RetentionStore} retention diff --git a/lib/app/prompt.js b/lib/app/prompt.js index 861db06..12f781f 100644 --- a/lib/app/prompt.js +++ b/lib/app/prompt.js @@ -19,12 +19,13 @@ const LATE_RECIPE_WAIT_MS = 6_000; * turn first so a re-judge sees the correction, and drop the candidate before removing drafts so a late write * after the removal still sees the candidate is gone. */ -async function labelFollowUp({ sessions, candidates, drafts, reports, scheduler }, sessionId, prompt) { +async function labelFollowUp({ sessions, candidates, drafts, draftCorrections, reports, scheduler }, sessionId, prompt) { const label = followUpLabelOf(prompt); if (label === 'correction') { - const { correctedTurn } = await sessions.update(sessionId, (stored) => correctedSession(sessionOf(stored))); + const { correctedTurn, sessionEpoch } = await sessions.update(sessionId, (stored) => correctedSession(sessionOf(stored))); + if (correctedTurn > 0) await draftCorrections.mark(sessionId, correctedTurn, sessionEpoch); await candidates.update((pending) => withoutCandidate(pending, sessionId)); - if (correctedTurn > 0) await drafts.update((list) => withoutDraftsForTurn(list, sessionId, correctedTurn)); + if (correctedTurn > 0) await drafts.update((list) => withoutDraftsForTurn(list, sessionId, correctedTurn, sessionEpoch)); } const queued = correctionReportsOf(sessionOf(await sessions.read(sessionId)).labelable, label, sessionId); if (queued.length === 0) return; @@ -101,7 +102,7 @@ export async function promptSubmitted(ports, { sessionId, prompt, turnId = null, if (!isCommand) await labelFollowUp(ports, sessionId, prompt); await ports.notes.clear(sessionId); const opened = await ports.sessions.update(sessionId, (stored) => - openedTurn(sessionOf(stored), { turnId, turnStart, hostTurn, prompt: prompt ? promptTextOf(prompt) : null, task: taskTextOf(prompt), pid: ports.owner.current(), isCommand })); + openedTurn(sessionOf(stored), { turnId, turnStart, hostTurn, prompt: prompt ? promptTextOf(prompt) : null, task: taskTextOf(prompt), pid: ports.owner.current(), isCommand, now: ports.clock.now() })); const wantsRecall = !isCommand && ports.settings?.recall !== false; const { injection, late } = wantsRecall ? await recalled(ports, sessionId, { turn: opened.turn, prompt, waitMs, keepLate, holdForToolResult }) : { injection: '' }; return { turn: opened.turn, injection, ...(late && { late }) }; diff --git a/lib/app/turn-end.js b/lib/app/turn-end.js index b1ef410..1891b31 100644 --- a/lib/app/turn-end.js +++ b/lib/app/turn-end.js @@ -39,7 +39,7 @@ async function reportTurn({ identity, sessions, reports, clock }, sessionId, ope async function rememberCandidate({ candidates, owner, clock }, sessionId, opened, { reason, record, model, written = withCandidate }) { if (opened.isCommand || skipReason(reason, record)) return false; - const candidate = candidateOf({ prompt: opened.prompt, record, model, turn: opened.turn, pid: owner.current(), now: clock.now() }); + const candidate = candidateOf({ prompt: opened.prompt, record, model, turn: opened.turn, sessionEpoch: opened.sessionEpoch, pid: owner.current(), now: clock.now() }); await candidates.update((pending) => written(pending, sessionId, candidate)); return true; } diff --git a/lib/distill.js b/lib/distill.js index 7be1418..cb3154a 100644 --- a/lib/distill.js +++ b/lib/distill.js @@ -151,7 +151,7 @@ export function withDraft(drafts, draft) { const incoming = draft.sourceCandidate; if (saved && incoming) { // Candidate order, not reply time, decides which draft can replace the other. - const sameSession = existing.sessionId === draft.sessionId; + const sameSession = existing.sessionId === draft.sessionId && saved.sessionEpoch === incoming.sessionEpoch; const newer = sameSession && saved.turn !== incoming.turn ? saved.turn > incoming.turn : saved.at > incoming.at || (!sameSession && saved.at === incoming.at); @@ -165,7 +165,14 @@ export function withDraft(drafts, draft) { export function sameDraftSource(draft, reference) { return draft.id === reference.id && draft.sessionId === reference.sessionId && draft.sourceCandidate?.at === reference.sourceCandidate.at - && draft.sourceCandidate?.turn === reference.sourceCandidate.turn; + && draft.sourceCandidate?.turn === reference.sourceCandidate.turn + && draft.sourceCandidate?.sessionEpoch === reference.sourceCandidate.sessionEpoch; +} + +export function withConfirmedDraft(drafts, draft) { + if (!Array.isArray(drafts)) return undefined; + const pending = drafts.find((row) => row.pendingCandidateCheck && sameDraftSource(row, draft)); + return pending ? drafts.map((row) => row === pending ? draft : row) : undefined; } export function withoutCorrectedDraft(drafts, corrected) { @@ -175,9 +182,10 @@ export function withoutCorrectedDraft(drafts, corrected) { return drafts.filter((draft) => !isCorrected(draft)); } -export function withoutDraftsForTurn(drafts, sessionId, turn) { +export function withoutDraftsForTurn(drafts, sessionId, turn, sessionEpoch) { if (!Array.isArray(drafts)) return undefined; - const kept = drafts.filter((draft) => draft.sessionId !== sessionId || draft.sourceCandidate?.turn !== turn); + const kept = drafts.filter((draft) => draft.sessionId !== sessionId || draft.sourceCandidate?.turn !== turn + || draft.sourceCandidate.sessionEpoch !== sessionEpoch); return kept.length === drafts.length ? undefined : kept; } diff --git a/lib/distill/candidate.js b/lib/distill/candidate.js index cb19b48..299ecc9 100644 --- a/lib/distill/candidate.js +++ b/lib/distill/candidate.js @@ -35,7 +35,7 @@ export function skipReason(reason, record) { } // Keys and tokens are stripped before anything is written down or sent to a model or the Proxy. -export function candidateOf({ prompt, record, model, turn, pid, now }) { +export function candidateOf({ prompt, record, model, turn, sessionEpoch, pid, now }) { return { prompt: withoutSecrets(String(prompt ?? '')).slice(0, PROMPT_MAX_CHARS), commands: record.commands.slice(-MAX_COMMANDS).map((command) => withoutSecrets(command).slice(0, COMMAND_MAX_CHARS)), @@ -44,6 +44,7 @@ export function candidateOf({ prompt, record, model, turn, pid, now }) { answer: withoutSecrets(record.answer).slice(0, ANSWER_MAX_CHARS), model, turn, + sessionEpoch, pid: pid ?? null, attempts: 0, at: now, @@ -58,7 +59,7 @@ export const withCandidate = (pending, sessionId, candidate) => ({ ...pending, [ */ export function withRejudgedCandidate(pending, sessionId, candidate) { const current = pending[sessionId]; - if (current?.turn !== candidate.turn) return withCandidate(pending, sessionId, candidate); + if (current?.turn !== candidate.turn || current.sessionEpoch !== candidate.sessionEpoch) return withCandidate(pending, sessionId, candidate); return { ...pending, [sessionId]: { ...current, ...candidate, at: current.at, attempts: current.attempts } }; } @@ -84,7 +85,7 @@ export const withClosedCandidate = (pending, sessionId) => /** Whether `candidate` is still the session's: not dropped by a correction, nor replaced by a newer turn. */ export function isCurrentCandidate(pending, sessionId, candidate) { const current = pending[sessionId]; - return Boolean(current) && current.at === candidate.at && current.turn === candidate.turn; + return Boolean(current) && current.at === candidate.at && current.turn === candidate.turn && current.sessionEpoch === candidate.sessionEpoch; } const isHeld = (candidate, isGone) => Boolean(candidate.claimedBy) && !isGone(candidate.claimedBy); diff --git a/lib/runtime/environment.js b/lib/runtime/environment.js index f4892ba..2d3662d 100644 --- a/lib/runtime/environment.js +++ b/lib/runtime/environment.js @@ -34,6 +34,7 @@ export function environmentFor({ profile, env = process.env, lifetime, runBackgr baseline: stores.baseline, candidates: stores.candidates, drafts: stores.drafts, + draftCorrections: stores.draftCorrections, reports: stores.reports, outbox: stores.outbox, retention: stores.retention, diff --git a/lib/runtime/store/file-stores.js b/lib/runtime/store/file-stores.js index d638713..615d3f7 100644 --- a/lib/runtime/store/file-stores.js +++ b/lib/runtime/store/file-stores.js @@ -63,6 +63,17 @@ export function fileStoresOf({ dir, agent }) { }; const stamped = (sessionId, next) => (next === undefined ? undefined : { ...next, sessionId, touchedAt: Date.now() }); + // Drafts outlive session files. Record correction independently of both session pruning and draft cleanup. + const corrections = path.join(dir, `${agent}-distill-corrections`); + const correctionFile = (sessionId, turn, sessionEpoch) => path.join(corrections, digestOf(JSON.stringify([sessionId, turn, sessionEpoch]))); + const draftCorrections = { + mark: async (sessionId, turn, sessionEpoch) => { + fs.mkdirSync(corrections, { recursive: true, mode: 0o700 }); + fs.closeSync(fs.openSync(correctionFile(sessionId, turn, sessionEpoch), 'a', 0o600)); + }, + has: async (sessionId, turn, sessionEpoch) => Boolean(fs.statSync(correctionFile(sessionId, turn, sessionEpoch), { throwIfNoEntry: false })), + }; + const sessions = { read: async (sessionId) => readState(sessionFile('session', sessionId), {}), update: async (sessionId, mutate) => updateState(sessionFile('session', sessionId), {}, (stored) => stamped(sessionId, mutate(stored))), @@ -163,6 +174,7 @@ export function fileStoresOf({ dir, agent }) { baseline, candidates: { ...tableStore(file('distill-pending'), {}), referenceOf: candidateReferenceOf }, drafts: tableStore(file('distill-drafts'), []), + draftCorrections, reports, outbox, retention, diff --git a/lib/session/state.js b/lib/session/state.js index 3ec618f..78f06d8 100644 --- a/lib/session/state.js +++ b/lib/session/state.js @@ -28,10 +28,12 @@ export const sessionOf = (stored) => ({ ...EMPTY_SESSION, ...stored }); * that numbers its own turns (`hostTurn`) keeps the count at least there, so a session resumed after its state was * gone never repeats a task id. The session's `task` is its first typed prompt, kept for the Hub to judge each later * recall's fit against; a resumed session keeps the task it had. + * A fresh session file gets an epoch so retained drafts cannot share a correction key with reused turn numbers. */ -export function openedTurn(session, { turnId = null, turnStart = null, hostTurn = null, prompt = null, task = '', pid = null, isCommand = false }) { +export function openedTurn(session, { turnId = null, turnStart = null, hostTurn = null, prompt = null, task = '', pid = null, isCommand = false, now }) { return { ...session, + sessionEpoch: session.turn === 0 ? now : session.sessionEpoch, turn: Math.max(session.turn + 1, Number.isInteger(hostTurn) ? hostTurn : 0), turnId, turnIds: turnId ? [...session.turnIds.filter((known) => known !== turnId), turnId].slice(-MAX_TURN_IDS) : session.turnIds, @@ -162,5 +164,5 @@ export const isEarlierTurn = (session, turnId) => Boolean(turnId) && turnId !== /** What an ended session keeps: its turn counter, owner and ledger, so a resume neither repeats a task id nor re-injects. */ export function endedSession(session) { - return { ...EMPTY_SESSION, turn: session.turn, pid: session.pid, injected: session.injected, turnIds: session.turnIds, shown: session.shown, task: session.task }; + return { ...EMPTY_SESSION, turn: session.turn, sessionEpoch: session.sessionEpoch, pid: session.pid, injected: session.injected, turnIds: session.turnIds, shown: session.shown, task: session.task }; } diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index 2bef602..87e5aa5 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -381,7 +381,7 @@ for (const other of ['legacy draft', 'same candidate', 'newer turn', 'other sess const { ports } = await withCandidate({ model: modelAnswering(GENE) }); const candidate = (await ports.candidates.read()).s1; const previous = { id: DRAFTED.distill_id, title: 'Earlier method', createdAt: ports.clock.now(), sessionId: 's1' }; - if (other === 'same candidate') previous.sourceCandidate = { at: candidate.at, turn: candidate.turn }; + if (other === 'same candidate') previous.sourceCandidate = { at: candidate.at, turn: candidate.turn, sessionEpoch: candidate.sessionEpoch }; const replacement = { ...previous, title: 'Concurrent method', sourceCandidate: { at: candidate.at, turn: candidate.turn + 1 } }; if (other === 'other session') Object.assign(replacement, { sessionId: 's2', sourceCandidate: { at: candidate.at, turn: candidate.turn } }); const before = { ...previous, id: 'before', sourceCandidate: { at: candidate.at - 1, turn: candidate.turn - 1 } }; @@ -446,6 +446,27 @@ for (const entry of ['headless model', 'agent Gene']) { }); } +test('a superseded draft stays blocked when distillation cannot undo its write', async () => { + const { ports, requests } = await withCandidate({ + model: modelAnswering(GENE), + distillAnswer: { ...DRAFTED, publishable: true, publish_receipt: { receiptId: 'proxy-issued-receipt' } }, + }); + const updateDrafts = ports.drafts.update; + ports.drafts.update = async (mutate) => { + const written = await updateDrafts(mutate); + await ports.candidates.update((pending) => ({ ...pending, s1: { ...pending.s1, turn: pending.s1.turn + 1 } })); + ports.drafts.update = async () => { throw new Error('draft cleanup denied'); }; + return written; + }; + assert.equal((await distillSession(ports, { sessionId: 's1' })).status, 'cleanup_failed'); + ports.drafts.update = updateDrafts; + assert.equal((await ports.drafts.read()).length, 1); + const result = await runCommand(ports, { sessionId: 's1', name: 'evolver-drafts', args: 'publish 1' }); + assert.equal(result.ok, false); + assert.match(result.text, /not confirmed by distillation/i); + assert.equal(requests.some(({ path }) => path === '/asset/reverify' || path === '/asset/submit'), false); +}); + test('a failing headless model keeps the candidate, and three failed tries give it up', async () => { const { ports } = await withCandidate({ model: { complete: async () => { throw new Error('Authentication required'); } } }); await sessionEnded(ports, { sessionId: 's1', projectDir: '/proj' }); diff --git a/test/app/draft-publication.test.js b/test/app/draft-publication.test.js index f87402d..984f9b0 100644 --- a/test/app/draft-publication.test.js +++ b/test/app/draft-publication.test.js @@ -4,6 +4,8 @@ import { test } from 'node:test'; import { hub } from '@evomap/evolver-core'; import { runCommand } from '../../lib/app/commands/index.js'; +import { promptSubmitted } from '../../lib/app/prompt.js'; +import { sessionEnded } from '../../lib/app/session-end.js'; import { draftOf } from '../../lib/distill.js'; import { memoryPorts } from './memory-ports.js'; @@ -23,6 +25,57 @@ async function withDraft(result) { return { ...made, draft }; } +for (const publishable of [false, true]) { + test(`a correction blocks a lingering ${publishable ? 'verified' : 'local-only'} draft after cleanup fails`, async () => { + const { ports, requests, draft } = await withDraft({ ...RESULT, publishable, ...(publishable && { publish_receipt: RECEIPT }) }); + const corrected = { ...draft, sourceCandidate: { turn: 2, at: 20 } }; + const earlier = { ...draft, id: 'earlier', publishable: true, publish_receipt: RECEIPT, sourceCandidate: { turn: 1, at: 10 } }; + await ports.sessions.update('s1', () => ({ turn: 2 })); + await ports.drafts.update(() => [corrected, earlier]); + const updateDrafts = ports.drafts.update; + ports.drafts.update = async () => { throw new Error('draft cleanup denied'); }; + await assert.rejects(promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }), /draft cleanup denied/); + ports.drafts.update = updateDrafts; + await sessionEnded(ports, { sessionId: 's1', projectDir: '/proj' }); + + const publish = (args) => runCommand(ports, { sessionId: 'another-session', name: 'evolver-drafts', args }); + const result = await publish('publish 1'); + assert.equal(result.ok, false); + assert.match(result.text, /invalidated by a correction/i); + assert.deepEqual(requests, []); + assert.deepEqual(await ports.drafts.read(), [corrected, earlier]); + assert.equal((await publish('publish 2')).ok, true); + assert.deepEqual(requests.map(({ path }) => path), ['/asset/submit']); + assert.equal((await publish('discard 1')).ok, true); + assert.deepEqual(await ports.drafts.read(), []); + }); +} + +test('a correction during reverification blocks submit even if it cannot remove the draft', async () => { + const { ports, requests, draft } = await withDraft({ ...RESULT, publishable: false }); + const source = { ...draft, sourceCandidate: { turn: 1, at: 10 } }; + await ports.sessions.update('s1', () => ({ turn: 1 })); + await ports.drafts.update(() => [source]); + const updateDrafts = ports.drafts.update; + ports.proxy.request = async (method, path, body) => { + requests.push({ method, path, body }); + if (path !== '/asset/reverify') return { ok: true }; + ports.drafts.update = async () => { throw new Error('draft cleanup denied'); }; + try { + await assert.rejects(promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }), /draft cleanup denied/); + } finally { + ports.drafts.update = updateDrafts; + } + return { ok: true, data: { ok: true, publishable: true, assets: [GENE, CAPSULE], publish_receipt: RECEIPT } }; + }; + + const result = await runCommand(ports, { sessionId: 's1', name: 'evolver-drafts', args: 'publish 1' }); + assert.equal(result.ok, false); + assert.match(result.text, /invalidated by a correction/i); + assert.deepEqual(requests.map(({ path }) => path), ['/asset/reverify']); + assert.deepEqual(await ports.drafts.read(), [source]); +}); + test('explicit publication preserves the Proxy receipt and its complete bound bundle', async () => { const { ports, requests } = await withDraft({ ...RESULT, publishable: true, publish_receipt: RECEIPT, event: EVENT }); const publish = ports.proxy.request; diff --git a/test/app/memory-ports.js b/test/app/memory-ports.js index ba973df..f6097fe 100644 --- a/test/app/memory-ports.js +++ b/test/app/memory-ports.js @@ -67,6 +67,7 @@ const lines = () => { export function memoryPorts({ assets = [ASSET], recipes = [], stepAssets = [], distillAnswer = null, model = null, owner = 4242, gone = new Set(), now = 1_000_000 } = {}) { const sessions = new Map(); + const draftCorrections = new Set(); const notes = lines(); const edits = lines(); const requests = []; @@ -109,6 +110,10 @@ export function memoryPorts({ assets = [ASSET], recipes = [], stepAssets = [], d }, candidates: { ...table({}), referenceOf: candidateReferenceOf }, drafts: table([]), + draftCorrections: { + mark: async (id, turn, epoch) => { draftCorrections.add(JSON.stringify([id, turn, epoch])); }, + has: async (id, turn, epoch) => draftCorrections.has(JSON.stringify([id, turn, epoch])), + }, reports: { enqueue: async (entries) => queued.push(...entries) }, outbox: { drain: async () => true }, retention: { diff --git a/test/runtime/file-stores.test.js b/test/runtime/file-stores.test.js index 7f52193..4f32899 100644 --- a/test/runtime/file-stores.test.js +++ b/test/runtime/file-stores.test.js @@ -90,3 +90,18 @@ test('a session file outlives the outbox: a session resumed within a month still stores.prune(Date.now() + 31 * day); assert.deepEqual(await stores.sessions.read('s1'), {}); }); + +test('a recorded draft correction survives reopening and pruning the source session', async () => { + const { dir, stores: first } = stores(); + await first.sessions.update('s1', () => ({ turn: 2, sessionEpoch: 10 })); + await first.draftCorrections.mark('s1', 2, 10); + await first.draftCorrections.mark('s1', 2, 10); + first.prune(Date.now() + 31 * 24 * 60 * 60 * 1000); + + const reopened = fileStoresOf({ dir, agent: 'cursor' }); + assert.deepEqual(await reopened.sessions.read('s1'), {}); + assert.equal(await reopened.draftCorrections.has('s1', 2, 10), true); + assert.equal(await reopened.draftCorrections.has('s1', 2, 20), false); + assert.equal(await reopened.draftCorrections.has('s1', 1, 10), false); + assert.equal(await reopened.draftCorrections.has('s2', 2, 10), false); +}); From 792e4cbac7553022e8635ca7408ad6fcde6b40f8 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 14:10:52 +0700 Subject: [PATCH 20/22] Suppress distillation success for revoked drafts --- lib/app/commands/drafts.js | 7 +++---- lib/app/distill-session.js | 18 ++++++++++++------ test/app/distill-session.test.js | 22 ++++++++++++++++++++++ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/lib/app/commands/drafts.js b/lib/app/commands/drafts.js index 82a1a0b..b1d2673 100644 --- a/lib/app/commands/drafts.js +++ b/lib/app/commands/drafts.js @@ -2,7 +2,7 @@ // Copyright (c) 2026 EvoMap import { draftNamed, draftsListText, reverifiedOf, sameDraftSource, submitRequestOf, withoutDraft } from '../../distill.js'; -import { DISTILL_TIMEOUT_MS } from '../distill-session.js'; +import { DISTILL_TIMEOUT_MS, draftWasCorrected } from '../distill-session.js'; import { failure, success } from './result.js'; const USAGE = 'Usage: /evolver-drafts [publish|discard ]'; @@ -12,9 +12,8 @@ async function discard({ drafts }, draft) { return success(`Discarded "${draft.title}".`); } -async function publicationBlock({ draftCorrections }, draft) { - const source = draft.sourceCandidate; - if (source && await draftCorrections.has(draft.sessionId, source.turn, source.sessionEpoch)) { +async function publicationBlock(ports, draft) { + if (await draftWasCorrected(ports, draft)) { return failure(`"${draft.title}" was invalidated by a correction and cannot be published. You can discard it.`); } if (draft.pendingCandidateCheck) return failure(`"${draft.title}" was not confirmed by distillation and cannot be published. You can discard it.`); diff --git a/lib/app/distill-session.js b/lib/app/distill-session.js index 37a5436..f041fd9 100644 --- a/lib/app/distill-session.js +++ b/lib/app/distill-session.js @@ -25,14 +25,20 @@ function settle({ candidates }, { sessionId, candidate }, outcome, countsFailure return candidates.update((pending) => settledCandidate(pending, sessionId, candidate, { done: isSettled(outcome), counted: countsFailure })); } -async function confirmDraftOutcomes({ drafts }, outcomes) { +export function draftWasCorrected({ draftCorrections }, draft) { + const source = draft.sourceCandidate; + return source ? draftCorrections.has(draft.sessionId, source.turn, source.sessionEpoch) : false; +} + +async function confirmDraftOutcomes(ports, outcomes) { if (!outcomes.some((outcome) => outcome.status === 'drafted')) return outcomes; - // Correction can remove a draft during settlement or a later run in the same batch. - const list = await drafts.read(); - return outcomes.map((outcome) => { - if (outcome.status !== 'drafted' || (Array.isArray(list) && list.some((draft) => sameDraftSource(draft, outcome.draft)))) return outcome; + const list = await ports.drafts.read(); + return Promise.all(outcomes.map(async (outcome) => { + if (outcome.status !== 'drafted') return outcome; + const saved = Array.isArray(list) && list.some((draft) => !draft.pendingCandidateCheck && sameDraftSource(draft, outcome.draft)); + if (saved && !await draftWasCorrected(ports, outcome.draft)) return outcome; return { status: 'draft_missing', ...(outcome.usage && { usage: outcome.usage }) }; - }); + })); } async function saveDraft({ drafts, candidates }, { sessionId, candidate }, draft) { diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index 87e5aa5..8d65d6a 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -467,6 +467,28 @@ test('a superseded draft stays blocked when distillation cannot undo its write', assert.equal(requests.some(({ path }) => path === '/asset/reverify' || path === '/asset/submit'), false); }); +test('distillation does not advertise a corrected draft that cleanup could not remove', async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const updateDrafts = ports.drafts.update; + let writes = 0; + ports.drafts.update = async (mutate) => { + if (++writes === 2) { + ports.drafts.update = async () => { throw new Error('draft cleanup denied'); }; + try { + await assert.rejects(promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }), /draft cleanup denied/); + } finally { + ports.drafts.update = updateDrafts; + } + } + return updateDrafts(mutate); + }; + + const outcome = await distillSession(ports, { sessionId: 's1' }); + assert.equal((await ports.drafts.read()).length, 1); + assert.equal(outcome.status, 'draft_missing'); + assert.equal(distillText(outcome).ok, false); +}); + test('a failing headless model keeps the candidate, and three failed tries give it up', async () => { const { ports } = await withCandidate({ model: { complete: async () => { throw new Error('Authentication required'); } } }); await sessionEnded(ports, { sessionId: 's1', projectDir: '/proj' }); From 0d59f0263e637c30bfd832487c3e37c3db0a95b9 Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 14:28:49 +0700 Subject: [PATCH 21/22] Serialize draft correction with submit dispatch --- lib/app/commands/drafts.js | 14 +++++-- lib/app/ports.js | 9 +++- lib/app/prompt.js | 7 +++- lib/runtime/proxy.js | 1 + lib/runtime/store/file-stores.js | 12 ++++++ test/app/draft-publication.test.js | 66 ++++++++++++++++++++++++++++++ test/app/memory-ports.js | 1 + test/runtime/file-stores.test.js | 32 +++++++++++++++ 8 files changed, 134 insertions(+), 8 deletions(-) diff --git a/lib/app/commands/drafts.js b/lib/app/commands/drafts.js index b1d2673..efbf052 100644 --- a/lib/app/commands/drafts.js +++ b/lib/app/commands/drafts.js @@ -12,10 +12,10 @@ async function discard({ drafts }, draft) { return success(`Discarded "${draft.title}".`); } +const invalidatedDraft = (draft) => failure(`"${draft.title}" was invalidated by a correction and cannot be published. You can discard it.`); + async function publicationBlock(ports, draft) { - if (await draftWasCorrected(ports, draft)) { - return failure(`"${draft.title}" was invalidated by a correction and cannot be published. You can discard it.`); - } + if (await draftWasCorrected(ports, draft)) return invalidatedDraft(draft); if (draft.pendingCandidateCheck) return failure(`"${draft.title}" was not confirmed by distillation and cannot be published. You can discard it.`); return null; } @@ -48,7 +48,13 @@ async function publish(ports, draft, signal) { const blocked = await publicationBlock(ports, draft); if (blocked) return blocked; const request = submitRequestOf(draft); - const answered = await ports.proxy.request(request.method, request.path, request.body, { signal }); + // Hold the correction lock through dispatch only. Wrap the request promise so the lock never waits for HTTP. + const dispatch = () => ({ response: ports.proxy.request(request.method, request.path, request.body, { signal }) }); + const started = draft.sourceCandidate + ? await ports.draftCorrections.withSession(draft.sessionId, async () => (await draftWasCorrected(ports, draft)) ? null : dispatch(), signal) + : dispatch(); + if (!started) return invalidatedDraft(draft); + const answered = await started.response; if (!answered.ok) return failure(`"${draft.title}" was not submitted: ${answered.error}`); await ports.drafts.update((list) => withoutDraft(list, draft.id)); return success(`Submitted "${draft.title}" to the EvoMap Hub for review (message ${answered.data?.message_id ?? 'pending'}).`); diff --git a/lib/app/ports.js b/lib/app/ports.js index 2d64d1b..c6294ab 100644 --- a/lib/app/ports.js +++ b/lib/app/ports.js @@ -32,6 +32,11 @@ * @property {() => Promise} read * @property {(mutate: Function) => Promise} update * + * @typedef {object} DraftCorrections + * @property {(sessionId: string, work: () => T|Promise, signal?: AbortSignal) => Promise} withSession + * @property {(sessionId: string, turn: number, sessionEpoch?: number) => Promise} mark + * @property {(sessionId: string, turn: number, sessionEpoch?: number) => Promise} has + * * @typedef {object} ReportQueue * @property {(entries: { report: object, sessionId: string }[]) => Promise} enqueue * @@ -42,7 +47,7 @@ * @property {(entry: { sessionId, projectDir, reuses, editedBefore: Map }) => Promise} remember * @property {(due: { sessionId, projectDir }) => Promise<{ due: object[], commit: () => Promise }>} takeDue * - * @typedef {object} Proxy + * @typedef {object} Proxy request initiates the transport before returning its promise. * @property {(method: string, path: string, body?: object, options?: { timeoutMs?: number, signal?: AbortSignal }) => * Promise<{ ok: true, data: any } | { ok: false, error: string }>} request * @@ -101,7 +106,7 @@ * @property {EditBaseline} baseline * @property {TableStore & { referenceOf: (sessionId: string, candidate: object) => string }} candidates * @property {TableStore} drafts - * @property {{ mark: (sessionId: string, turn: number, sessionEpoch?: number) => Promise, has: (sessionId: string, turn: number, sessionEpoch?: number) => Promise }} draftCorrections + * @property {DraftCorrections} draftCorrections * @property {ReportQueue} reports * @property {Outbox} outbox * @property {RetentionStore} retention diff --git a/lib/app/prompt.js b/lib/app/prompt.js index 12f781f..16caf1e 100644 --- a/lib/app/prompt.js +++ b/lib/app/prompt.js @@ -22,8 +22,11 @@ const LATE_RECIPE_WAIT_MS = 6_000; async function labelFollowUp({ sessions, candidates, drafts, draftCorrections, reports, scheduler }, sessionId, prompt) { const label = followUpLabelOf(prompt); if (label === 'correction') { - const { correctedTurn, sessionEpoch } = await sessions.update(sessionId, (stored) => correctedSession(sessionOf(stored))); - if (correctedTurn > 0) await draftCorrections.mark(sessionId, correctedTurn, sessionEpoch); + const { correctedTurn, sessionEpoch } = await draftCorrections.withSession(sessionId, async () => { + const corrected = await sessions.update(sessionId, (stored) => correctedSession(sessionOf(stored))); + if (corrected.correctedTurn > 0) await draftCorrections.mark(sessionId, corrected.correctedTurn, corrected.sessionEpoch); + return corrected; + }); await candidates.update((pending) => withoutCandidate(pending, sessionId)); if (correctedTurn > 0) await drafts.update((list) => withoutDraftsForTurn(list, sessionId, correctedTurn, sessionEpoch)); } diff --git a/lib/runtime/proxy.js b/lib/runtime/proxy.js index 53489bc..3c63e55 100644 --- a/lib/runtime/proxy.js +++ b/lib/runtime/proxy.js @@ -24,6 +24,7 @@ export function proxyFetchOf(env = process.env) { const headers = { ...(body !== undefined && { 'Content-Type': 'application/json' }), ...(token && { Authorization: `Bearer ${token}` }) }; const timeout = AbortSignal.timeout(timeoutMs); try { + // Draft publication releases its correction lock once this call has started fetch. const response = await fetch(base + path, { method, headers, diff --git a/lib/runtime/store/file-stores.js b/lib/runtime/store/file-stores.js index 615d3f7..8e972d8 100644 --- a/lib/runtime/store/file-stores.js +++ b/lib/runtime/store/file-stores.js @@ -4,6 +4,7 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; import { OUTBOX_TTL_MS, withQueued } from '../../report-outbox.js'; import { ABSENT } from '../../retention.js'; @@ -67,6 +68,17 @@ export function fileStoresOf({ dir, agent }) { const corrections = path.join(dir, `${agent}-distill-corrections`); const correctionFile = (sessionId, turn, sessionEpoch) => path.join(corrections, digestOf(JSON.stringify([sessionId, turn, sessionEpoch]))); const draftCorrections = { + withSession: async (sessionId, work, signal) => { + const lock = path.join(corrections, `${digestOf(sessionId)}.lock`); + const deadline = Date.now() + 2_000; + let result; + for (;;) { + signal?.throwIfAborted(); + if (await withLock(lock, async () => { result = await work(); })) return result; + if (Date.now() >= deadline) throw new Error('Timed out waiting for the draft correction lock. Please retry.'); + await delay(20, undefined, { signal }); + } + }, mark: async (sessionId, turn, sessionEpoch) => { fs.mkdirSync(corrections, { recursive: true, mode: 0o700 }); fs.closeSync(fs.openSync(correctionFile(sessionId, turn, sessionEpoch), 'a', 0o600)); diff --git a/test/app/draft-publication.test.js b/test/app/draft-publication.test.js index 984f9b0..b93d6a8 100644 --- a/test/app/draft-publication.test.js +++ b/test/app/draft-publication.test.js @@ -7,6 +7,9 @@ import { runCommand } from '../../lib/app/commands/index.js'; import { promptSubmitted } from '../../lib/app/prompt.js'; import { sessionEnded } from '../../lib/app/session-end.js'; import { draftOf } from '../../lib/distill.js'; +import { proxyFetchOf } from '../../lib/runtime/proxy.js'; +import { fileStoresOf } from '../../lib/runtime/store/file-stores.js'; +import { fakeProxy } from '../helpers/fake-proxy.js'; import { memoryPorts } from './memory-ports.js'; const GENE = { type: 'Gene', asset_id: 'sha256:gene' }; @@ -76,6 +79,69 @@ test('a correction during reverification blocks submit even if it cannot remove assert.deepEqual(await ports.drafts.read(), [source]); }); +test('a correction after the publication check cannot slip into submit dispatch', async (t) => { + const { ports, draft } = await withDraft({ ...RESULT, publishable: true, publish_receipt: RECEIPT }); + const proxy = await fakeProxy(); + t.after(proxy.close); + ports.proxy.request = proxyFetchOf(proxy.env); + ports.draftCorrections = fileStoresOf({ dir: proxy.env.HOME, agent: 'testagent' }).draftCorrections; + const correctionPorts = { ...ports, draftCorrections: fileStoresOf({ dir: proxy.env.HOME, agent: 'testagent' }).draftCorrections }; + const source = { ...draft, sourceCandidate: { turn: 1, at: 10 } }; + await ports.sessions.update('s1', () => ({ turn: 1 })); + await ports.drafts.update(() => [source]); + const hasCorrection = ports.draftCorrections.has; + const updateDrafts = ports.drafts.update; + let checked = false; + ports.draftCorrections.has = async (...args) => { + const observed = await hasCorrection(...args); + if (!checked) { + checked = true; + ports.drafts.update = async () => { throw new Error('draft cleanup denied'); }; + try { + await assert.rejects(promptSubmitted(correctionPorts, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }), /draft cleanup denied/); + } finally { + ports.drafts.update = updateDrafts; + } + } + return observed; + }; + + const result = await runCommand(ports, { sessionId: 'another-session', name: 'evolver-drafts', args: 'publish 1' }); + assert.equal(checked, true); + assert.deepEqual(proxy.requests, []); + assert.equal(result.ok, false); + assert.match(result.text, /invalidated by a correction/i); + assert.deepEqual(await ports.drafts.read(), [source]); +}); + +test('a correction can finish while a dispatched submit waits for the Proxy reply', async (t) => { + const { ports, draft } = await withDraft({ ...RESULT, publishable: true, publish_receipt: RECEIPT }); + const proxy = await fakeProxy(); + t.after(proxy.close); + ports.proxy.request = proxyFetchOf(proxy.env); + ports.draftCorrections = fileStoresOf({ dir: proxy.env.HOME, agent: 'testagent' }).draftCorrections; + const correctionPorts = { ...ports, settings: { recall: false }, draftCorrections: fileStoresOf({ dir: proxy.env.HOME, agent: 'testagent' }).draftCorrections }; + await ports.sessions.update('s1', () => ({ turn: 1 })); + await ports.drafts.update(() => [{ ...draft, sourceCandidate: { turn: 1, at: 10 } }]); + + const dispatched = Promise.withResolvers(); + const reply = Promise.withResolvers(); + t.mock.method(globalThis, 'fetch', (url) => { + dispatched.resolve(new URL(url).pathname); + return reply.promise; + }); + const publishing = runCommand(ports, { sessionId: 'another-session', name: 'evolver-drafts', args: 'publish 1' }); + try { + assert.equal(await dispatched.promise, '/asset/submit'); + await promptSubmitted(correctionPorts, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + assert.equal(await correctionPorts.draftCorrections.has('s1', 1), true); + assert.deepEqual(await ports.drafts.read(), []); + } finally { + reply.resolve(new Response('{"message_id":"fixture-submit"}')); + assert.equal((await publishing).ok, true); + } +}); + test('explicit publication preserves the Proxy receipt and its complete bound bundle', async () => { const { ports, requests } = await withDraft({ ...RESULT, publishable: true, publish_receipt: RECEIPT, event: EVENT }); const publish = ports.proxy.request; diff --git a/test/app/memory-ports.js b/test/app/memory-ports.js index f6097fe..b810752 100644 --- a/test/app/memory-ports.js +++ b/test/app/memory-ports.js @@ -111,6 +111,7 @@ export function memoryPorts({ assets = [ASSET], recipes = [], stepAssets = [], d candidates: { ...table({}), referenceOf: candidateReferenceOf }, drafts: table([]), draftCorrections: { + withSession: async (_id, work) => work(), mark: async (id, turn, epoch) => { draftCorrections.add(JSON.stringify([id, turn, epoch])); }, has: async (id, turn, epoch) => draftCorrections.has(JSON.stringify([id, turn, epoch])), }, diff --git a/test/runtime/file-stores.test.js b/test/runtime/file-stores.test.js index 4f32899..d50abb2 100644 --- a/test/runtime/file-stores.test.js +++ b/test/runtime/file-stores.test.js @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; +import { once } from 'node:events'; import { mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -105,3 +106,34 @@ test('a recorded draft correction survives reopening and pruning the source sess assert.equal(await reopened.draftCorrections.has('s1', 1, 10), false); assert.equal(await reopened.draftCorrections.has('s2', 2, 10), false); }); + +test('a publish gate in another process waits for the source session correction', async (t) => { + const { dir, stores: parent } = stores(); + const module = new URL('../../lib/runtime/store/file-stores.js', import.meta.url).href; + const script = `import { once } from 'node:events'; +import { fileStoresOf } from ${JSON.stringify(module)}; +const { draftCorrections } = fileStoresOf({ dir: ${JSON.stringify(dir)}, agent: 'cursor' }); +await draftCorrections.withSession('s1', async () => { + const release = once(process, 'message'); + process.send('locked'); + await release; + await draftCorrections.mark('s1', 2, 10); +}); +process.disconnect();`; + const child = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }); + t.after(() => child.kill()); + const exited = once(child, 'exit'); + const locked = once(child, 'message'); + const [ready] = await Promise.race([locked, exited]); + assert.equal(ready, 'locked'); + + let checked = false; + const publication = parent.draftCorrections.withSession('s1', async () => { + checked = true; + return parent.draftCorrections.has('s1', 2, 10); + }); + assert.equal(checked, false); + child.send('release'); + assert.equal(await publication, true); + assert.deepEqual(await exited, [0, null]); +}); From a551020be7e3ae589a6b4b723660d4b9c9af51fa Mon Sep 17 00:00:00 2001 From: "xyanua." Date: Wed, 4 Nov 2026 17:10:41 +0700 Subject: [PATCH 22/22] Cover draft publication during concurrent stale-lock recovery --- test/app/draft-publication.test.js | 97 +++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/test/app/draft-publication.test.js b/test/app/draft-publication.test.js index b93d6a8..6ccc9ed 100644 --- a/test/app/draft-publication.test.js +++ b/test/app/draft-publication.test.js @@ -1,5 +1,9 @@ import assert from 'node:assert/strict'; -import { randomBytes } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { createHash, randomBytes } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { test } from 'node:test'; import { hub } from '@evomap/evolver-core'; @@ -10,6 +14,7 @@ import { draftOf } from '../../lib/distill.js'; import { proxyFetchOf } from '../../lib/runtime/proxy.js'; import { fileStoresOf } from '../../lib/runtime/store/file-stores.js'; import { fakeProxy } from '../helpers/fake-proxy.js'; +import { lockContenders, until } from '../helpers/lock-contenders.js'; import { memoryPorts } from './memory-ports.js'; const GENE = { type: 'Gene', asset_id: 'sha256:gene' }; @@ -142,6 +147,96 @@ test('a correction can finish while a dispatched submit waits for the Proxy repl } }); +test('two dead-owner recoverers cannot dispatch a draft after correction', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-draft-recovery-')); + const state = fileStoresOf({ dir, agent: 'testagent' }); + const url = (path) => JSON.stringify(new URL(path, import.meta.url).href); + const script = ` + import assert from 'node:assert/strict'; + import fs from 'node:fs'; + import { join } from 'node:path'; + import { runCommand } from ${url('../../lib/app/commands/index.js')}; + import { promptSubmitted } from ${url('../../lib/app/prompt.js')}; + import { proxyFetchOf } from ${url('../../lib/runtime/proxy.js')}; + import { fileStoresOf } from ${url('../../lib/runtime/store/file-stores.js')}; + import { memoryPorts } from ${url('./memory-ports.js')}; + import { until } from ${url('../helpers/lock-contenders.js')}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => join(dir, name); + const { sessions, drafts, draftCorrections } = fileStoresOf({ dir, agent: 'testagent' }); + const { ports } = memoryPorts(); + Object.assign(ports, { sessions, drafts, draftCorrections, settings: { recall: false } }); + if (role === 'correction') { + const stat = fs.statSync; + fs.statSync = (file, ...args) => { + const snapshot = stat(file, ...args); + if (file === lock) { + fs.statSync = stat; + fs.writeFileSync(marker('stale-read'), ''); + const deadline = Date.now() + 10_000; + while (!fs.existsSync(marker('resume'))) { + if (Date.now() >= deadline) throw new Error('timed out resuming correction'); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + } + return snapshot; + }; + ports.drafts.update = async () => { throw new Error('draft cleanup denied'); }; + await assert.rejects(promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }), /draft cleanup denied/); + fs.writeFileSync(marker('corrected'), ''); + } else { + const { has, withSession } = draftCorrections; + let inGate = false; + draftCorrections.has = async (...args) => { + const corrected = await has(...args); + if (inGate && !corrected) await until(() => fs.existsSync(marker('corrected'))); + return corrected; + }; + draftCorrections.withSession = (id, work, signal) => { + const result = withSession(id, () => { + inGate = true; + return work(); + }, signal); + fs.writeFileSync(marker('publish-attempted'), ''); + return result; + }; + globalThis.fetch = async (url) => { + assert.equal(new URL(url).pathname, '/asset/submit'); + fs.writeFileSync(marker('submitted'), ''); + return new Response('{}'); + }; + ports.proxy.request = proxyFetchOf({ HOME: dir }); + const result = await runCommand(ports, { sessionId: 'another-session', name: 'evolver-drafts', args: 'publish 1' }); + fs.writeFileSync(marker('result'), JSON.stringify(result)); + } + `; + const lockName = createHash('sha256').update('s1').digest('hex').slice(0, 16); + const lock = join(dir, 'testagent-distill-corrections', `${lockName}.lock`); + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['resume', 'corrected']); + const dead = spawnSync(process.execPath, ['-e', ''], { windowsHide: true }); + assert.equal(dead.status, 0); + mkdirSync(lock, { recursive: true }); + writeFileSync(join(lock, 'owner'), `${dead.pid}:deadbeef`); + utimesSync(lock, new Date(0), new Date(0)); + await state.sessions.update('s1', () => ({ turn: 1 })); + const draft = { ...draftOf({ ...RESULT, publishable: true, publish_receipt: RECEIPT }, { title: 'Verified method', sessionId: 's1' }, 100), sourceCandidate: { turn: 1, at: 10 } }; + await state.drafts.update(() => [draft]); + + run('correction'); + await until(() => existsSync(join(dir, 'stale-read'))); + run('publish'); + await until(() => existsSync(join(dir, 'publish-attempted'))); + writeFileSync(join(dir, 'resume'), ''); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); + assert.equal(await state.draftCorrections.has('s1', 1), true); + assert.equal(existsSync(join(dir, 'submitted')), false); + const result = JSON.parse(readFileSync(join(dir, 'result'), 'utf8')); + assert.equal(result.ok, false); + assert.match(result.text, /invalidated by a correction/i); + assert.deepEqual(await state.drafts.read(), [draft]); +}); + test('explicit publication preserves the Proxy receipt and its complete bound bundle', async () => { const { ports, requests } = await withDraft({ ...RESULT, publishable: true, publish_receipt: RECEIPT, event: EVENT }); const publish = ports.proxy.request;