diff --git a/hosts/workbuddy/commands.js b/hosts/workbuddy/commands.js index 06914ec..4d4901e 100644 --- a/hosts/workbuddy/commands.js +++ b/hosts/workbuddy/commands.js @@ -48,7 +48,7 @@ function agentGeneRequest({ text, agentGene }, payload) { 'Write the Gene with your own model: answer the request below with the JSON object only, put it between the', 'two marker lines of this command, run the command with your shell tool, and show the user what it prints.', '', - `${env.join(' ')} node ${shellQuoted(payload.script)} ${AGENT_NAME} evolver-distill --post ${agentGene.writtenFor} <<'${end}'`, + `${env.join(' ')} node ${shellQuoted(payload.script)} ${AGENT_NAME} evolver-distill --post ${shellQuoted(agentGene.writtenFor)} <<'${end}'`, '', end, '', diff --git a/lib/agent/defaults/mcp.js b/lib/agent/defaults/mcp.js index f89ce7c..0dcbc19 100644 --- a/lib/agent/defaults/mcp.js +++ b/lib/agent/defaults/mcp.js @@ -77,7 +77,7 @@ export const mcpReply = Object.freeze({ empty: (hostEvent) => asContent(EMPTY[hostEvent] ?? ''), command: (result) => asContent(result.text, result.ok === false), turnEnd: (ended) => asContent(ended?.agentGene - ? `Reported. This task qualifies for distillation: call ${MCP_TOOLS.DISTILL} with written_for ${ended.agentGene.writtenFor} and the JSON the request below asks for, or with {"skip": ""} if it holds no method worth reusing.\n\n${ended.agentGene.prompt}` + ? `Reported. This task qualifies for distillation: call ${MCP_TOOLS.DISTILL} with written_for "${ended.agentGene.writtenFor}" and the JSON the request below asks for, or with {"skip": ""} if it holds no method worth reusing.\n\n${ended.agentGene.prompt}` : 'Reported.'), lifetime: 'turn', }); diff --git a/lib/agent/profile.js b/lib/agent/profile.js index e743ab2..f7994dd 100644 --- a/lib/agent/profile.js +++ b/lib/agent/profile.js @@ -25,7 +25,7 @@ const COMMON = Object.freeze({ stateDir: defaultStateDir, legacy: Object.freeze([]), reply: Object.freeze({ strategyAt: STRATEGY_AT.PROMPT, toolResultContext: false }), - payload: Object.freeze({ transcriptPath: (payload) => payload.transcript_path ?? null, gene: (payload) => payload.gene, writtenFor: (payload) => Number(payload.written_for), hostTurn: () => null }), + payload: Object.freeze({ transcriptPath: (payload) => payload.transcript_path ?? null, gene: (payload) => payload.gene, writtenFor: (payload) => payload.written_for, hostTurn: () => null }), commands: Object.freeze({ entry: COMMAND_ENTRIES.CLI_ONLY, parse: typedFromPrompt, scriptArgs: (text) => text, scriptAnswer: (result) => result.text }), }); diff --git a/lib/app/commands/distill.js b/lib/app/commands/distill.js index b96346b..f2906b6 100644 --- a/lib/app/commands/distill.js +++ b/lib/app/commands/distill.js @@ -10,10 +10,13 @@ export function distillText(outcome) { case 'busy': return failure('Another process is distilling this session right now.'); case 'declined': return success(`No Gene: the turn holds no method worth reusing (${outcome.reason}).`); case 'drafted': return success(`Distilled "${outcome.draft.title}" into a draft. Publish it with /evolver-drafts publish 1, or list drafts with /evolver-drafts.`); + case 'draft_missing': return failure('The distilled draft is no longer available.'); case 'stored': return success(shown(outcome.data ?? 'distilled')); case 'skipped': return failure(`The Proxy kept nothing: ${outcome.reason}.`); case 'unparsed': return failure('The Gene must be one JSON object with at least a "summary".'); - case 'dropped': return failure('A newer turn replaced the one this Gene was written for; nothing was sent.'); + case 'dropped': return failure('A newer turn replaced the one this Gene was written for.'); + case 'superseded': return failure('A draft with the same ID is already saved.'); + case 'cleanup_failed': return failure('A correction invalidated the draft. Evolver could not remove it. Check /evolver-drafts and discard it if it is still listed.'); default: return failure(`Distillation ${outcome.status}${outcome.error ? `: ${outcome.error}` : ''}.`); } } diff --git a/lib/app/commands/drafts.js b/lib/app/commands/drafts.js index 895a8a3..efbf052 100644 --- a/lib/app/commands/drafts.js +++ b/lib/app/commands/drafts.js @@ -1,8 +1,8 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 EvoMap -import { draftNamed, draftsListText, reverifiedOf, submitRequestOf, withoutDraft } from '../../distill.js'; -import { DISTILL_TIMEOUT_MS } from '../distill-session.js'; +import { draftNamed, draftsListText, reverifiedOf, sameDraftSource, submitRequestOf, withoutDraft } from '../../distill.js'; +import { DISTILL_TIMEOUT_MS, draftWasCorrected } from '../distill-session.js'; import { failure, success } from './result.js'; const USAGE = 'Usage: /evolver-drafts [publish|discard ]'; @@ -12,9 +12,21 @@ async function discard({ drafts }, draft) { return success(`Discarded "${draft.title}".`); } +const invalidatedDraft = (draft) => failure(`"${draft.title}" was invalidated by a correction and cannot be published. You can discard it.`); + +async function publicationBlock(ports, draft) { + if (await draftWasCorrected(ports, draft)) return invalidatedDraft(draft); + if (draft.pendingCandidateCheck) return failure(`"${draft.title}" was not confirmed by distillation and cannot be published. You can discard it.`); + return null; +} + async function publish(ports, draft, signal) { if (draft.publishable === false || !draft.publish_receipt) { + const blocked = await publicationBlock(ports, draft); + if (blocked) return blocked; const verified = await ports.proxy.request('POST', '/asset/reverify', { assets: draft.assets, persist: false }, { timeoutMs: DISTILL_TIMEOUT_MS, signal }); + const invalidated = await publicationBlock(ports, draft); + if (invalidated) return invalidated; const promoted = verified.ok && reverifiedOf(verified.data); if (!promoted) { return failure(`"${draft.title}" needs trusted Proxy verification before publication: ${verified.error ?? verified.data?.error ?? 'invalid reverification response'}. The draft was kept.`); @@ -24,6 +36,7 @@ async function publish(ports, draft, signal) { await ports.drafts.update((list) => { const current = list.find((existing) => existing.id === draft.id); if (!current || current.createdAt !== draft.createdAt + || (draft.sourceCandidate ? !sameDraftSource(current, draft) : current.sourceCandidate) || current.publishable !== draft.publishable || Boolean(current.publish_receipt) !== Boolean(draft.publish_receipt)) return undefined; saved = true; @@ -32,8 +45,16 @@ async function publish(ports, draft, signal) { if (!saved) return failure(`"${draft.title}" changed or was discarded during verification; it was not submitted.`); draft = next; } + const blocked = await publicationBlock(ports, draft); + if (blocked) return blocked; const request = submitRequestOf(draft); - const answered = await ports.proxy.request(request.method, request.path, request.body, { signal }); + // Hold the correction lock through dispatch only. Wrap the request promise so the lock never waits for HTTP. + const dispatch = () => ({ response: ports.proxy.request(request.method, request.path, request.body, { signal }) }); + const started = draft.sourceCandidate + ? await ports.draftCorrections.withSession(draft.sessionId, async () => (await draftWasCorrected(ports, draft)) ? null : dispatch(), signal) + : dispatch(); + if (!started) return invalidatedDraft(draft); + const answered = await started.response; if (!answered.ok) return failure(`"${draft.title}" was not submitted: ${answered.error}`); await ports.drafts.update((list) => withoutDraft(list, draft.id)); return success(`Submitted "${draft.title}" to the EvoMap Hub for review (message ${answered.data?.message_id ?? 'pending'}).`); diff --git a/lib/app/distill-session.js b/lib/app/distill-session.js index f82ad92..e70ce16 100644 --- a/lib/app/distill-session.js +++ b/lib/app/distill-session.js @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 EvoMap -import { withDraft } from '../distill.js'; +import { sameDraftSource, withConfirmedDraft, withDraft, withoutCorrectedDraft } from '../distill.js'; import { candidateSessionOf, claimedDue, claimedOwn, hasDueCandidate, isCurrentCandidate, nextDueAt, settledCandidate, withoutExpiredLeftovers } from '../distill/candidate.js'; import { argumentsOf, declineOf, distillBody, distillPrompt } from '../distill/prompt.js'; import { distillOutcomeOf, isSettled, needsAgentModel } from '../distill/result.js'; @@ -25,14 +25,54 @@ async function claim({ candidates, owner, clock }, claimOf) { return claimed; } -const settle = ({ candidates, clock }, { sessionId, candidate }, outcome, countsFailure) => - candidates.update((pending) => settledCandidate(pending, sessionId, candidate, { done: isSettled(outcome), counted: countsFailure, now: clock.now() })); +function settle({ candidates, clock }, { sessionId, candidate }, outcome, countsFailure) { + if (outcome?.status === 'dropped' || outcome?.status === 'cleanup_failed') return; + return candidates.update((pending) => settledCandidate(pending, sessionId, candidate, { done: isSettled(outcome), counted: countsFailure, now: clock.now() })); +} + +export function draftWasCorrected({ draftCorrections }, draft) { + const source = draft.sourceCandidate; + return source ? draftCorrections.has(draft.sessionId, source.turn, source.sessionEpoch) : false; +} + +async function confirmDraftOutcomes(ports, outcomes) { + if (!outcomes.some((outcome) => outcome.status === 'drafted')) return outcomes; + const list = await ports.drafts.read(); + return Promise.all(outcomes.map(async (outcome) => { + if (outcome.status !== 'drafted') return outcome; + const saved = Array.isArray(list) && list.some((draft) => !draft.pendingCandidateCheck && sameDraftSource(draft, outcome.draft)); + if (saved && !await draftWasCorrected(ports, outcome.draft)) return outcome; + return { status: 'draft_missing', ...(outcome.usage && { usage: outcome.usage }) }; + })); +} + +async function saveDraft({ drafts, candidates }, { sessionId, candidate }, draft) { + const written = await drafts.update((list) => withDraft(list, { ...draft, pendingCandidateCheck: true })); + if (isCurrentCandidate(await candidates.read(), sessionId, candidate)) { + if (!written) return { status: 'superseded' }; + const confirmed = await drafts.update((list) => withConfirmedDraft(list, draft)); + return confirmed ? null : { status: 'draft_missing' }; + } + if (!written) return { status: 'dropped' }; + try { + await drafts.update((list) => withoutCorrectedDraft(list, draft)); + return { status: 'dropped' }; + } catch { + return { status: 'cleanup_failed' }; + } +} async function post(ports, { sessionId, candidate, trigger }, args, usage, signal) { const body = distillBody({ platform: ports.identity.agent, sessionId: taskSessionId(ports.identity, sessionId), candidate, args, usage, trigger }); const result = await ports.proxy.request('POST', '/conversation/distill', body, { timeoutMs: DISTILL_TIMEOUT_MS, signal }); const outcome = distillOutcomeOf(result, { title: args.title, sessionId }, ports.clock.now()); - if (outcome.draft) await ports.drafts.update((drafts) => withDraft(drafts, outcome.draft)); + if (outcome.draft) { + if (!isCurrentCandidate(await ports.candidates.read(), sessionId, candidate)) return { status: 'dropped' }; + const draft = { ...outcome.draft, sourceCandidate: { at: candidate.at, turn: candidate.turn, sessionEpoch: candidate.sessionEpoch } }; + const correction = await saveDraft(ports, { sessionId, candidate }, draft); + if (correction) return correction; + return { ...outcome, draft, ...(usage && { usage }) }; + } return { ...outcome, ...(usage && { usage }) }; } @@ -71,15 +111,17 @@ async function distillClaimed(ports, claimed, focus) { async function run(ports, claimed, { focus = '', countModelFailures = true } = {}) { const outcome = await distillClaimed(ports, claimed, focus); const counts = outcome.status !== 'stopped' && (countModelFailures || !needsAgentModel(outcome)); - if (outcome.status !== 'dropped') await settle(ports, claimed, outcome, counts); + await settle(ports, claimed, outcome, counts); return outcome; } // A command run outside any session (a terminal) acts on the newest candidate, or the one a Gene was written for. -async function sessionToDistill({ candidates }, { sessionId, anySession, at }) { +async function sessionToDistill({ candidates }, { sessionId, anySession, writtenFor }) { + if (!anySession) return sessionId; const pending = await candidates.read(); - if (pending[sessionId] || !anySession) return sessionId; - return candidateSessionOf(pending, { at }) ?? sessionId; + if (writtenFor === undefined) return pending[sessionId] ? sessionId : candidateSessionOf(pending) ?? sessionId; + const match = Object.entries(pending).find(([id, candidate]) => candidate && candidates.referenceOf(id, candidate) === writtenFor); + return match?.[0] ?? sessionId; } /** @@ -94,15 +136,16 @@ export async function distillSession(ports, { sessionId: asked, focus = '', anyS if (!claimed) return { status: 'busy' }; const outcome = await run(ports, claimed, { focus, countModelFailures: false }); await armDistillWakeUp(ports); + if (!needsAgentModel(outcome)) return (await confirmDraftOutcomes(ports, [outcome]))[0]; const due = (await ports.candidates.read())[sessionId]; - return needsAgentModel(outcome) && due ? { ...outcome, agentGene: { writtenFor: due.at, prompt: distillPrompt(due, focus) } } : outcome; + return due ? { ...outcome, agentGene: { writtenFor: ports.candidates.referenceOf(sessionId, due), prompt: distillPrompt(due, focus) } } : outcome; } /** Distills the candidates that are due: their session is over (ended, or killed with its owner) or quiet. */ export async function distillDue(ports) { const outcomes = []; for (const claimed of (await claim(ports, claimedDue)) ?? []) outcomes.push(await run(ports, claimed)); - return outcomes; + return confirmDraftOutcomes(ports, outcomes); } /** Whether distillation runs in the background here: a model outside the conversation, and the person did not turn it off. */ @@ -136,11 +179,11 @@ export async function dropExpiredLeftovers({ candidates, owner, clock }) { } /** - * Posts the Gene the agent's own model wrote for the candidate stamped `writtenFor`, or settles it when the agent + * Posts the Gene the agent's own model wrote for the candidate identified by `writtenFor`, or settles it when the agent * judged it not worth one. A Gene written for a candidate since replaced is not sent. */ export async function postAgentGene(ports, { sessionId: asked, text, writtenFor, anySession = false }) { - const sessionId = await sessionToDistill(ports, { sessionId: asked, anySession, at: writtenFor }); + const sessionId = await sessionToDistill(ports, { sessionId: asked, anySession, writtenFor }); const declined = declineOf(text); const args = declined ? null : argumentsOf(text); if (!declined && !args) return { status: 'unparsed' }; @@ -148,11 +191,11 @@ export async function postAgentGene(ports, { sessionId: asked, text, writtenFor, if (!claimed) return (await ports.candidates.read())[sessionId] ? { status: 'busy' } : { status: 'nothing' }; let outcome; try { - if (claimed.candidate.at !== writtenFor) return { status: 'dropped' }; + if (ports.candidates.referenceOf(sessionId, claimed.candidate) !== writtenFor) return { status: 'dropped' }; outcome = declined ? { status: 'declined', reason: declined } : await post(ports, claimed, args, null, AbortSignal.timeout(DISTILL_TIMEOUT_MS)); - return outcome; } finally { await settle(ports, claimed, outcome, false); await armDistillWakeUp(ports); } + return (await confirmDraftOutcomes(ports, [outcome]))[0]; } diff --git a/lib/app/ports.js b/lib/app/ports.js index 3b6b0cb..c60dc9e 100644 --- a/lib/app/ports.js +++ b/lib/app/ports.js @@ -32,6 +32,11 @@ * @property {() => Promise} read * @property {(mutate: Function) => Promise} update * + * @typedef {object} DraftCorrections + * @property {(sessionId: string, work: () => T|Promise, signal?: AbortSignal) => Promise} withSession + * @property {(sessionId: string, turn: number, sessionEpoch?: number) => Promise} mark + * @property {(sessionId: string, turn: number, sessionEpoch?: number) => Promise} has + * * @typedef {object} ReportQueue * @property {(entries: { report: object, sessionId: string }[]) => Promise} enqueue * @@ -42,7 +47,7 @@ * @property {(entry: { sessionId, projectDir, reuses, editedBefore: Map }) => Promise} remember * @property {(due: { sessionId, projectDir }) => Promise<{ due: object[], commit: () => Promise }>} takeDue * - * @typedef {object} Proxy + * @typedef {object} Proxy request initiates the transport before returning its promise. * @property {(method: string, path: string, body?: object, options?: { timeoutMs?: number, signal?: AbortSignal }) => * Promise<{ ok: true, data: any } | { ok: false, error: string, status?: number, data?: any }>} request * @@ -104,8 +109,9 @@ * @property {SessionStore} sessions * @property {TurnNotes} notes * @property {EditBaseline} baseline - * @property {TableStore} candidates + * @property {TableStore & { referenceOf: (sessionId: string, candidate: object) => string }} candidates * @property {TableStore} drafts + * @property {DraftCorrections} draftCorrections * @property {ReportQueue} reports * @property {Outbox} outbox * @property {RetentionStore} retention diff --git a/lib/app/prompt.js b/lib/app/prompt.js index 6086f25..4da3514 100644 --- a/lib/app/prompt.js +++ b/lib/app/prompt.js @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 EvoMap +import { withoutDraftsForTurn } from '../distill.js'; import { withSessionActivity, withoutCandidate } from '../distill/candidate.js'; import { promptTextOf, recallStrategy, taskTextOf } from '../recall.js'; import { correctionReportsOf, followUpLabelOf } from '../session/follow-ups.js'; @@ -15,16 +16,22 @@ const LATE_RECIPE_WAIT_MS = 6_000; /** * The next prompt is the only verdict on a turn available without asking anyone: a correction drops the - * session's candidate first, then the last turn's reuses are reported again under `:correction`. The session is - * marked corrected before the drop, so a re-judge of that turn writing meanwhile sees the mark and takes its write back. + * session's candidate and that turn's drafts, then its reuses are reported again under `:correction`. Mark the + * turn first so a re-judge sees the correction, and drop the candidate before removing drafts so a late write + * after the removal still sees the candidate is gone. * Any other prompt is the session's activity, which a candidate waits out before it is distilled. */ async function labelFollowUp(ports, sessionId, prompt) { - const { sessions, candidates, reports, clock } = ports; + const { sessions, candidates, drafts, draftCorrections, reports, clock } = ports; const label = followUpLabelOf(prompt); if (label === 'correction') { - await sessions.update(sessionId, (stored) => correctedSession(sessionOf(stored))); + const { correctedTurn, sessionEpoch } = await draftCorrections.withSession(sessionId, async () => { + const corrected = await sessions.update(sessionId, (stored) => correctedSession(sessionOf(stored))); + if (corrected.correctedTurn > 0) await draftCorrections.mark(sessionId, corrected.correctedTurn, corrected.sessionEpoch); + return corrected; + }); await candidates.update((pending) => withoutCandidate(pending, sessionId)); + if (correctedTurn > 0) await drafts.update((list) => withoutDraftsForTurn(list, sessionId, correctedTurn, sessionEpoch)); } else { await candidates.update((pending) => withSessionActivity(pending, sessionId, clock.now())); } @@ -102,7 +109,7 @@ export async function promptSubmitted(ports, { sessionId, prompt, turnId = null, if (!isCommand) await labelFollowUp(ports, sessionId, prompt); await ports.notes.clear(sessionId); const opened = await ports.sessions.update(sessionId, (stored) => - openedTurn(sessionOf(stored), { turnId, turnStart, hostTurn, prompt: prompt ? promptTextOf(prompt) : null, task: taskTextOf(prompt), pid: ports.owner.current(), isCommand })); + openedTurn(sessionOf(stored), { turnId, turnStart, hostTurn, prompt: prompt ? promptTextOf(prompt) : null, task: taskTextOf(prompt), pid: ports.owner.current(), isCommand, now: ports.clock.now() })); const wantsRecall = !isCommand && ports.settings?.recall !== false; const { injection, late } = wantsRecall ? await recalled(ports, sessionId, { turn: opened.turn, prompt, waitMs, keepLate, holdForToolResult }) : { injection: '' }; return { turn: opened.turn, injection, ...(late && { late }) }; diff --git a/lib/app/turn-end.js b/lib/app/turn-end.js index 4b9e757..6102b94 100644 --- a/lib/app/turn-end.js +++ b/lib/app/turn-end.js @@ -49,7 +49,7 @@ async function snapshotIfReadable(project, projectDir) { async function rememberCandidate({ candidates, owner, clock, project }, sessionId, opened, { reason, record, model, projectDir, source, written = withCandidate }) { if (opened.isCommand || skipReason(reason, record)) return false; const snapshot = projectDir ? await snapshotIfReadable(project, projectDir) : null; - const candidate = candidateOf({ prompt: opened.prompt, record, model, turn: opened.turn, pid: owner.current(), now: clock.now(), source, project: snapshot }); + const candidate = candidateOf({ prompt: opened.prompt, record, model, turn: opened.turn, sessionEpoch: opened.sessionEpoch, pid: owner.current(), now: clock.now(), source, project: snapshot }); await candidates.update((pending) => written(pending, sessionId, candidate)); return true; } diff --git a/lib/channels/mcp/tools.js b/lib/channels/mcp/tools.js index 5fa1646..46932c8 100644 --- a/lib/channels/mcp/tools.js +++ b/lib/channels/mcp/tools.js @@ -28,7 +28,7 @@ export const TOOL_DEFINITIONS = Object.freeze([ { name: MCP_TOOLS.DISTILL, description: 'Only when a report asked for it: the reusable Gene you wrote for that turn, or {"skip": ""}.', - input: { written_for: z.number().int(), gene: z.union([z.string(), z.record(z.string(), z.unknown())]) }, + input: { written_for: z.union([z.string().max(64), z.number().int()]).describe('Copy the candidate reference from the report exactly.'), gene: z.union([z.string(), z.record(z.string(), z.unknown())]) }, }, { name: MCP_TOOLS.STATUS, description: 'Evolver health: Proxy, engine setup, Hub sign-in, distillation candidate and drafts.', input: {} }, { name: MCP_TOOLS.SEARCH, description: 'Find the EvoMap strategy that best fits a task.', input: { query: z.string().min(1) } }, diff --git a/lib/channels/script/runner.js b/lib/channels/script/runner.js index 25f3a78..7eaf5d5 100644 --- a/lib/channels/script/runner.js +++ b/lib/channels/script/runner.js @@ -60,7 +60,7 @@ export async function runCommandScript(agent, name, { argv = [], stdin = process const { ports } = environmentFor({ profile, env, lifetime: EPHEMERAL, runBackground: runBackgroundTasks }); const driver = createDriver({ profile, ports, transcripts, env, lifetime: EPHEMERAL }); const result = postAt >= 0 - ? await driver.direct(at, { gene: input, writtenFor: Number(argv[postAt + 1]) }) + ? await driver.direct(at, { gene: input, writtenFor: argv[postAt + 1] }) : await driver.direct(at, { command: { name, args: input } }); return write(profile.commands.scriptAnswer(result, script)); } diff --git a/lib/distill.js b/lib/distill.js index 472fafb..9d4e0d7 100644 --- a/lib/distill.js +++ b/lib/distill.js @@ -144,8 +144,49 @@ export function skipReasonOf(result) { } export function withDraft(drafts, draft) { - const kept = (Array.isArray(drafts) ? drafts : []).filter((existing) => existing.id !== draft.id); - return [...kept, draft]; + const list = Array.isArray(drafts) ? drafts : []; + const index = list.findIndex((existing) => existing.id === draft.id); + const existing = list[index]; + const saved = existing?.sourceCandidate; + const incoming = draft.sourceCandidate; + if (saved && incoming) { + // Candidate order, not reply time, decides which draft can replace the other. + const sameSession = existing.sessionId === draft.sessionId && saved.sessionEpoch === incoming.sessionEpoch; + const newer = sameSession && saved.turn !== incoming.turn + ? saved.turn > incoming.turn + : saved.at > incoming.at || (!sameSession && saved.at === incoming.at); + if (newer) return undefined; + } + const kept = list.filter((existing) => existing.id !== draft.id); + kept.splice(index < 0 ? kept.length : index, 0, draft); + return kept; +} + +export function sameDraftSource(draft, reference) { + return draft.id === reference.id && draft.sessionId === reference.sessionId + && draft.sourceCandidate?.at === reference.sourceCandidate.at + && draft.sourceCandidate?.turn === reference.sourceCandidate.turn + && draft.sourceCandidate?.sessionEpoch === reference.sourceCandidate.sessionEpoch; +} + +export function withConfirmedDraft(drafts, draft) { + if (!Array.isArray(drafts)) return undefined; + const pending = drafts.find((row) => row.pendingCandidateCheck && sameDraftSource(row, draft)); + return pending ? drafts.map((row) => row === pending ? draft : row) : undefined; +} + +export function withoutCorrectedDraft(drafts, corrected) { + if (!Array.isArray(drafts)) return undefined; + const isCorrected = (draft) => sameDraftSource(draft, corrected); + if (!drafts.some(isCorrected)) return undefined; + return drafts.filter((draft) => !isCorrected(draft)); +} + +export function withoutDraftsForTurn(drafts, sessionId, turn, sessionEpoch) { + if (!Array.isArray(drafts)) return undefined; + const kept = drafts.filter((draft) => draft.sessionId !== sessionId || draft.sourceCandidate?.turn !== turn + || draft.sourceCandidate.sessionEpoch !== sessionEpoch); + return kept.length === drafts.length ? undefined : kept; } export function withoutDraft(drafts, id) { diff --git a/lib/distill/candidate.js b/lib/distill/candidate.js index 0215495..c74eda8 100644 --- a/lib/distill/candidate.js +++ b/lib/distill/candidate.js @@ -45,7 +45,7 @@ export function skipReason(reason, record) { // Keys and tokens are stripped before anything is written down or sent to a model or the Proxy. `source` says who // saw the checks run (OBSERVED by the host, AGENT_REPORTED by the agent), and `project` is the git work tree they ran // in, whose diff its snapshot has already left out when it touches a secret. -export function candidateOf({ prompt, record, model, turn, pid, now, source, project = null }) { +export function candidateOf({ prompt, record, model, turn, sessionEpoch, pid, now, source, project = null }) { return { prompt: withoutSecrets(String(prompt ?? '')).slice(0, PROMPT_MAX_CHARS), commands: record.commands.slice(-MAX_COMMANDS).map((command) => withoutSecrets(command).slice(0, COMMAND_MAX_CHARS)), @@ -56,6 +56,7 @@ export function candidateOf({ prompt, record, model, turn, pid, now, source, pro answer: withoutSecrets(record.answer).slice(0, ANSWER_MAX_CHARS), model, turn, + sessionEpoch, pid: pid ?? null, attempts: 0, at: now, @@ -70,14 +71,13 @@ export const withCandidate = (pending, sessionId, candidate) => ({ ...pending, [ */ export function withRejudgedCandidate(pending, sessionId, candidate) { const current = pending[sessionId]; - if (current?.turn !== candidate.turn) return withCandidate(pending, sessionId, candidate); + if (current?.turn !== candidate.turn || current.sessionEpoch !== candidate.sessionEpoch) return withCandidate(pending, sessionId, candidate); return { ...pending, [sessionId]: { ...current, ...candidate, at: current.at, attempts: current.attempts } }; } -/** The session of the newest candidate, or of the one stamped `at`; null when there is none. */ -export function candidateSessionOf(pending, { at } = {}) { +/** The session of the newest candidate, or null when there is none. */ +export function candidateSessionOf(pending) { const entries = Object.entries(pending ?? {}); - if (at !== undefined) return entries.find(([, candidate]) => candidate?.at === at)?.[0] ?? null; return entries.sort(([, left], [, right]) => (right?.at ?? 0) - (left?.at ?? 0))[0]?.[0] ?? null; } @@ -95,7 +95,10 @@ export const withClosedCandidate = (pending, sessionId) => (pending[sessionId] ? { ...pending, [sessionId]: { ...pending[sessionId], closed: true } } : undefined); /** Whether `candidate` is still the session's: not dropped by a correction, nor replaced by a newer turn. */ -export const isCurrentCandidate = (pending, sessionId, candidate) => pending[sessionId]?.at === candidate.at; +export function isCurrentCandidate(pending, sessionId, candidate) { + const current = pending[sessionId]; + return Boolean(current) && current.at === candidate.at && current.turn === candidate.turn && current.sessionEpoch === candidate.sessionEpoch; +} const isHeld = (candidate, isGone) => Boolean(candidate.claimedBy) && !isGone(candidate.claimedBy); @@ -179,7 +182,7 @@ const retryDelayOf = (attempts) => Math.min(RETRY_BASE_MS * 2 ** (attempts - 1), */ export function settledCandidate(pending, sessionId, candidate, { done, counted, now }) { const current = pending[sessionId]; - if (!current || current.at !== candidate.at) return undefined; + if (!isCurrentCandidate(pending, sessionId, candidate)) return undefined; const attempts = (current.attempts ?? 0) + (counted ? 1 : 0); if (done || attempts >= MAX_ATTEMPTS) return without(pending, sessionId); const retry = counted ? { retryAt: now + retryDelayOf(attempts) } : {}; diff --git a/lib/distill/result.js b/lib/distill/result.js index 406cfac..47f67be 100644 --- a/lib/distill/result.js +++ b/lib/distill/result.js @@ -3,7 +3,7 @@ import { draftOf, skipReasonOf } from '../distill.js'; -const SETTLED = new Set(['stored', 'drafted', 'skipped', 'declined']); +const SETTLED = new Set(['stored', 'drafted', 'skipped', 'declined', 'superseded']); const MODEL_FAILURES = new Set(['no_model', 'model_failed', 'unparsed']); /** diff --git a/lib/runtime/environment.js b/lib/runtime/environment.js index d8e925d..5363cfa 100644 --- a/lib/runtime/environment.js +++ b/lib/runtime/environment.js @@ -36,6 +36,7 @@ export function environmentFor({ profile, env = process.env, lifetime, runBackgr baseline: stores.baseline, candidates: stores.candidates, drafts: stores.drafts, + draftCorrections: stores.draftCorrections, reports: stores.reports, outbox: stores.outbox, retention: stores.retention, diff --git a/lib/runtime/process/owner.js b/lib/runtime/process/owner.js index 8d39208..5ea48fa 100644 --- a/lib/runtime/process/owner.js +++ b/lib/runtime/process/owner.js @@ -6,8 +6,32 @@ import { readlinkSync } from 'node:fs'; import path from 'node:path'; const SHELLS = new Set(['sh', 'bash', 'zsh', 'dash', 'ksh', 'fish', 'cmd.exe', 'powershell', 'pwsh']); -const PS_TIMEOUT_MS = 1000; +export const PROCESS_START_TIMEOUT_MS = 1_000; +// POSIX ps reports start times to the nearest second. +export const PROCESS_START_TOLERANCE_MS = 2_000; const MAX_ANCESTORS = 16; +const ownStartTime = Math.round(Date.now() - process.uptime() * 1000); + +/** A bounded process start-time probe for lock recovery. The hot path needs no subprocess. */ +export function processStartTime(pid, timeout = PROCESS_START_TIMEOUT_MS) { + if (pid === process.pid) return ownStartTime; + if (!Number.isSafeInteger(pid) || pid <= 0 || timeout <= 0) return null; + const windows = process.platform === 'win32'; + const systemRoot = process.env.SystemRoot ?? 'C:\\Windows'; + if (windows && !path.isAbsolute(systemRoot)) return null; + const executable = windows ? path.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') : '/bin/ps'; + const args = windows + ? ['-NoProfile', '-NonInteractive', '-Command', `([DateTimeOffset] (Get-Process -Id ${pid} -ErrorAction Stop).StartTime).ToUnixTimeMilliseconds()`] + : ['-p', String(pid), '-o', 'lstart=']; + const env = windows ? process.env : { ...process.env, LC_ALL: 'C', TZ: 'UTC' }; + const result = spawnSync(executable, args, { + env, encoding: 'utf8', windowsHide: true, timeout, killSignal: 'SIGKILL', + }); + const started = result.status === 0 ? result.stdout.trim() : ''; + if (!started) return null; + const time = windows ? Number(started) : Date.parse(started + ' GMT'); + return Number.isSafeInteger(time) && time > 0 ? time : null; +} /** * Whether the process that wrote a state entry has exited. Signal 0 only probes: ESRCH means no such process; @@ -35,7 +59,7 @@ function executablePathOf(pid, command) { } export function processInfo(pid) { - const probe = spawnSync('ps', ['-o', 'ppid=,comm=', '-p', String(pid)], { encoding: 'utf8', timeout: PS_TIMEOUT_MS }); + const probe = spawnSync('ps', ['-o', 'ppid=,comm=', '-p', String(pid)], { encoding: 'utf8', timeout: PROCESS_START_TIMEOUT_MS }); const match = /^\s*(\d+)\s+(.+?)\s*$/.exec(probe.stdout ?? ''); if (!match) return null; const command = match[2].replace(/^-/, ''); diff --git a/lib/runtime/proxy.js b/lib/runtime/proxy.js index 53489bc..3c63e55 100644 --- a/lib/runtime/proxy.js +++ b/lib/runtime/proxy.js @@ -24,6 +24,7 @@ export function proxyFetchOf(env = process.env) { const headers = { ...(body !== undefined && { 'Content-Type': 'application/json' }), ...(token && { Authorization: `Bearer ${token}` }) }; const timeout = AbortSignal.timeout(timeoutMs); try { + // Draft publication releases its correction lock once this call has started fetch. const response = await fetch(base + path, { method, headers, diff --git a/lib/runtime/store/admission.js b/lib/runtime/store/admission.js new file mode 100644 index 0000000..b416d0b --- /dev/null +++ b/lib/runtime/store/admission.js @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 EvoMap + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { ownerIsGone, processStartTime, PROCESS_START_TIMEOUT_MS, PROCESS_START_TOLERANCE_MS } from '../process/owner.js'; + +export const LOCK_WAIT_MS = 2_000; +export const LOCK_RETRY_MS = 20; +export const LOCK_ACCESS_ERRORS = new Set(['EPERM', 'EBUSY', 'EACCES']); +const ADMISSION_IDENTITY_CHECK_MS = 10_000; +const ABANDONED_PREFIX = 'abandoned-'; +const abandonedAdmissions = new Map(); + +export function pause(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function reclaimAdmission(admission, deadline, starts) { + try { + const info = fs.lstatSync(admission); + if (!info.isDirectory()) return false; + const entries = fs.readdirSync(admission); + if (entries.length > 0) { + const marker = entries.find((name) => name.startsWith(ABANDONED_PREFIX)); + const ownerName = marker ? marker.slice(ABANDONED_PREFIX.length) : entries[0]; + if (entries.some((name) => name !== ownerName && name !== marker)) return false; + const owner = /^owner-(\d+)-[a-f0-9]+(?:-([a-f0-9]{24}|[0-9]{13}))?$/.exec(ownerName); + if (!owner) return false; + const pid = Number(owner[1]); + if (!Number.isSafeInteger(pid) || pid <= 0) return false; + const abandonedHere = abandonedAdmissions.get(admission) === ownerName; + const markerPath = marker && path.join(admission, marker); + if (markerPath && !fs.lstatSync(markerPath).isFile()) return false; + if (!markerPath && !abandonedHere && !ownerIsGone(pid)) { + const birth = Number(owner[2]); + const remaining = deadline - Date.now(); + if (owner[2]?.length !== 13 || !Number.isSafeInteger(birth) || Date.now() - info.mtimeMs < ADMISSION_IDENTITY_CHECK_MS || remaining <= 0) return false; + if (!starts.has(pid)) starts.set(pid, processStartTime(pid, Math.min(remaining, PROCESS_START_TIMEOUT_MS))); + const actualBirth = starts.get(pid); + // Only a clearly newer process proves reuse. ps timestamps round to whole seconds. + if (!actualBirth || actualBirth <= birth + PROCESS_START_TOLERANCE_MS) return false; + } + if (entries.includes(ownerName)) fs.unlinkSync(path.join(admission, ownerName)); + if (markerPath) fs.unlinkSync(markerPath); + if (abandonedHere) abandonedAdmissions.delete(admission); + } + // Only the inspected unique owner is unlinked. A fresh guard is already nonempty, so rmdir cannot + // remove it even if another reclaimer replaced the directory after our read. + fs.rmdirSync(admission); + return true; + } catch { + return false; + } +} + +function enterAdmission(lockDir, deadline, starts) { + const admission = `${lockDir}.admission`; + const owner = `owner-${process.pid}-${crypto.randomBytes(12).toString('hex')}-${processStartTime(process.pid)}`; + const staging = `${admission}.${owner}.claim`; + fs.mkdirSync(path.dirname(admission), { recursive: true, mode: 0o700 }); + fs.mkdirSync(staging, { mode: 0o700 }); + let published = false; + try { + fs.writeFileSync(path.join(staging, owner), '', { mode: 0o600, flag: 'wx' }); + // Publishing a prepopulated directory never exposes an empty fresh guard to a stale reclaimer. + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + fs.renameSync(staging, admission); + published = true; + abandonedAdmissions.delete(admission); + return { admission, owner }; + } catch { + if (attempt > 0 || !reclaimAdmission(admission, deadline, starts)) return null; + } + } + return null; + } finally { + if (!published) fs.rmSync(staging, { recursive: true, force: true }); + } +} + +export function retryLockAccess(access) { + for (let attempt = 0; ; attempt += 1) { + try { + return access(); + } catch (error) { + if (attempt === 2 || !LOCK_ACCESS_ERRORS.has(error?.code)) throw error; + pause(LOCK_RETRY_MS); + } + } +} + +function releaseAdmission({ admission, owner }) { + try { + retryLockAccess(() => fs.unlinkSync(path.join(admission, owner))); + } catch (cause) { + if (cause?.code !== 'ENOENT') { + throw Object.assign(new Error(`Could not release admission ${admission}`, { cause }), { code: 'ELOCKED' }); + } + } + // The guard is released when its unique owner is gone. An empty directory can be reclaimed. + try { retryLockAccess(() => fs.rmdirSync(admission)); } catch {} +} + +function abandonAdmission({ admission, owner }) { + // A separate marker can be written even when Windows denies access to the owner file. + try { + retryLockAccess(() => fs.writeFileSync(path.join(admission, ABANDONED_PREFIX + owner), '', { mode: 0o600, flag: 'wx' })); + } catch { + abandonedAdmissions.set(admission, owner); + } +} + +// Every directory mutation uses this short, nonblocking admission. No work or await belongs inside it. +export function admitted(lockDir, mutate, rollback, deadline = Date.now() + LOCK_WAIT_MS, starts = new Map()) { + const admission = enterAdmission(lockDir, deadline, starts); + if (!admission) return null; + let result; + try { + result = mutate(); + } catch (error) { + try { releaseAdmission(admission); } catch { abandonAdmission(admission); } + throw error; + } + try { + releaseAdmission(admission); + } catch (error) { + // Publish only after rollback, so another process cannot enter while it runs. + try { rollback?.(); } catch (rollbackError) { error.rollbackError = rollbackError; } + abandonAdmission(admission); + if (rollback) throw error; + } + return result; +} diff --git a/lib/runtime/store/file-lock.js b/lib/runtime/store/file-lock.js index f730098..40829fd 100644 --- a/lib/runtime/store/file-lock.js +++ b/lib/runtime/store/file-lock.js @@ -5,12 +5,15 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import { ownerIsGone } from '../process/owner.js'; +import { ownerIsGone, processStartTime, PROCESS_START_TIMEOUT_MS, PROCESS_START_TOLERANCE_MS } from '../process/owner.js'; + +import { admitted, LOCK_WAIT_MS, LOCK_RETRY_MS, pause, retryLockAccess } from './admission.js'; -const LOCK_WAIT_MS = 2_000; -const LOCK_RETRY_MS = 20; const WORK_LOCK_STALE_MS = 2 * 60 * 1000; +const WORK_OWNER_PROBE_MS = process.platform === 'win32' ? 5_000 : PROCESS_START_TIMEOUT_MS; +const WORK_OWNER_WITH_START = /^\d+:[a-f0-9]+:([1-9]\d{12})$/; const CLAIM_STALE_MS = 60 * 1000; +const abandonedWork = new Map(); export function readState(target, fallback) { try { @@ -41,11 +44,6 @@ export function removeState(target) { } } -// Atomics.wait is the only sleep that holds a synchronous read-modify-write together. -function pause(ms) { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); -} - function contentOf(file) { try { return fs.readFileSync(file, 'utf8'); @@ -180,38 +178,107 @@ function isStale(lockDir, now) { } } +function workOwnerOf(lockDir) { + try { + return retryLockAccess(() => fs.readFileSync(path.join(lockDir, 'owner'), 'utf8')); + } catch (cause) { + if (cause?.code === 'ENOENT') return null; + throw Object.assign(new Error(`Could not read lock owner ${lockDir}`, { cause }), { code: 'ELOCKED' }); + } +} + function ownsLock(lockDir, token) { - return contentOf(path.join(lockDir, 'owner')) === token; + return workOwnerOf(lockDir) === token; } -function tryLock(lockDir, now) { - fs.mkdirSync(path.dirname(lockDir), { recursive: true, mode: 0o700 }); - if (isStale(lockDir, now)) fs.rmSync(lockDir, { recursive: true, force: true }); - try { - fs.mkdirSync(lockDir); - } catch { - return null; - } - const token = crypto.randomBytes(12).toString('hex'); - fs.writeFileSync(path.join(lockDir, 'owner'), token); - return { - refresh: () => { - if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); - }, - release: () => { +async function releaseWorkLock(lockDir, token) { + const deadline = Date.now() + LOCK_WAIT_MS; + const starts = new Map(); + for (;;) { + if (admitted(lockDir, () => { if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); - }, - }; + return true; + }, null, deadline, starts)) return; + if (Date.now() >= deadline) throw Object.assign(new Error(`Could not release lock ${lockDir}`), { code: 'ELOCKED' }); + await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS)); + } } -/** Runs slow `work` while holding a lock directory no other process gets meanwhile; false when it is held. */ +function tryLock(lockDir, now) { + let token, suspect, reusedToken; + const take = () => admitted(lockDir, () => { + const stale = isStale(lockDir, now); + if (abandonedWork.has(lockDir) || stale) { + let held; + try { held = workOwnerOf(lockDir); } catch { return null; } + if (held !== abandonedWork.get(lockDir)) { + if (!stale) return null; + if (held !== reusedToken && /^\d+:/.test(held ?? '')) { + const pid = ownerPidOf(held); + if (!ownerIsGone(pid)) { + const birth = WORK_OWNER_WITH_START.exec(held)?.[1]; + if (birth) suspect = { token: held, pid, birth: Number(birth) }; + return null; + } + } + } + fs.rmSync(lockDir, { recursive: true, force: true }); + abandonedWork.delete(lockDir); + } + try { + fs.mkdirSync(lockDir); + } catch { + return null; + } + token = `${process.pid}:${crypto.randomBytes(12).toString('hex')}:${processStartTime(process.pid)}`; + fs.writeFileSync(path.join(lockDir, 'owner'), token); + return { + refresh: () => { + try { + return admitted(lockDir, () => { + if (ownsLock(lockDir, token)) fs.utimesSync(lockDir, new Date(), new Date()); + }); + } catch { + // Timer-driven refresh is best effort; a live pid stays protected and final release still surfaces errors. + return null; + } + }, + release: async () => { + try { await releaseWorkLock(lockDir, token); } + catch (error) { abandonedWork.set(lockDir, token); throw error; } + }, + }; + }, () => { + if (!token) return; + abandonedWork.set(lockDir, token); + if (ownsLock(lockDir, token)) fs.rmSync(lockDir, { recursive: true, force: true }); + abandonedWork.delete(lockDir); + }); + const lock = take(); + if (lock || !suspect) return lock; + // A foreign lookup can be slow. Release admission, then recheck staleness and the exact token. + const actualBirth = processStartTime(suspect.pid, WORK_OWNER_PROBE_MS); + if (!actualBirth || actualBirth <= suspect.birth + PROCESS_START_TOLERANCE_MS) return null; + reusedToken = suspect.token; + return take(); +} + +/** Runs slow `work` under one owner's lock; dead or legacy directories recover after their lease expires. */ export async function withLock(lockDir, work, now = Date.now()) { const lock = tryLock(lockDir, now); if (!lock) return false; + let workFailed = false; try { await work(lock); return true; + } catch (error) { + workFailed = true; + throw error; } finally { - lock.release(); + try { + await lock.release(); + } catch (error) { + if (!workFailed) throw error; + } } } diff --git a/lib/runtime/store/file-stores.js b/lib/runtime/store/file-stores.js index 1266f9a..12493cd 100644 --- a/lib/runtime/store/file-stores.js +++ b/lib/runtime/store/file-stores.js @@ -4,6 +4,7 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; import { OUTBOX_TTL_MS, withQueued } from '../../report-outbox.js'; import { ABSENT } from '../../retention.js'; @@ -19,6 +20,10 @@ const MAX_DRAIN_ROUNDS = 5; const SESSION_TTL_MS = 30 * 24 * 60 * 60 * 1000; const digestOf = (key) => crypto.createHash('sha256').update(String(key)).digest('hex').slice(0, 16); +export function candidateReferenceOf(sessionId, { at, turn }) { + return `v1-${crypto.createHash('sha256').update(JSON.stringify([sessionId, at, turn])).digest('base64url')}`; +} + export function contentHashOf(file) { try { return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); @@ -59,6 +64,28 @@ export function fileStoresOf({ dir, agent }) { }; const stamped = (sessionId, next) => (next === undefined ? undefined : { ...next, sessionId, touchedAt: Date.now() }); + // Drafts outlive session files. Record correction independently of both session pruning and draft cleanup. + const corrections = path.join(dir, `${agent}-distill-corrections`); + const correctionFile = (sessionId, turn, sessionEpoch) => path.join(corrections, digestOf(JSON.stringify([sessionId, turn, sessionEpoch]))); + const draftCorrections = { + withSession: async (sessionId, work, signal) => { + const lock = path.join(corrections, `${digestOf(sessionId)}.lock`); + const deadline = Date.now() + 2_000; + let result; + for (;;) { + signal?.throwIfAborted(); + if (await withLock(lock, async () => { result = await work(); })) return result; + if (Date.now() >= deadline) throw new Error('Timed out waiting for the draft correction lock. Please retry.'); + await delay(20, undefined, { signal }); + } + }, + mark: async (sessionId, turn, sessionEpoch) => { + fs.mkdirSync(corrections, { recursive: true, mode: 0o700 }); + fs.closeSync(fs.openSync(correctionFile(sessionId, turn, sessionEpoch), 'a', 0o600)); + }, + has: async (sessionId, turn, sessionEpoch) => Boolean(fs.statSync(correctionFile(sessionId, turn, sessionEpoch), { throwIfNoEntry: false })), + }; + const sessions = { read: async (sessionId) => readState(sessionFile('session', sessionId), {}), update: async (sessionId, mutate) => updateState(sessionFile('session', sessionId), {}, (stored) => stamped(sessionId, mutate(stored))), @@ -157,8 +184,9 @@ export function fileStoresOf({ dir, agent }) { sessions, notes, baseline, - candidates: tableStore(file('distill-pending'), {}), + candidates: { ...tableStore(file('distill-pending'), {}), referenceOf: candidateReferenceOf }, drafts: tableStore(file('distill-drafts'), []), + draftCorrections, reports, outbox, retention, diff --git a/lib/session/state.js b/lib/session/state.js index 3ec618f..78f06d8 100644 --- a/lib/session/state.js +++ b/lib/session/state.js @@ -28,10 +28,12 @@ export const sessionOf = (stored) => ({ ...EMPTY_SESSION, ...stored }); * that numbers its own turns (`hostTurn`) keeps the count at least there, so a session resumed after its state was * gone never repeats a task id. The session's `task` is its first typed prompt, kept for the Hub to judge each later * recall's fit against; a resumed session keeps the task it had. + * A fresh session file gets an epoch so retained drafts cannot share a correction key with reused turn numbers. */ -export function openedTurn(session, { turnId = null, turnStart = null, hostTurn = null, prompt = null, task = '', pid = null, isCommand = false }) { +export function openedTurn(session, { turnId = null, turnStart = null, hostTurn = null, prompt = null, task = '', pid = null, isCommand = false, now }) { return { ...session, + sessionEpoch: session.turn === 0 ? now : session.sessionEpoch, turn: Math.max(session.turn + 1, Number.isInteger(hostTurn) ? hostTurn : 0), turnId, turnIds: turnId ? [...session.turnIds.filter((known) => known !== turnId), turnId].slice(-MAX_TURN_IDS) : session.turnIds, @@ -162,5 +164,5 @@ export const isEarlierTurn = (session, turnId) => Boolean(turnId) && turnId !== /** What an ended session keeps: its turn counter, owner and ledger, so a resume neither repeats a task id nor re-injects. */ export function endedSession(session) { - return { ...EMPTY_SESSION, turn: session.turn, pid: session.pid, injected: session.injected, turnIds: session.turnIds, shown: session.shown, task: session.task }; + return { ...EMPTY_SESSION, turn: session.turn, sessionEpoch: session.sessionEpoch, pid: session.pid, injected: session.injected, turnIds: session.turnIds, shown: session.shown, task: session.task }; } diff --git a/test/agent/driver.test.js b/test/agent/driver.test.js index 7602888..dd2cf70 100644 --- a/test/agent/driver.test.js +++ b/test/agent/driver.test.js @@ -100,7 +100,7 @@ test('the MCP profile turns tool calls into a turn whose report is agent-reporte const recalled = await driver.handle('evolver_recall', { session_id: 'm1', turn_id: 't1', task: PROMPT }); assert.match(recalled.content[0].text, /Apply them where they fit\.\n\nturn_id: t1 /); const reported = await driver.handle('evolver_report', { session_id: 'm1', turn_id: 't1', outcome: 'success', summary: 'Fixed the retries.', checks: [{ command: 'npm test', exit: 0 }], files: ['lib/redis.js'] }); - assert.match(reported.content[0].text, /call evolver_distill with written_for \d+/); + assert.match(reported.content[0].text, /call evolver_distill with written_for "v1-[A-Za-z0-9_-]{43}"/); assert.equal(queued[0].report.evidence.reported_by, 'agent'); assert.equal(queued[0].report.evidence.tier, 'none'); const status = await driver.handle('evolver_status', { session_id: 'm1' }); diff --git a/test/app/distill-session.test.js b/test/app/distill-session.test.js index 0f4110d..c4ede62 100644 --- a/test/app/distill-session.test.js +++ b/test/app/distill-session.test.js @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import { runBackgroundTasks } from '../../lib/app/background.js'; +import { distillText } from '../../lib/app/commands/distill.js'; import { runCommand } from '../../lib/app/commands/index.js'; import { MIN_WAKE_DELAY_MS, armDistillWakeUp, distillDue, distillSession, postAgentGene } from '../../lib/app/distill-session.js'; import { adoptStrategy, promptSubmitted } from '../../lib/app/prompt.js'; @@ -30,14 +31,17 @@ const modelAnswering = (answer, onCall = () => {}) => ({ async function withCandidate(options = {}) { const made = memoryPorts({ distillAnswer: DRAFTED, project: PROJECT, ...options }); - const { ports } = made; - await promptSubmitted(ports, { sessionId: 's1', prompt: 'fix the flaky redis retries', waitMs: 1_000 }); - await toolFinished(ports, { sessionId: 's1', note: { kind: 'edit', paths: ['/proj/lib/redis.js'] } }); - await toolFinished(ports, { sessionId: 's1', note: { kind: 'shell', command: 'npm test', exit: 0, output: 'ok' } }); - await turnEnded(ports, { sessionId: 's1', reason: 'answer', answer: 'Fixed the retries.', model: 'model-x', projectDir: '/proj' }); + await verifiedTurn(made.ports, 's1'); return made; } +async function verifiedTurn(ports, sessionId, prompt = 'fix the flaky redis retries') { + await promptSubmitted(ports, { sessionId, prompt, waitMs: 1_000 }); + await toolFinished(ports, { sessionId, note: { kind: 'edit', paths: ['/proj/lib/redis.js'] } }); + await toolFinished(ports, { sessionId, note: { kind: 'shell', command: 'npm test', exit: 0, output: 'ok' } }); + await turnEnded(ports, { sessionId, reason: 'answer', answer: 'Fixed the retries.', model: 'model-x', projectDir: '/proj' }); +} + test('a session-end distillation the Proxy answers as a draft is kept as a draft and settles the candidate', async () => { const { ports, requests } = await withCandidate({ model: modelAnswering(GENE) }); await sessionEnded(ports, { sessionId: 's1', projectDir: '/proj' }); @@ -158,6 +162,362 @@ test('a candidate dropped by a correction while the model answers is not posted' assert.equal(requests.some((request) => request.path === '/conversation/distill'), false); }); +test('a correction while a draft request is pending discards its late reply', async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const started = Promise.withResolvers(); + const finish = Promise.withResolvers(); + const request = ports.proxy.request; + ports.proxy.request = async (...args) => { + const reply = await request(...args); + if (args[1] === '/conversation/distill') { + started.resolve(); + await finish.promise; + } + return reply; + }; + const running = distillSession(ports, { sessionId: 's1' }); + await started.promise; + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + finish.resolve(); + const outcome = await running; + assert.equal(outcome.status, 'dropped'); + assert.equal(distillText(outcome).text, 'A newer turn replaced the one this Gene was written for.'); + assert.deepEqual(await ports.drafts.read(), []); + assert.equal((await ports.candidates.read()).s1, undefined); +}); + +test('a correction while a request is pending still reports a Gene the Proxy stored', async () => { + const stored = { status: 'stored', gene: DRAFTED.gene }; + const { ports, requests } = await withCandidate({ model: modelAnswering(GENE), distillAnswer: stored }); + const started = Promise.withResolvers(); + const finish = Promise.withResolvers(); + const request = ports.proxy.request; + ports.proxy.request = async (...args) => { + const reply = await request(...args); + if (args[1] === '/conversation/distill') { + started.resolve(); + await finish.promise; + } + return reply; + }; + + const running = runCommand(ports, { sessionId: 's1', name: 'evolver-distill' }); + await started.promise; + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + finish.resolve(); + const answer = await running; + assert.equal(answer.ok, true); + assert.deepEqual(JSON.parse(answer.text), stored); + assert.equal(requests.filter(({ path }) => path === '/conversation/distill').length, 1); + assert.deepEqual(await ports.drafts.read(), []); + assert.equal((await ports.candidates.read()).s1, undefined); +}); + +test('a correction while an agent Gene request is pending discards its late draft', async () => { + const { ports } = await withCandidate(); + const writtenFor = ports.candidates.referenceOf('s1', (await ports.candidates.read()).s1); + const started = Promise.withResolvers(); + const finish = Promise.withResolvers(); + const request = ports.proxy.request; + ports.proxy.request = async (...args) => { + const reply = await request(...args); + if (args[1] === '/conversation/distill') { + started.resolve(); + await finish.promise; + } + return reply; + }; + const running = postAgentGene(ports, { sessionId: 's1', text: JSON.stringify(GENE), writtenFor }); + await started.promise; + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + finish.resolve(); + assert.equal((await running).status, 'dropped'); + assert.deepEqual(await ports.drafts.read(), []); + assert.equal((await ports.candidates.read()).s1, undefined); +}); + +test('a corrected request reaching the draft store late preserves a newer draft with the same ID', async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const earlier = (await ports.candidates.read()).s1; + const started = Promise.withResolvers(); + const finish = Promise.withResolvers(); + const update = ports.drafts.update; + let pauseFirst = true; + ports.drafts.update = async (mutate) => { + if (pauseFirst) { + pauseFirst = false; + started.resolve(); + await finish.promise; + } + return update(mutate); + }; + + const late = distillSession(ports, { sessionId: 's1' }); + await started.promise; + let newer; + try { + await verifiedTurn(ports, 's1', 'still failing'); + ports.model = modelAnswering({ ...GENE, title: 'Newer verified retries' }); + newer = await distillSession(ports, { sessionId: 's1' }); + assert.equal(newer.status, 'drafted'); + assert.equal(newer.draft.id, DRAFTED.distill_id); + assert.equal(newer.draft.sourceCandidate.at, earlier.at); + assert.equal(newer.draft.sourceCandidate.turn, earlier.turn + 1); + assert.deepEqual(await ports.drafts.read(), [newer.draft]); + } finally { + finish.resolve(); + } + + assert.equal((await late).status, 'dropped'); + assert.deepEqual(await ports.drafts.read(), [newer.draft]); +}); + +test('an older candidate from another session settles without replacing a newer same-ID draft', async () => { + const { ports, advance } = await withCandidate({ model: modelAnswering(GENE) }); + advance(1); + await verifiedTurn(ports, 's2'); + const newer = await distillSession(ports, { sessionId: 's2' }); + assert.equal(newer.status, 'drafted'); + + const outcome = await distillSession(ports, { sessionId: 's1' }); + assert.equal(outcome.status, 'superseded'); + assert.match(distillText(outcome).text, /same ID is already saved/); + assert.equal((await ports.candidates.read()).s1, undefined); + assert.deepEqual(await ports.drafts.read(), [newer.draft]); +}); + +test('a correction after the final candidate snapshot removes only the corrected turn draft', async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const candidate = (await ports.candidates.read()).s1; + const kept = [ + { id: 'earlier', sessionId: 's1', sourceCandidate: { at: candidate.at - 1, turn: candidate.turn - 1 } }, + { id: 'other-session', sessionId: 's2', sourceCandidate: { at: candidate.at, turn: candidate.turn } }, + { id: 'legacy', sessionId: 's1' }, + ]; + await ports.drafts.update(() => kept); + const captured = Promise.withResolvers(); + const resume = Promise.withResolvers(); + const read = ports.candidates.read; + let paused = false; + ports.candidates.read = async () => { + const snapshot = await read(); + if (!paused && (await ports.drafts.read()).some(({ id }) => id === DRAFTED.distill_id)) { + paused = true; + captured.resolve(); + await resume.promise; + } + return snapshot; + }; + + const running = distillSession(ports, { sessionId: 's1' }); + await captured.promise; + try { + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + } finally { + resume.resolve(); + } + const outcome = await running; + assert.equal(outcome.status, 'draft_missing'); + assert.equal(distillText(outcome).ok, false); + assert.match(distillText(outcome).text, /no longer available/); + assert.equal((await ports.candidates.read()).s1, undefined); + assert.deepEqual(await ports.drafts.read(), kept); +}); + +test('an agent Gene does not report a draft replaced with the same ID during settlement', async () => { + const { ports } = await withCandidate(); + const writtenFor = ports.candidates.referenceOf('s1', (await ports.candidates.read()).s1); + const update = ports.candidates.update; + let replacement; + ports.candidates.update = async (mutate) => { + const next = await update(mutate); + if (!replacement && next && !next.s1) { + const [draft] = await ports.drafts.read(); + replacement = { ...draft, title: 'Draft from another session', sessionId: 's2' }; + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + await ports.drafts.update(() => [replacement]); + } + return next; + }; + + const outcome = await postAgentGene(ports, { sessionId: 's1', text: JSON.stringify(GENE), writtenFor }); + assert.equal(outcome.status, 'draft_missing'); + assert.equal(distillText(outcome).ok, false); + assert.equal(replacement.id, DRAFTED.distill_id); + assert.deepEqual(await ports.drafts.read(), [replacement]); +}); + +test('a correction removes the saved draft even after its candidate has settled', async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const saved = await distillSession(ports, { sessionId: 's1' }); + assert.equal(saved.status, 'drafted'); + assert.equal((await ports.candidates.read()).s1, undefined); + + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + assert.deepEqual(await ports.drafts.read(), []); +}); + +test('background distillation checks an idle draft again before returning the batch', async () => { + const { ports, advance, requests } = await withCandidate(); + advance(1); + await verifiedTurn(ports, 's2'); + await sessionEnded(ports, { sessionId: 's2', projectDir: '/proj' }); + advance(IDLE_DISTILL_MS - 1); + let completions = 0; + ports.model = modelAnswering(GENE, async () => { + if (++completions === 2) await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + }); + + const outcomes = await distillDue(ports); + assert.deepEqual(requests.filter(({ path }) => path === '/conversation/distill').map(({ body }) => body.trigger), ['idle', 'session_end']); + assert.deepEqual(outcomes.map(({ status }) => status), ['draft_missing', 'drafted']); + assert.deepEqual(await ports.drafts.read(), [outcomes[1].draft]); +}); + +for (const [count, replace] of [[1, false], [25, false], [25, true]]) test(`a correction crossing a draft write keeps ${count} earlier drafts${replace ? ' and a concurrent same-id replacement' : ''}`, async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const kept = Array.from({ length: count }, (_, index) => ({ id: `earlier-${index}`, title: 'Keep this method', createdAt: ports.clock.now() })); + await ports.drafts.update(() => kept); + const started = Promise.withResolvers(); + const finish = Promise.withResolvers(); + const update = ports.drafts.update; + let firstWrite = true; + ports.drafts.update = async (mutate) => { + const written = await update(mutate); + if (firstWrite) { + firstWrite = false; + started.resolve(); + await finish.promise; + } + return written; + }; + const running = distillSession(ports, { sessionId: 's1' }); + await started.promise; + let replacement; + if (replace) await update((drafts) => drafts.map((draft) => { + if (draft.id !== DRAFTED.distill_id) return draft; + replacement = { ...draft, title: 'Concurrent method', sourceCandidate: { ...draft.sourceCandidate, turn: draft.sourceCandidate.turn + 1 } }; + return replacement; + })); + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + finish.resolve(); + assert.equal((await running).status, 'dropped'); + assert.deepEqual(await ports.drafts.read(), replace ? [...kept, replacement] : kept); +}); + +for (const other of ['legacy draft', 'same candidate', 'newer turn', 'other session', 'discarded draft']) { + test(`draft compensation never restores a displaced draft and preserves concurrent replacements (${other})`, async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const candidate = (await ports.candidates.read()).s1; + const previous = { id: DRAFTED.distill_id, title: 'Earlier method', createdAt: ports.clock.now(), sessionId: 's1' }; + if (other === 'same candidate') previous.sourceCandidate = { at: candidate.at, turn: candidate.turn, sessionEpoch: candidate.sessionEpoch }; + const replacement = { ...previous, title: 'Concurrent method', sourceCandidate: { at: candidate.at, turn: candidate.turn + 1 } }; + if (other === 'other session') Object.assign(replacement, { sessionId: 's2', sourceCandidate: { at: candidate.at, turn: candidate.turn } }); + const before = { ...previous, id: 'before', sourceCandidate: { at: candidate.at - 1, turn: candidate.turn - 1 } }; + const after = { ...previous, id: 'after', sessionId: 's2' }; + await ports.drafts.update(() => [before, previous, after]); + const update = ports.drafts.update; + let firstWrite = true; + ports.drafts.update = async (mutate) => { + const written = await update(mutate); + if (firstWrite) { + firstWrite = false; + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + if (other === 'newer turn' || other === 'other session') await update(() => [before, replacement, after]); + if (other === 'discarded draft') await update(() => [before, after]); + } + return written; + }; + assert.equal((await distillSession(ports, { sessionId: 's1' })).status, 'dropped'); + const expected = other === 'newer turn' || other === 'other session' ? [replacement] : []; + assert.deepEqual(await ports.drafts.read(), [before, ...expected, after]); + }); +} + +for (const entry of ['headless model', 'agent Gene']) { + test(`a failed correction cleanup is reported for a draft from the ${entry}`, async (t) => { + const { ports, requests } = await withCandidate({ model: modelAnswering(GENE) }); + const writtenFor = ports.candidates.referenceOf('s1', (await ports.candidates.read()).s1); + const update = ports.drafts.update; + let writes = 0; + let signal; + if (entry === 'headless model') { + t.mock.timers.enable({ apis: ['setInterval'] }); + const request = ports.proxy.request; + ports.proxy.request = (...args) => { + if (args[1] === '/conversation/distill') signal = args[3].signal; + return request(...args); + }; + } + ports.drafts.update = async (mutate) => { + if (++writes > 1) throw Object.assign(new Error('fixture cleanup denied'), { code: 'EACCES' }); + const written = await update(mutate); + await assert.rejects(promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }), { code: 'EACCES' }); + if (signal) t.mock.timers.tick(1_000); + return written; + }; + + const outcome = entry === 'headless model' + ? await distillSession(ports, { sessionId: 's1' }) + : await postAgentGene(ports, { sessionId: 's1', text: JSON.stringify(GENE), writtenFor }); + if (entry === 'headless model') assert.equal(signal.aborted, true); + assert.equal(writes, 3); + assert.equal(requests.filter(({ path }) => path === '/conversation/distill').length, 1); + assert.equal((await ports.candidates.read()).s1, undefined); + assert.deepEqual((await ports.drafts.read()).map(({ id }) => id), [DRAFTED.distill_id]); + assert.equal(outcome.status, 'cleanup_failed'); + assert.equal(distillText(outcome).ok, false); + assert.match(distillText(outcome).text, /could not remove it/i); + + ports.drafts.update = update; + await promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + assert.deepEqual(await ports.drafts.read(), []); + }); +} + +test('a superseded draft stays blocked when distillation cannot undo its write', async () => { + const { ports, requests } = await withCandidate({ + model: modelAnswering(GENE), + distillAnswer: { ...DRAFTED, publishable: true, publish_receipt: { receiptId: 'proxy-issued-receipt' } }, + }); + const updateDrafts = ports.drafts.update; + ports.drafts.update = async (mutate) => { + const written = await updateDrafts(mutate); + await ports.candidates.update((pending) => ({ ...pending, s1: { ...pending.s1, turn: pending.s1.turn + 1 } })); + ports.drafts.update = async () => { throw new Error('draft cleanup denied'); }; + return written; + }; + assert.equal((await distillSession(ports, { sessionId: 's1' })).status, 'cleanup_failed'); + ports.drafts.update = updateDrafts; + assert.equal((await ports.drafts.read()).length, 1); + const result = await runCommand(ports, { sessionId: 's1', name: 'evolver-drafts', args: 'publish 1' }); + assert.equal(result.ok, false); + assert.match(result.text, /not confirmed by distillation/i); + assert.equal(requests.some(({ path }) => path === '/asset/reverify' || path === '/asset/submit'), false); +}); + +test('distillation does not advertise a corrected draft that cleanup could not remove', async () => { + const { ports } = await withCandidate({ model: modelAnswering(GENE) }); + const updateDrafts = ports.drafts.update; + let writes = 0; + ports.drafts.update = async (mutate) => { + if (++writes === 2) { + ports.drafts.update = async () => { throw new Error('draft cleanup denied'); }; + try { + await assert.rejects(promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }), /draft cleanup denied/); + } finally { + ports.drafts.update = updateDrafts; + } + } + return updateDrafts(mutate); + }; + + const outcome = await distillSession(ports, { sessionId: 's1' }); + assert.equal((await ports.drafts.read()).length, 1); + assert.equal(outcome.status, 'draft_missing'); + assert.equal(distillText(outcome).ok, false); +}); + test('a failing headless model keeps the candidate, tries again only after a doubling wait, and three failed tries give it up', async () => { const { ports, advance } = await withCandidate({ model: { complete: async () => { throw new Error('Authentication required'); } } }); await sessionEnded(ports, { sessionId: 's1', projectDir: '/proj' }); @@ -236,6 +596,38 @@ test('a Gene written for a candidate since replaced is not sent', async () => { assert.equal(requests.some((request) => request.path === '/conversation/distill'), false); }); +test('a Gene from an older turn cannot post against a new candidate with the same timestamp', async () => { + const { ports, requests } = await withCandidate(); + const earlier = (await ports.candidates.read()).s1; + const oldRequest = await distillSession(ports, { sessionId: 's1' }); + await verifiedTurn(ports, 's1', 'still failing'); + const newer = (await ports.candidates.read()).s1; + assert.equal(newer.at, earlier.at); + assert.equal(newer.turn, earlier.turn + 1); + + const gene = { sessionId: 's1', text: JSON.stringify(GENE) }; + assert.equal((await postAgentGene(ports, { ...gene, writtenFor: oldRequest.agentGene.writtenFor })).status, 'dropped'); + assert.equal((await postAgentGene(ports, { ...gene, writtenFor: earlier.at })).status, 'dropped'); + assert.equal(requests.some(({ path }) => path === '/conversation/distill'), false); + const fresh = await distillSession(ports, { sessionId: 's1' }); + assert.notEqual(fresh.agentGene.writtenFor, oldRequest.agentGene.writtenFor); + assert.equal((await postAgentGene(ports, { ...gene, writtenFor: fresh.agentGene.writtenFor })).status, 'drafted'); +}); + +test('a terminal Gene selects its session even when another candidate has the same timestamp and turn', async () => { + const { ports, requests } = await withCandidate(); + await verifiedTurn(ports, 's2'); + const { s1, s2 } = await ports.candidates.read(); + assert.deepEqual([s1.at, s1.turn], [s2.at, s2.turn]); + const { agentGene } = await distillSession(ports, { sessionId: 's2' }); + const gene = { text: JSON.stringify(GENE), writtenFor: agentGene.writtenFor }; + + assert.equal((await postAgentGene(ports, { ...gene, sessionId: 's1' })).status, 'dropped'); + assert.equal((await postAgentGene(ports, { ...gene, sessionId: 'terminal', anySession: true })).status, 'drafted'); + assert.deepEqual(requests.filter(({ path }) => path === '/conversation/distill').map(({ body }) => body.session_id), ['testagent:s2']); + assert.equal((await ports.candidates.read()).s1.turn, s1.turn); +}); + test('every command answers from the plugin; evolve is the one prompt for the agent', async () => { const { ports } = memoryPorts(); assert.equal((await runCommand(ports, { sessionId: 's1', name: 'evolver-sync', args: '--dry-run "two words"' })).text, 'ran sync --dry-run two words'); diff --git a/test/app/draft-publication.test.js b/test/app/draft-publication.test.js index f87402d..6ccc9ed 100644 --- a/test/app/draft-publication.test.js +++ b/test/app/draft-publication.test.js @@ -1,10 +1,20 @@ import assert from 'node:assert/strict'; -import { randomBytes } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { createHash, randomBytes } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { test } from 'node:test'; import { hub } from '@evomap/evolver-core'; import { runCommand } from '../../lib/app/commands/index.js'; +import { promptSubmitted } from '../../lib/app/prompt.js'; +import { sessionEnded } from '../../lib/app/session-end.js'; import { draftOf } from '../../lib/distill.js'; +import { proxyFetchOf } from '../../lib/runtime/proxy.js'; +import { fileStoresOf } from '../../lib/runtime/store/file-stores.js'; +import { fakeProxy } from '../helpers/fake-proxy.js'; +import { lockContenders, until } from '../helpers/lock-contenders.js'; import { memoryPorts } from './memory-ports.js'; const GENE = { type: 'Gene', asset_id: 'sha256:gene' }; @@ -23,6 +33,210 @@ async function withDraft(result) { return { ...made, draft }; } +for (const publishable of [false, true]) { + test(`a correction blocks a lingering ${publishable ? 'verified' : 'local-only'} draft after cleanup fails`, async () => { + const { ports, requests, draft } = await withDraft({ ...RESULT, publishable, ...(publishable && { publish_receipt: RECEIPT }) }); + const corrected = { ...draft, sourceCandidate: { turn: 2, at: 20 } }; + const earlier = { ...draft, id: 'earlier', publishable: true, publish_receipt: RECEIPT, sourceCandidate: { turn: 1, at: 10 } }; + await ports.sessions.update('s1', () => ({ turn: 2 })); + await ports.drafts.update(() => [corrected, earlier]); + const updateDrafts = ports.drafts.update; + ports.drafts.update = async () => { throw new Error('draft cleanup denied'); }; + await assert.rejects(promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }), /draft cleanup denied/); + ports.drafts.update = updateDrafts; + await sessionEnded(ports, { sessionId: 's1', projectDir: '/proj' }); + + const publish = (args) => runCommand(ports, { sessionId: 'another-session', name: 'evolver-drafts', args }); + const result = await publish('publish 1'); + assert.equal(result.ok, false); + assert.match(result.text, /invalidated by a correction/i); + assert.deepEqual(requests, []); + assert.deepEqual(await ports.drafts.read(), [corrected, earlier]); + assert.equal((await publish('publish 2')).ok, true); + assert.deepEqual(requests.map(({ path }) => path), ['/asset/submit']); + assert.equal((await publish('discard 1')).ok, true); + assert.deepEqual(await ports.drafts.read(), []); + }); +} + +test('a correction during reverification blocks submit even if it cannot remove the draft', async () => { + const { ports, requests, draft } = await withDraft({ ...RESULT, publishable: false }); + const source = { ...draft, sourceCandidate: { turn: 1, at: 10 } }; + await ports.sessions.update('s1', () => ({ turn: 1 })); + await ports.drafts.update(() => [source]); + const updateDrafts = ports.drafts.update; + ports.proxy.request = async (method, path, body) => { + requests.push({ method, path, body }); + if (path !== '/asset/reverify') return { ok: true }; + ports.drafts.update = async () => { throw new Error('draft cleanup denied'); }; + try { + await assert.rejects(promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }), /draft cleanup denied/); + } finally { + ports.drafts.update = updateDrafts; + } + return { ok: true, data: { ok: true, publishable: true, assets: [GENE, CAPSULE], publish_receipt: RECEIPT } }; + }; + + const result = await runCommand(ports, { sessionId: 's1', name: 'evolver-drafts', args: 'publish 1' }); + assert.equal(result.ok, false); + assert.match(result.text, /invalidated by a correction/i); + assert.deepEqual(requests.map(({ path }) => path), ['/asset/reverify']); + assert.deepEqual(await ports.drafts.read(), [source]); +}); + +test('a correction after the publication check cannot slip into submit dispatch', async (t) => { + const { ports, draft } = await withDraft({ ...RESULT, publishable: true, publish_receipt: RECEIPT }); + const proxy = await fakeProxy(); + t.after(proxy.close); + ports.proxy.request = proxyFetchOf(proxy.env); + ports.draftCorrections = fileStoresOf({ dir: proxy.env.HOME, agent: 'testagent' }).draftCorrections; + const correctionPorts = { ...ports, draftCorrections: fileStoresOf({ dir: proxy.env.HOME, agent: 'testagent' }).draftCorrections }; + const source = { ...draft, sourceCandidate: { turn: 1, at: 10 } }; + await ports.sessions.update('s1', () => ({ turn: 1 })); + await ports.drafts.update(() => [source]); + const hasCorrection = ports.draftCorrections.has; + const updateDrafts = ports.drafts.update; + let checked = false; + ports.draftCorrections.has = async (...args) => { + const observed = await hasCorrection(...args); + if (!checked) { + checked = true; + ports.drafts.update = async () => { throw new Error('draft cleanup denied'); }; + try { + await assert.rejects(promptSubmitted(correctionPorts, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }), /draft cleanup denied/); + } finally { + ports.drafts.update = updateDrafts; + } + } + return observed; + }; + + const result = await runCommand(ports, { sessionId: 'another-session', name: 'evolver-drafts', args: 'publish 1' }); + assert.equal(checked, true); + assert.deepEqual(proxy.requests, []); + assert.equal(result.ok, false); + assert.match(result.text, /invalidated by a correction/i); + assert.deepEqual(await ports.drafts.read(), [source]); +}); + +test('a correction can finish while a dispatched submit waits for the Proxy reply', async (t) => { + const { ports, draft } = await withDraft({ ...RESULT, publishable: true, publish_receipt: RECEIPT }); + const proxy = await fakeProxy(); + t.after(proxy.close); + ports.proxy.request = proxyFetchOf(proxy.env); + ports.draftCorrections = fileStoresOf({ dir: proxy.env.HOME, agent: 'testagent' }).draftCorrections; + const correctionPorts = { ...ports, settings: { recall: false }, draftCorrections: fileStoresOf({ dir: proxy.env.HOME, agent: 'testagent' }).draftCorrections }; + await ports.sessions.update('s1', () => ({ turn: 1 })); + await ports.drafts.update(() => [{ ...draft, sourceCandidate: { turn: 1, at: 10 } }]); + + const dispatched = Promise.withResolvers(); + const reply = Promise.withResolvers(); + t.mock.method(globalThis, 'fetch', (url) => { + dispatched.resolve(new URL(url).pathname); + return reply.promise; + }); + const publishing = runCommand(ports, { sessionId: 'another-session', name: 'evolver-drafts', args: 'publish 1' }); + try { + assert.equal(await dispatched.promise, '/asset/submit'); + await promptSubmitted(correctionPorts, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }); + assert.equal(await correctionPorts.draftCorrections.has('s1', 1), true); + assert.deepEqual(await ports.drafts.read(), []); + } finally { + reply.resolve(new Response('{"message_id":"fixture-submit"}')); + assert.equal((await publishing).ok, true); + } +}); + +test('two dead-owner recoverers cannot dispatch a draft after correction', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-draft-recovery-')); + const state = fileStoresOf({ dir, agent: 'testagent' }); + const url = (path) => JSON.stringify(new URL(path, import.meta.url).href); + const script = ` + import assert from 'node:assert/strict'; + import fs from 'node:fs'; + import { join } from 'node:path'; + import { runCommand } from ${url('../../lib/app/commands/index.js')}; + import { promptSubmitted } from ${url('../../lib/app/prompt.js')}; + import { proxyFetchOf } from ${url('../../lib/runtime/proxy.js')}; + import { fileStoresOf } from ${url('../../lib/runtime/store/file-stores.js')}; + import { memoryPorts } from ${url('./memory-ports.js')}; + import { until } from ${url('../helpers/lock-contenders.js')}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => join(dir, name); + const { sessions, drafts, draftCorrections } = fileStoresOf({ dir, agent: 'testagent' }); + const { ports } = memoryPorts(); + Object.assign(ports, { sessions, drafts, draftCorrections, settings: { recall: false } }); + if (role === 'correction') { + const stat = fs.statSync; + fs.statSync = (file, ...args) => { + const snapshot = stat(file, ...args); + if (file === lock) { + fs.statSync = stat; + fs.writeFileSync(marker('stale-read'), ''); + const deadline = Date.now() + 10_000; + while (!fs.existsSync(marker('resume'))) { + if (Date.now() >= deadline) throw new Error('timed out resuming correction'); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + } + return snapshot; + }; + ports.drafts.update = async () => { throw new Error('draft cleanup denied'); }; + await assert.rejects(promptSubmitted(ports, { sessionId: 's1', prompt: 'still failing', waitMs: 0 }), /draft cleanup denied/); + fs.writeFileSync(marker('corrected'), ''); + } else { + const { has, withSession } = draftCorrections; + let inGate = false; + draftCorrections.has = async (...args) => { + const corrected = await has(...args); + if (inGate && !corrected) await until(() => fs.existsSync(marker('corrected'))); + return corrected; + }; + draftCorrections.withSession = (id, work, signal) => { + const result = withSession(id, () => { + inGate = true; + return work(); + }, signal); + fs.writeFileSync(marker('publish-attempted'), ''); + return result; + }; + globalThis.fetch = async (url) => { + assert.equal(new URL(url).pathname, '/asset/submit'); + fs.writeFileSync(marker('submitted'), ''); + return new Response('{}'); + }; + ports.proxy.request = proxyFetchOf({ HOME: dir }); + const result = await runCommand(ports, { sessionId: 'another-session', name: 'evolver-drafts', args: 'publish 1' }); + fs.writeFileSync(marker('result'), JSON.stringify(result)); + } + `; + const lockName = createHash('sha256').update('s1').digest('hex').slice(0, 16); + const lock = join(dir, 'testagent-distill-corrections', `${lockName}.lock`); + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['resume', 'corrected']); + const dead = spawnSync(process.execPath, ['-e', ''], { windowsHide: true }); + assert.equal(dead.status, 0); + mkdirSync(lock, { recursive: true }); + writeFileSync(join(lock, 'owner'), `${dead.pid}:deadbeef`); + utimesSync(lock, new Date(0), new Date(0)); + await state.sessions.update('s1', () => ({ turn: 1 })); + const draft = { ...draftOf({ ...RESULT, publishable: true, publish_receipt: RECEIPT }, { title: 'Verified method', sessionId: 's1' }, 100), sourceCandidate: { turn: 1, at: 10 } }; + await state.drafts.update(() => [draft]); + + run('correction'); + await until(() => existsSync(join(dir, 'stale-read'))); + run('publish'); + await until(() => existsSync(join(dir, 'publish-attempted'))); + writeFileSync(join(dir, 'resume'), ''); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); + assert.equal(await state.draftCorrections.has('s1', 1), true); + assert.equal(existsSync(join(dir, 'submitted')), false); + const result = JSON.parse(readFileSync(join(dir, 'result'), 'utf8')); + assert.equal(result.ok, false); + assert.match(result.text, /invalidated by a correction/i); + assert.deepEqual(await state.drafts.read(), [draft]); +}); + test('explicit publication preserves the Proxy receipt and its complete bound bundle', async () => { const { ports, requests } = await withDraft({ ...RESULT, publishable: true, publish_receipt: RECEIPT, event: EVENT }); const publish = ports.proxy.request; diff --git a/test/app/memory-ports.js b/test/app/memory-ports.js index eb8d290..4b7d48f 100644 --- a/test/app/memory-ports.js +++ b/test/app/memory-ports.js @@ -1,5 +1,6 @@ // In-memory ports: the use cases run against these without a filesystem, a process or a network. +import { candidateReferenceOf } from '../../lib/runtime/store/file-stores.js'; import { recipeSearchAnswer } from '../helpers/recipe-search.js'; export const ASSET = { @@ -66,6 +67,7 @@ const lines = () => { export function memoryPorts({ assets = [ASSET], recipes = [], stepAssets = [], distillAnswer = null, model = null, owner = 4242, gone = new Set(), now = 1_000_000, project = null } = {}) { const sessions = new Map(); + const draftCorrections = new Set(); const notes = lines(); const edits = lines(); const requests = []; @@ -107,8 +109,13 @@ export function memoryPorts({ assets = [ASSET], recipes = [], stepAssets = [], d return { before: new Map(taken.filter(ofTurn).map(({ file, before }) => [file, before])), commit }; }, }, - candidates: table({}), + candidates: { ...table({}), referenceOf: candidateReferenceOf }, drafts: table([]), + draftCorrections: { + withSession: async (_id, work) => work(), + mark: async (id, turn, epoch) => { draftCorrections.add(JSON.stringify([id, turn, epoch])); }, + has: async (id, turn, epoch) => draftCorrections.has(JSON.stringify([id, turn, epoch])), + }, reports: { enqueue: async (entries) => queued.push(...entries) }, outbox: { drain: async () => true }, retention: { diff --git a/test/channels/mcp.test.js b/test/channels/mcp.test.js index 9253eb0..b65d5ce 100644 --- a/test/channels/mcp.test.js +++ b/test/channels/mcp.test.js @@ -46,9 +46,10 @@ test('a recalled turn is reported under its turn_id, agent-reported, and its Gen assert.match(recalled, /Apply them where they fit\./); const turnId = turnIdOf(recalled); const reported = textOf(await call(client, 'evolver_report', { turn_id: turnId, outcome: 'success', summary: 'Fixed the retries.', checks: [{ command: 'npm test', exit: 0 }], files: ['lib/redis.js'] })); - const writtenFor = Number(/written_for (\d+)/.exec(reported)?.[1]); - assert.ok(writtenFor > 0); - assert.match(textOf(await call(client, 'evolver_distill', { written_for: writtenFor + 1, gene: GENE })), /newer turn replaced/); + const writtenFor = /written_for "(v1-[A-Za-z0-9_-]{43})"/.exec(reported)?.[1]; + assert.ok(writtenFor); + assert.match(textOf(await call(client, 'evolver_distill', { written_for: 'wrong-reference', gene: GENE })), /newer turn replaced/); + assert.match(textOf(await call(client, 'evolver_distill', { written_for: 123, gene: GENE })), /newer turn replaced/); assert.match(textOf(await call(client, 'evolver_distill', { written_for: writtenFor, gene: GENE })), /Distilled "Stabilise Redis retries" into a draft/); assert.match(textOf(await call(client, 'evolver_drafts')), /Stabilise Redis retries/); const delivered = () => proxy.requests.find((request) => request.path === '/asset/reuse-result')?.body; diff --git a/test/channels/script.test.js b/test/channels/script.test.js index f176e80..9d7c72e 100644 --- a/test/channels/script.test.js +++ b/test/channels/script.test.js @@ -1,9 +1,11 @@ import assert from 'node:assert/strict'; +import { join } from 'node:path'; import { Readable } from 'node:stream'; import { after, before, test } from 'node:test'; import { defineProfile } from '../../lib/agent/profile.js'; import { runCommandScript } from '../../lib/channels/script/runner.js'; +import { fileStoresOf } from '../../lib/runtime/store/file-stores.js'; import { fakeProxy } from '../helpers/fake-proxy.js'; // A host whose own readers know nothing of a script: the channel must not need them. @@ -16,7 +18,7 @@ const profile = defineProfile({ commands: { entry: 'inline-script', sessionOf: (env) => env.TEST_SESSION_ID ?? null, - scriptAnswer: (result, script) => `${result.text}\n[${script.session_id} script ${script.script.endsWith('bin/command.mjs')} nonce ${script.nonce.length}]`, + scriptAnswer: (result, script) => `${result.text}\n[${script.session_id} script ${script.script.endsWith(join('bin', 'command.mjs'))} nonce ${script.nonce.length}]`, }, }); @@ -57,6 +59,19 @@ test('a Gene posted for no candidate says there is nothing to distill', async () assert.match(await run('evolver-distill', { input: '{"summary":"s"}', argv: ['--post', '123'] }), /^Nothing to distill yet/); }); +test('the terminal sends a Gene reference unchanged to the candidate it names', async () => { + const candidate = { at: 1_000, turn: 1, attempts: 0, prompt: 'Repair retries', answer: 'Done', paths: [], commands: [], checks: [] }; + const { candidates } = fileStoresOf({ dir: profile.stateDir(proxy.env), agent: profile.name }); + await candidates.update(() => ({ s1: candidate })); + const input = '{"summary":"Retry safely","strategy":["Retry once"]}'; + const posted = () => proxy.requests.filter(({ path }) => path === '/conversation/distill'); + + assert.match(await run('evolver-distill', { input, argv: ['--post', String(candidate.at)], env: { TEST_SESSION_ID: 's1' } }), /^A newer turn replaced/); + assert.equal(posted().length, 0); + assert.match(await run('evolver-distill', { input, argv: ['--post', candidates.referenceOf('s1', candidate)] }), /not_in_this_test/); + assert.deepEqual(posted().map(({ body }) => body.session_id), ['testscript:s1']); +}); + test('a host whose commands run only from a terminal never waits on a stdin held open', async () => { const terminalOnly = defineProfile({ ...profile, commands: { entry: 'cli-only' } }); const heldOpen = () => new Readable({ read() {} }); diff --git a/test/distill.test.js b/test/distill.test.js index ac67d34..2ca8c4c 100644 --- a/test/distill.test.js +++ b/test/distill.test.js @@ -5,6 +5,7 @@ import { distillPromptOf, distillRequestOf, draftNamed, draftOf, draftsListText, isDistillableTurn, lessonOf, skipReasonOf, submitRequestOf, withDraft, withoutDraft, } from '../lib/distill.js'; +import { isCurrentCandidate, settledCandidate } from '../lib/distill/candidate.js'; const LESSON = { reusable: true, @@ -14,6 +15,15 @@ const LESSON = { signals: ['timeout'], }; +test('settling an earlier turn cannot remove a candidate with the same timestamp', () => { + const earlier = { at: 1_000, turn: 1 }; + const current = { at: 1_000, turn: 2 }; + const pending = { session: current }; + assert.equal(isCurrentCandidate(pending, 'session', earlier), false); + assert.equal(settledCandidate(pending, 'session', earlier, { done: true }), undefined); + assert.equal(isCurrentCandidate(pending, 'session', current), true); +}); + test('only an answered person turn that edited and ended on a passing check is distillable', () => { const turn = { reason: 'answer', agentId: undefined, isPersonTurn: true, changedLines: 3, checks: [{ command: 'npm test', failed: false }] }; assert.equal(isDistillableTurn(turn), true); diff --git a/test/helpers/lock-contenders.js b/test/helpers/lock-contenders.js new file mode 100644 index 0000000..1e412a7 --- /dev/null +++ b/test/helpers/lock-contenders.js @@ -0,0 +1,39 @@ +import { spawn } from 'node:child_process'; +import { rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export async function until(check) { + const deadline = Date.now() + 10_000; + while (!check()) { + if (Date.now() >= deadline) throw new Error('timed out waiting for a lock contender'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +export function lockContenders(t, dir, script, args, cleanupMarkers) { + const children = []; + const results = () => Promise.all(children); + t.after(async () => { + try { + for (const name of cleanupMarkers) writeFileSync(join(dir, name), ''); + } finally { + await results(); + rmSync(dir, { recursive: true, force: true }); + } + }); + return { + run(role) { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, ...args, role], { windowsHide: true }); + let stderr = ''; + const watchdog = setTimeout(() => child.kill('SIGKILL'), 15_000); + child.stderr.on('data', (data) => { stderr += data; }); + child.on('error', (error) => { stderr += String(error); }); + children.push(new Promise((resolve) => child.on('close', (code) => { + clearTimeout(watchdog); + resolve({ code, stderr }); + }))); + return child; + }, + results, + }; +} diff --git a/test/hosts/workbuddy/commands.test.js b/test/hosts/workbuddy/commands.test.js index 56475a5..9a3a930 100644 --- a/test/hosts/workbuddy/commands.test.js +++ b/test/hosts/workbuddy/commands.test.js @@ -46,11 +46,11 @@ test('a command script prints the answer\'s text; a hook with nothing to say ans test('a Gene the headless model could not write is handed to the agent with a command whose heredoc ends on the nonce', () => { const payload = { session_id: "s'1", script: '/plugin/bin/command.mjs', nonce: '0123456789abcdef', state_dir: '/data/evolver' }; - const result = { ok: false, text: 'Distillation model_failed: spawn codebuddy ENOENT.', agentGene: { writtenFor: 1700, prompt: 'Below is the record…' } }; + const result = { ok: false, text: 'Distillation model_failed: spawn codebuddy ENOENT.', agentGene: { writtenFor: 'v1-fixture', prompt: 'Below is the record…' } }; const lines = profile.commands.scriptAnswer(result, payload).split('\n'); assert.equal(lines[0], 'EVOLVER: WRITE THE GENE YOURSELF'); assert.match(lines[1], /\(Distillation model_failed: spawn codebuddy ENOENT\)\.$/); - assert.equal(lines[5], "CODEBUDDY_SESSION_ID='s'\\''1' CODEBUDDY_PLUGIN_DATA='/data/evolver' node '/plugin/bin/command.mjs' workbuddy evolver-distill --post 1700 <<'EVOLVER_GENE_0123456789abcdef'"); + assert.equal(lines[5], "CODEBUDDY_SESSION_ID='s'\\''1' CODEBUDDY_PLUGIN_DATA='/data/evolver' node '/plugin/bin/command.mjs' workbuddy evolver-distill --post 'v1-fixture' <<'EVOLVER_GENE_0123456789abcdef'"); assert.deepEqual(lines.slice(6, 8), ['', 'EVOLVER_GENE_0123456789abcdef']); assert.equal(lines.at(-1), 'Below is the record…'); }); diff --git a/test/runtime/file-lock.test.js b/test/runtime/file-lock.test.js new file mode 100644 index 0000000..0e40c47 --- /dev/null +++ b/test/runtime/file-lock.test.js @@ -0,0 +1,621 @@ +import assert from 'node:assert/strict'; +import childProcess, { spawnSync } from 'node:child_process'; +import { syncBuiltinESMExports } from 'node:module'; +import fs from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, rmdirSync, statSync, unlinkSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; + +import { processStartTime } from '../../lib/runtime/process/owner.js'; +import { withLock } from '../../lib/runtime/store/file-lock.js'; +import { lockContenders, until } from '../helpers/lock-contenders.js'; + +const RUNTIME = new URL('../../lib/runtime/store/file-lock.js', import.meta.url).href; +const STALE = new Date(0); + +function scratch(t, token = null) { + const dir = mkdtempSync(join(tmpdir(), 'evolver-work-lock-')); + const lock = join(dir, 'work.lock'); + if (token !== null) { + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), token); + } + t.after(() => rmSync(dir, { recursive: true, force: true })); + return { dir, lock }; +} + +function holdAdmission(lock, pid = process.pid) { + const admission = `${lock}.admission`; + const owner = join(admission, `owner-${pid}-deadbeef`); + mkdirSync(admission); + writeFileSync(owner, 'fixture owner'); + return { admission, owner, release: () => { unlinkSync(owner); rmdirSync(admission); } }; +} + +test('taking an uncontended lock does not launch a process lookup', async (t) => { + const { lock } = scratch(t); + let launches = 0; + const spawn = childProcess.spawnSync; + childProcess.spawnSync = (...args) => { launches += 1; return spawn(...args); }; + syncBuiltinESMExports(); + try { + assert.equal(await withLock(lock, () => {}), true); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(launches, 0); +}); + +test('stale legacy and dead-owner directories recover, but a fresh legacy lock stays held', async (t) => { + const { lock } = scratch(t, 'legacy token'); + let entered = 0; + assert.equal(await withLock(lock, async () => { entered += 1; }), false); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), `${exited.pid}:deadbeef`); + holdAdmission(lock, exited.pid); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + assert.equal(entered, 2); + assert.equal(existsSync(lock), false); + assert.equal(existsSync(`${lock}.admission`), false); +}); + +test('an old mtime never evicts a live owner, which can refresh and release its lock', async (t) => { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + utimesSync(lock, STALE, STALE); + let entered = false; + assert.equal(await withLock(lock, async () => { entered = true; }), false); + assert.equal(entered, false); + held.refresh(); + assert.ok(statSync(lock).mtimeMs > STALE.getTime()); + }), true); + assert.equal(existsSync(lock), false); + assert.deepEqual(readdirSync(dir), []); +}); + +test('only a stale worker lock with proof of PID reuse can displace a live PID', async (t) => { + const birth = processStartTime(process.pid); + const prefix = `${process.pid}:deadbeef`; + for (const token of [prefix, `${prefix}:${birth}`, `${prefix}:${birth - 1_000}`, `${prefix}:invalid`]) { + const { lock } = scratch(t, token); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, () => assert.fail('owner identity is not disproved')), false); + assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), token); + } + + const { dir, lock } = scratch(t, `${prefix}:${birth - 10_000}`); + assert.equal(await withLock(lock, () => assert.fail('the lease is still fresh')), false); + utimesSync(lock, STALE, STALE); + assert.equal(await withLock(lock, () => { + assert.match(readFileSync(join(lock, 'owner'), 'utf8'), new RegExp(`^${process.pid}:[a-f0-9]+:${birth}$`)); + }), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('a foreign PID probe leaves admission free and cannot remove a different worker token', async (t) => { + const pid = process.ppid; + const birth = Date.now(); + const old = `${pid}:aaaa:${birth - 10_000}`; + const replacement = `${pid}:bbbb:${birth - 10_000}`; + const { lock } = scratch(t, old); + utimesSync(lock, STALE, STALE); + const spawn = childProcess.spawnSync; + let probes = 0; + childProcess.spawnSync = () => { + probes += 1; + assert.equal(existsSync(`${lock}.admission`), false); + writeFileSync(join(lock, 'owner'), replacement); + const stdout = process.platform === 'win32' ? String(birth) : new Date(birth).toUTCString().replace(/ GMT$/, ''); + return { status: 0, stdout }; + }; + syncBuiltinESMExports(); + try { + assert.equal(await withLock(lock, () => assert.fail('the observed token was replaced')), false); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(probes, 1); + assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), replacement); +}); + +test('a transient owner read error never turns a stale live lock into a legacy lock', async (t) => { + for (const code of ['EBUSY', 'EPERM', 'EACCES', 'EIO']) { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async () => { + const owner = join(lock, 'owner'); + const token = readFileSync(owner, 'utf8'); + utimesSync(lock, STALE, STALE); + const read = fs.readFileSync; + let refused = false; + let entered = false; + fs.readFileSync = (target, ...args) => { + if (target === owner && !refused) { + refused = true; + throw Object.assign(new Error('transient owner read refusal'), { code }); + } + return read(target, ...args); + }; + try { + assert.equal(await withLock(lock, async () => { entered = true; }), false, code); + } finally { + fs.readFileSync = read; + } + assert.equal(refused, true); + assert.equal(entered, false, code); + assert.equal(readFileSync(owner, 'utf8'), token, 'the live owner stays intact'); + }), true); + assert.deepEqual(readdirSync(dir), [], 'the original owner still releases normally'); + } +}); + +test('a stale directory left before its owner was written is recoverable', async (t) => { + const { dir, lock } = scratch(t); + mkdirSync(lock); + utimesSync(lock, STALE, STALE); + let entered = false; + assert.equal(await withLock(lock, async () => { entered = true; }), true); + assert.equal(entered, true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release retries a transient owner read failure and really removes its lock', async (t) => { + const { dir, lock } = scratch(t); + const owner = join(lock, 'owner'); + const read = fs.readFileSync; + let refused = false; + try { + assert.equal(await withLock(lock, async () => { + fs.readFileSync = (target, ...args) => { + if (target === owner && !refused) { + refused = true; + throw Object.assign(new Error('transient read refusal'), { code: 'EBUSY' }); + } + return read(target, ...args); + }; + }), true); + } finally { + fs.readFileSync = read; + } + assert.equal(refused, true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('release refuses an unreadable owner explicitly without claiming cleanup', async (t) => { + for (const code of ['EBUSY', 'EPERM', 'EIO']) { + const { lock } = scratch(t); + const owner = join(lock, 'owner'); + const read = fs.readFileSync; + let token; + try { + await assert.rejects(withLock(lock, async () => { + token = read(owner, 'utf8'); + fs.readFileSync = (target, ...args) => { + if (target === owner) throw Object.assign(new Error('persistent read refusal'), { code }); + return read(target, ...args); + }; + }), { code: 'ELOCKED' }); + } finally { + fs.readFileSync = read; + } + assert.equal(readFileSync(owner, 'utf8'), token, code); + assert.equal(await withLock(lock, () => {}), true, 'known abandoned work is recoverable after access returns'); + } +}); + +test('a stale foreign admission is probed once within the release deadline', async (t) => { + const { lock } = scratch(t); + const spawn = childProcess.spawnSync; + const timeouts = []; + try { + await assert.rejects(withLock(lock, () => { + const guard = holdAdmission(lock, process.ppid); + unlinkSync(guard.owner); + writeFileSync(join(guard.admission, `owner-${process.ppid}-a1-${Date.now() - 30_000}`), ''); + utimesSync(guard.admission, STALE, STALE); + childProcess.spawnSync = (_command, _args, { timeout }) => { + timeouts.push(timeout); + return { status: null, stdout: '' }; + }; + syncBuiltinESMExports(); + }), { code: 'ELOCKED' }); + } finally { + childProcess.spawnSync = spawn; + syncBuiltinESMExports(); + } + assert.equal(timeouts.length, 1); + assert.ok(timeouts[0] > 0 && timeouts[0] <= 1_000); + assert.equal(existsSync(lock), true, 'an inconclusive identity probe does not reclaim a live guard'); +}); + +test('a release failure preserves the original work error', async (t) => { + const { lock } = scratch(t); + const read = fs.readFileSync; + const workError = new Error('work failed'); + try { + await assert.rejects(withLock(lock, async () => { + fs.readFileSync = (file, ...args) => { + if (String(file) === join(lock, 'owner')) throw Object.assign(new Error('unreadable owner'), { code: 'EIO' }); + return read(file, ...args); + }; + throw workError; + }), (error) => error === workError); + } finally { + fs.readFileSync = read; + } +}); + +test('refresh leaves an unreadable live lock untouched without crashing its timer caller', async (t) => { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + const owner = join(lock, 'owner'); + utimesSync(lock, STALE, STALE); + const read = fs.readFileSync; + fs.readFileSync = (target, ...args) => { + if (target === owner) throw Object.assign(new Error('unreadable owner'), { code: 'EPERM' }); + return read(target, ...args); + }; + try { + assert.doesNotThrow(() => held.refresh()); + assert.equal(statSync(lock).mtimeMs, STALE.getTime()); + } finally { + fs.readFileSync = read; + } + }), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('refresh tolerates file access refusals and the next refresh and release still work', async (t) => { + for (const method of ['utimesSync', 'mkdirSync', 'readFileSync']) { + const { dir, lock } = scratch(t); + assert.equal(await withLock(lock, async (held) => { + utimesSync(lock, STALE, STALE); + const access = fs[method]; + let refused = false; + fs[method] = (target, ...args) => { + const matches = method === 'mkdirSync' ? String(target).startsWith(`${lock}.admission.`) + : target === (method === 'readFileSync' ? join(lock, 'owner') : lock); + if (matches && !refused) { + refused = true; + throw Object.assign(new Error('transient access refusal'), { code: 'EPERM' }); + } + return access(target, ...args); + }; + try { + assert.doesNotThrow(() => held.refresh(), method); + if (method !== 'readFileSync') assert.equal(statSync(lock).mtimeMs, STALE.getTime()); + } finally { + fs[method] = access; + } + assert.equal(refused, true, method); + held.refresh(); + assert.ok(statSync(lock).mtimeMs > STALE.getTime()); + }), true); + assert.deepEqual(readdirSync(dir), []); + } +}); + +test('release does not delete a missing or different owner', async (t) => { + for (const replacement of [null, `${process.pid}:another-owner`]) { + const { lock } = scratch(t); + assert.equal(await withLock(lock, async () => { + if (replacement === null) unlinkSync(join(lock, 'owner')); + else writeFileSync(join(lock, 'owner'), replacement); + }), true); + assert.equal(existsSync(lock), true); + assert.equal(existsSync(join(lock, 'owner')), replacement !== null); + if (replacement !== null) assert.equal(readFileSync(join(lock, 'owner'), 'utf8'), replacement); + } +}); + +test('release waits asynchronously for transient admission contention', async (t) => { + const { lock } = scratch(t); + let guard; + let finished = false; + let refreshedAt; + const running = withLock(lock, async (held) => { + guard = holdAdmission(lock); + utimesSync(lock, STALE, STALE); + held.refresh(); + refreshedAt = statSync(lock).mtimeMs; + }).then(() => { finished = true; }); + try { + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(refreshedAt, STALE.getTime(), 'refresh also respects admission'); + assert.equal(finished, false); + assert.equal(existsSync(lock), true); + } finally { + guard.release(); + await running; + } + assert.equal(existsSync(lock), false); +}); + +test('release refuses bounded admission contention without deleting the held directory', async (t) => { + const { lock } = scratch(t); + let guard; + const started = Date.now(); + await assert.rejects(withLock(lock, async () => { + guard = holdAdmission(lock); + }), { code: 'ELOCKED' }); + assert.ok(Date.now() - started < 5_000, 'cleanup cannot wait forever'); + assert.equal(existsSync(lock), true); + assert.equal(readFileSync(guard.owner, 'utf8'), 'fixture owner'); +}); + +test('an admission cleanup failure is surfaced instead of silently orphaning a live guard', async (t) => { + const { lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + return unlink(target); + }; + let entered = false; + try { + await assert.rejects(withLock(lock, async () => { entered = true; }), { code: 'ELOCKED' }); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(entered, false); + assert.equal(existsSync(lock), false, 'the failed acquire must not leave a work lock'); + assert.ok(readdirSync(admission).some((name) => name.startsWith(`owner-${process.pid}-`))); + assert.equal(await withLock(lock, async () => { entered = true; }), true); + assert.equal(entered, true); +}); + +test('another process reclaims a failed admission release while the original owner is alive', async (t) => { + const { dir, lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + try { + assert.equal(await withLock(lock, () => { + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('fixture cleanup refusal'), { code: 'EPERM' }); + return unlink(target); + }; + }), true); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(existsSync(lock), false); + assert.ok(readdirSync(admission).some((name) => name.startsWith(`owner-${process.pid}-`))); + + const other = spawnSync(process.execPath, ['--input-type=module', '-e', ` + import assert from 'node:assert/strict'; + const [runtime, lock, owner] = process.argv.slice(1); + const { withLock } = await import(runtime); + process.kill(Number(owner), 0); + assert.equal(await withLock(lock, () => {}), true, 'the stranded admission can be reclaimed'); + `, RUNTIME, lock, String(process.pid)], { encoding: 'utf8', timeout: 10_000, windowsHide: true }); + assert.ifError(other.error); + assert.equal(other.status, 0, other.stderr); + assert.deepEqual(readdirSync(dir), []); +}); + +test('admission recovery resumes after the abandoned owner has already been removed', async (t) => { + const { dir, lock } = scratch(t); + const guard = holdAdmission(lock); + const marker = join(guard.admission, `abandoned-owner-${process.pid}-abcd`); + writeFileSync(marker, ''); + assert.equal(await withLock(lock, () => assert.fail('a different owner is still active')), false); + assert.equal(existsSync(guard.owner), true); + + unlinkSync(guard.owner); + assert.equal(await withLock(lock, () => {}), true); + assert.deepEqual(readdirSync(dir), []); +}); + +test('a transient admission cleanup refusal retries and leaves later work unblocked', async (t) => { + const { dir, lock } = scratch(t); + const unlink = fs.unlinkSync; + let refused = false; + fs.unlinkSync = (target) => { + if (dirname(target) === `${lock}.admission` && !refused) { + refused = true; + throw Object.assign(new Error('temporarily refused by fixture'), { code: 'EPERM' }); + } + return unlink(target); + }; + let entered = 0; + try { + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + } finally { + fs.unlinkSync = unlink; + } + assert.equal(refused, true); + assert.equal(entered, 1); + assert.deepEqual(readdirSync(dir), []); + assert.equal(await withLock(lock, async () => { entered += 1; }), true); + assert.equal(entered, 2); + assert.deepEqual(readdirSync(dir), []); +}); + +test('an old admission stays held by the same process, but a reused PID can be recovered', async (t) => { + const { lock } = scratch(t); + const admission = `${lock}.admission`; + const unlink = fs.unlinkSync; + fs.unlinkSync = (target) => { + if (dirname(target) === admission) throw Object.assign(new Error('refused by fixture'), { code: 'EPERM' }); + return unlink(target); + }; + try { + await assert.rejects(withLock(lock, async () => assert.fail()), { code: 'ELOCKED' }); + } finally { + fs.unlinkSync = unlink; + } + const original = readdirSync(admission).find((name) => name.startsWith(`owner-${process.pid}-`)); + const birth = original.split('-')[3]; + assert.match(birth ?? '', /^\d{13}$/); + const replaceOwner = (name) => { + for (const name of readdirSync(admission)) unlinkSync(join(admission, name)); + writeFileSync(join(admission, name), ''); + utimesSync(admission, STALE, STALE); + }; + replaceOwner(`owner-${process.pid}-a1-${birth}`); + assert.equal(await withLock(lock, async () => assert.fail('same process still owns the guard')), false); + const differentBirth = Number(birth) - 10_000; + replaceOwner(`owner-${process.pid}-a2-${differentBirth}`); + assert.equal(await withLock(lock, async () => {}), true); +}); + +test('busy and unrecognised admission guards stay intact and leave no staging directories', async (t) => { + const { dir, lock } = scratch(t); + const guard = holdAdmission(lock); + assert.equal(await withLock(lock, async () => assert.fail('a live guard is held')), false); + assert.deepEqual(readdirSync(dir), ['work.lock.admission']); + writeFileSync(join(guard.admission, 'unexpected'), 'keep'); + assert.equal(await withLock(lock, async () => assert.fail('multiple owners are unknown')), false); + assert.equal(readdirSync(guard.admission).length, 2); + unlinkSync(guard.owner); + assert.equal(await withLock(lock, async () => assert.fail('an unknown owner is held')), false); + assert.equal(readFileSync(join(guard.admission, 'unexpected'), 'utf8'), 'keep'); + assert.deepEqual(readdirSync(dir), ['work.lock.admission']); +}); + +test('two contenders cannot replace the fresh lock while another reclaims a stale directory', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-work-lock-')); + const lock = join(dir, 'work.lock'); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), 'legacy token'); + utimesSync(lock, STALE, STALE); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { withLock } from ${JSON.stringify(RUNTIME)}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const wait = async (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + }; + if (role === 'A') { + const stat = fs.statSync; + let paused = false; + fs.statSync = (file, ...args) => { + const result = stat(file, ...args); + if (file === lock && !paused) { + paused = true; + fs.writeFileSync(marker('stale-read'), ''); + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker('resume'))) { + if (Date.now() > deadline) throw new Error('timed out waiting to resume'); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + } + return result; + }; + } + const result = await withLock(lock, async () => { + fs.writeFileSync(marker(role + '-entered'), ''); + await wait('release'); + }); + fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); + `; + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['resume', 'release']); + run('A'); + await until(() => existsSync(marker('stale-read'))); + run('B'); + run('C'); + await until(() => ['B', 'C'].every((role) => existsSync(marker(role + '-entered')) || existsSync(marker(role + '-result')))); + writeFileSync(marker('resume'), ''); + await until(() => existsSync(marker('A-entered'))); + const entered = ['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))); + writeFileSync(marker('release'), ''); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); + assert.deepEqual(entered, ['A'], 'only the process reclaiming the inspected stale directory may enter'); + assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [true, false, false]); +}); + +test('a delayed dead-admission reclaimer cannot unlink the fresh owner another contender published', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'evolver-admission-race-')); + const lock = join(dir, 'work.lock'); + mkdirSync(lock); + writeFileSync(join(lock, 'owner'), 'legacy token'); + utimesSync(lock, STALE, STALE); + const exited = spawnSync(process.execPath, ['-e', '']); + assert.equal(exited.status, 0); + holdAdmission(lock, exited.pid); + const marker = (name) => join(dir, name); + const script = ` + import fs from 'node:fs'; + import path from 'node:path'; + import { withLock } from ${JSON.stringify(RUNTIME)}; + const [dir, lock, role] = process.argv.slice(1); + const marker = (name) => path.join(dir, name); + const waitSync = (name) => { + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker(name))) { + if (Date.now() > deadline) throw new Error('timed out waiting for ' + name); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + }; + if (role === 'A') { + const list = fs.readdirSync; + let paused = false; + fs.readdirSync = (file, ...args) => { + const result = list(file, ...args); + if (file === lock + '.admission' && !paused) { + paused = true; + fs.writeFileSync(marker('dead-inspected'), ''); + waitSync('A-resume'); + } + return result; + }; + } + if (role === 'B') { + const rename = fs.renameSync; + let paused = false; + fs.renameSync = (source, target) => { + const result = rename(source, target); + if (target === lock + '.admission' && !paused) { + paused = true; + fs.writeFileSync(marker('fresh-held'), ''); + waitSync('B-resume'); + } + return result; + }; + } + const result = await withLock(lock, async () => { + fs.writeFileSync(marker(role + '-entered'), ''); + const deadline = Date.now() + 10000; + while (!fs.existsSync(marker('release'))) { + if (Date.now() > deadline) throw new Error('timed out waiting to release'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + }); + fs.writeFileSync(marker(role + '-result'), JSON.stringify(result)); + `; + const { run, results } = lockContenders(t, dir, script, [dir, lock], ['A-resume', 'B-resume', 'release']); + run('A'); + await until(() => existsSync(marker('dead-inspected'))); + const fresh = run('B'); + await until(() => existsSync(marker('fresh-held'))); + run('C'); + await until(() => existsSync(marker('C-result')) || existsSync(marker('C-entered'))); + assert.equal(existsSync(marker('C-entered')), false, 'publishing a fresh guard never exposes an empty directory'); + writeFileSync(marker('A-resume'), ''); + await until(() => existsSync(marker('A-result')) || existsSync(marker('A-entered'))); + assert.equal(existsSync(marker('A-entered')), false, 'the delayed reclaimer cannot displace the fresh guard'); + assert.equal(readdirSync(`${lock}.admission`).length, 1); + assert.match(readdirSync(`${lock}.admission`)[0], new RegExp(`^owner-${fresh.pid}-`)); + writeFileSync(marker('B-resume'), ''); + await until(() => existsSync(marker('B-entered'))); + assert.deepEqual(['A', 'B', 'C'].filter((role) => existsSync(marker(role + '-entered'))), ['B']); + writeFileSync(marker('release'), ''); + const outcomes = await results(); + assert.ok(outcomes.every(({ code }) => code === 0), JSON.stringify(outcomes)); + assert.deepEqual(['A', 'B', 'C'].map((role) => JSON.parse(readFileSync(marker(role + '-result'), 'utf8'))), [false, true, false]); +}); diff --git a/test/runtime/file-stores.test.js b/test/runtime/file-stores.test.js index 7f52193..d50abb2 100644 --- a/test/runtime/file-stores.test.js +++ b/test/runtime/file-stores.test.js @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; +import { once } from 'node:events'; import { mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -90,3 +91,49 @@ test('a session file outlives the outbox: a session resumed within a month still stores.prune(Date.now() + 31 * day); assert.deepEqual(await stores.sessions.read('s1'), {}); }); + +test('a recorded draft correction survives reopening and pruning the source session', async () => { + const { dir, stores: first } = stores(); + await first.sessions.update('s1', () => ({ turn: 2, sessionEpoch: 10 })); + await first.draftCorrections.mark('s1', 2, 10); + await first.draftCorrections.mark('s1', 2, 10); + first.prune(Date.now() + 31 * 24 * 60 * 60 * 1000); + + const reopened = fileStoresOf({ dir, agent: 'cursor' }); + assert.deepEqual(await reopened.sessions.read('s1'), {}); + assert.equal(await reopened.draftCorrections.has('s1', 2, 10), true); + assert.equal(await reopened.draftCorrections.has('s1', 2, 20), false); + assert.equal(await reopened.draftCorrections.has('s1', 1, 10), false); + assert.equal(await reopened.draftCorrections.has('s2', 2, 10), false); +}); + +test('a publish gate in another process waits for the source session correction', async (t) => { + const { dir, stores: parent } = stores(); + const module = new URL('../../lib/runtime/store/file-stores.js', import.meta.url).href; + const script = `import { once } from 'node:events'; +import { fileStoresOf } from ${JSON.stringify(module)}; +const { draftCorrections } = fileStoresOf({ dir: ${JSON.stringify(dir)}, agent: 'cursor' }); +await draftCorrections.withSession('s1', async () => { + const release = once(process, 'message'); + process.send('locked'); + await release; + await draftCorrections.mark('s1', 2, 10); +}); +process.disconnect();`; + const child = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }); + t.after(() => child.kill()); + const exited = once(child, 'exit'); + const locked = once(child, 'message'); + const [ready] = await Promise.race([locked, exited]); + assert.equal(ready, 'locked'); + + let checked = false; + const publication = parent.draftCorrections.withSession('s1', async () => { + checked = true; + return parent.draftCorrections.has('s1', 2, 10); + }); + assert.equal(checked, false); + child.send('release'); + assert.equal(await publication, true); + assert.deepEqual(await exited, [0, null]); +});